1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 11.605793952941895,
|
|
"kl": 38.56059265136719,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37154164.36363637,
|
|
"logits/rejected": -37756561.341935486,
|
|
"logps/chosen": -471.05530303030304,
|
|
"logps/rejected": -359.22752016129033,
|
|
"loss": 0.4618,
|
|
"loss/base_kto": 3.694525957107544,
|
|
"metrics/advantage_var": 740.17822265625,
|
|
"rewards/chosen": 0.8659122351444128,
|
|
"rewards/margins": 0.3264073287054008,
|
|
"rewards/rejected": 0.539504906439012,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 10.088749885559082,
|
|
"kl": 30.996259689331055,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -38410347.78947368,
|
|
"logits/rejected": -39412351.375609756,
|
|
"logps/chosen": -490.34473684210525,
|
|
"logps/rejected": -391.3777947154472,
|
|
"loss": 0.4519,
|
|
"loss/base_kto": 3.6148488521575928,
|
|
"metrics/advantage_var": 912.3471069335938,
|
|
"rewards/chosen": 0.8235496578359962,
|
|
"rewards/margins": 0.35158057437563034,
|
|
"rewards/rejected": 0.47196908346036587,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 10.01998233795166,
|
|
"kl": 30.216571807861328,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35869769.86229508,
|
|
"logits/rejected": -37921590.256716415,
|
|
"logps/chosen": -471.6993852459016,
|
|
"logps/rejected": -353.15212220149255,
|
|
"loss": 0.4291,
|
|
"loss/base_kto": 3.4325599670410156,
|
|
"metrics/advantage_var": 872.025390625,
|
|
"rewards/chosen": 0.8260520059554304,
|
|
"rewards/margins": 0.6611124444247051,
|
|
"rewards/rejected": 0.16493956153072528,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 8.345759391784668,
|
|
"kl": 5.388225078582764,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36489209.467304625,
|
|
"logits/rejected": -38324117.36600307,
|
|
"logps/chosen": -461.50956937799043,
|
|
"logps/rejected": -364.3822741194487,
|
|
"loss": 0.4193,
|
|
"loss/base_kto": 3.354257345199585,
|
|
"metrics/advantage_var": 1499.1781005859375,
|
|
"rewards/chosen": 0.2019310910165595,
|
|
"rewards/margins": 0.8813837972761307,
|
|
"rewards/rejected": -0.6794527062595712,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 8.309015274047852,
|
|
"kl": 13.195367813110352,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -38952421.558685444,
|
|
"logits/rejected": -39376905.5850234,
|
|
"logps/chosen": -480.94776995305165,
|
|
"logps/rejected": -377.9949297971919,
|
|
"loss": 0.4222,
|
|
"loss/base_kto": 3.3775384426116943,
|
|
"metrics/advantage_var": 2635.339599609375,
|
|
"rewards/chosen": 0.5415884862669943,
|
|
"rewards/margins": 0.8321847921260646,
|
|
"rewards/rejected": -0.2905963058590703,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 10.182024002075195,
|
|
"kl": 18.867149353027344,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -39828811.294117644,
|
|
"logits/rejected": -40465155.413333334,
|
|
"logps/chosen": -488.21875,
|
|
"logps/rejected": -362.84151041666667,
|
|
"loss": 0.3905,
|
|
"loss/base_kto": 3.1243832111358643,
|
|
"metrics/advantage_var": 2918.470458984375,
|
|
"rewards/chosen": 1.039443790211397,
|
|
"rewards/margins": 1.222436903411267,
|
|
"rewards/rejected": -0.1829931131998698,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 8.444199562072754,
|
|
"kl": 30.96662712097168,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -39468761.98019802,
|
|
"logits/rejected": -39347196.961424336,
|
|
"logps/chosen": -481.0873556105611,
|
|
"logps/rejected": -369.1377271884273,
|
|
"loss": 0.4128,
|
|
"loss/base_kto": 3.3024284839630127,
|
|
"metrics/advantage_var": 3886.193115234375,
|
|
"rewards/chosen": 0.9114466500360974,
|
|
"rewards/margins": 1.037914289721482,
|
|
"rewards/rejected": -0.1264676396853846,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 7.57260799407959,
|
|
"kl": 19.63751983642578,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -39679900.75770925,
|
|
"logits/rejected": -40483971.6327212,
|
|
"logps/chosen": -465.3614629221733,
|
|
"logps/rejected": -387.42336185308847,
|
|
"loss": 0.3935,
|
|
"loss/base_kto": 3.1482582092285156,
|
|
"metrics/advantage_var": 4115.23583984375,
|
|
"rewards/chosen": 1.1309302689920613,
|
|
"rewards/margins": 1.4825149821207133,
|
|
"rewards/rejected": -0.3515847131286519,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 7.263091564178467,
|
|
"kl": 27.852466583251953,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -38640721.2192846,
|
|
"logits/rejected": -39732961.85871272,
|
|
"logps/chosen": -457.130783437014,
|
|
"logps/rejected": -364.8397272370487,
|
|
"loss": 0.3727,
|
|
"loss/base_kto": 2.9812400341033936,
|
|
"metrics/advantage_var": 4626.76513671875,
|
|
"rewards/chosen": 1.418452664815805,
|
|
"rewards/margins": 1.6763056952256852,
|
|
"rewards/rejected": -0.2578530304098803,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 9.630581855773926,
|
|
"kl": 23.25460433959961,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -39927047.7824,
|
|
"logits/rejected": -40623030.5221374,
|
|
"logps/chosen": -479.1677,
|
|
"logps/rejected": -378.52118320610685,
|
|
"loss": 0.3835,
|
|
"loss/base_kto": 3.0679073333740234,
|
|
"metrics/advantage_var": 4547.76513671875,
|
|
"rewards/chosen": 1.1745533203125,
|
|
"rewards/margins": 1.6070852449606394,
|
|
"rewards/rejected": -0.43253192464813933,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 9.11620044708252,
|
|
"kl": 10.112515449523926,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -41367302.24390244,
|
|
"logits/rejected": -39530563.75338346,
|
|
"logps/chosen": -471.4418699186992,
|
|
"logps/rejected": -374.84586466165416,
|
|
"loss": 0.3788,
|
|
"loss/base_kto": 3.0303921699523926,
|
|
"metrics/advantage_var": 4945.99169921875,
|
|
"rewards/chosen": 0.36794150747903964,
|
|
"rewards/margins": 1.698686062959538,
|
|
"rewards/rejected": -1.3307445554804982,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 7.173008918762207,
|
|
"kl": 11.464367866516113,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -40763076.18691589,
|
|
"logits/rejected": -41477347.91222571,
|
|
"logps/chosen": -462.00783683800626,
|
|
"logps/rejected": -361.60555446708463,
|
|
"loss": 0.3659,
|
|
"loss/base_kto": 2.9271907806396484,
|
|
"metrics/advantage_var": 5027.3837890625,
|
|
"rewards/chosen": 0.8695794696748442,
|
|
"rewards/margins": 1.992290364316302,
|
|
"rewards/rejected": -1.1227108946414577,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 8.966463088989258,
|
|
"kl": 21.945667266845703,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -42089467.266563945,
|
|
"logits/rejected": -42016605.717908084,
|
|
"logps/chosen": -477.8919491525424,
|
|
"logps/rejected": -397.86717511885894,
|
|
"loss": 0.3576,
|
|
"loss/base_kto": 2.860548257827759,
|
|
"metrics/advantage_var": 5913.638671875,
|
|
"rewards/chosen": 1.2006039376986228,
|
|
"rewards/margins": 2.0745871364902233,
|
|
"rewards/rejected": -0.8739831987916006,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 7.794249534606934,
|
|
"kl": 19.930723190307617,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -41426398.53594771,
|
|
"logits/rejected": -41161758.65868264,
|
|
"logps/chosen": -449.8214358660131,
|
|
"logps/rejected": -375.6300056137725,
|
|
"loss": 0.3565,
|
|
"loss/base_kto": 2.852098226547241,
|
|
"metrics/advantage_var": 6078.57275390625,
|
|
"rewards/chosen": 0.9442857729843239,
|
|
"rewards/margins": 2.0589253293208696,
|
|
"rewards/rejected": -1.1146395563365457,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 7.703754901885986,
|
|
"kl": 38.4437370300293,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -41185273.68875193,
|
|
"logits/rejected": -40902857.22979398,
|
|
"logps/chosen": -444.9513193374422,
|
|
"logps/rejected": -354.3673236925515,
|
|
"loss": 0.3444,
|
|
"loss/base_kto": 2.75480318069458,
|
|
"metrics/advantage_var": 5763.5478515625,
|
|
"rewards/chosen": 1.9258929753707628,
|
|
"rewards/margins": 2.2856331464858703,
|
|
"rewards/rejected": -0.3597401711151075,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 9.472077369689941,
|
|
"kl": 20.64752197265625,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -40008365.798165135,
|
|
"logits/rejected": -42599021.59744409,
|
|
"logps/chosen": -482.6004873853211,
|
|
"logps/rejected": -369.16458666134184,
|
|
"loss": 0.3551,
|
|
"loss/base_kto": 2.8407864570617676,
|
|
"metrics/advantage_var": 6040.44482421875,
|
|
"rewards/chosen": 1.5773067182721712,
|
|
"rewards/margins": 2.260341206840462,
|
|
"rewards/rejected": -0.6830344885682907,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 10.506298065185547,
|
|
"kl": 13.261344909667969,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -42238106.0815047,
|
|
"logits/rejected": -42170939.015576325,
|
|
"logps/chosen": -505.4585619122257,
|
|
"logps/rejected": -399.87091121495325,
|
|
"loss": 0.3522,
|
|
"loss/base_kto": 2.8176186084747314,
|
|
"metrics/advantage_var": 6590.22998046875,
|
|
"rewards/chosen": 1.1125284511841202,
|
|
"rewards/margins": 2.436326514943612,
|
|
"rewards/rejected": -1.3237980637594917,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 10.129120826721191,
|
|
"kl": 21.043310165405273,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -42286177.60258482,
|
|
"logits/rejected": -43014422.85022693,
|
|
"logps/chosen": -467.0582592891761,
|
|
"logps/rejected": -390.2524111195159,
|
|
"loss": 0.3483,
|
|
"loss/base_kto": 2.7863080501556396,
|
|
"metrics/advantage_var": 7763.60009765625,
|
|
"rewards/chosen": 1.3456361736735158,
|
|
"rewards/margins": 2.490973715631723,
|
|
"rewards/rejected": -1.1453375419582073,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 8.79063606262207,
|
|
"kl": 13.356651306152344,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -44623424.59076923,
|
|
"logits/rejected": -43544953.092063494,
|
|
"logps/chosen": -497.1355288461538,
|
|
"logps/rejected": -378.4875248015873,
|
|
"loss": 0.3515,
|
|
"loss/base_kto": 2.812063217163086,
|
|
"metrics/advantage_var": 8117.02294921875,
|
|
"rewards/chosen": 1.206925330528846,
|
|
"rewards/margins": 2.5688414275340543,
|
|
"rewards/rejected": -1.3619160970052084,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 6.636849403381348,
|
|
"kl": 31.743444442749023,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -42751870.06646526,
|
|
"logits/rejected": -43807303.24919094,
|
|
"logps/chosen": -484.7029833836858,
|
|
"logps/rejected": -392.5766332928803,
|
|
"loss": 0.3452,
|
|
"loss/base_kto": 2.7615013122558594,
|
|
"metrics/advantage_var": 7189.68896484375,
|
|
"rewards/chosen": 2.0912890477483006,
|
|
"rewards/margins": 2.4713890881223652,
|
|
"rewards/rejected": -0.38010004037406453,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 8.101085662841797,
|
|
"kl": 31.998029708862305,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -44528687.362041466,
|
|
"logits/rejected": -43389398.444104135,
|
|
"logps/chosen": -480.5172448165869,
|
|
"logps/rejected": -392.3862940275651,
|
|
"loss": 0.3334,
|
|
"loss/base_kto": 2.666898488998413,
|
|
"metrics/advantage_var": 7660.3828125,
|
|
"rewards/chosen": 1.7667409601774322,
|
|
"rewards/margins": 2.757692241224044,
|
|
"rewards/rejected": -0.9909512810466118,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 11.130863189697266,
|
|
"kl": 12.283018112182617,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -44233614.58227848,
|
|
"logits/rejected": -44120013.43209877,
|
|
"logps/chosen": -465.97962816455697,
|
|
"logps/rejected": -396.02430555555554,
|
|
"loss": 0.3415,
|
|
"loss/base_kto": 2.7321603298187256,
|
|
"metrics/advantage_var": 8213.9638671875,
|
|
"rewards/chosen": 1.2179762441900712,
|
|
"rewards/margins": 2.7348437994080497,
|
|
"rewards/rejected": -1.5168675552179784,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 10.0189208984375,
|
|
"kl": 22.358829498291016,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -45440364.307692304,
|
|
"logits/rejected": -45431271.02439024,
|
|
"logps/chosen": -463.77383814102564,
|
|
"logps/rejected": -389.2611471036585,
|
|
"loss": 0.329,
|
|
"loss/base_kto": 2.632397174835205,
|
|
"metrics/advantage_var": 8176.68212890625,
|
|
"rewards/chosen": 1.6049115107609675,
|
|
"rewards/margins": 2.86886492902745,
|
|
"rewards/rejected": -1.2639534182664824,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 9.430086135864258,
|
|
"kl": 11.885652542114258,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -45028388.218649514,
|
|
"logits/rejected": -45278267.136778116,
|
|
"logps/chosen": -477.4973874598071,
|
|
"logps/rejected": -379.5108045212766,
|
|
"loss": 0.3388,
|
|
"loss/base_kto": 2.7101292610168457,
|
|
"metrics/advantage_var": 9187.2099609375,
|
|
"rewards/chosen": 1.0469594875715937,
|
|
"rewards/margins": 2.83426569631257,
|
|
"rewards/rejected": -1.7873062087409763,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 9.097721099853516,
|
|
"kl": 13.10331916809082,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -45285132.962025315,
|
|
"logits/rejected": -45354138.86419753,
|
|
"logps/chosen": -491.37193433544303,
|
|
"logps/rejected": -402.21766493055554,
|
|
"loss": 0.324,
|
|
"loss/base_kto": 2.592097043991089,
|
|
"metrics/advantage_var": 9418.1435546875,
|
|
"rewards/chosen": 1.265615921986254,
|
|
"rewards/margins": 3.227010432137681,
|
|
"rewards/rejected": -1.9613945101514274,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.278778076171875,
|
|
"kl": 24.34718894958496,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -45963127.23287671,
|
|
"logits/rejected": -46308840.45977011,
|
|
"logps/chosen": -502.2434717465753,
|
|
"logps/rejected": -399.21125179597703,
|
|
"loss": 0.327,
|
|
"loss/base_kto": 2.616192579269409,
|
|
"metrics/advantage_var": 9953.5107421875,
|
|
"rewards/chosen": 1.5700516374143836,
|
|
"rewards/margins": 3.2479978815774584,
|
|
"rewards/rejected": -1.6779462441630748,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 7.010895252227783,
|
|
"kl": 13.371597290039062,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -43210697.06110283,
|
|
"logits/rejected": -44957669.09688013,
|
|
"logps/chosen": -475.28055141579733,
|
|
"logps/rejected": -381.8127052545156,
|
|
"loss": 0.3057,
|
|
"loss/base_kto": 2.4459750652313232,
|
|
"metrics/advantage_var": 10097.4755859375,
|
|
"rewards/chosen": 1.9163489078800298,
|
|
"rewards/margins": 3.5759380019699822,
|
|
"rewards/rejected": -1.6595890940899527,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 7.648643970489502,
|
|
"kl": 42.0706787109375,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -45007123.15114504,
|
|
"logits/rejected": -44185783.5008,
|
|
"logps/chosen": -448.98234732824426,
|
|
"logps/rejected": -368.3568,
|
|
"loss": 0.3176,
|
|
"loss/base_kto": 2.5408151149749756,
|
|
"metrics/advantage_var": 9049.8310546875,
|
|
"rewards/chosen": 2.615083045085878,
|
|
"rewards/margins": 3.058985632976503,
|
|
"rewards/rejected": -0.443902587890625,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 7.628378391265869,
|
|
"kl": 39.29732894897461,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -45669481.82370821,
|
|
"logits/rejected": -44295375.4340836,
|
|
"logps/chosen": -457.95146276595744,
|
|
"logps/rejected": -362.8493770096463,
|
|
"loss": 0.3251,
|
|
"loss/base_kto": 2.600756883621216,
|
|
"metrics/advantage_var": 9290.9599609375,
|
|
"rewards/chosen": 2.090501222929331,
|
|
"rewards/margins": 2.8993852333669317,
|
|
"rewards/rejected": -0.8088840104376005,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 7.913888454437256,
|
|
"kl": 23.4282283782959,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -44830290.13493253,
|
|
"logits/rejected": -43444036.90701468,
|
|
"logps/chosen": -460.875,
|
|
"logps/rejected": -383.3435715742251,
|
|
"loss": 0.3012,
|
|
"loss/base_kto": 2.409473180770874,
|
|
"metrics/advantage_var": 11249.1708984375,
|
|
"rewards/chosen": 2.4155743783381745,
|
|
"rewards/margins": 3.6628631429308536,
|
|
"rewards/rejected": -1.2472887645926793,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 6.445258140563965,
|
|
"kl": 29.516122817993164,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -46369467.09172932,
|
|
"logits/rejected": -47186552.71544716,
|
|
"logps/chosen": -422.66809210526316,
|
|
"logps/rejected": -364.4777693089431,
|
|
"loss": 0.3262,
|
|
"loss/base_kto": 2.609204053878784,
|
|
"metrics/advantage_var": 10683.3701171875,
|
|
"rewards/chosen": 1.9281864940671993,
|
|
"rewards/margins": 3.128518763503175,
|
|
"rewards/rejected": -1.2003322694359757,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 9.627001762390137,
|
|
"kl": 23.947446823120117,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -46729614.133748055,
|
|
"logits/rejected": -47353798.12872841,
|
|
"logps/chosen": -487.88039463452566,
|
|
"logps/rejected": -393.484693877551,
|
|
"loss": 0.3246,
|
|
"loss/base_kto": 2.596665382385254,
|
|
"metrics/advantage_var": 11245.66796875,
|
|
"rewards/chosen": 2.035088475347492,
|
|
"rewards/margins": 3.350969458900769,
|
|
"rewards/rejected": -1.315880983553277,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 5.452735900878906,
|
|
"kl": 20.10293197631836,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -45706389.36541353,
|
|
"logits/rejected": -44925420.85203252,
|
|
"logps/chosen": -478.0436090225564,
|
|
"logps/rejected": -420.4835619918699,
|
|
"loss": 0.3199,
|
|
"loss/base_kto": 2.5589563846588135,
|
|
"metrics/advantage_var": 12959.5068359375,
|
|
"rewards/chosen": 2.0530989338580827,
|
|
"rewards/margins": 3.7097156410163654,
|
|
"rewards/rejected": -1.6566167071582825,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 11.135366439819336,
|
|
"kl": 31.90445327758789,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -45910517.47945205,
|
|
"logits/rejected": -46397469.42528736,
|
|
"logps/chosen": -491.58063998287673,
|
|
"logps/rejected": -392.5621408045977,
|
|
"loss": 0.3016,
|
|
"loss/base_kto": 2.4128031730651855,
|
|
"metrics/advantage_var": 11997.140625,
|
|
"rewards/chosen": 2.2658833542915238,
|
|
"rewards/margins": 3.9077030999362794,
|
|
"rewards/rejected": -1.6418197456447559,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 6.5614848136901855,
|
|
"kl": 30.111474990844727,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -45139300.37004405,
|
|
"logits/rejected": -45934034.697829716,
|
|
"logps/chosen": -459.43658223201174,
|
|
"logps/rejected": -384.7371139398998,
|
|
"loss": 0.3333,
|
|
"loss/base_kto": 2.6663873195648193,
|
|
"metrics/advantage_var": 11191.8955078125,
|
|
"rewards/chosen": 2.1196790966868577,
|
|
"rewards/margins": 3.0193248686114194,
|
|
"rewards/rejected": -0.8996457719245617,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 9.026055335998535,
|
|
"kl": 38.45805740356445,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -45909858.461538464,
|
|
"logits/rejected": -46176119.46666667,
|
|
"logps/chosen": -476.51875,
|
|
"logps/rejected": -382.14977678571427,
|
|
"loss": 0.3181,
|
|
"loss/base_kto": 2.544774055480957,
|
|
"metrics/advantage_var": 11464.9521484375,
|
|
"rewards/chosen": 2.5570633638822113,
|
|
"rewards/margins": 3.5192292206220426,
|
|
"rewards/rejected": -0.9621658567398313,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 9.854619979858398,
|
|
"kl": 32.282257080078125,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -48232252.631578945,
|
|
"logits/rejected": -47521709.71428572,
|
|
"logps/chosen": -477.9053248355263,
|
|
"logps/rejected": -400.0725911458333,
|
|
"loss": 0.3232,
|
|
"loss/base_kto": 2.585928440093994,
|
|
"metrics/advantage_var": 12165.7529296875,
|
|
"rewards/chosen": 1.7689299332468134,
|
|
"rewards/margins": 3.367724052945474,
|
|
"rewards/rejected": -1.5987941196986608,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 9.233141899108887,
|
|
"kl": 37.69820785522461,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -46655029.39877301,
|
|
"logits/rejected": -47521289.78343949,
|
|
"logps/chosen": -435.96841449386505,
|
|
"logps/rejected": -400.7941132563694,
|
|
"loss": 0.3245,
|
|
"loss/base_kto": 2.595954418182373,
|
|
"metrics/advantage_var": 10885.6875,
|
|
"rewards/chosen": 2.3360288655099692,
|
|
"rewards/margins": 3.291977616194185,
|
|
"rewards/rejected": -0.9559487506842158,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 6.974427700042725,
|
|
"kl": 27.17978858947754,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -47464819.11409396,
|
|
"logits/rejected": -48631063.27272727,
|
|
"logps/chosen": -476.66013003355704,
|
|
"logps/rejected": -390.7624175219941,
|
|
"loss": 0.2932,
|
|
"loss/base_kto": 2.345959424972534,
|
|
"metrics/advantage_var": 13561.8515625,
|
|
"rewards/chosen": 2.032148322803062,
|
|
"rewards/margins": 4.105728262032809,
|
|
"rewards/rejected": -2.073579939229747,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 8.006534576416016,
|
|
"kl": 12.020552635192871,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -47671432.53333333,
|
|
"logits/rejected": -47465428.45984252,
|
|
"logps/chosen": -442.7193798449612,
|
|
"logps/rejected": -396.0220964566929,
|
|
"loss": 0.2713,
|
|
"loss/base_kto": 2.1700031757354736,
|
|
"metrics/advantage_var": 14014.9755859375,
|
|
"rewards/chosen": 2.201185879966085,
|
|
"rewards/margins": 4.629551175856833,
|
|
"rewards/rejected": -2.4283652958907482,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 7.6743483543396,
|
|
"kl": 24.583898544311523,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -49097158.05271318,
|
|
"logits/rejected": -47500434.74645669,
|
|
"logps/chosen": -466.3345930232558,
|
|
"logps/rejected": -376.74753937007875,
|
|
"loss": 0.2633,
|
|
"loss/base_kto": 2.1062209606170654,
|
|
"metrics/advantage_var": 13870.4033203125,
|
|
"rewards/chosen": 2.9003382010053294,
|
|
"rewards/margins": 4.841445715153951,
|
|
"rewards/rejected": -1.941107514148622,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 8.210561752319336,
|
|
"kl": 15.757326126098633,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -46189912.631239936,
|
|
"logits/rejected": -45744180.83156297,
|
|
"logps/chosen": -455.30495169082127,
|
|
"logps/rejected": -347.65063069044004,
|
|
"loss": 0.2604,
|
|
"loss/base_kto": 2.0829906463623047,
|
|
"metrics/advantage_var": 13050.2421875,
|
|
"rewards/chosen": 2.3093481877767714,
|
|
"rewards/margins": 4.568288032119526,
|
|
"rewards/rejected": -2.258939844342754,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 8.713674545288086,
|
|
"kl": 13.271387100219727,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -45598088.13643411,
|
|
"logits/rejected": -45550296.894488186,
|
|
"logps/chosen": -445.2339631782946,
|
|
"logps/rejected": -374.74773622047246,
|
|
"loss": 0.2738,
|
|
"loss/base_kto": 2.1901986598968506,
|
|
"metrics/advantage_var": 13676.1171875,
|
|
"rewards/chosen": 2.0202163578003876,
|
|
"rewards/margins": 4.501684930942612,
|
|
"rewards/rejected": -2.4814685731422244,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 7.964659690856934,
|
|
"kl": 18.402326583862305,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -45798747.37463127,
|
|
"logits/rejected": -47414003.242524914,
|
|
"logps/chosen": -449.796552359882,
|
|
"logps/rejected": -371.562032807309,
|
|
"loss": 0.2526,
|
|
"loss/base_kto": 2.020669937133789,
|
|
"metrics/advantage_var": 14040.15625,
|
|
"rewards/chosen": 2.557764464071718,
|
|
"rewards/margins": 4.88743947315602,
|
|
"rewards/rejected": -2.329675009084302,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 7.827051162719727,
|
|
"kl": 17.168731689453125,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -47639918.85358255,
|
|
"logits/rejected": -47987541.868338555,
|
|
"logps/chosen": -488.3009151090343,
|
|
"logps/rejected": -416.904927507837,
|
|
"loss": 0.2435,
|
|
"loss/base_kto": 1.9479866027832031,
|
|
"metrics/advantage_var": 14400.1591796875,
|
|
"rewards/chosen": 2.3399764681902258,
|
|
"rewards/margins": 5.139182284094419,
|
|
"rewards/rejected": -2.799205815904193,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 7.2465667724609375,
|
|
"kl": 25.497581481933594,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -48523415.194630876,
|
|
"logits/rejected": -48706350.40935673,
|
|
"logps/chosen": -467.25388003355704,
|
|
"logps/rejected": -402.67114400584796,
|
|
"loss": 0.2546,
|
|
"loss/base_kto": 2.0367116928100586,
|
|
"metrics/advantage_var": 14426.1474609375,
|
|
"rewards/chosen": 2.1353075680316693,
|
|
"rewards/margins": 4.538409260454458,
|
|
"rewards/rejected": -2.403101692422789,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 6.00705099105835,
|
|
"kl": 21.413923263549805,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -47991043.65134707,
|
|
"logits/rejected": -48706687.8027735,
|
|
"logps/chosen": -492.87371236133123,
|
|
"logps/rejected": -417.9573382126348,
|
|
"loss": 0.2415,
|
|
"loss/base_kto": 1.9320722818374634,
|
|
"metrics/advantage_var": 14509.7939453125,
|
|
"rewards/chosen": 2.263974826044968,
|
|
"rewards/margins": 5.147939564464652,
|
|
"rewards/rejected": -2.8839647384196843,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 6.559169769287109,
|
|
"kl": 21.99832534790039,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -47288352.05007825,
|
|
"logits/rejected": -48029686.4149766,
|
|
"logps/chosen": -456.8280516431925,
|
|
"logps/rejected": -395.6653666146646,
|
|
"loss": 0.2501,
|
|
"loss/base_kto": 2.000561475753784,
|
|
"metrics/advantage_var": 12681.5341796875,
|
|
"rewards/chosen": 2.232018603592038,
|
|
"rewards/margins": 4.566518682813957,
|
|
"rewards/rejected": -2.334500079221919,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 7.573378562927246,
|
|
"kl": 24.434301376342773,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -48100737.87928222,
|
|
"logits/rejected": -46305408.95952024,
|
|
"logps/chosen": -460.87418433931487,
|
|
"logps/rejected": -382.82650862068965,
|
|
"loss": 0.2611,
|
|
"loss/base_kto": 2.0884008407592773,
|
|
"metrics/advantage_var": 12721.8154296875,
|
|
"rewards/chosen": 2.5049300077105423,
|
|
"rewards/margins": 4.615244235362342,
|
|
"rewards/rejected": -2.110314227651799,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 9.15713882446289,
|
|
"kl": 9.266432762145996,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -50749891.62633997,
|
|
"logits/rejected": -49394278.073365234,
|
|
"logps/chosen": -472.1126052833078,
|
|
"logps/rejected": -413.897826953748,
|
|
"loss": 0.2668,
|
|
"loss/base_kto": 2.1340107917785645,
|
|
"metrics/advantage_var": 12453.4423828125,
|
|
"rewards/chosen": 1.885936976574464,
|
|
"rewards/margins": 4.484020959391948,
|
|
"rewards/rejected": -2.598083982817484,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 6.648115634918213,
|
|
"kl": 19.018064498901367,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -48953454.3518225,
|
|
"logits/rejected": -49181296.81355932,
|
|
"logps/chosen": -481.27902139461173,
|
|
"logps/rejected": -391.85891756548534,
|
|
"loss": 0.2595,
|
|
"loss/base_kto": 2.075740098953247,
|
|
"metrics/advantage_var": 12773.25390625,
|
|
"rewards/chosen": 2.3078029046156896,
|
|
"rewards/margins": 4.5615917865494335,
|
|
"rewards/rejected": -2.2537888819337444,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 8.432963371276855,
|
|
"kl": 15.786555290222168,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -50043944.09638554,
|
|
"logits/rejected": -49340396.051948056,
|
|
"logps/chosen": -471.32958396084337,
|
|
"logps/rejected": -410.0531655844156,
|
|
"loss": 0.2574,
|
|
"loss/base_kto": 2.0592541694641113,
|
|
"metrics/advantage_var": 12785.0068359375,
|
|
"rewards/chosen": 2.132786578442677,
|
|
"rewards/margins": 4.651474433556313,
|
|
"rewards/rejected": -2.5186878551136362,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 7.463342666625977,
|
|
"kl": 18.011547088623047,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -50799278.9068323,
|
|
"logits/rejected": -50387240.251572326,
|
|
"logps/chosen": -484.422748447205,
|
|
"logps/rejected": -387.8544123427673,
|
|
"loss": 0.2459,
|
|
"loss/base_kto": 1.9671169519424438,
|
|
"metrics/advantage_var": 13662.9599609375,
|
|
"rewards/chosen": 2.752448230056289,
|
|
"rewards/margins": 4.902260748481996,
|
|
"rewards/rejected": -2.1498125184257075,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 7.461705207824707,
|
|
"kl": 19.686004638671875,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -48579801.8031746,
|
|
"logits/rejected": -48981808.04923077,
|
|
"logps/chosen": -444.07172619047617,
|
|
"logps/rejected": -402.4366105769231,
|
|
"loss": 0.2576,
|
|
"loss/base_kto": 2.0607800483703613,
|
|
"metrics/advantage_var": 15960.1123046875,
|
|
"rewards/chosen": 2.096119326636905,
|
|
"rewards/margins": 4.8182899245936355,
|
|
"rewards/rejected": -2.7221705979567306,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 5.755481719970703,
|
|
"kl": 23.16046714782715,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -50075429.79604262,
|
|
"logits/rejected": -49548141.71428572,
|
|
"logps/chosen": -451.0539383561644,
|
|
"logps/rejected": -415.2268007624398,
|
|
"loss": 0.2499,
|
|
"loss/base_kto": 1.999585747718811,
|
|
"metrics/advantage_var": 14338.078125,
|
|
"rewards/chosen": 2.7706828356877855,
|
|
"rewards/margins": 4.9420303665365015,
|
|
"rewards/rejected": -2.171347530848716,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 7.263795375823975,
|
|
"kl": 20.749311447143555,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -51008243.731629394,
|
|
"logits/rejected": -50280040.905198775,
|
|
"logps/chosen": -457.93106030351436,
|
|
"logps/rejected": -404.38876146788994,
|
|
"loss": 0.2574,
|
|
"loss/base_kto": 2.0591797828674316,
|
|
"metrics/advantage_var": 14394.5234375,
|
|
"rewards/chosen": 2.1054878600489215,
|
|
"rewards/margins": 4.947445068991486,
|
|
"rewards/rejected": -2.841957208942565,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 8.185635566711426,
|
|
"kl": 20.86648941040039,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -50875951.299837925,
|
|
"logits/rejected": -50029618.968325794,
|
|
"logps/chosen": -462.942868719611,
|
|
"logps/rejected": -397.45456259426845,
|
|
"loss": 0.2628,
|
|
"loss/base_kto": 2.102466344833374,
|
|
"metrics/advantage_var": 14819.9130859375,
|
|
"rewards/chosen": 2.3890321505900527,
|
|
"rewards/margins": 4.7555759736481225,
|
|
"rewards/rejected": -2.3665438230580693,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 7.5429463386535645,
|
|
"kl": 18.411264419555664,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -51579456.402515724,
|
|
"logits/rejected": -49579478.65838509,
|
|
"logps/chosen": -472.2454304245283,
|
|
"logps/rejected": -389.61934685559004,
|
|
"loss": 0.2593,
|
|
"loss/base_kto": 2.0742015838623047,
|
|
"metrics/advantage_var": 13896.4990234375,
|
|
"rewards/chosen": 2.2443824624115565,
|
|
"rewards/margins": 4.678256367569747,
|
|
"rewards/rejected": -2.433873905158191,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 7.785284042358398,
|
|
"kl": 21.28897476196289,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -49997334.6835443,
|
|
"logits/rejected": -50460479.209876545,
|
|
"logps/chosen": -468.59083267405066,
|
|
"logps/rejected": -376.14158950617286,
|
|
"loss": 0.2469,
|
|
"loss/base_kto": 1.9749466180801392,
|
|
"metrics/advantage_var": 13357.8466796875,
|
|
"rewards/chosen": 2.6586543215981013,
|
|
"rewards/margins": 4.838992298124433,
|
|
"rewards/rejected": -2.180337976526331,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 12.808547973632812,
|
|
"kl": 12.648378372192383,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -50732009.80804954,
|
|
"logits/rejected": -51210214.15772871,
|
|
"logps/chosen": -462.1233552631579,
|
|
"logps/rejected": -389.2183063880126,
|
|
"loss": 0.2536,
|
|
"loss/base_kto": 2.0289154052734375,
|
|
"metrics/advantage_var": 14579.5732421875,
|
|
"rewards/chosen": 2.117305791045859,
|
|
"rewards/margins": 4.87371756309091,
|
|
"rewards/rejected": -2.7564117720450514,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 9.873778343200684,
|
|
"kl": 26.791122436523438,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -49176047.06771654,
|
|
"logits/rejected": -49040658.65426356,
|
|
"logps/chosen": -465.3425688976378,
|
|
"logps/rejected": -400.659253875969,
|
|
"loss": 0.2663,
|
|
"loss/base_kto": 2.130403518676758,
|
|
"metrics/advantage_var": 14569.2568359375,
|
|
"rewards/chosen": 2.50025817390502,
|
|
"rewards/margins": 4.679842339966067,
|
|
"rewards/rejected": -2.1795841660610464,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 11.098130226135254,
|
|
"kl": 17.67889404296875,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -50631510.12713178,
|
|
"logits/rejected": -49922744.64251968,
|
|
"logps/chosen": -460.33478682170545,
|
|
"logps/rejected": -392.7937992125984,
|
|
"loss": 0.2534,
|
|
"loss/base_kto": 2.027090072631836,
|
|
"metrics/advantage_var": 12999.9736328125,
|
|
"rewards/chosen": 2.4095879133357556,
|
|
"rewards/margins": 4.742966358217645,
|
|
"rewards/rejected": -2.3333784448818897,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 10.580224990844727,
|
|
"kl": 17.041147232055664,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -50606938.11838006,
|
|
"logits/rejected": -50023391.89968652,
|
|
"logps/chosen": -484.9173481308411,
|
|
"logps/rejected": -402.11265673981194,
|
|
"loss": 0.2528,
|
|
"loss/base_kto": 2.0220911502838135,
|
|
"metrics/advantage_var": 13346.0146484375,
|
|
"rewards/chosen": 2.3297069704049846,
|
|
"rewards/margins": 4.816585751140682,
|
|
"rewards/rejected": -2.4868787807356973,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 9.04914379119873,
|
|
"kl": 25.278024673461914,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -50172554.938530736,
|
|
"logits/rejected": -50815196.50244698,
|
|
"logps/chosen": -453.6979947526237,
|
|
"logps/rejected": -378.65576570146817,
|
|
"loss": 0.2627,
|
|
"loss/base_kto": 2.1016016006469727,
|
|
"metrics/advantage_var": 13008.0537109375,
|
|
"rewards/chosen": 2.8642087647582457,
|
|
"rewards/margins": 4.541724488529657,
|
|
"rewards/rejected": -1.6775157237714111,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 8.332275390625,
|
|
"kl": 34.81646728515625,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -49631567.01234568,
|
|
"logits/rejected": -49031887.39240506,
|
|
"logps/chosen": -466.037037037037,
|
|
"logps/rejected": -399.32730913765823,
|
|
"loss": 0.2689,
|
|
"loss/base_kto": 2.1509957313537598,
|
|
"metrics/advantage_var": 13046.8349609375,
|
|
"rewards/chosen": 2.996243323808835,
|
|
"rewards/margins": 4.272110447955888,
|
|
"rewards/rejected": -1.275867124147053,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 9.706599235534668,
|
|
"kl": 21.504140853881836,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -50742327.97427653,
|
|
"logits/rejected": -52272604.20668693,
|
|
"logps/chosen": -450.47432676848877,
|
|
"logps/rejected": -411.2093939969605,
|
|
"loss": 0.2669,
|
|
"loss/base_kto": 2.1349241733551025,
|
|
"metrics/advantage_var": 13198.6015625,
|
|
"rewards/chosen": 2.2765420502788385,
|
|
"rewards/margins": 4.696065805384272,
|
|
"rewards/rejected": -2.419523755105433,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 9.024945259094238,
|
|
"kl": 20.595943450927734,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -50928485.52410576,
|
|
"logits/rejected": -51772176.477237046,
|
|
"logps/chosen": -463.4435264385692,
|
|
"logps/rejected": -410.42481848508635,
|
|
"loss": 0.2558,
|
|
"loss/base_kto": 2.0465526580810547,
|
|
"metrics/advantage_var": 15188.572265625,
|
|
"rewards/chosen": 2.496919955287714,
|
|
"rewards/margins": 4.9222173316000175,
|
|
"rewards/rejected": -2.4252973763123036,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 8.380735397338867,
|
|
"kl": 22.653018951416016,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -50397374.511627905,
|
|
"logits/rejected": -50354895.21889764,
|
|
"logps/chosen": -444.8803294573643,
|
|
"logps/rejected": -395.8338336614173,
|
|
"loss": 0.2498,
|
|
"loss/base_kto": 1.998310923576355,
|
|
"metrics/advantage_var": 13499.3720703125,
|
|
"rewards/chosen": 2.6756109193313953,
|
|
"rewards/margins": 4.87968326185108,
|
|
"rewards/rejected": -2.204072342519685,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 12.051867485046387,
|
|
"kl": 19.444766998291016,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -52307059.86544342,
|
|
"logits/rejected": -52765738.53035144,
|
|
"logps/chosen": -442.72348050458714,
|
|
"logps/rejected": -422.0098841853035,
|
|
"loss": 0.2538,
|
|
"loss/base_kto": 2.0302066802978516,
|
|
"metrics/advantage_var": 14113.2998046875,
|
|
"rewards/chosen": 2.5834022078674503,
|
|
"rewards/margins": 4.8570046782897744,
|
|
"rewards/rejected": -2.273602470422324,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 6.080167293548584,
|
|
"kl": 7.740408420562744,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -52097834.53776435,
|
|
"logits/rejected": -52385685.95469256,
|
|
"logps/chosen": -476.92942787009065,
|
|
"logps/rejected": -413.64504955501616,
|
|
"loss": 0.2637,
|
|
"loss/base_kto": 2.1094841957092285,
|
|
"metrics/advantage_var": 14161.9091796875,
|
|
"rewards/chosen": 1.8997557487372545,
|
|
"rewards/margins": 4.792029290530337,
|
|
"rewards/rejected": -2.8922735417930827,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 7.666107654571533,
|
|
"kl": 29.72540283203125,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -50770696.827586204,
|
|
"logits/rejected": -52469633.04404568,
|
|
"logps/chosen": -465.53841829085457,
|
|
"logps/rejected": -401.9165477161501,
|
|
"loss": 0.246,
|
|
"loss/base_kto": 1.9683994054794312,
|
|
"metrics/advantage_var": 13659.8037109375,
|
|
"rewards/chosen": 2.673970509862256,
|
|
"rewards/margins": 4.902071932170576,
|
|
"rewards/rejected": -2.2281014223083195,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 8.589899063110352,
|
|
"kl": 30.167007446289062,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -51137649.777777776,
|
|
"logits/rejected": -51158989.587692305,
|
|
"logps/chosen": -476.984375,
|
|
"logps/rejected": -403.0909375,
|
|
"loss": 0.2636,
|
|
"loss/base_kto": 2.1090667247772217,
|
|
"metrics/advantage_var": 12947.015625,
|
|
"rewards/chosen": 2.508792937748016,
|
|
"rewards/margins": 4.470145289010035,
|
|
"rewards/rejected": -1.9613523512620192,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 9.947898864746094,
|
|
"kl": 24.149383544921875,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -50633531.65781711,
|
|
"logits/rejected": -50599163.7475083,
|
|
"logps/chosen": -453.01069321533924,
|
|
"logps/rejected": -398.06753529900334,
|
|
"loss": 0.2631,
|
|
"loss/base_kto": 2.1051042079925537,
|
|
"metrics/advantage_var": 14002.7099609375,
|
|
"rewards/chosen": 2.5346550055309733,
|
|
"rewards/margins": 4.710280727408569,
|
|
"rewards/rejected": -2.1756257218775956,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 7.734704494476318,
|
|
"kl": 35.1544303894043,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -51763093.56850394,
|
|
"logits/rejected": -51378245.85426357,
|
|
"logps/chosen": -463.19566929133856,
|
|
"logps/rejected": -394.7056201550388,
|
|
"loss": 0.24,
|
|
"loss/base_kto": 1.9199455976486206,
|
|
"metrics/advantage_var": 13500.6142578125,
|
|
"rewards/chosen": 2.862628414124016,
|
|
"rewards/margins": 4.883309963905992,
|
|
"rewards/rejected": -2.0206815497819766,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 6.12605619430542,
|
|
"kl": 18.320152282714844,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -49078500.644338116,
|
|
"logits/rejected": -50882374.95865238,
|
|
"logps/chosen": -414.313446969697,
|
|
"logps/rejected": -398.36633805513014,
|
|
"loss": 0.2708,
|
|
"loss/base_kto": 2.166231155395508,
|
|
"metrics/advantage_var": 12967.21875,
|
|
"rewards/chosen": 1.996106015438098,
|
|
"rewards/margins": 4.480647016108083,
|
|
"rewards/rejected": -2.4845410006699846,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 5.955198287963867,
|
|
"kl": 15.756006240844727,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -50458820.608,
|
|
"logits/rejected": -51340717.92366412,
|
|
"logps/chosen": -462.9027,
|
|
"logps/rejected": -390.4223282442748,
|
|
"loss": 0.2493,
|
|
"loss/base_kto": 1.9940131902694702,
|
|
"metrics/advantage_var": 14228.1376953125,
|
|
"rewards/chosen": 2.121536328125,
|
|
"rewards/margins": 5.070816392831584,
|
|
"rewards/rejected": -2.949280064706584,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 7.135191440582275,
|
|
"kl": 24.769649505615234,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -51178060.88178914,
|
|
"logits/rejected": -52667457.76146789,
|
|
"logps/chosen": -469.5570087859425,
|
|
"logps/rejected": -387.4057482798165,
|
|
"loss": 0.2443,
|
|
"loss/base_kto": 1.9541826248168945,
|
|
"metrics/advantage_var": 13435.5263671875,
|
|
"rewards/chosen": 2.4769127208965656,
|
|
"rewards/margins": 4.854215258167208,
|
|
"rewards/rejected": -2.377302537270642,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 7.013718605041504,
|
|
"kl": 30.34478187561035,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -51477564.23529412,
|
|
"logits/rejected": -52709886.46246246,
|
|
"logps/chosen": -484.40109272875816,
|
|
"logps/rejected": -392.9382272897898,
|
|
"loss": 0.2,
|
|
"loss/base_kto": 1.5998648405075073,
|
|
"metrics/advantage_var": 14826.869140625,
|
|
"rewards/chosen": 3.1217575571895426,
|
|
"rewards/margins": 5.812387762590256,
|
|
"rewards/rejected": -2.6906302054007134,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 5.8692803382873535,
|
|
"kl": 12.55923080444336,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -51684074.43797856,
|
|
"logits/rejected": -52261406.21371611,
|
|
"logps/chosen": -428.73631316998467,
|
|
"logps/rejected": -383.90814393939394,
|
|
"loss": 0.2111,
|
|
"loss/base_kto": 1.6889790296554565,
|
|
"metrics/advantage_var": 14125.8115234375,
|
|
"rewards/chosen": 2.474229143735643,
|
|
"rewards/margins": 5.531732632571369,
|
|
"rewards/rejected": -3.0575034888357258,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 5.495901584625244,
|
|
"kl": 18.747608184814453,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -51583710.500807755,
|
|
"logits/rejected": -52984393.58547655,
|
|
"logps/chosen": -455.1819466882068,
|
|
"logps/rejected": -395.01366301059,
|
|
"loss": 0.2167,
|
|
"loss/base_kto": 1.7334028482437134,
|
|
"metrics/advantage_var": 14336.228515625,
|
|
"rewards/chosen": 2.454308036841175,
|
|
"rewards/margins": 5.4658964061443145,
|
|
"rewards/rejected": -3.0115883693031393,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 9.811450958251953,
|
|
"kl": 15.564459800720215,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -51759881.68229955,
|
|
"logits/rejected": -52201898.80452342,
|
|
"logps/chosen": -459.4629349470499,
|
|
"logps/rejected": -398.67245557350566,
|
|
"loss": 0.2064,
|
|
"loss/base_kto": 1.6514919996261597,
|
|
"metrics/advantage_var": 14630.4482421875,
|
|
"rewards/chosen": 3.066073649950359,
|
|
"rewards/margins": 5.556259906765787,
|
|
"rewards/rejected": -2.490186256815428,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 5.7088398933410645,
|
|
"kl": 17.640766143798828,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -50637759.902290076,
|
|
"logits/rejected": -52251328.512,
|
|
"logps/chosen": -458.0880248091603,
|
|
"logps/rejected": -429.6307,
|
|
"loss": 0.2117,
|
|
"loss/base_kto": 1.6933727264404297,
|
|
"metrics/advantage_var": 15645.4248046875,
|
|
"rewards/chosen": 2.766318098461355,
|
|
"rewards/margins": 5.662018098461355,
|
|
"rewards/rejected": -2.8957,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 9.062532424926758,
|
|
"kl": 18.433141708374023,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -51133872.56528418,
|
|
"logits/rejected": -49807802.81081081,
|
|
"logps/chosen": -442.15101766513055,
|
|
"logps/rejected": -396.4219495230525,
|
|
"loss": 0.2287,
|
|
"loss/base_kto": 1.8292301893234253,
|
|
"metrics/advantage_var": 13636.6064453125,
|
|
"rewards/chosen": 2.8211182390673004,
|
|
"rewards/margins": 5.241429419388247,
|
|
"rewards/rejected": -2.420311180320946,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 8.166321754455566,
|
|
"kl": 22.650665283203125,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -52599490.43037975,
|
|
"logits/rejected": -51828483.16049383,
|
|
"logps/chosen": -466.92306170886076,
|
|
"logps/rejected": -387.64144483024694,
|
|
"loss": 0.2115,
|
|
"loss/base_kto": 1.6923620700836182,
|
|
"metrics/advantage_var": 14835.9453125,
|
|
"rewards/chosen": 2.4578896051720727,
|
|
"rewards/margins": 5.504484295565591,
|
|
"rewards/rejected": -3.0465946903935186,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.062451362609863,
|
|
"kl": 12.976425170898438,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -52049936.07535322,
|
|
"logits/rejected": -53151976.510108866,
|
|
"logps/chosen": -446.8129905808477,
|
|
"logps/rejected": -401.06497861586314,
|
|
"loss": 0.2076,
|
|
"loss/base_kto": 1.660894751548767,
|
|
"metrics/advantage_var": 13747.203125,
|
|
"rewards/chosen": 2.508091517857143,
|
|
"rewards/margins": 5.53510038133887,
|
|
"rewards/rejected": -3.0270088634817265,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 7.792516231536865,
|
|
"kl": 20.18115234375,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -53751346.09618574,
|
|
"logits/rejected": -53658386.52880354,
|
|
"logps/chosen": -464.70885157545604,
|
|
"logps/rejected": -407.9797359675037,
|
|
"loss": 0.2025,
|
|
"loss/base_kto": 1.6203575134277344,
|
|
"metrics/advantage_var": 15640.3388671875,
|
|
"rewards/chosen": 2.468496547211857,
|
|
"rewards/margins": 5.659715995374707,
|
|
"rewards/rejected": -3.1912194481628506,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 6.621832847595215,
|
|
"kl": 16.571104049682617,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -54200557.00890208,
|
|
"logits/rejected": -53209642.24422442,
|
|
"logps/chosen": -462.1706695103858,
|
|
"logps/rejected": -402.0129950495049,
|
|
"loss": 0.2237,
|
|
"loss/base_kto": 1.7895901203155518,
|
|
"metrics/advantage_var": 14439.8701171875,
|
|
"rewards/chosen": 2.625101061178598,
|
|
"rewards/margins": 5.253301478876618,
|
|
"rewards/rejected": -2.62820041769802,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 8.105810165405273,
|
|
"kl": 21.278722763061523,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -50443359.44391785,
|
|
"logits/rejected": -49563052.90880989,
|
|
"logps/chosen": -468.43725315955766,
|
|
"logps/rejected": -405.7663736476043,
|
|
"loss": 0.2185,
|
|
"loss/base_kto": 1.7482216358184814,
|
|
"metrics/advantage_var": 15048.6787109375,
|
|
"rewards/chosen": 2.739806068263724,
|
|
"rewards/margins": 5.470672537303523,
|
|
"rewards/rejected": -2.7308664690397992,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.075442314147949,
|
|
"kl": 19.994430541992188,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -52378354.526315786,
|
|
"logits/rejected": -51290253.13323124,
|
|
"logps/chosen": -460.5764553429027,
|
|
"logps/rejected": -422.2306661562021,
|
|
"loss": 0.214,
|
|
"loss/base_kto": 1.7116119861602783,
|
|
"metrics/advantage_var": 16243.4814453125,
|
|
"rewards/chosen": 2.764156651839115,
|
|
"rewards/margins": 5.658843060122997,
|
|
"rewards/rejected": -2.894686408283882,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 7.0332512855529785,
|
|
"kl": 19.325130462646484,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -51644251.14551084,
|
|
"logits/rejected": -52120175.44479495,
|
|
"logps/chosen": -464.09950657894734,
|
|
"logps/rejected": -402.872535488959,
|
|
"loss": 0.2107,
|
|
"loss/base_kto": 1.6855430603027344,
|
|
"metrics/advantage_var": 13973.416015625,
|
|
"rewards/chosen": 2.7761113311484134,
|
|
"rewards/margins": 5.570257077710421,
|
|
"rewards/rejected": -2.794145746562007,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 7.986169815063477,
|
|
"kl": 16.947446823120117,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -54452428.475435816,
|
|
"logits/rejected": -54813858.5146379,
|
|
"logps/chosen": -476.34974247226626,
|
|
"logps/rejected": -396.2220242681048,
|
|
"loss": 0.1982,
|
|
"loss/base_kto": 1.5859140157699585,
|
|
"metrics/advantage_var": 14368.3193359375,
|
|
"rewards/chosen": 2.7504950521865097,
|
|
"rewards/margins": 5.752678587252476,
|
|
"rewards/rejected": -3.0021835350659667,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 7.128055572509766,
|
|
"kl": 20.3778133392334,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -50613619.450980395,
|
|
"logits/rejected": -53207653.17365269,
|
|
"logps/chosen": -435.92953431372547,
|
|
"logps/rejected": -404.85291916167665,
|
|
"loss": 0.2143,
|
|
"loss/base_kto": 1.7144848108291626,
|
|
"metrics/advantage_var": 14200.7333984375,
|
|
"rewards/chosen": 2.464855717677696,
|
|
"rewards/margins": 5.405907452099477,
|
|
"rewards/rejected": -2.9410517344217815,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 8.116265296936035,
|
|
"kl": 16.64581298828125,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -52379571.53246753,
|
|
"logits/rejected": -51955798.361445785,
|
|
"logps/chosen": -464.8143262987013,
|
|
"logps/rejected": -407.6978539156627,
|
|
"loss": 0.2168,
|
|
"loss/base_kto": 1.734110713005066,
|
|
"metrics/advantage_var": 14600.6748046875,
|
|
"rewards/chosen": 2.50759253563819,
|
|
"rewards/margins": 5.451606935522885,
|
|
"rewards/rejected": -2.944014399884695,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 9.05835247039795,
|
|
"kl": 18.716712951660156,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -52313060.8424337,
|
|
"logits/rejected": -52677460.53208138,
|
|
"logps/chosen": -454.5990639625585,
|
|
"logps/rejected": -390.97911776212834,
|
|
"loss": 0.2116,
|
|
"loss/base_kto": 1.6926889419555664,
|
|
"metrics/advantage_var": 14277.234375,
|
|
"rewards/chosen": 2.606424554833756,
|
|
"rewards/margins": 5.508694756992892,
|
|
"rewards/rejected": -2.9022702021591353,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 5.412971019744873,
|
|
"kl": 21.103687286376953,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -52296739.194762684,
|
|
"logits/rejected": -52193589.18983558,
|
|
"logps/chosen": -446.3010433715221,
|
|
"logps/rejected": -400.5539985052317,
|
|
"loss": 0.2085,
|
|
"loss/base_kto": 1.6677101850509644,
|
|
"metrics/advantage_var": 15533.890625,
|
|
"rewards/chosen": 2.562060866535393,
|
|
"rewards/margins": 5.68401731973046,
|
|
"rewards/rejected": -3.121956453195067,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 7.837034702301025,
|
|
"kl": 12.999594688415527,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -52166433.04510109,
|
|
"logits/rejected": -52585486.467817895,
|
|
"logps/chosen": -468.9892107309487,
|
|
"logps/rejected": -407.98569956828885,
|
|
"loss": 0.2153,
|
|
"loss/base_kto": 1.7224851846694946,
|
|
"metrics/advantage_var": 14251.125,
|
|
"rewards/chosen": 2.384062439249611,
|
|
"rewards/margins": 5.407675475209187,
|
|
"rewards/rejected": -3.0236130359595763,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 7.596892356872559,
|
|
"kl": 17.91208839416504,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -52285226.17944535,
|
|
"logits/rejected": -52072183.940029986,
|
|
"logps/chosen": -451.25785073409463,
|
|
"logps/rejected": -392.07093328335833,
|
|
"loss": 0.2083,
|
|
"loss/base_kto": 1.6665111780166626,
|
|
"metrics/advantage_var": 14959.923828125,
|
|
"rewards/chosen": 2.6295397013279977,
|
|
"rewards/margins": 5.66366997505832,
|
|
"rewards/rejected": -3.034130273730322,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 8.364079475402832,
|
|
"kl": 9.80460262298584,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -52477008.22119816,
|
|
"logits/rejected": -53476327.58028617,
|
|
"logps/chosen": -435.32574884792626,
|
|
"logps/rejected": -392.80703497615264,
|
|
"loss": 0.226,
|
|
"loss/base_kto": 1.8082932233810425,
|
|
"metrics/advantage_var": 14568.2607421875,
|
|
"rewards/chosen": 2.3125780049923197,
|
|
"rewards/margins": 5.337056110740531,
|
|
"rewards/rejected": -3.0244781057482113,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 11.61428165435791,
|
|
"kl": 17.37249755859375,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -53554920.149068326,
|
|
"logits/rejected": -52665005.88679245,
|
|
"logps/chosen": -458.90052406832297,
|
|
"logps/rejected": -417.7639544025157,
|
|
"loss": 0.2136,
|
|
"loss/base_kto": 1.7086185216903687,
|
|
"metrics/advantage_var": 13907.859375,
|
|
"rewards/chosen": 2.4842216539086763,
|
|
"rewards/margins": 5.431352540922237,
|
|
"rewards/rejected": -2.9471308870135613,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.054996490478516,
|
|
"kl": 9.886092185974121,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -52787035.56204379,
|
|
"logits/rejected": -52475398.02352941,
|
|
"logps/chosen": -484.2533759124088,
|
|
"logps/rejected": -387.08613445378154,
|
|
"loss": 0.2097,
|
|
"loss/base_kto": 1.6778475046157837,
|
|
"metrics/advantage_var": 13511.6064453125,
|
|
"rewards/chosen": 2.6910379006044707,
|
|
"rewards/margins": 5.576327709887559,
|
|
"rewards/rejected": -2.8852898092830883,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 7.358473777770996,
|
|
"kl": 18.090463638305664,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -50325776.0,
|
|
"logits/rejected": -50889488.0,
|
|
"logps/chosen": -437.87919921875,
|
|
"logps/rejected": -388.3118896484375,
|
|
"loss": 0.208,
|
|
"loss/base_kto": 1.6639856100082397,
|
|
"metrics/advantage_var": 13564.1123046875,
|
|
"rewards/chosen": 2.6645410537719725,
|
|
"rewards/margins": 5.443995475769043,
|
|
"rewards/rejected": -2.7794544219970705,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.689790725708008,
|
|
"kl": 18.853412628173828,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -53538429.46727549,
|
|
"logits/rejected": -54475780.93097913,
|
|
"logps/chosen": -470.3403729071537,
|
|
"logps/rejected": -404.3622592295345,
|
|
"loss": 0.2138,
|
|
"loss/base_kto": 1.7103630304336548,
|
|
"metrics/advantage_var": 14769.6220703125,
|
|
"rewards/chosen": 2.7466041413979263,
|
|
"rewards/margins": 5.500908540474972,
|
|
"rewards/rejected": -2.7543043990770464,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 7.575942516326904,
|
|
"kl": 18.5459041595459,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -52107129.6,
|
|
"logits/rejected": -52743897.6,
|
|
"logps/chosen": -443.063037109375,
|
|
"logps/rejected": -400.88740234375,
|
|
"loss": 0.2097,
|
|
"loss/base_kto": 1.677259087562561,
|
|
"metrics/advantage_var": 13134.6123046875,
|
|
"rewards/chosen": 2.7101999282836915,
|
|
"rewards/margins": 5.353364753723145,
|
|
"rewards/rejected": -2.6431648254394533,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 7.75359582901001,
|
|
"kl": 19.799182891845703,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -49708657.50558214,
|
|
"logits/rejected": -51144722.81776416,
|
|
"logps/chosen": -456.9265350877193,
|
|
"logps/rejected": -399.776440467075,
|
|
"loss": 0.2149,
|
|
"loss/base_kto": 1.7193249464035034,
|
|
"metrics/advantage_var": 13961.0068359375,
|
|
"rewards/chosen": 2.6514213111792264,
|
|
"rewards/margins": 5.558279980302504,
|
|
"rewards/rejected": -2.9068586691232774,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 5.188368320465088,
|
|
"kl": 15.995924949645996,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -52745606.757164404,
|
|
"logits/rejected": -52081904.64829822,
|
|
"logps/chosen": -503.8714177978884,
|
|
"logps/rejected": -408.71950972447326,
|
|
"loss": 0.2177,
|
|
"loss/base_kto": 1.7417219877243042,
|
|
"metrics/advantage_var": 14489.0693359375,
|
|
"rewards/chosen": 2.7078634638951735,
|
|
"rewards/margins": 5.375946963945822,
|
|
"rewards/rejected": -2.668083500050648,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 6.56077241897583,
|
|
"kl": 16.229629516601562,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -51885137.2192846,
|
|
"logits/rejected": -52316156.78492936,
|
|
"logps/chosen": -480.2950038880249,
|
|
"logps/rejected": -415.40894819466246,
|
|
"loss": 0.2057,
|
|
"loss/base_kto": 1.6456931829452515,
|
|
"metrics/advantage_var": 14930.6376953125,
|
|
"rewards/chosen": 2.668261490936042,
|
|
"rewards/margins": 5.4684896876834905,
|
|
"rewards/rejected": -2.800228196747449,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 7.278511047363281,
|
|
"kl": 17.13551139831543,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -53035779.32467532,
|
|
"logits/rejected": -52839140.240963854,
|
|
"logps/chosen": -471.993455762987,
|
|
"logps/rejected": -399.45218373493975,
|
|
"loss": 0.2193,
|
|
"loss/base_kto": 1.7541418075561523,
|
|
"metrics/advantage_var": 14101.2490234375,
|
|
"rewards/chosen": 2.4638329047661323,
|
|
"rewards/margins": 5.324497915884826,
|
|
"rewards/rejected": -2.8606650111186935,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 7.59853458404541,
|
|
"kl": 16.550039291381836,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -51503687.3114241,
|
|
"logits/rejected": -50366601.385335416,
|
|
"logps/chosen": -455.4739827856025,
|
|
"logps/rejected": -414.71514235569424,
|
|
"loss": 0.2146,
|
|
"loss/base_kto": 1.7168140411376953,
|
|
"metrics/advantage_var": 15569.7490234375,
|
|
"rewards/chosen": 2.7565891224080596,
|
|
"rewards/margins": 5.4543376964135195,
|
|
"rewards/rejected": -2.6977485740054603,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 6.622338771820068,
|
|
"kl": 15.1036376953125,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -52360348.09756097,
|
|
"logits/rejected": -53833321.025641024,
|
|
"logps/chosen": -446.0986566310976,
|
|
"logps/rejected": -419.4545272435897,
|
|
"loss": 0.2128,
|
|
"loss/base_kto": 1.7021082639694214,
|
|
"metrics/advantage_var": 13941.78125,
|
|
"rewards/chosen": 2.581070504537443,
|
|
"rewards/margins": 5.448736504512403,
|
|
"rewards/rejected": -2.86766599997496,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 7.87120246887207,
|
|
"kl": 16.005062103271484,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -54459303.96380091,
|
|
"logits/rejected": -53815627.92868719,
|
|
"logps/chosen": -476.5891308446455,
|
|
"logps/rejected": -403.3553737844408,
|
|
"loss": 0.2189,
|
|
"loss/base_kto": 1.7509472370147705,
|
|
"metrics/advantage_var": 14989.7529296875,
|
|
"rewards/chosen": 2.553857863758484,
|
|
"rewards/margins": 5.529682006536037,
|
|
"rewards/rejected": -2.9758241427775527,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 8.618688583374023,
|
|
"kl": 20.370731353759766,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -51821293.021416806,
|
|
"logits/rejected": -51338192.832095094,
|
|
"logps/chosen": -461.3043657331137,
|
|
"logps/rejected": -396.511794205052,
|
|
"loss": 0.2111,
|
|
"loss/base_kto": 1.6885474920272827,
|
|
"metrics/advantage_var": 15565.265625,
|
|
"rewards/chosen": 2.3771085786269563,
|
|
"rewards/margins": 5.516415291804891,
|
|
"rewards/rejected": -3.1393067131779344,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 10.641010284423828,
|
|
"kl": 15.09222412109375,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -51623777.486068115,
|
|
"logits/rejected": -51611396.03785489,
|
|
"logps/chosen": -458.0194465944272,
|
|
"logps/rejected": -377.268089511041,
|
|
"loss": 0.227,
|
|
"loss/base_kto": 1.8159443140029907,
|
|
"metrics/advantage_var": 14810.2236328125,
|
|
"rewards/chosen": 2.501199917158475,
|
|
"rewards/margins": 5.438771927090455,
|
|
"rewards/rejected": -2.9375720099319795,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 7.686944484710693,
|
|
"kl": 10.616701126098633,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -52331420.31858407,
|
|
"logits/rejected": -52707416.45182724,
|
|
"logps/chosen": -463.57586651917404,
|
|
"logps/rejected": -415.9006955980066,
|
|
"loss": 0.21,
|
|
"loss/base_kto": 1.6796455383300781,
|
|
"metrics/advantage_var": 14238.4287109375,
|
|
"rewards/chosen": 2.7552421817385695,
|
|
"rewards/margins": 5.631798071286431,
|
|
"rewards/rejected": -2.8765558895478613,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 8.286816596984863,
|
|
"kl": 21.288183212280273,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -50794508.94154818,
|
|
"logits/rejected": -50980807.02318393,
|
|
"logps/chosen": -460.2579976303318,
|
|
"logps/rejected": -390.54897604327664,
|
|
"loss": 0.2197,
|
|
"loss/base_kto": 1.7577959299087524,
|
|
"metrics/advantage_var": 13955.8486328125,
|
|
"rewards/chosen": 2.697458854783768,
|
|
"rewards/margins": 5.36005537900614,
|
|
"rewards/rejected": -2.6625965242223724,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 8.0601806640625,
|
|
"kl": 20.204782485961914,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -53413062.298412696,
|
|
"logits/rejected": -53059010.56,
|
|
"logps/chosen": -463.35287698412696,
|
|
"logps/rejected": -399.71653846153845,
|
|
"loss": 0.2183,
|
|
"loss/base_kto": 1.746720314025879,
|
|
"metrics/advantage_var": 15232.736328125,
|
|
"rewards/chosen": 2.51821773468502,
|
|
"rewards/margins": 5.441813362689828,
|
|
"rewards/rejected": -2.923595628004808,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.989338484496138e+17,
|
|
"train_loss": 0.27494916788242846,
|
|
"train_runtime": 11062.9074,
|
|
"train_samples_per_second": 13.398,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.989338484496138e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|