1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 9.878138542175293,
|
||
|
|
"kl": 1.9441527128219604,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -36347301.26582278,
|
||
|
|
"logits/rejected": -37878834.56790123,
|
||
|
|
"logps/chosen": -506.0238825158228,
|
||
|
|
"logps/rejected": -381.65121045524694,
|
||
|
|
"loss": 0.4676,
|
||
|
|
"loss/base_kto": 3.740809679031372,
|
||
|
|
"metrics/advantage_var": 518.8466186523438,
|
||
|
|
"rewards/chosen": -0.3824396978450727,
|
||
|
|
"rewards/margins": 0.3086688173880967,
|
||
|
|
"rewards/rejected": -0.6911085152331694,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 9.061779975891113,
|
||
|
|
"kl": 36.309051513671875,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -37144238.348228045,
|
||
|
|
"logits/rejected": -38884373.09667195,
|
||
|
|
"logps/chosen": -485.66212442218796,
|
||
|
|
"logps/rejected": -357.98058637083994,
|
||
|
|
"loss": 0.4366,
|
||
|
|
"loss/base_kto": 3.492482900619507,
|
||
|
|
"metrics/advantage_var": 762.9915161132812,
|
||
|
|
"rewards/chosen": 0.8913009949200693,
|
||
|
|
"rewards/margins": 0.49241496874014656,
|
||
|
|
"rewards/rejected": 0.39888602617992275,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 9.688044548034668,
|
||
|
|
"kl": 15.731460571289062,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -36332691.692307696,
|
||
|
|
"logits/rejected": -36808822.63414634,
|
||
|
|
"logps/chosen": -465.1143329326923,
|
||
|
|
"logps/rejected": -365.1838557545732,
|
||
|
|
"loss": 0.4249,
|
||
|
|
"loss/base_kto": 3.398885726928711,
|
||
|
|
"metrics/advantage_var": 1136.6014404296875,
|
||
|
|
"rewards/chosen": 0.5513690068171575,
|
||
|
|
"rewards/margins": 0.6965122831247985,
|
||
|
|
"rewards/rejected": -0.145143276307641,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 9.593290328979492,
|
||
|
|
"kl": 7.7284698486328125,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -39501343.44566929,
|
||
|
|
"logits/rejected": -39199355.03875969,
|
||
|
|
"logps/chosen": -503.3968996062992,
|
||
|
|
"logps/rejected": -388.6171511627907,
|
||
|
|
"loss": 0.4287,
|
||
|
|
"loss/base_kto": 3.4294159412384033,
|
||
|
|
"metrics/advantage_var": 1320.1468505859375,
|
||
|
|
"rewards/chosen": 0.357440521961122,
|
||
|
|
"rewards/margins": 0.7577194857450368,
|
||
|
|
"rewards/rejected": -0.40027896378391475,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 11.081856727600098,
|
||
|
|
"kl": 4.866374492645264,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -38675630.06431853,
|
||
|
|
"logits/rejected": -39611403.432216905,
|
||
|
|
"logps/chosen": -491.76311255742723,
|
||
|
|
"logps/rejected": -380.7640051834131,
|
||
|
|
"loss": 0.4353,
|
||
|
|
"loss/base_kto": 3.482758045196533,
|
||
|
|
"metrics/advantage_var": 2065.534423828125,
|
||
|
|
"rewards/chosen": -0.05095923804944726,
|
||
|
|
"rewards/margins": 0.79693024996391,
|
||
|
|
"rewards/rejected": -0.8478894880133573,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 7.145315170288086,
|
||
|
|
"kl": 42.45624923706055,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -40589290.1129771,
|
||
|
|
"logits/rejected": -39799858.7904,
|
||
|
|
"logps/chosen": -479.55500954198476,
|
||
|
|
"logps/rejected": -358.55725,
|
||
|
|
"loss": 0.4017,
|
||
|
|
"loss/base_kto": 3.213905096054077,
|
||
|
|
"metrics/advantage_var": 3067.258544921875,
|
||
|
|
"rewards/chosen": 1.443365897841126,
|
||
|
|
"rewards/margins": 1.0941520306536259,
|
||
|
|
"rewards/rejected": 0.3492138671875,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 9.371260643005371,
|
||
|
|
"kl": 19.413406372070312,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -39658741.00613497,
|
||
|
|
"logits/rejected": -41928746.39490446,
|
||
|
|
"logps/chosen": -479.15423696319016,
|
||
|
|
"logps/rejected": -389.13002587579615,
|
||
|
|
"loss": 0.4023,
|
||
|
|
"loss/base_kto": 3.2183144092559814,
|
||
|
|
"metrics/advantage_var": 3842.186767578125,
|
||
|
|
"rewards/chosen": 0.8319151708684815,
|
||
|
|
"rewards/margins": 1.3273419562828674,
|
||
|
|
"rewards/rejected": -0.49542678541438595,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 7.865825176239014,
|
||
|
|
"kl": 42.96455001831055,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -41660416.0,
|
||
|
|
"logits/rejected": -40298753.25285481,
|
||
|
|
"logps/chosen": -480.15606259370315,
|
||
|
|
"logps/rejected": -371.54175163132135,
|
||
|
|
"loss": 0.3747,
|
||
|
|
"loss/base_kto": 2.997523069381714,
|
||
|
|
"metrics/advantage_var": 4104.15380859375,
|
||
|
|
"rewards/chosen": 1.8185332382636807,
|
||
|
|
"rewards/margins": 1.4788250500340467,
|
||
|
|
"rewards/rejected": 0.33970818822963395,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 7.099153518676758,
|
||
|
|
"kl": 48.65797805786133,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -40652091.898658715,
|
||
|
|
"logits/rejected": -40906916.78160919,
|
||
|
|
"logps/chosen": -451.25363263785397,
|
||
|
|
"logps/rejected": -357.00413074712645,
|
||
|
|
"loss": 0.3747,
|
||
|
|
"loss/base_kto": 2.9979875087738037,
|
||
|
|
"metrics/advantage_var": 4341.8828125,
|
||
|
|
"rewards/chosen": 1.846562892953614,
|
||
|
|
"rewards/margins": 1.5223601283516794,
|
||
|
|
"rewards/rejected": 0.32420276460193453,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 6.9005632400512695,
|
||
|
|
"kl": 35.48698043823242,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -43210083.51335312,
|
||
|
|
"logits/rejected": -43863675.35313531,
|
||
|
|
"logps/chosen": -473.6956602373887,
|
||
|
|
"logps/rejected": -381.14686468646863,
|
||
|
|
"loss": 0.3735,
|
||
|
|
"loss/base_kto": 2.988220453262329,
|
||
|
|
"metrics/advantage_var": 4227.18359375,
|
||
|
|
"rewards/chosen": 1.754835360835497,
|
||
|
|
"rewards/margins": 1.5429234385946888,
|
||
|
|
"rewards/rejected": 0.21191192224080807,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 11.100954055786133,
|
||
|
|
"kl": 13.9872407913208,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -42800803.18381113,
|
||
|
|
"logits/rejected": -42655946.713245995,
|
||
|
|
"logps/chosen": -471.9735455311973,
|
||
|
|
"logps/rejected": -391.8405203784571,
|
||
|
|
"loss": 0.3627,
|
||
|
|
"loss/base_kto": 2.901885747909546,
|
||
|
|
"metrics/advantage_var": 5783.517578125,
|
||
|
|
"rewards/chosen": 0.8845758293370574,
|
||
|
|
"rewards/margins": 2.0644699782066898,
|
||
|
|
"rewards/rejected": -1.1798941488696324,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 6.646932125091553,
|
||
|
|
"kl": 10.82445240020752,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -41599086.3518225,
|
||
|
|
"logits/rejected": -41190625.62711865,
|
||
|
|
"logps/chosen": -460.8179477020602,
|
||
|
|
"logps/rejected": -390.94602272727275,
|
||
|
|
"loss": 0.3665,
|
||
|
|
"loss/base_kto": 2.9323196411132812,
|
||
|
|
"metrics/advantage_var": 5771.20068359375,
|
||
|
|
"rewards/chosen": 0.5926709545395453,
|
||
|
|
"rewards/margins": 2.086920859742261,
|
||
|
|
"rewards/rejected": -1.4942499052027156,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 8.745616912841797,
|
||
|
|
"kl": 12.242945671081543,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -41989474.093457945,
|
||
|
|
"logits/rejected": -41297611.836990595,
|
||
|
|
"logps/chosen": -471.8758761682243,
|
||
|
|
"logps/rejected": -389.6364615987461,
|
||
|
|
"loss": 0.3707,
|
||
|
|
"loss/base_kto": 2.965242862701416,
|
||
|
|
"metrics/advantage_var": 5933.3544921875,
|
||
|
|
"rewards/chosen": 0.8538461845611858,
|
||
|
|
"rewards/margins": 2.0028409412777317,
|
||
|
|
"rewards/rejected": -1.148994756716546,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 7.0273518562316895,
|
||
|
|
"kl": 19.097814559936523,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -42896259.92891761,
|
||
|
|
"logits/rejected": -44195567.34644478,
|
||
|
|
"logps/chosen": -496.05331179321485,
|
||
|
|
"logps/rejected": -374.2357696671709,
|
||
|
|
"loss": 0.3581,
|
||
|
|
"loss/base_kto": 2.864837646484375,
|
||
|
|
"metrics/advantage_var": 5337.912109375,
|
||
|
|
"rewards/chosen": 1.04482228613439,
|
||
|
|
"rewards/margins": 2.0973767874817804,
|
||
|
|
"rewards/rejected": -1.0525545013473903,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 5.808931827545166,
|
||
|
|
"kl": 9.683107376098633,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -43473398.80239521,
|
||
|
|
"logits/rejected": -44432296.99346405,
|
||
|
|
"logps/chosen": -459.32185628742513,
|
||
|
|
"logps/rejected": -388.02706290849676,
|
||
|
|
"loss": 0.373,
|
||
|
|
"loss/base_kto": 2.98384952545166,
|
||
|
|
"metrics/advantage_var": 5935.91357421875,
|
||
|
|
"rewards/chosen": 0.4494812559938716,
|
||
|
|
"rewards/margins": 1.9479031182283488,
|
||
|
|
"rewards/rejected": -1.4984218622344772,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 8.083940505981445,
|
||
|
|
"kl": 32.34806442260742,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -44761008.59450727,
|
||
|
|
"logits/rejected": -43924521.82753404,
|
||
|
|
"logps/chosen": -506.91735662358644,
|
||
|
|
"logps/rejected": -400.2647976550681,
|
||
|
|
"loss": 0.3519,
|
||
|
|
"loss/base_kto": 2.8153538703918457,
|
||
|
|
"metrics/advantage_var": 7049.96728515625,
|
||
|
|
"rewards/chosen": 1.8274850675863288,
|
||
|
|
"rewards/margins": 2.3638658065530933,
|
||
|
|
"rewards/rejected": -0.5363807389667644,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 7.935533046722412,
|
||
|
|
"kl": 11.448060989379883,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -41366528.0,
|
||
|
|
"logits/rejected": -42194198.70336392,
|
||
|
|
"logps/chosen": -462.01277955271564,
|
||
|
|
"logps/rejected": -379.04319571865443,
|
||
|
|
"loss": 0.3455,
|
||
|
|
"loss/base_kto": 2.763831377029419,
|
||
|
|
"metrics/advantage_var": 7012.91552734375,
|
||
|
|
"rewards/chosen": 0.9539188470322484,
|
||
|
|
"rewards/margins": 2.6009064354305664,
|
||
|
|
"rewards/rejected": -1.646987588398318,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 8.790483474731445,
|
||
|
|
"kl": 13.847442626953125,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -41813509.789983846,
|
||
|
|
"logits/rejected": -42020693.59152799,
|
||
|
|
"logps/chosen": -485.4153877221325,
|
||
|
|
"logps/rejected": -379.9791036308623,
|
||
|
|
"loss": 0.3311,
|
||
|
|
"loss/base_kto": 2.648519992828369,
|
||
|
|
"metrics/advantage_var": 7620.4296875,
|
||
|
|
"rewards/chosen": 1.1262229709902565,
|
||
|
|
"rewards/margins": 2.844358187222764,
|
||
|
|
"rewards/rejected": -1.7181352162325076,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 8.30214786529541,
|
||
|
|
"kl": 15.4894380569458,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -43492787.642072216,
|
||
|
|
"logits/rejected": -42438321.56765163,
|
||
|
|
"logps/chosen": -472.0441522762951,
|
||
|
|
"logps/rejected": -367.4388851088647,
|
||
|
|
"loss": 0.3532,
|
||
|
|
"loss/base_kto": 2.82547926902771,
|
||
|
|
"metrics/advantage_var": 7682.71875,
|
||
|
|
"rewards/chosen": 1.021260643304798,
|
||
|
|
"rewards/margins": 2.4276647024939018,
|
||
|
|
"rewards/rejected": -1.4064040591891038,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 8.737303733825684,
|
||
|
|
"kl": 27.308744430541992,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -44707931.02222222,
|
||
|
|
"logits/rejected": -43532987.470769234,
|
||
|
|
"logps/chosen": -465.1904761904762,
|
||
|
|
"logps/rejected": -398.69610576923077,
|
||
|
|
"loss": 0.3264,
|
||
|
|
"loss/base_kto": 2.6114189624786377,
|
||
|
|
"metrics/advantage_var": 8271.7685546875,
|
||
|
|
"rewards/chosen": 1.7188228546626985,
|
||
|
|
"rewards/margins": 2.914910876748035,
|
||
|
|
"rewards/rejected": -1.1960880220853365,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 7.870279788970947,
|
||
|
|
"kl": 13.653708457946777,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -46274614.9148265,
|
||
|
|
"logits/rejected": -45654735.653250776,
|
||
|
|
"logps/chosen": -470.99980283911674,
|
||
|
|
"logps/rejected": -372.77447755417955,
|
||
|
|
"loss": 0.3567,
|
||
|
|
"loss/base_kto": 2.853562831878662,
|
||
|
|
"metrics/advantage_var": 7688.44921875,
|
||
|
|
"rewards/chosen": 0.8195493680069499,
|
||
|
|
"rewards/margins": 2.347240999501242,
|
||
|
|
"rewards/rejected": -1.5276916314942919,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 9.080231666564941,
|
||
|
|
"kl": 19.786767959594727,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -43581364.14814815,
|
||
|
|
"logits/rejected": -43813042.2278481,
|
||
|
|
"logps/chosen": -442.10792824074076,
|
||
|
|
"logps/rejected": -387.9043710443038,
|
||
|
|
"loss": 0.3467,
|
||
|
|
"loss/base_kto": 2.7739083766937256,
|
||
|
|
"metrics/advantage_var": 9266.0595703125,
|
||
|
|
"rewards/chosen": 1.423888971776138,
|
||
|
|
"rewards/margins": 2.5969124370896273,
|
||
|
|
"rewards/rejected": -1.173023465313489,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 8.503353118896484,
|
||
|
|
"kl": 19.65529441833496,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -43856134.64427481,
|
||
|
|
"logits/rejected": -43512653.4144,
|
||
|
|
"logps/chosen": -452.02118320610685,
|
||
|
|
"logps/rejected": -370.3334,
|
||
|
|
"loss": 0.3258,
|
||
|
|
"loss/base_kto": 2.6065726280212402,
|
||
|
|
"metrics/advantage_var": 9236.5703125,
|
||
|
|
"rewards/chosen": 1.5968776091364503,
|
||
|
|
"rewards/margins": 3.0344336638239504,
|
||
|
|
"rewards/rejected": -1.4375560546875,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 6.571924686431885,
|
||
|
|
"kl": 33.894203186035156,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -46261658.5437788,
|
||
|
|
"logits/rejected": -45714837.3672496,
|
||
|
|
"logps/chosen": -473.17857142857144,
|
||
|
|
"logps/rejected": -392.7427464228935,
|
||
|
|
"loss": 0.341,
|
||
|
|
"loss/base_kto": 2.7280545234680176,
|
||
|
|
"metrics/advantage_var": 9399.193359375,
|
||
|
|
"rewards/chosen": 2.450667655229935,
|
||
|
|
"rewards/margins": 2.9155829201904377,
|
||
|
|
"rewards/rejected": -0.4649152649605028,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 9.13626766204834,
|
||
|
|
"kl": 20.319122314453125,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -45136019.85867895,
|
||
|
|
"logits/rejected": -46194492.64228935,
|
||
|
|
"logps/chosen": -452.5185291858679,
|
||
|
|
"logps/rejected": -397.5857015103339,
|
||
|
|
"loss": 0.3428,
|
||
|
|
"loss/base_kto": 2.7425696849823,
|
||
|
|
"metrics/advantage_var": 9139.390625,
|
||
|
|
"rewards/chosen": 1.4733294180827572,
|
||
|
|
"rewards/margins": 2.7737037023509408,
|
||
|
|
"rewards/rejected": -1.3003742842681836,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 8.005231857299805,
|
||
|
|
"kl": 21.759937286376953,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -46905621.8044164,
|
||
|
|
"logits/rejected": -46833293.07739938,
|
||
|
|
"logps/chosen": -450.3179219242902,
|
||
|
|
"logps/rejected": -391.17748645510835,
|
||
|
|
"loss": 0.3391,
|
||
|
|
"loss/base_kto": 2.71299147605896,
|
||
|
|
"metrics/advantage_var": 9798.2314453125,
|
||
|
|
"rewards/chosen": 1.6295716679810726,
|
||
|
|
"rewards/margins": 2.907357781141386,
|
||
|
|
"rewards/rejected": -1.2777861131603134,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 8.948205947875977,
|
||
|
|
"kl": 21.718387603759766,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -46783021.97869102,
|
||
|
|
"logits/rejected": -45139701.72712681,
|
||
|
|
"logps/chosen": -484.298848934551,
|
||
|
|
"logps/rejected": -372.73266954253614,
|
||
|
|
"loss": 0.3111,
|
||
|
|
"loss/base_kto": 2.489197015762329,
|
||
|
|
"metrics/advantage_var": 9011.4482421875,
|
||
|
|
"rewards/chosen": 1.8880607802392504,
|
||
|
|
"rewards/margins": 3.203403274658893,
|
||
|
|
"rewards/rejected": -1.3153424944196428,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 7.8195061683654785,
|
||
|
|
"kl": 30.360706329345703,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -46205451.61875945,
|
||
|
|
"logits/rejected": -45409423.92245557,
|
||
|
|
"logps/chosen": -478.77704236006053,
|
||
|
|
"logps/rejected": -396.2744850565428,
|
||
|
|
"loss": 0.3272,
|
||
|
|
"loss/base_kto": 2.6175248622894287,
|
||
|
|
"metrics/advantage_var": 10326.212890625,
|
||
|
|
"rewards/chosen": 2.0666980368286687,
|
||
|
|
"rewards/margins": 3.1215515840065766,
|
||
|
|
"rewards/rejected": -1.054853547177908,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 6.63793420791626,
|
||
|
|
"kl": 52.894161224365234,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -46940785.777777776,
|
||
|
|
"logits/rejected": -46880508.75949367,
|
||
|
|
"logps/chosen": -480.3076774691358,
|
||
|
|
"logps/rejected": -374.3922320015823,
|
||
|
|
"loss": 0.3066,
|
||
|
|
"loss/base_kto": 2.4528987407684326,
|
||
|
|
"metrics/advantage_var": 10011.7216796875,
|
||
|
|
"rewards/chosen": 2.773547137225116,
|
||
|
|
"rewards/margins": 3.3432198786925555,
|
||
|
|
"rewards/rejected": -0.5696727414674396,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 8.211414337158203,
|
||
|
|
"kl": 24.846866607666016,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -46749095.506172836,
|
||
|
|
"logits/rejected": -46511726.17721519,
|
||
|
|
"logps/chosen": -505.8790509259259,
|
||
|
|
"logps/rejected": -378.9453619462025,
|
||
|
|
"loss": 0.3305,
|
||
|
|
"loss/base_kto": 2.643768787384033,
|
||
|
|
"metrics/advantage_var": 11103.37109375,
|
||
|
|
"rewards/chosen": 2.026964352454668,
|
||
|
|
"rewards/margins": 3.399310482053165,
|
||
|
|
"rewards/rejected": -1.3723461295984969,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 10.186355590820312,
|
||
|
|
"kl": 35.13449478149414,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -43608122.05354331,
|
||
|
|
"logits/rejected": -44839264.446511626,
|
||
|
|
"logps/chosen": -456.92529527559054,
|
||
|
|
"logps/rejected": -393.90515988372096,
|
||
|
|
"loss": 0.3321,
|
||
|
|
"loss/base_kto": 2.657120943069458,
|
||
|
|
"metrics/advantage_var": 10315.302734375,
|
||
|
|
"rewards/chosen": 2.076808178518701,
|
||
|
|
"rewards/margins": 3.154547190298013,
|
||
|
|
"rewards/rejected": -1.077739011779312,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 8.631143569946289,
|
||
|
|
"kl": 36.79061508178711,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -45510366.791461416,
|
||
|
|
"logits/rejected": -46906311.535022356,
|
||
|
|
"logps/chosen": -460.31958128078816,
|
||
|
|
"logps/rejected": -377.0022820417288,
|
||
|
|
"loss": 0.3293,
|
||
|
|
"loss/base_kto": 2.634350299835205,
|
||
|
|
"metrics/advantage_var": 10736.0703125,
|
||
|
|
"rewards/chosen": 2.139907010083128,
|
||
|
|
"rewards/margins": 3.1054689919665113,
|
||
|
|
"rewards/rejected": -0.965561981883383,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 6.811507225036621,
|
||
|
|
"kl": 16.8293399810791,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -45059225.12149533,
|
||
|
|
"logits/rejected": -45106856.526645765,
|
||
|
|
"logps/chosen": -440.123734423676,
|
||
|
|
"logps/rejected": -401.2125048981191,
|
||
|
|
"loss": 0.3373,
|
||
|
|
"loss/base_kto": 2.69854474067688,
|
||
|
|
"metrics/advantage_var": 12008.3115234375,
|
||
|
|
"rewards/chosen": 1.570432193554079,
|
||
|
|
"rewards/margins": 3.2257186416829975,
|
||
|
|
"rewards/rejected": -1.6552864481289185,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 7.607090950012207,
|
||
|
|
"kl": 25.78407859802246,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -45679349.567398116,
|
||
|
|
"logits/rejected": -46405013.13395639,
|
||
|
|
"logps/chosen": -476.6412617554859,
|
||
|
|
"logps/rejected": -389.7828563084112,
|
||
|
|
"loss": 0.3237,
|
||
|
|
"loss/base_kto": 2.5895638465881348,
|
||
|
|
"metrics/advantage_var": 11293.404296875,
|
||
|
|
"rewards/chosen": 2.03739312151009,
|
||
|
|
"rewards/margins": 3.2593509848156783,
|
||
|
|
"rewards/rejected": -1.221957863305588,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 6.921399116516113,
|
||
|
|
"kl": 16.107458114624023,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -45514713.2976378,
|
||
|
|
"logits/rejected": -45708515.02635659,
|
||
|
|
"logps/chosen": -475.71624015748034,
|
||
|
|
"logps/rejected": -367.9354651162791,
|
||
|
|
"loss": 0.3122,
|
||
|
|
"loss/base_kto": 2.4977047443389893,
|
||
|
|
"metrics/advantage_var": 12347.6845703125,
|
||
|
|
"rewards/chosen": 1.3382462629183072,
|
||
|
|
"rewards/margins": 3.6617011744977646,
|
||
|
|
"rewards/rejected": -2.323454911579457,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 9.065701484680176,
|
||
|
|
"kl": 24.047327041625977,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -46841501.47758887,
|
||
|
|
"logits/rejected": -47183131.09636651,
|
||
|
|
"logps/chosen": -479.98551004636784,
|
||
|
|
"logps/rejected": -390.7399289099526,
|
||
|
|
"loss": 0.324,
|
||
|
|
"loss/base_kto": 2.5916531085968018,
|
||
|
|
"metrics/advantage_var": 12090.9658203125,
|
||
|
|
"rewards/chosen": 2.2090130530332304,
|
||
|
|
"rewards/margins": 3.4206983060138287,
|
||
|
|
"rewards/rejected": -1.2116852529805984,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 7.208115577697754,
|
||
|
|
"kl": 21.597766876220703,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -47840753.08737864,
|
||
|
|
"logits/rejected": -47484934.18731118,
|
||
|
|
"logps/chosen": -478.4880663430421,
|
||
|
|
"logps/rejected": -403.3761329305136,
|
||
|
|
"loss": 0.3316,
|
||
|
|
"loss/base_kto": 2.652467727661133,
|
||
|
|
"metrics/advantage_var": 13014.21875,
|
||
|
|
"rewards/chosen": 1.5764168057241101,
|
||
|
|
"rewards/margins": 3.3739137742498846,
|
||
|
|
"rewards/rejected": -1.7974969685257742,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 7.711939811706543,
|
||
|
|
"kl": 28.2313175201416,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -46367084.08888889,
|
||
|
|
"logits/rejected": -46945661.24307692,
|
||
|
|
"logps/chosen": -453.5967261904762,
|
||
|
|
"logps/rejected": -374.50927884615385,
|
||
|
|
"loss": 0.297,
|
||
|
|
"loss/base_kto": 2.3756587505340576,
|
||
|
|
"metrics/advantage_var": 13752.7392578125,
|
||
|
|
"rewards/chosen": 2.4194787403893847,
|
||
|
|
"rewards/margins": 4.005452673682654,
|
||
|
|
"rewards/rejected": -1.5859739332932692,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 7.007646560668945,
|
||
|
|
"kl": 30.02141761779785,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -45276892.37540453,
|
||
|
|
"logits/rejected": -45065901.76969697,
|
||
|
|
"logps/chosen": -446.30157766990294,
|
||
|
|
"logps/rejected": -402.1261837121212,
|
||
|
|
"loss": 0.3006,
|
||
|
|
"loss/base_kto": 2.4051828384399414,
|
||
|
|
"metrics/advantage_var": 13163.7060546875,
|
||
|
|
"rewards/chosen": 2.2396643184921117,
|
||
|
|
"rewards/margins": 4.016955356311714,
|
||
|
|
"rewards/rejected": -1.7772910378196023,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 8.173478126525879,
|
||
|
|
"kl": 17.41115379333496,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -46656193.6661562,
|
||
|
|
"logits/rejected": -47212380.68261563,
|
||
|
|
"logps/chosen": -445.1758231240429,
|
||
|
|
"logps/rejected": -404.7448664274322,
|
||
|
|
"loss": 0.279,
|
||
|
|
"loss/base_kto": 2.232347249984741,
|
||
|
|
"metrics/advantage_var": 14762.0546875,
|
||
|
|
"rewards/chosen": 2.299037495214395,
|
||
|
|
"rewards/margins": 4.4196192118461735,
|
||
|
|
"rewards/rejected": -2.1205817166317784,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 6.80936861038208,
|
||
|
|
"kl": 17.929452896118164,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -45475994.56603774,
|
||
|
|
"logits/rejected": -46844374.65838509,
|
||
|
|
"logps/chosen": -426.599597091195,
|
||
|
|
"logps/rejected": -415.8588412267081,
|
||
|
|
"loss": 0.2593,
|
||
|
|
"loss/base_kto": 2.0747039318084717,
|
||
|
|
"metrics/advantage_var": 14235.2373046875,
|
||
|
|
"rewards/chosen": 2.141762595506584,
|
||
|
|
"rewards/margins": 4.803825341163611,
|
||
|
|
"rewards/rejected": -2.6620627456570265,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 4.544503211975098,
|
||
|
|
"kl": 22.975421905517578,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -47021896.24796085,
|
||
|
|
"logits/rejected": -46036727.940029986,
|
||
|
|
"logps/chosen": -491.61449836867865,
|
||
|
|
"logps/rejected": -387.09075149925036,
|
||
|
|
"loss": 0.2567,
|
||
|
|
"loss/base_kto": 2.0539605617523193,
|
||
|
|
"metrics/advantage_var": 14759.0185546875,
|
||
|
|
"rewards/chosen": 2.3486837912928222,
|
||
|
|
"rewards/margins": 5.012761664465611,
|
||
|
|
"rewards/rejected": -2.6640778731727885,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 6.556363582611084,
|
||
|
|
"kl": 15.817468643188477,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -48574518.3151326,
|
||
|
|
"logits/rejected": -47992947.38028169,
|
||
|
|
"logps/chosen": -441.83502340093605,
|
||
|
|
"logps/rejected": -393.63113751956183,
|
||
|
|
"loss": 0.2591,
|
||
|
|
"loss/base_kto": 2.072817087173462,
|
||
|
|
"metrics/advantage_var": 13400.6279296875,
|
||
|
|
"rewards/chosen": 2.281471288148401,
|
||
|
|
"rewards/margins": 4.700414117214813,
|
||
|
|
"rewards/rejected": -2.4189428290664123,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 7.777688503265381,
|
||
|
|
"kl": 14.315707206726074,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -48459913.57557252,
|
||
|
|
"logits/rejected": -48805127.7824,
|
||
|
|
"logps/chosen": -440.69608778625957,
|
||
|
|
"logps/rejected": -391.97955,
|
||
|
|
"loss": 0.2579,
|
||
|
|
"loss/base_kto": 2.063169479370117,
|
||
|
|
"metrics/advantage_var": 14690.3310546875,
|
||
|
|
"rewards/chosen": 2.5303023243678435,
|
||
|
|
"rewards/margins": 5.0792437306178435,
|
||
|
|
"rewards/rejected": -2.54894140625,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 12.152023315429688,
|
||
|
|
"kl": 22.047971725463867,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -50287049.86102719,
|
||
|
|
"logits/rejected": -48711812.55663431,
|
||
|
|
"logps/chosen": -487.5616503021148,
|
||
|
|
"logps/rejected": -401.6143810679612,
|
||
|
|
"loss": 0.2559,
|
||
|
|
"loss/base_kto": 2.04754638671875,
|
||
|
|
"metrics/advantage_var": 13613.640625,
|
||
|
|
"rewards/chosen": 2.5347214436603096,
|
||
|
|
"rewards/margins": 4.730875794262049,
|
||
|
|
"rewards/rejected": -2.1961543506017396,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 6.6538896560668945,
|
||
|
|
"kl": 11.32634162902832,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -47334103.578947365,
|
||
|
|
"logits/rejected": -46812358.0952381,
|
||
|
|
"logps/chosen": -472.7476356907895,
|
||
|
|
"logps/rejected": -401.02167038690476,
|
||
|
|
"loss": 0.2566,
|
||
|
|
"loss/base_kto": 2.052682638168335,
|
||
|
|
"metrics/advantage_var": 13816.6689453125,
|
||
|
|
"rewards/chosen": 2.0561051619680306,
|
||
|
|
"rewards/margins": 4.973276821891766,
|
||
|
|
"rewards/rejected": -2.917171659923735,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 6.753261566162109,
|
||
|
|
"kl": 20.307422637939453,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -49337350.56410257,
|
||
|
|
"logits/rejected": -49350955.70731708,
|
||
|
|
"logps/chosen": -468.1454326923077,
|
||
|
|
"logps/rejected": -390.9633193597561,
|
||
|
|
"loss": 0.2608,
|
||
|
|
"loss/base_kto": 2.086202621459961,
|
||
|
|
"metrics/advantage_var": 13973.15625,
|
||
|
|
"rewards/chosen": 2.541408049754607,
|
||
|
|
"rewards/margins": 4.750835860051387,
|
||
|
|
"rewards/rejected": -2.2094278102967797,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 6.562922954559326,
|
||
|
|
"kl": 18.85896873474121,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -51021522.356589146,
|
||
|
|
"logits/rejected": -51042939.36377953,
|
||
|
|
"logps/chosen": -473.3996124031008,
|
||
|
|
"logps/rejected": -400.0395915354331,
|
||
|
|
"loss": 0.2626,
|
||
|
|
"loss/base_kto": 2.1009585857391357,
|
||
|
|
"metrics/advantage_var": 15106.8251953125,
|
||
|
|
"rewards/chosen": 2.31993777252907,
|
||
|
|
"rewards/margins": 4.723193493493637,
|
||
|
|
"rewards/rejected": -2.403255720964567,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 8.071813583374023,
|
||
|
|
"kl": 22.20347785949707,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -49203014.97819315,
|
||
|
|
"logits/rejected": -49981135.04702194,
|
||
|
|
"logps/chosen": -453.9005062305296,
|
||
|
|
"logps/rejected": -412.33821512539186,
|
||
|
|
"loss": 0.2573,
|
||
|
|
"loss/base_kto": 2.0587565898895264,
|
||
|
|
"metrics/advantage_var": 14139.6328125,
|
||
|
|
"rewards/chosen": 2.6057438835548092,
|
||
|
|
"rewards/margins": 4.811828801633768,
|
||
|
|
"rewards/rejected": -2.206084918078958,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 8.019198417663574,
|
||
|
|
"kl": 24.09480094909668,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -48533296.93522907,
|
||
|
|
"logits/rejected": -47720226.42349304,
|
||
|
|
"logps/chosen": -449.9792654028436,
|
||
|
|
"logps/rejected": -422.0738021638331,
|
||
|
|
"loss": 0.2723,
|
||
|
|
"loss/base_kto": 2.1783738136291504,
|
||
|
|
"metrics/advantage_var": 13183.6513671875,
|
||
|
|
"rewards/chosen": 2.459912340787915,
|
||
|
|
"rewards/margins": 4.491786842695759,
|
||
|
|
"rewards/rejected": -2.031874501907844,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 8.52418041229248,
|
||
|
|
"kl": 16.32106590270996,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -49187318.80239521,
|
||
|
|
"logits/rejected": -48029050.14379085,
|
||
|
|
"logps/chosen": -460.259309505988,
|
||
|
|
"logps/rejected": -408.9749285130719,
|
||
|
|
"loss": 0.2512,
|
||
|
|
"loss/base_kto": 2.009221315383911,
|
||
|
|
"metrics/advantage_var": 14127.5673828125,
|
||
|
|
"rewards/chosen": 2.2250684178517965,
|
||
|
|
"rewards/margins": 4.883991997049101,
|
||
|
|
"rewards/rejected": -2.658923579197304,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 5.944615840911865,
|
||
|
|
"kl": 19.7237548828125,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -48926827.78947368,
|
||
|
|
"logits/rejected": -48995842.49756098,
|
||
|
|
"logps/chosen": -488.859022556391,
|
||
|
|
"logps/rejected": -380.9029471544715,
|
||
|
|
"loss": 0.2604,
|
||
|
|
"loss/base_kto": 2.0830843448638916,
|
||
|
|
"metrics/advantage_var": 12477.3466796875,
|
||
|
|
"rewards/chosen": 2.6787015757166355,
|
||
|
|
"rewards/margins": 4.541950670609928,
|
||
|
|
"rewards/rejected": -1.8632490948932927,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 8.016765594482422,
|
||
|
|
"kl": 26.5404052734375,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -49199443.15923567,
|
||
|
|
"logits/rejected": -48270034.45398773,
|
||
|
|
"logps/chosen": -470.86912818471336,
|
||
|
|
"logps/rejected": -393.27779907975463,
|
||
|
|
"loss": 0.2685,
|
||
|
|
"loss/base_kto": 2.1477668285369873,
|
||
|
|
"metrics/advantage_var": 14028.2255859375,
|
||
|
|
"rewards/chosen": 2.6989994899482483,
|
||
|
|
"rewards/margins": 4.560636347993206,
|
||
|
|
"rewards/rejected": -1.8616368580449578,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 7.640358924865723,
|
||
|
|
"kl": 22.410078048706055,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -50904264.24165342,
|
||
|
|
"logits/rejected": -50952012.68202765,
|
||
|
|
"logps/chosen": -484.72043918918916,
|
||
|
|
"logps/rejected": -387.44239631336404,
|
||
|
|
"loss": 0.2606,
|
||
|
|
"loss/base_kto": 2.0846333503723145,
|
||
|
|
"metrics/advantage_var": 15510.6748046875,
|
||
|
|
"rewards/chosen": 2.4037286367249604,
|
||
|
|
"rewards/margins": 4.797877362243394,
|
||
|
|
"rewards/rejected": -2.3941487255184333,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 6.453657627105713,
|
||
|
|
"kl": 26.574853897094727,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -48059187.83800623,
|
||
|
|
"logits/rejected": -48764972.940438874,
|
||
|
|
"logps/chosen": -447.25292056074767,
|
||
|
|
"logps/rejected": -395.70219435736675,
|
||
|
|
"loss": 0.245,
|
||
|
|
"loss/base_kto": 1.9602657556533813,
|
||
|
|
"metrics/advantage_var": 13271.8349609375,
|
||
|
|
"rewards/chosen": 2.6893097589320485,
|
||
|
|
"rewards/margins": 4.852744687419509,
|
||
|
|
"rewards/rejected": -2.1634349284874608,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 11.55324649810791,
|
||
|
|
"kl": 17.889604568481445,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -48340237.647819065,
|
||
|
|
"logits/rejected": -48361713.670196675,
|
||
|
|
"logps/chosen": -455.0253937802908,
|
||
|
|
"logps/rejected": -400.8607696671709,
|
||
|
|
"loss": 0.2562,
|
||
|
|
"loss/base_kto": 2.049893856048584,
|
||
|
|
"metrics/advantage_var": 12405.05078125,
|
||
|
|
"rewards/chosen": 1.9586449840342286,
|
||
|
|
"rewards/margins": 4.61523220096407,
|
||
|
|
"rewards/rejected": -2.6565872169298412,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 7.1196184158325195,
|
||
|
|
"kl": 20.197782516479492,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -47066458.75993641,
|
||
|
|
"logits/rejected": -47454664.159754224,
|
||
|
|
"logps/chosen": -467.860393481717,
|
||
|
|
"logps/rejected": -399.22633448540705,
|
||
|
|
"loss": 0.244,
|
||
|
|
"loss/base_kto": 1.9522743225097656,
|
||
|
|
"metrics/advantage_var": 12976.525390625,
|
||
|
|
"rewards/chosen": 2.20633880663752,
|
||
|
|
"rewards/margins": 4.915033288084333,
|
||
|
|
"rewards/rejected": -2.7086944814468126,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 8.951573371887207,
|
||
|
|
"kl": 20.724227905273438,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -47757560.14216478,
|
||
|
|
"logits/rejected": -48288456.6172466,
|
||
|
|
"logps/chosen": -464.625,
|
||
|
|
"logps/rejected": -395.11682110438727,
|
||
|
|
"loss": 0.2513,
|
||
|
|
"loss/base_kto": 2.010521411895752,
|
||
|
|
"metrics/advantage_var": 13302.541015625,
|
||
|
|
"rewards/chosen": 2.286919663141155,
|
||
|
|
"rewards/margins": 4.8331484845883,
|
||
|
|
"rewards/rejected": -2.5462288214471447,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 9.4291353225708,
|
||
|
|
"kl": 22.72542381286621,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -47091389.32475884,
|
||
|
|
"logits/rejected": -47570663.87841945,
|
||
|
|
"logps/chosen": -478.8822347266881,
|
||
|
|
"logps/rejected": -365.6098736702128,
|
||
|
|
"loss": 0.239,
|
||
|
|
"loss/base_kto": 1.9119118452072144,
|
||
|
|
"metrics/advantage_var": 12811.125,
|
||
|
|
"rewards/chosen": 2.425092004119775,
|
||
|
|
"rewards/margins": 4.811326925282389,
|
||
|
|
"rewards/rejected": -2.386234921162614,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 6.64006233215332,
|
||
|
|
"kl": 19.150110244750977,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -47568567.346210994,
|
||
|
|
"logits/rejected": -49767010.68863262,
|
||
|
|
"logps/chosen": -483.8380850668648,
|
||
|
|
"logps/rejected": -384.9492637973641,
|
||
|
|
"loss": 0.2549,
|
||
|
|
"loss/base_kto": 2.039292812347412,
|
||
|
|
"metrics/advantage_var": 14007.6123046875,
|
||
|
|
"rewards/chosen": 2.5177968720978825,
|
||
|
|
"rewards/margins": 4.919014035787133,
|
||
|
|
"rewards/rejected": -2.4012171636892505,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 8.059784889221191,
|
||
|
|
"kl": 21.792648315429688,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -47696774.7788162,
|
||
|
|
"logits/rejected": -48640969.42946708,
|
||
|
|
"logps/chosen": -445.69314641744546,
|
||
|
|
"logps/rejected": -396.8174471003135,
|
||
|
|
"loss": 0.2522,
|
||
|
|
"loss/base_kto": 2.0174732208251953,
|
||
|
|
"metrics/advantage_var": 12939.958984375,
|
||
|
|
"rewards/chosen": 2.2641485576689058,
|
||
|
|
"rewards/margins": 4.6352878448701205,
|
||
|
|
"rewards/rejected": -2.3711392872012147,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 8.086423873901367,
|
||
|
|
"kl": 20.496225357055664,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -47647981.307936504,
|
||
|
|
"logits/rejected": -49093603.64307693,
|
||
|
|
"logps/chosen": -490.843501984127,
|
||
|
|
"logps/rejected": -401.04983173076926,
|
||
|
|
"loss": 0.2477,
|
||
|
|
"loss/base_kto": 1.9819554090499878,
|
||
|
|
"metrics/advantage_var": 13979.4912109375,
|
||
|
|
"rewards/chosen": 2.251154048859127,
|
||
|
|
"rewards/margins": 5.037465084015377,
|
||
|
|
"rewards/rejected": -2.78631103515625,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 8.792497634887695,
|
||
|
|
"kl": 30.228851318359375,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -47447484.43879173,
|
||
|
|
"logits/rejected": -48123139.5391705,
|
||
|
|
"logps/chosen": -460.79113672496027,
|
||
|
|
"logps/rejected": -372.7542722734255,
|
||
|
|
"loss": 0.2564,
|
||
|
|
"loss/base_kto": 2.0514838695526123,
|
||
|
|
"metrics/advantage_var": 13797.9755859375,
|
||
|
|
"rewards/chosen": 2.6598088638960653,
|
||
|
|
"rewards/margins": 4.687813406186772,
|
||
|
|
"rewards/rejected": -2.0280045422907067,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 7.578653335571289,
|
||
|
|
"kl": 25.090009689331055,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -49122948.62166405,
|
||
|
|
"logits/rejected": -49207139.93157076,
|
||
|
|
"logps/chosen": -454.2995486656201,
|
||
|
|
"logps/rejected": -390.2356629082426,
|
||
|
|
"loss": 0.2587,
|
||
|
|
"loss/base_kto": 2.069631576538086,
|
||
|
|
"metrics/advantage_var": 13317.5,
|
||
|
|
"rewards/chosen": 2.299419888147567,
|
||
|
|
"rewards/margins": 4.770764689129294,
|
||
|
|
"rewards/rejected": -2.4713448009817265,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 14.963940620422363,
|
||
|
|
"kl": 17.4893856048584,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -47609887.65378671,
|
||
|
|
"logits/rejected": -47908110.154818326,
|
||
|
|
"logps/chosen": -450.29921754250387,
|
||
|
|
"logps/rejected": -389.2090244865719,
|
||
|
|
"loss": 0.2503,
|
||
|
|
"loss/base_kto": 2.002574920654297,
|
||
|
|
"metrics/advantage_var": 13172.140625,
|
||
|
|
"rewards/chosen": 2.4579678141904946,
|
||
|
|
"rewards/margins": 4.867302355499243,
|
||
|
|
"rewards/rejected": -2.409334541308748,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 9.611598014831543,
|
||
|
|
"kl": 19.83905601501465,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -49030802.28571428,
|
||
|
|
"logits/rejected": -49205513.253012046,
|
||
|
|
"logps/chosen": -464.34293831168833,
|
||
|
|
"logps/rejected": -408.77179028614455,
|
||
|
|
"loss": 0.2459,
|
||
|
|
"loss/base_kto": 1.9675463438034058,
|
||
|
|
"metrics/advantage_var": 13988.8271484375,
|
||
|
|
"rewards/chosen": 2.4432099577668427,
|
||
|
|
"rewards/margins": 4.986179626971003,
|
||
|
|
"rewards/rejected": -2.5429696692041603,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 7.579370975494385,
|
||
|
|
"kl": 14.656468391418457,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -49270376.28177641,
|
||
|
|
"logits/rejected": -49555152.22966507,
|
||
|
|
"logps/chosen": -466.990141653905,
|
||
|
|
"logps/rejected": -396.08756977671453,
|
||
|
|
"loss": 0.2616,
|
||
|
|
"loss/base_kto": 2.0930798053741455,
|
||
|
|
"metrics/advantage_var": 12933.798828125,
|
||
|
|
"rewards/chosen": 2.0334977388016844,
|
||
|
|
"rewards/margins": 4.576472913554974,
|
||
|
|
"rewards/rejected": -2.5429751747532894,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 6.426728248596191,
|
||
|
|
"kl": 21.355215072631836,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -48487235.53374233,
|
||
|
|
"logits/rejected": -48920758.62420382,
|
||
|
|
"logps/chosen": -463.2795245398773,
|
||
|
|
"logps/rejected": -401.8110569267516,
|
||
|
|
"loss": 0.2608,
|
||
|
|
"loss/base_kto": 2.086047410964966,
|
||
|
|
"metrics/advantage_var": 13996.095703125,
|
||
|
|
"rewards/chosen": 2.549413762940951,
|
||
|
|
"rewards/margins": 4.723553283865513,
|
||
|
|
"rewards/rejected": -2.174139520924562,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 8.00083065032959,
|
||
|
|
"kl": 29.541763305664062,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -49463453.538461536,
|
||
|
|
"logits/rejected": -49750379.09797823,
|
||
|
|
"logps/chosen": -454.42327315541604,
|
||
|
|
"logps/rejected": -412.99912519440124,
|
||
|
|
"loss": 0.2473,
|
||
|
|
"loss/base_kto": 1.9786356687545776,
|
||
|
|
"metrics/advantage_var": 13827.681640625,
|
||
|
|
"rewards/chosen": 2.412167631475667,
|
||
|
|
"rewards/margins": 4.759135503632549,
|
||
|
|
"rewards/rejected": -2.3469678721568816,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 8.693033218383789,
|
||
|
|
"kl": 15.414921760559082,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -49728912.02413273,
|
||
|
|
"logits/rejected": -49519315.60453809,
|
||
|
|
"logps/chosen": -455.81061463046757,
|
||
|
|
"logps/rejected": -374.9399311183144,
|
||
|
|
"loss": 0.2398,
|
||
|
|
"loss/base_kto": 1.9187382459640503,
|
||
|
|
"metrics/advantage_var": 14239.2685546875,
|
||
|
|
"rewards/chosen": 2.254514392145079,
|
||
|
|
"rewards/margins": 5.03609360608349,
|
||
|
|
"rewards/rejected": -2.7815792139384117,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 6.900964736938477,
|
||
|
|
"kl": 20.708576202392578,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -50192860.88676236,
|
||
|
|
"logits/rejected": -49531403.7611026,
|
||
|
|
"logps/chosen": -466.3460925039872,
|
||
|
|
"logps/rejected": -379.18082408116385,
|
||
|
|
"loss": 0.2688,
|
||
|
|
"loss/base_kto": 2.150141716003418,
|
||
|
|
"metrics/advantage_var": 13455.515625,
|
||
|
|
"rewards/chosen": 2.2594463342304625,
|
||
|
|
"rewards/margins": 4.589688216648151,
|
||
|
|
"rewards/rejected": -2.330241882417688,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 8.05775260925293,
|
||
|
|
"kl": 35.416603088378906,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -47505559.004608296,
|
||
|
|
"logits/rejected": -47748973.48171701,
|
||
|
|
"logps/chosen": -449.55587557603684,
|
||
|
|
"logps/rejected": -367.85159976152624,
|
||
|
|
"loss": 0.2338,
|
||
|
|
"loss/base_kto": 1.8703714609146118,
|
||
|
|
"metrics/advantage_var": 12229.8173828125,
|
||
|
|
"rewards/chosen": 3.1599852390552994,
|
||
|
|
"rewards/margins": 5.009129039047847,
|
||
|
|
"rewards/rejected": -1.8491437999925477,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 10.813697814941406,
|
||
|
|
"kl": 20.462011337280273,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -49673536.19814242,
|
||
|
|
"logits/rejected": -50293795.53312303,
|
||
|
|
"logps/chosen": -452.44615905572755,
|
||
|
|
"logps/rejected": -416.8113170347003,
|
||
|
|
"loss": 0.2643,
|
||
|
|
"loss/base_kto": 2.1144533157348633,
|
||
|
|
"metrics/advantage_var": 14252.142578125,
|
||
|
|
"rewards/chosen": 2.1609387471579913,
|
||
|
|
"rewards/margins": 4.722467575775894,
|
||
|
|
"rewards/rejected": -2.5615288286179023,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 7.989013195037842,
|
||
|
|
"kl": 17.5428466796875,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -50056393.64923077,
|
||
|
|
"logits/rejected": -49990984.33015873,
|
||
|
|
"logps/chosen": -474.66769230769233,
|
||
|
|
"logps/rejected": -413.96671626984124,
|
||
|
|
"loss": 0.2435,
|
||
|
|
"loss/base_kto": 1.9483126401901245,
|
||
|
|
"metrics/advantage_var": 14465.009765625,
|
||
|
|
"rewards/chosen": 2.5189391150841347,
|
||
|
|
"rewards/margins": 5.101884536091079,
|
||
|
|
"rewards/rejected": -2.5829454210069445,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 8.70841121673584,
|
||
|
|
"kl": 32.39929962158203,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -50555225.681528665,
|
||
|
|
"logits/rejected": -52477776.09815951,
|
||
|
|
"logps/chosen": -451.25084593949043,
|
||
|
|
"logps/rejected": -388.8933569785276,
|
||
|
|
"loss": 0.2402,
|
||
|
|
"loss/base_kto": 1.921789526939392,
|
||
|
|
"metrics/advantage_var": 12992.45703125,
|
||
|
|
"rewards/chosen": 2.8987142963773884,
|
||
|
|
"rewards/margins": 4.943832757003348,
|
||
|
|
"rewards/rejected": -2.0451184606259587,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 8.169147491455078,
|
||
|
|
"kl": 25.643451690673828,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -49150005.58139535,
|
||
|
|
"logits/rejected": -49291436.972972974,
|
||
|
|
"logps/chosen": -454.4925054505814,
|
||
|
|
"logps/rejected": -391.3290223817568,
|
||
|
|
"loss": 0.2583,
|
||
|
|
"loss/base_kto": 2.0667595863342285,
|
||
|
|
"metrics/advantage_var": 12927.4765625,
|
||
|
|
"rewards/chosen": 2.863457968068677,
|
||
|
|
"rewards/margins": 4.700496162879699,
|
||
|
|
"rewards/rejected": -1.837038194811022,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 7.993652820587158,
|
||
|
|
"kl": 25.68218994140625,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -51835381.418124005,
|
||
|
|
"logits/rejected": -51349979.03533026,
|
||
|
|
"logps/chosen": -465.550079491256,
|
||
|
|
"logps/rejected": -408.51060867895546,
|
||
|
|
"loss": 0.2639,
|
||
|
|
"loss/base_kto": 2.1111161708831787,
|
||
|
|
"metrics/advantage_var": 14188.736328125,
|
||
|
|
"rewards/chosen": 2.582842464477345,
|
||
|
|
"rewards/margins": 4.783263541426269,
|
||
|
|
"rewards/rejected": -2.2004210769489245,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 7.127334117889404,
|
||
|
|
"kl": 16.307735443115234,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -49595238.06535948,
|
||
|
|
"logits/rejected": -49896564.85285285,
|
||
|
|
"logps/chosen": -468.554891748366,
|
||
|
|
"logps/rejected": -403.67328265765764,
|
||
|
|
"loss": 0.2215,
|
||
|
|
"loss/base_kto": 1.7723366022109985,
|
||
|
|
"metrics/advantage_var": 15443.9345703125,
|
||
|
|
"rewards/chosen": 2.419900931564032,
|
||
|
|
"rewards/margins": 5.460144910894925,
|
||
|
|
"rewards/rejected": -3.0402439793308935,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 6.784162521362305,
|
||
|
|
"kl": 10.496627807617188,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -49782327.20827179,
|
||
|
|
"logits/rejected": -49491845.731343284,
|
||
|
|
"logps/chosen": -436.35293574593794,
|
||
|
|
"logps/rejected": -407.7500777363184,
|
||
|
|
"loss": 0.212,
|
||
|
|
"loss/base_kto": 1.6957463026046753,
|
||
|
|
"metrics/advantage_var": 14359.087890625,
|
||
|
|
"rewards/chosen": 2.7411344857828657,
|
||
|
|
"rewards/margins": 5.641210521619309,
|
||
|
|
"rewards/rejected": -2.9000760358364426,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 7.4223833084106445,
|
||
|
|
"kl": 18.540119171142578,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -49057408.80503145,
|
||
|
|
"logits/rejected": -50834272.99378882,
|
||
|
|
"logps/chosen": -478.2899960691824,
|
||
|
|
"logps/rejected": -385.6640139751553,
|
||
|
|
"loss": 0.2166,
|
||
|
|
"loss/base_kto": 1.7324780225753784,
|
||
|
|
"metrics/advantage_var": 14026.9814453125,
|
||
|
|
"rewards/chosen": 2.643333003206073,
|
||
|
|
"rewards/margins": 5.287066626070979,
|
||
|
|
"rewards/rejected": -2.643733622864907,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 7.456380844116211,
|
||
|
|
"kl": 19.18268394470215,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -50089974.59111792,
|
||
|
|
"logits/rejected": -51495983.362041466,
|
||
|
|
"logps/chosen": -460.1427067381317,
|
||
|
|
"logps/rejected": -408.0125598086124,
|
||
|
|
"loss": 0.2258,
|
||
|
|
"loss/base_kto": 1.8063572645187378,
|
||
|
|
"metrics/advantage_var": 15073.0185546875,
|
||
|
|
"rewards/chosen": 2.8757813995501533,
|
||
|
|
"rewards/margins": 5.380927821624516,
|
||
|
|
"rewards/rejected": -2.505146422074362,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 7.515079498291016,
|
||
|
|
"kl": 15.434982299804688,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -50601090.21036107,
|
||
|
|
"logits/rejected": -49526793.55520996,
|
||
|
|
"logps/chosen": -463.3057299843014,
|
||
|
|
"logps/rejected": -399.95956454121307,
|
||
|
|
"loss": 0.2194,
|
||
|
|
"loss/base_kto": 1.7549737691879272,
|
||
|
|
"metrics/advantage_var": 13595.9375,
|
||
|
|
"rewards/chosen": 2.455097863213795,
|
||
|
|
"rewards/margins": 5.31824090607927,
|
||
|
|
"rewards/rejected": -2.8631430428654743,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 6.709864616394043,
|
||
|
|
"kl": 11.128249168395996,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -50074304.83116883,
|
||
|
|
"logits/rejected": -49689544.48192771,
|
||
|
|
"logps/chosen": -454.7149959415584,
|
||
|
|
"logps/rejected": -400.812311746988,
|
||
|
|
"loss": 0.2063,
|
||
|
|
"loss/base_kto": 1.6505447626113892,
|
||
|
|
"metrics/advantage_var": 14310.376953125,
|
||
|
|
"rewards/chosen": 2.2841396579494724,
|
||
|
|
"rewards/margins": 5.6345535204306465,
|
||
|
|
"rewards/rejected": -3.3504138624811746,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 6.725008010864258,
|
||
|
|
"kl": 10.414727210998535,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -49975073.39130435,
|
||
|
|
"logits/rejected": -51824623.89937107,
|
||
|
|
"logps/chosen": -471.26674107142856,
|
||
|
|
"logps/rejected": -390.125393081761,
|
||
|
|
"loss": 0.2155,
|
||
|
|
"loss/base_kto": 1.723760962486267,
|
||
|
|
"metrics/advantage_var": 14411.134765625,
|
||
|
|
"rewards/chosen": 2.309495629731172,
|
||
|
|
"rewards/margins": 5.520320272272446,
|
||
|
|
"rewards/rejected": -3.2108246425412736,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 6.696305751800537,
|
||
|
|
"kl": 19.083173751831055,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -51039629.06122449,
|
||
|
|
"logits/rejected": -52006752.746500775,
|
||
|
|
"logps/chosen": -500.1241660125589,
|
||
|
|
"logps/rejected": -397.9802196734059,
|
||
|
|
"loss": 0.1986,
|
||
|
|
"loss/base_kto": 1.5888489484786987,
|
||
|
|
"metrics/advantage_var": 15714.0361328125,
|
||
|
|
"rewards/chosen": 3.0086947789933283,
|
||
|
|
"rewards/margins": 5.956221630665179,
|
||
|
|
"rewards/rejected": -2.9475268516718507,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 7.990821838378906,
|
||
|
|
"kl": 16.3265323638916,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -50054883.46870229,
|
||
|
|
"logits/rejected": -51335384.2688,
|
||
|
|
"logps/chosen": -452.41808206106873,
|
||
|
|
"logps/rejected": -417.467625,
|
||
|
|
"loss": 0.2201,
|
||
|
|
"loss/base_kto": 1.7606159448623657,
|
||
|
|
"metrics/advantage_var": 15616.5,
|
||
|
|
"rewards/chosen": 2.714349505009542,
|
||
|
|
"rewards/margins": 5.649598333134542,
|
||
|
|
"rewards/rejected": -2.935248828125,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 6.282646656036377,
|
||
|
|
"kl": 14.768630981445312,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -50328071.035114504,
|
||
|
|
"logits/rejected": -50948728.4224,
|
||
|
|
"logps/chosen": -472.78134541984736,
|
||
|
|
"logps/rejected": -390.61135,
|
||
|
|
"loss": 0.2196,
|
||
|
|
"loss/base_kto": 1.7569602727890015,
|
||
|
|
"metrics/advantage_var": 14497.3505859375,
|
||
|
|
"rewards/chosen": 2.513952170801527,
|
||
|
|
"rewards/margins": 5.425104905176527,
|
||
|
|
"rewards/rejected": -2.911152734375,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 7.860062599182129,
|
||
|
|
"kl": 18.362916946411133,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -49646857.01684533,
|
||
|
|
"logits/rejected": -49474414.64752791,
|
||
|
|
"logps/chosen": -423.1053790199081,
|
||
|
|
"logps/rejected": -397.5823116028708,
|
||
|
|
"loss": 0.2189,
|
||
|
|
"loss/base_kto": 1.750977873802185,
|
||
|
|
"metrics/advantage_var": 15536.078125,
|
||
|
|
"rewards/chosen": 2.5981626642060682,
|
||
|
|
"rewards/margins": 5.376900780359378,
|
||
|
|
"rewards/rejected": -2.7787381161533093,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 8.946330070495605,
|
||
|
|
"kl": 24.330366134643555,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -51386275.489096574,
|
||
|
|
"logits/rejected": -50967632.2507837,
|
||
|
|
"logps/chosen": -464.68092873831773,
|
||
|
|
"logps/rejected": -414.86162813479626,
|
||
|
|
"loss": 0.203,
|
||
|
|
"loss/base_kto": 1.6239101886749268,
|
||
|
|
"metrics/advantage_var": 13769.6953125,
|
||
|
|
"rewards/chosen": 2.8734166986103,
|
||
|
|
"rewards/margins": 5.41564441676028,
|
||
|
|
"rewards/rejected": -2.5422277181499804,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 7.575611114501953,
|
||
|
|
"kl": 14.73845386505127,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -52410310.66874028,
|
||
|
|
"logits/rejected": -50013941.14913658,
|
||
|
|
"logps/chosen": -467.96491057542767,
|
||
|
|
"logps/rejected": -395.2499754709576,
|
||
|
|
"loss": 0.2125,
|
||
|
|
"loss/base_kto": 1.699928879737854,
|
||
|
|
"metrics/advantage_var": 15204.1005859375,
|
||
|
|
"rewards/chosen": 2.6341455913442844,
|
||
|
|
"rewards/margins": 5.696759613064751,
|
||
|
|
"rewards/rejected": -3.062614021720467,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 8.54968547821045,
|
||
|
|
"kl": 20.515256881713867,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -50502933.50237718,
|
||
|
|
"logits/rejected": -51343561.95993837,
|
||
|
|
"logps/chosen": -443.70106973058637,
|
||
|
|
"logps/rejected": -378.1875481510015,
|
||
|
|
"loss": 0.2262,
|
||
|
|
"loss/base_kto": 1.8092983961105347,
|
||
|
|
"metrics/advantage_var": 14298.4443359375,
|
||
|
|
"rewards/chosen": 2.5944340580427894,
|
||
|
|
"rewards/margins": 5.181517965677131,
|
||
|
|
"rewards/rejected": -2.5870839076343413,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 8.126795768737793,
|
||
|
|
"kl": 19.532196044921875,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -51061946.03475513,
|
||
|
|
"logits/rejected": -51500096.89026275,
|
||
|
|
"logps/chosen": -433.7949249605055,
|
||
|
|
"logps/rejected": -422.66653786707883,
|
||
|
|
"loss": 0.2046,
|
||
|
|
"loss/base_kto": 1.637192964553833,
|
||
|
|
"metrics/advantage_var": 14092.0693359375,
|
||
|
|
"rewards/chosen": 2.652627616508689,
|
||
|
|
"rewards/margins": 5.654991478866204,
|
||
|
|
"rewards/rejected": -3.0023638623575155,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 5.897149085998535,
|
||
|
|
"kl": 21.163227081298828,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -51688446.40746501,
|
||
|
|
"logits/rejected": -51002498.21036107,
|
||
|
|
"logps/chosen": -450.73950233281494,
|
||
|
|
"logps/rejected": -417.90124607535324,
|
||
|
|
"loss": 0.2114,
|
||
|
|
"loss/base_kto": 1.6915730237960815,
|
||
|
|
"metrics/advantage_var": 13375.6953125,
|
||
|
|
"rewards/chosen": 2.561840858281493,
|
||
|
|
"rewards/margins": 5.410429595158456,
|
||
|
|
"rewards/rejected": -2.8485887368769625,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 8.426025390625,
|
||
|
|
"kl": 22.875829696655273,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -47735147.663551405,
|
||
|
|
"logits/rejected": -50488265.42946708,
|
||
|
|
"logps/chosen": -445.261292834891,
|
||
|
|
"logps/rejected": -409.0168740203762,
|
||
|
|
"loss": 0.2198,
|
||
|
|
"loss/base_kto": 1.7581509351730347,
|
||
|
|
"metrics/advantage_var": 13349.6689453125,
|
||
|
|
"rewards/chosen": 2.66365179614486,
|
||
|
|
"rewards/margins": 5.254863766841862,
|
||
|
|
"rewards/rejected": -2.5912119706970023,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 7.257813930511475,
|
||
|
|
"kl": 19.541242599487305,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -49885532.455858745,
|
||
|
|
"logits/rejected": -50699527.40334856,
|
||
|
|
"logps/chosen": -451.54303772070625,
|
||
|
|
"logps/rejected": -408.70933219178085,
|
||
|
|
"loss": 0.2043,
|
||
|
|
"loss/base_kto": 1.6344120502471924,
|
||
|
|
"metrics/advantage_var": 13755.4189453125,
|
||
|
|
"rewards/chosen": 2.823837647660012,
|
||
|
|
"rewards/margins": 5.598703946294906,
|
||
|
|
"rewards/rejected": -2.7748662986348935,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 7.962888240814209,
|
||
|
|
"kl": 10.586474418640137,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -51680913.762517884,
|
||
|
|
"logits/rejected": -51276505.66609294,
|
||
|
|
"logps/chosen": -461.40334406294704,
|
||
|
|
"logps/rejected": -414.9785660499139,
|
||
|
|
"loss": 0.214,
|
||
|
|
"loss/base_kto": 1.7116488218307495,
|
||
|
|
"metrics/advantage_var": 14825.0048828125,
|
||
|
|
"rewards/chosen": 2.555452404104077,
|
||
|
|
"rewards/margins": 5.493936950548784,
|
||
|
|
"rewards/rejected": -2.9384845464447076,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 7.1761298179626465,
|
||
|
|
"kl": 15.605143547058105,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -51403234.077922076,
|
||
|
|
"logits/rejected": -51802593.15662651,
|
||
|
|
"logps/chosen": -507.03059050324674,
|
||
|
|
"logps/rejected": -390.7486351656627,
|
||
|
|
"loss": 0.1924,
|
||
|
|
"loss/base_kto": 1.5393751859664917,
|
||
|
|
"metrics/advantage_var": 15480.7099609375,
|
||
|
|
"rewards/chosen": 2.8869965788605927,
|
||
|
|
"rewards/margins": 6.081190591238228,
|
||
|
|
"rewards/rejected": -3.1941940123776353,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 7.228409767150879,
|
||
|
|
"kl": 15.267898559570312,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -48605112.933753945,
|
||
|
|
"logits/rejected": -49457862.14241486,
|
||
|
|
"logps/chosen": -452.8354199526814,
|
||
|
|
"logps/rejected": -385.3189821981424,
|
||
|
|
"loss": 0.2023,
|
||
|
|
"loss/base_kto": 1.6183490753173828,
|
||
|
|
"metrics/advantage_var": 14819.5283203125,
|
||
|
|
"rewards/chosen": 2.5347646237923898,
|
||
|
|
"rewards/margins": 5.6482885027107335,
|
||
|
|
"rewards/rejected": -3.1135238789183437,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 6.922834396362305,
|
||
|
|
"kl": 16.767065048217773,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -50873499.4428795,
|
||
|
|
"logits/rejected": -49571018.88299532,
|
||
|
|
"logps/chosen": -442.87377738654146,
|
||
|
|
"logps/rejected": -409.1465483619345,
|
||
|
|
"loss": 0.2299,
|
||
|
|
"loss/base_kto": 1.8388410806655884,
|
||
|
|
"metrics/advantage_var": 13165.7158203125,
|
||
|
|
"rewards/chosen": 2.2737991261370305,
|
||
|
|
"rewards/margins": 5.173699946388591,
|
||
|
|
"rewards/rejected": -2.89990082025156,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 6.087769508361816,
|
||
|
|
"kl": 17.345020294189453,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -50924598.05731523,
|
||
|
|
"logits/rejected": -51076071.10534846,
|
||
|
|
"logps/chosen": -451.93999811463044,
|
||
|
|
"logps/rejected": -398.00182333873585,
|
||
|
|
"loss": 0.1927,
|
||
|
|
"loss/base_kto": 1.5418506860733032,
|
||
|
|
"metrics/advantage_var": 15268.994140625,
|
||
|
|
"rewards/chosen": 2.669620841876414,
|
||
|
|
"rewards/margins": 5.844617834633707,
|
||
|
|
"rewards/rejected": -3.1749969927572934,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 7.265406608581543,
|
||
|
|
"kl": 17.250513076782227,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -50358195.913312696,
|
||
|
|
"logits/rejected": -50521524.08832808,
|
||
|
|
"logps/chosen": -470.1187113003096,
|
||
|
|
"logps/rejected": -375.2814964511041,
|
||
|
|
"loss": 0.2118,
|
||
|
|
"loss/base_kto": 1.6946674585342407,
|
||
|
|
"metrics/advantage_var": 13960.9951171875,
|
||
|
|
"rewards/chosen": 2.7360907870549536,
|
||
|
|
"rewards/margins": 5.317020323603652,
|
||
|
|
"rewards/rejected": -2.5809295365486986,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 7.966547966003418,
|
||
|
|
"kl": 12.166990280151367,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -51448348.86646884,
|
||
|
|
"logits/rejected": -50085110.70627063,
|
||
|
|
"logps/chosen": -448.09375,
|
||
|
|
"logps/rejected": -415.8252887788779,
|
||
|
|
"loss": 0.2227,
|
||
|
|
"loss/base_kto": 1.7817531824111938,
|
||
|
|
"metrics/advantage_var": 15057.634765625,
|
||
|
|
"rewards/chosen": 2.597703701664503,
|
||
|
|
"rewards/margins": 5.362192601726385,
|
||
|
|
"rewards/rejected": -2.764488900061881,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 6.479636192321777,
|
||
|
|
"kl": 21.107120513916016,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -51183870.3483871,
|
||
|
|
"logits/rejected": -51802614.69090909,
|
||
|
|
"logps/chosen": -458.6647681451613,
|
||
|
|
"logps/rejected": -418.3832859848485,
|
||
|
|
"loss": 0.2142,
|
||
|
|
"loss/base_kto": 1.7134571075439453,
|
||
|
|
"metrics/advantage_var": 14570.9716796875,
|
||
|
|
"rewards/chosen": 2.7536784510458667,
|
||
|
|
"rewards/margins": 5.491471789705905,
|
||
|
|
"rewards/rejected": -2.7377933386600377,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 6.600201606750488,
|
||
|
|
"kl": 18.848928451538086,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -51338816.99071208,
|
||
|
|
"logits/rejected": -51748482.82649842,
|
||
|
|
"logps/chosen": -448.76683436532505,
|
||
|
|
"logps/rejected": -409.4468404968454,
|
||
|
|
"loss": 0.2105,
|
||
|
|
"loss/base_kto": 1.6841938495635986,
|
||
|
|
"metrics/advantage_var": 14403.134765625,
|
||
|
|
"rewards/chosen": 2.503087471882256,
|
||
|
|
"rewards/margins": 5.406129756730442,
|
||
|
|
"rewards/rejected": -2.903042284848186,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 6.502203941345215,
|
||
|
|
"kl": 19.20033836364746,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -49168930.98136646,
|
||
|
|
"logits/rejected": -50254120.251572326,
|
||
|
|
"logps/chosen": -479.9908288043478,
|
||
|
|
"logps/rejected": -378.2589426100629,
|
||
|
|
"loss": 0.2073,
|
||
|
|
"loss/base_kto": 1.658690094947815,
|
||
|
|
"metrics/advantage_var": 14569.7236328125,
|
||
|
|
"rewards/chosen": 2.8965670188761647,
|
||
|
|
"rewards/margins": 5.645065323711071,
|
||
|
|
"rewards/rejected": -2.748498304834906,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 9.374115943908691,
|
||
|
|
"kl": 16.311094284057617,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -50253843.629392974,
|
||
|
|
"logits/rejected": -52141566.434250765,
|
||
|
|
"logps/chosen": -469.2500499201278,
|
||
|
|
"logps/rejected": -407.09193425076455,
|
||
|
|
"loss": 0.2019,
|
||
|
|
"loss/base_kto": 1.6155580282211304,
|
||
|
|
"metrics/advantage_var": 14986.9287109375,
|
||
|
|
"rewards/chosen": 2.7423072303065097,
|
||
|
|
"rewards/margins": 5.90321232754466,
|
||
|
|
"rewards/rejected": -3.16090509723815,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 7.66606330871582,
|
||
|
|
"kl": 20.138084411621094,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -50138638.5362776,
|
||
|
|
"logits/rejected": -50570610.92260062,
|
||
|
|
"logps/chosen": -468.341285488959,
|
||
|
|
"logps/rejected": -399.02713815789474,
|
||
|
|
"loss": 0.1985,
|
||
|
|
"loss/base_kto": 1.5876652002334595,
|
||
|
|
"metrics/advantage_var": 14451.103515625,
|
||
|
|
"rewards/chosen": 2.6708017824576102,
|
||
|
|
"rewards/margins": 5.763690676553489,
|
||
|
|
"rewards/rejected": -3.0928888940958785,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 7.446461200714111,
|
||
|
|
"kl": 21.64815330505371,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -50349630.0355412,
|
||
|
|
"logits/rejected": -50701945.6096823,
|
||
|
|
"logps/chosen": -476.92790791599356,
|
||
|
|
"logps/rejected": -398.7458396369138,
|
||
|
|
"loss": 0.2051,
|
||
|
|
"loss/base_kto": 1.6408064365386963,
|
||
|
|
"metrics/advantage_var": 14884.3896484375,
|
||
|
|
"rewards/chosen": 2.6866137576989093,
|
||
|
|
"rewards/margins": 5.743195840657968,
|
||
|
|
"rewards/rejected": -3.0565820829590584,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 8.375565528869629,
|
||
|
|
"kl": 23.32106590270996,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -51635112.70819672,
|
||
|
|
"logits/rejected": -52851250.43582089,
|
||
|
|
"logps/chosen": -470.435143442623,
|
||
|
|
"logps/rejected": -406.24967350746266,
|
||
|
|
"loss": 0.1974,
|
||
|
|
"loss/base_kto": 1.5795862674713135,
|
||
|
|
"metrics/advantage_var": 15001.556640625,
|
||
|
|
"rewards/chosen": 2.739492347592213,
|
||
|
|
"rewards/margins": 5.774384779961617,
|
||
|
|
"rewards/rejected": -3.034892432369403,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 6.117384433746338,
|
||
|
|
"kl": 19.464855194091797,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -51135410.499194846,
|
||
|
|
"logits/rejected": -52004487.96358118,
|
||
|
|
"logps/chosen": -449.73661433172305,
|
||
|
|
"logps/rejected": -397.84754362670714,
|
||
|
|
"loss": 0.2139,
|
||
|
|
"loss/base_kto": 1.7114804983139038,
|
||
|
|
"metrics/advantage_var": 13869.8876953125,
|
||
|
|
"rewards/chosen": 2.5128948316098025,
|
||
|
|
"rewards/margins": 5.446001411773877,
|
||
|
|
"rewards/rejected": -2.9331065801640746,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 7.1891303062438965,
|
||
|
|
"kl": 23.11102867126465,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -51682678.87163233,
|
||
|
|
"logits/rejected": -51322785.33127889,
|
||
|
|
"logps/chosen": -445.8949088748019,
|
||
|
|
"logps/rejected": -414.2937692604006,
|
||
|
|
"loss": 0.2002,
|
||
|
|
"loss/base_kto": 1.6013821363449097,
|
||
|
|
"metrics/advantage_var": 15572.7216796875,
|
||
|
|
"rewards/chosen": 2.7213179260350633,
|
||
|
|
"rewards/margins": 5.883317080383099,
|
||
|
|
"rewards/rejected": -3.1619991543480355,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 7.130632400512695,
|
||
|
|
"kl": 19.88947105407715,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -52005329.45454545,
|
||
|
|
"logits/rejected": -52686891.18072289,
|
||
|
|
"logps/chosen": -470.73787540584414,
|
||
|
|
"logps/rejected": -424.3311370481928,
|
||
|
|
"loss": 0.2134,
|
||
|
|
"loss/base_kto": 1.7074934244155884,
|
||
|
|
"metrics/advantage_var": 14958.361328125,
|
||
|
|
"rewards/chosen": 2.825003091391031,
|
||
|
|
"rewards/margins": 5.454456267866464,
|
||
|
|
"rewards/rejected": -2.629453176475433,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 9.399202346801758,
|
||
|
|
"kl": 17.426616668701172,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -50207483.987460814,
|
||
|
|
"logits/rejected": -51213363.04049844,
|
||
|
|
"logps/chosen": -443.9810442789969,
|
||
|
|
"logps/rejected": -393.54239680685356,
|
||
|
|
"loss": 0.213,
|
||
|
|
"loss/base_kto": 1.7043474912643433,
|
||
|
|
"metrics/advantage_var": 14574.78125,
|
||
|
|
"rewards/chosen": 2.522138733101489,
|
||
|
|
"rewards/margins": 5.6771853327777935,
|
||
|
|
"rewards/rejected": -3.1550465996763046,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 6.621180534362793,
|
||
|
|
"kl": 12.879165649414062,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -51292347.33643411,
|
||
|
|
"logits/rejected": -51794236.06929134,
|
||
|
|
"logps/chosen": -472.0147286821705,
|
||
|
|
"logps/rejected": -419.4152559055118,
|
||
|
|
"loss": 0.2145,
|
||
|
|
"loss/base_kto": 1.7159721851348877,
|
||
|
|
"metrics/advantage_var": 14325.78125,
|
||
|
|
"rewards/chosen": 2.685220218628876,
|
||
|
|
"rewards/margins": 5.562849055673659,
|
||
|
|
"rewards/rejected": -2.8776288370447833,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 8.082818031311035,
|
||
|
|
"kl": 14.013216972351074,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -50113596.60091047,
|
||
|
|
"logits/rejected": -48584469.84863124,
|
||
|
|
"logps/chosen": -449.9308611532625,
|
||
|
|
"logps/rejected": -370.71844806763283,
|
||
|
|
"loss": 0.2059,
|
||
|
|
"loss/base_kto": 1.6473382711410522,
|
||
|
|
"metrics/advantage_var": 14097.734375,
|
||
|
|
"rewards/chosen": 2.5490370708459786,
|
||
|
|
"rewards/margins": 5.652958614034888,
|
||
|
|
"rewards/rejected": -3.103921543188909,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.975174191323546e+17,
|
||
|
|
"train_loss": 0.274414178620966,
|
||
|
|
"train_runtime": 11069.1641,
|
||
|
|
"train_samples_per_second": 13.39,
|
||
|
|
"train_steps_per_second": 0.209
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.975174191323546e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|