7a46807b36
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed1337 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 14.088885307312012,
|
|
"kl": 23.88652801513672,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37528123.6809816,
|
|
"logits/rejected": -38386909.75796178,
|
|
"logps/chosen": -457.04021280674846,
|
|
"logps/rejected": -378.969247611465,
|
|
"loss": 0.5291,
|
|
"loss/base_kto": 3.8639419078826904,
|
|
"loss/total_with_kl": 4.232768535614014,
|
|
"metrics/advantage_var": 221.517333984375,
|
|
"regularization/advantage_var": 221.517333984375,
|
|
"regularization/kl": 0.36882632970809937,
|
|
"rewards/chosen": 0.3737228253136383,
|
|
"rewards/margins": 0.10674999800969726,
|
|
"rewards/rejected": 0.26697282730394106,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 18.572708129882812,
|
|
"kl": 5.136453628540039,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36810364.3659306,
|
|
"logits/rejected": -37871863.281733744,
|
|
"logps/chosen": -480.0217369873817,
|
|
"logps/rejected": -364.0091669891641,
|
|
"loss": 0.5372,
|
|
"loss/base_kto": 3.9049298763275146,
|
|
"loss/total_with_kl": 4.297947883605957,
|
|
"metrics/advantage_var": 236.04702758789062,
|
|
"regularization/advantage_var": 236.04702758789062,
|
|
"regularization/kl": 0.39301830530166626,
|
|
"rewards/chosen": 0.02810211662990438,
|
|
"rewards/margins": 0.09888574945752145,
|
|
"rewards/rejected": -0.07078363282761707,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 14.051726341247559,
|
|
"kl": 13.226417541503906,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35373757.74117647,
|
|
"logits/rejected": -36958098.77333333,
|
|
"logps/chosen": -462.99545036764704,
|
|
"logps/rejected": -382.065546875,
|
|
"loss": 0.5205,
|
|
"loss/base_kto": 3.847067356109619,
|
|
"loss/total_with_kl": 4.164321422576904,
|
|
"metrics/advantage_var": 190.5430450439453,
|
|
"regularization/advantage_var": 190.5430450439453,
|
|
"regularization/kl": 0.3172541558742523,
|
|
"rewards/chosen": 0.24383798487046185,
|
|
"rewards/margins": 0.13585693882960898,
|
|
"rewards/rejected": 0.10798104604085286,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 15.613126754760742,
|
|
"kl": 12.781538009643555,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35698037.996810205,
|
|
"logits/rejected": -37063073.12710567,
|
|
"logps/chosen": -485.55352870813397,
|
|
"logps/rejected": -384.21927641653906,
|
|
"loss": 0.5216,
|
|
"loss/base_kto": 3.8480918407440186,
|
|
"loss/total_with_kl": 4.172784328460693,
|
|
"metrics/advantage_var": 195.01004028320312,
|
|
"regularization/advantage_var": 195.01004028320312,
|
|
"regularization/kl": 0.3246917128562927,
|
|
"rewards/chosen": 0.2367199376059111,
|
|
"rewards/margins": 0.1433791091616215,
|
|
"rewards/rejected": 0.09334082844428958,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 16.93861961364746,
|
|
"kl": 25.310453414916992,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -37539468.76132931,
|
|
"logits/rejected": -37722582.57605178,
|
|
"logps/chosen": -478.8783515861027,
|
|
"logps/rejected": -368.5616403721683,
|
|
"loss": 0.5153,
|
|
"loss/base_kto": 3.77561354637146,
|
|
"loss/total_with_kl": 4.122737884521484,
|
|
"metrics/advantage_var": 208.4832305908203,
|
|
"regularization/advantage_var": 208.4832305908203,
|
|
"regularization/kl": 0.3471245765686035,
|
|
"rewards/chosen": 0.48323478583482815,
|
|
"rewards/margins": 0.17729996624605893,
|
|
"rewards/rejected": 0.3059348195887692,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 11.267261505126953,
|
|
"kl": 20.408489227294922,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36446131.913312696,
|
|
"logits/rejected": -36282477.829652995,
|
|
"logps/chosen": -468.5743034055728,
|
|
"logps/rejected": -342.97466482649844,
|
|
"loss": 0.5129,
|
|
"loss/base_kto": 3.792896270751953,
|
|
"loss/total_with_kl": 4.103132724761963,
|
|
"metrics/advantage_var": 186.32827758789062,
|
|
"regularization/advantage_var": 186.32827758789062,
|
|
"regularization/kl": 0.3102365732192993,
|
|
"rewards/chosen": 0.34514804521212267,
|
|
"rewards/margins": 0.195202837261302,
|
|
"rewards/rejected": 0.14994520795082067,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 16.725078582763672,
|
|
"kl": 9.431735038757324,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36302867.47385103,
|
|
"logits/rejected": -37263016.036979966,
|
|
"logps/chosen": -472.6939381933439,
|
|
"logps/rejected": -379.2325211864407,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.7329156398773193,
|
|
"loss/total_with_kl": 4.099330902099609,
|
|
"metrics/advantage_var": 220.06900024414062,
|
|
"regularization/advantage_var": 220.06900024414062,
|
|
"regularization/kl": 0.3664149045944214,
|
|
"rewards/chosen": 0.231383358430938,
|
|
"rewards/margins": 0.25438596824813725,
|
|
"rewards/rejected": -0.023002609817199236,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 19.889673233032227,
|
|
"kl": 11.57478141784668,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36108709.26582278,
|
|
"logits/rejected": -37462275.16049383,
|
|
"logps/chosen": -489.14492681962025,
|
|
"logps/rejected": -368.169319058642,
|
|
"loss": 0.5113,
|
|
"loss/base_kto": 3.741086721420288,
|
|
"loss/total_with_kl": 4.090657711029053,
|
|
"metrics/advantage_var": 209.95278930664062,
|
|
"regularization/advantage_var": 209.95278930664062,
|
|
"regularization/kl": 0.3495713770389557,
|
|
"rewards/chosen": 0.2500881967665274,
|
|
"rewards/margins": 0.23948889116623154,
|
|
"rewards/rejected": 0.010599305600295832,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 14.76732063293457,
|
|
"kl": 7.241082191467285,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36247020.55696203,
|
|
"logits/rejected": -38475520.0,
|
|
"logps/chosen": -520.4291930379746,
|
|
"logps/rejected": -396.0368441358025,
|
|
"loss": 0.5165,
|
|
"loss/base_kto": 3.7427737712860107,
|
|
"loss/total_with_kl": 4.132046222686768,
|
|
"metrics/advantage_var": 233.79727172851562,
|
|
"regularization/advantage_var": 233.79727172851562,
|
|
"regularization/kl": 0.38927245140075684,
|
|
"rewards/chosen": 0.0891423225402832,
|
|
"rewards/margins": 0.24869954733201016,
|
|
"rewards/rejected": -0.15955722479172696,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 13.966711044311523,
|
|
"kl": 5.442081928253174,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37434309.299363054,
|
|
"logits/rejected": -38459379.435582824,
|
|
"logps/chosen": -519.813594745223,
|
|
"logps/rejected": -362.97804831288346,
|
|
"loss": 0.5212,
|
|
"loss/base_kto": 3.753812551498413,
|
|
"loss/total_with_kl": 4.16942024230957,
|
|
"metrics/advantage_var": 249.6142578125,
|
|
"regularization/advantage_var": 249.6142578125,
|
|
"regularization/kl": 0.4156077802181244,
|
|
"rewards/chosen": 0.06246923944752687,
|
|
"rewards/margins": 0.24750974263932857,
|
|
"rewards/rejected": -0.18504050319180168,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 19.831310272216797,
|
|
"kl": 4.0591559410095215,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37643101.974683546,
|
|
"logits/rejected": -37689195.45679013,
|
|
"logps/chosen": -472.71415150316454,
|
|
"logps/rejected": -414.88290895061726,
|
|
"loss": 0.5285,
|
|
"loss/base_kto": 3.799222707748413,
|
|
"loss/total_with_kl": 4.227824687957764,
|
|
"metrics/advantage_var": 257.4183044433594,
|
|
"regularization/advantage_var": 257.4183044433594,
|
|
"regularization/kl": 0.42860147356987,
|
|
"rewards/chosen": -0.03937487662593021,
|
|
"rewards/margins": 0.1872332584561734,
|
|
"rewards/rejected": -0.2266081350821036,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 14.725709915161133,
|
|
"kl": 15.463202476501465,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35475091.33227345,
|
|
"logits/rejected": -34877540.66973887,
|
|
"logps/chosen": -491.9404312400636,
|
|
"logps/rejected": -388.3087077572965,
|
|
"loss": 0.5144,
|
|
"loss/base_kto": 3.704681873321533,
|
|
"loss/total_with_kl": 4.115037441253662,
|
|
"metrics/advantage_var": 246.46006774902344,
|
|
"regularization/advantage_var": 246.46006774902344,
|
|
"regularization/kl": 0.4103560149669647,
|
|
"rewards/chosen": 0.26734863669390896,
|
|
"rewards/margins": 0.2802479930144322,
|
|
"rewards/rejected": -0.012899356320523263,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.967681884765625,
|
|
"kl": 10.818643569946289,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36582384.670658685,
|
|
"logits/rejected": -37477309.07189543,
|
|
"logps/chosen": -477.23521706586826,
|
|
"logps/rejected": -353.39276960784315,
|
|
"loss": 0.5248,
|
|
"loss/base_kto": 3.752309799194336,
|
|
"loss/total_with_kl": 4.198126792907715,
|
|
"metrics/advantage_var": 267.7579040527344,
|
|
"regularization/advantage_var": 267.7579040527344,
|
|
"regularization/kl": 0.44581690430641174,
|
|
"rewards/chosen": 0.18722849406168132,
|
|
"rewards/margins": 0.23522973668061609,
|
|
"rewards/rejected": -0.04800124261893478,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 14.118301391601562,
|
|
"kl": 19.1561336517334,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -37597438.448484845,
|
|
"logits/rejected": -37940686.451612905,
|
|
"logps/chosen": -484.2588068181818,
|
|
"logps/rejected": -359.1635080645161,
|
|
"loss": 0.514,
|
|
"loss/base_kto": 3.6524834632873535,
|
|
"loss/total_with_kl": 4.1116042137146,
|
|
"metrics/advantage_var": 275.748291015625,
|
|
"regularization/advantage_var": 275.748291015625,
|
|
"regularization/kl": 0.45912089943885803,
|
|
"rewards/chosen": 0.41523123076467805,
|
|
"rewards/margins": 0.3117529036595558,
|
|
"rewards/rejected": 0.10347832710512223,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 13.877946853637695,
|
|
"kl": 12.911961555480957,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37545008.129659645,
|
|
"logits/rejected": -37747849.46003017,
|
|
"logps/chosen": -494.53251620745544,
|
|
"logps/rejected": -369.32909125188536,
|
|
"loss": 0.5202,
|
|
"loss/base_kto": 3.7146193981170654,
|
|
"loss/total_with_kl": 4.161755084991455,
|
|
"metrics/advantage_var": 268.5498962402344,
|
|
"regularization/advantage_var": 268.5498962402344,
|
|
"regularization/kl": 0.4471355974674225,
|
|
"rewards/chosen": 0.18139129094598358,
|
|
"rewards/margins": 0.25910562699027206,
|
|
"rewards/rejected": -0.07771433604428851,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 21.11815071105957,
|
|
"kl": 8.587013244628906,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37360628.5312,
|
|
"logits/rejected": -37771964.38473283,
|
|
"logps/chosen": -467.6982,
|
|
"logps/rejected": -396.3062977099237,
|
|
"loss": 0.5307,
|
|
"loss/base_kto": 3.743504762649536,
|
|
"loss/total_with_kl": 4.245852947235107,
|
|
"metrics/advantage_var": 301.710693359375,
|
|
"regularization/advantage_var": 301.710693359375,
|
|
"regularization/kl": 0.5023483037948608,
|
|
"rewards/chosen": 0.129776025390625,
|
|
"rewards/margins": 0.2512573369848819,
|
|
"rewards/rejected": -0.12148131159425692,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 20.89030647277832,
|
|
"kl": 16.20552635192871,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -38006194.37672282,
|
|
"logits/rejected": -38407630.18819777,
|
|
"logps/chosen": -465.12992917304746,
|
|
"logps/rejected": -362.1497707336523,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.702958345413208,
|
|
"loss/total_with_kl": 4.200279235839844,
|
|
"metrics/advantage_var": 298.6911315917969,
|
|
"regularization/advantage_var": 298.6911315917969,
|
|
"regularization/kl": 0.4973207116127014,
|
|
"rewards/chosen": 0.3274364383808504,
|
|
"rewards/margins": 0.2768062491757598,
|
|
"rewards/rejected": 0.05063018920509058,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 17.321910858154297,
|
|
"kl": 8.329656600952148,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -37467050.81154499,
|
|
"logits/rejected": -37957904.671490595,
|
|
"logps/chosen": -507.00100806451616,
|
|
"logps/rejected": -369.32122829233,
|
|
"loss": 0.5128,
|
|
"loss/base_kto": 3.6502671241760254,
|
|
"loss/total_with_kl": 4.102503299713135,
|
|
"metrics/advantage_var": 271.6133728027344,
|
|
"regularization/advantage_var": 271.6133728027344,
|
|
"regularization/kl": 0.4522361755371094,
|
|
"rewards/chosen": 0.09033037324915111,
|
|
"rewards/margins": 0.31583184639580225,
|
|
"rewards/rejected": -0.22550147314665114,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 17.043107986450195,
|
|
"kl": 4.269077777862549,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36541597.28104575,
|
|
"logits/rejected": -36786960.86227545,
|
|
"logps/chosen": -454.10442197712416,
|
|
"logps/rejected": -359.67482223053895,
|
|
"loss": 0.514,
|
|
"loss/base_kto": 3.698291063308716,
|
|
"loss/total_with_kl": 4.112191677093506,
|
|
"metrics/advantage_var": 248.5888671875,
|
|
"regularization/advantage_var": 248.5888671875,
|
|
"regularization/kl": 0.4139004647731781,
|
|
"rewards/chosen": -0.13552749234866474,
|
|
"rewards/margins": 0.2849269783486126,
|
|
"rewards/rejected": -0.4204544706972773,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 12.396024703979492,
|
|
"kl": 0.5512941479682922,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35668370.40255591,
|
|
"logits/rejected": -36740099.13149847,
|
|
"logps/chosen": -481.2458067092652,
|
|
"logps/rejected": -379.31426796636083,
|
|
"loss": 0.5234,
|
|
"loss/base_kto": 3.7515289783477783,
|
|
"loss/total_with_kl": 4.186891078948975,
|
|
"metrics/advantage_var": 261.47918701171875,
|
|
"regularization/advantage_var": 261.47918701171875,
|
|
"regularization/kl": 0.4353628158569336,
|
|
"rewards/chosen": -0.25150243009622103,
|
|
"rewards/margins": 0.25027948849071996,
|
|
"rewards/rejected": -0.501781918586941,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 17.49666404724121,
|
|
"kl": 5.36279296875,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36949414.92284418,
|
|
"logits/rejected": -37455438.57835218,
|
|
"logps/chosen": -484.5858547655068,
|
|
"logps/rejected": -374.9609753634895,
|
|
"loss": 0.5224,
|
|
"loss/base_kto": 3.6900017261505127,
|
|
"loss/total_with_kl": 4.179175853729248,
|
|
"metrics/advantage_var": 293.7980041503906,
|
|
"regularization/advantage_var": 293.7980041503906,
|
|
"regularization/kl": 0.4891737103462219,
|
|
"rewards/chosen": 0.16721020812526594,
|
|
"rewards/margins": 0.3163328869158745,
|
|
"rewards/rejected": -0.1491226787906086,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 11.672085762023926,
|
|
"kl": 6.16011381149292,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36321454.013071895,
|
|
"logits/rejected": -36864312.71856287,
|
|
"logps/chosen": -487.15206290849676,
|
|
"logps/rejected": -380.71191991017963,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.7490785121917725,
|
|
"loss/total_with_kl": 4.144867897033691,
|
|
"metrics/advantage_var": 237.7109375,
|
|
"regularization/advantage_var": 237.7109375,
|
|
"regularization/kl": 0.3957887291908264,
|
|
"rewards/chosen": -0.037287263309254366,
|
|
"rewards/margins": 0.21023126469820727,
|
|
"rewards/rejected": -0.24751852800746163,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 31.15283966064453,
|
|
"kl": 5.339746475219727,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36557604.340694,
|
|
"logits/rejected": -37723589.3498452,
|
|
"logps/chosen": -496.4055599369085,
|
|
"logps/rejected": -359.9879789086687,
|
|
"loss": 0.5166,
|
|
"loss/base_kto": 3.6446235179901123,
|
|
"loss/total_with_kl": 4.132524013519287,
|
|
"metrics/advantage_var": 293.0332946777344,
|
|
"regularization/advantage_var": 293.0332946777344,
|
|
"regularization/kl": 0.4879004657268524,
|
|
"rewards/chosen": 0.05749362500308064,
|
|
"rewards/margins": 0.343972334882386,
|
|
"rewards/rejected": -0.28647870987930535,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 16.87729263305664,
|
|
"kl": 7.568799018859863,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37066449.41835148,
|
|
"logits/rejected": -37377951.547880694,
|
|
"logps/chosen": -466.99679237947123,
|
|
"logps/rejected": -378.9932790423862,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.704345703125,
|
|
"loss/total_with_kl": 4.199629783630371,
|
|
"metrics/advantage_var": 297.46783447265625,
|
|
"regularization/advantage_var": 297.46783447265625,
|
|
"regularization/kl": 0.49528390169143677,
|
|
"rewards/chosen": 0.09906621075714728,
|
|
"rewards/margins": 0.2816256448184822,
|
|
"rewards/rejected": -0.18255943406133487,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.461814880371094,
|
|
"kl": 7.9263763427734375,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -37181132.47284345,
|
|
"logits/rejected": -38937051.987767585,
|
|
"logps/chosen": -506.9655051916933,
|
|
"logps/rejected": -362.3202886085627,
|
|
"loss": 0.5281,
|
|
"loss/base_kto": 3.7497482299804688,
|
|
"loss/total_with_kl": 4.224986553192139,
|
|
"metrics/advantage_var": 285.4281311035156,
|
|
"regularization/advantage_var": 285.4281311035156,
|
|
"regularization/kl": 0.4752378463745117,
|
|
"rewards/chosen": 0.07912487724718575,
|
|
"rewards/margins": 0.2165681630813553,
|
|
"rewards/rejected": -0.13744328583416954,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 14.819686889648438,
|
|
"kl": 2.916916608810425,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -35531653.901430845,
|
|
"logits/rejected": -36233669.01382489,
|
|
"logps/chosen": -494.38284976152624,
|
|
"logps/rejected": -366.886232718894,
|
|
"loss": 0.5225,
|
|
"loss/base_kto": 3.692812442779541,
|
|
"loss/total_with_kl": 4.179948329925537,
|
|
"metrics/advantage_var": 292.57440185546875,
|
|
"regularization/advantage_var": 292.57440185546875,
|
|
"regularization/kl": 0.4871363639831543,
|
|
"rewards/chosen": -0.0795598962538193,
|
|
"rewards/margins": 0.3158700987658619,
|
|
"rewards/rejected": -0.39542999501968124,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 13.16500473022461,
|
|
"kl": 3.9249343872070312,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -38203268.66561014,
|
|
"logits/rejected": -38793834.502311245,
|
|
"logps/chosen": -505.7901148969889,
|
|
"logps/rejected": -372.7730643297381,
|
|
"loss": 0.5178,
|
|
"loss/base_kto": 3.6519525051116943,
|
|
"loss/total_with_kl": 4.142402172088623,
|
|
"metrics/advantage_var": 294.5643005371094,
|
|
"regularization/advantage_var": 294.5643005371094,
|
|
"regularization/kl": 0.4904495179653168,
|
|
"rewards/chosen": -0.017278565468765477,
|
|
"rewards/margins": 0.35273578390587335,
|
|
"rewards/rejected": -0.37001434937463884,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 15.714888572692871,
|
|
"kl": 5.598243236541748,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35762880.20189274,
|
|
"logits/rejected": -37315216.24767802,
|
|
"logps/chosen": -473.1084384858044,
|
|
"logps/rejected": -364.20989744582045,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.6598050594329834,
|
|
"loss/total_with_kl": 4.099398136138916,
|
|
"metrics/advantage_var": 264.0198059082031,
|
|
"regularization/advantage_var": 264.0198059082031,
|
|
"regularization/kl": 0.4395929276943207,
|
|
"rewards/chosen": 0.0033412837079646834,
|
|
"rewards/margins": 0.3203957251777063,
|
|
"rewards/rejected": -0.31705444146974165,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 15.652578353881836,
|
|
"kl": 10.228057861328125,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36725028.571428575,
|
|
"logits/rejected": -37858987.197511666,
|
|
"logps/chosen": -470.73297684458396,
|
|
"logps/rejected": -384.79359447900464,
|
|
"loss": 0.5161,
|
|
"loss/base_kto": 3.6901605129241943,
|
|
"loss/total_with_kl": 4.129108905792236,
|
|
"metrics/advantage_var": 263.6327209472656,
|
|
"regularization/advantage_var": 263.6327209472656,
|
|
"regularization/kl": 0.43894848227500916,
|
|
"rewards/chosen": 0.2029308516713476,
|
|
"rewards/margins": 0.28676627549833356,
|
|
"rewards/rejected": -0.08383542382698593,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 10.454867362976074,
|
|
"kl": 8.26036262512207,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36549298.15673981,
|
|
"logits/rejected": -38308752.34890965,
|
|
"logps/chosen": -501.919230015674,
|
|
"logps/rejected": -381.7785241433022,
|
|
"loss": 0.524,
|
|
"loss/base_kto": 3.6981353759765625,
|
|
"loss/total_with_kl": 4.191633701324463,
|
|
"metrics/advantage_var": 296.3955078125,
|
|
"regularization/advantage_var": 296.3955078125,
|
|
"regularization/kl": 0.4934985339641571,
|
|
"rewards/chosen": 0.15806019941467475,
|
|
"rewards/margins": 0.289453834056328,
|
|
"rewards/rejected": -0.13139363464165327,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 17.957624435424805,
|
|
"kl": 7.0264892578125,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -36025242.86419753,
|
|
"logits/rejected": -37315324.75949367,
|
|
"logps/chosen": -511.977237654321,
|
|
"logps/rejected": -355.42380340189874,
|
|
"loss": 0.5186,
|
|
"loss/base_kto": 3.6867783069610596,
|
|
"loss/total_with_kl": 4.148876190185547,
|
|
"metrics/advantage_var": 277.5362854003906,
|
|
"regularization/advantage_var": 277.5362854003906,
|
|
"regularization/kl": 0.4620979428291321,
|
|
"rewards/chosen": 0.13489515987443335,
|
|
"rewards/margins": 0.31560371912798113,
|
|
"rewards/rejected": -0.18070855925354776,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 15.093327522277832,
|
|
"kl": 20.08521270751953,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35954897.1051051,
|
|
"logits/rejected": -37164088.70358306,
|
|
"logps/chosen": -475.45026276276275,
|
|
"logps/rejected": -369.4489515472313,
|
|
"loss": 0.5125,
|
|
"loss/base_kto": 3.623955488204956,
|
|
"loss/total_with_kl": 4.100308895111084,
|
|
"metrics/advantage_var": 286.0978698730469,
|
|
"regularization/advantage_var": 286.0978698730469,
|
|
"regularization/kl": 0.4763529896736145,
|
|
"rewards/chosen": 0.4123509495823949,
|
|
"rewards/margins": 0.3309227883092214,
|
|
"rewards/rejected": 0.08142816127317348,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 11.913163185119629,
|
|
"kl": 15.90697956085205,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36560309.554896146,
|
|
"logits/rejected": -36841049.557755776,
|
|
"logps/chosen": -470.7982195845697,
|
|
"logps/rejected": -381.52307652640263,
|
|
"loss": 0.5164,
|
|
"loss/base_kto": 3.6569411754608154,
|
|
"loss/total_with_kl": 4.13106107711792,
|
|
"metrics/advantage_var": 284.7563171386719,
|
|
"regularization/advantage_var": 284.7563171386719,
|
|
"regularization/kl": 0.47411924600601196,
|
|
"rewards/chosen": 0.37493629342370643,
|
|
"rewards/margins": 0.3042059141113611,
|
|
"rewards/rejected": 0.0707303793123453,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 17.52887535095215,
|
|
"kl": 25.070058822631836,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36559771.12835821,
|
|
"logits/rejected": -36813088.73442623,
|
|
"logps/chosen": -486.43358208955226,
|
|
"logps/rejected": -373.98263319672134,
|
|
"loss": 0.514,
|
|
"loss/base_kto": 3.657564640045166,
|
|
"loss/total_with_kl": 4.112019062042236,
|
|
"metrics/advantage_var": 272.9454040527344,
|
|
"regularization/advantage_var": 272.9454040527344,
|
|
"regularization/kl": 0.4544541537761688,
|
|
"rewards/chosen": 0.4803892676510028,
|
|
"rewards/margins": 0.2836826896620812,
|
|
"rewards/rejected": 0.19670657798892163,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 14.99044132232666,
|
|
"kl": 18.93796730041504,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36955543.67398119,
|
|
"logits/rejected": -38141888.19937695,
|
|
"logps/chosen": -503.7553879310345,
|
|
"logps/rejected": -397.38517328660436,
|
|
"loss": 0.5239,
|
|
"loss/base_kto": 3.693930149078369,
|
|
"loss/total_with_kl": 4.190843105316162,
|
|
"metrics/advantage_var": 298.4464416503906,
|
|
"regularization/advantage_var": 298.4464416503906,
|
|
"regularization/kl": 0.496913343667984,
|
|
"rewards/chosen": 0.3072308148709958,
|
|
"rewards/margins": 0.2573570052942185,
|
|
"rewards/rejected": 0.049873809576777284,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 19.618785858154297,
|
|
"kl": 21.14823341369629,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35984314.469135806,
|
|
"logits/rejected": -37689862.48101266,
|
|
"logps/chosen": -488.61863425925924,
|
|
"logps/rejected": -360.02071795886076,
|
|
"loss": 0.5082,
|
|
"loss/base_kto": 3.5900628566741943,
|
|
"loss/total_with_kl": 4.065627574920654,
|
|
"metrics/advantage_var": 285.6246337890625,
|
|
"regularization/advantage_var": 285.6246337890625,
|
|
"regularization/kl": 0.4755650460720062,
|
|
"rewards/chosen": 0.365076983416522,
|
|
"rewards/margins": 0.3575620056595872,
|
|
"rewards/rejected": 0.00751497775693483,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 12.12169075012207,
|
|
"kl": 11.233113288879395,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -37096621.94871795,
|
|
"logits/rejected": -36576309.07317073,
|
|
"logps/chosen": -485.01362179487177,
|
|
"logps/rejected": -383.921112804878,
|
|
"loss": 0.5211,
|
|
"loss/base_kto": 3.6869747638702393,
|
|
"loss/total_with_kl": 4.169081211090088,
|
|
"metrics/advantage_var": 289.5533142089844,
|
|
"regularization/advantage_var": 289.5533142089844,
|
|
"regularization/kl": 0.48210620880126953,
|
|
"rewards/chosen": 0.15414186624380258,
|
|
"rewards/margins": 0.2610718779000287,
|
|
"rewards/rejected": -0.10693001165622618,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 42.802669525146484,
|
|
"kl": 30.603002548217773,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -36386325.80323055,
|
|
"logits/rejected": -37259578.5509182,
|
|
"logps/chosen": -485.596640969163,
|
|
"logps/rejected": -366.5489618113522,
|
|
"loss": 0.5143,
|
|
"loss/base_kto": 3.6019227504730225,
|
|
"loss/total_with_kl": 4.1142096519470215,
|
|
"metrics/advantage_var": 307.6795349121094,
|
|
"regularization/advantage_var": 307.6795349121094,
|
|
"regularization/kl": 0.512286365032196,
|
|
"rewards/chosen": 0.5539134469501422,
|
|
"rewards/margins": 0.33295757119041247,
|
|
"rewards/rejected": 0.22095587575972975,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 18.507246017456055,
|
|
"kl": 32.631935119628906,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -37053597.41362916,
|
|
"logits/rejected": -37193072.766615145,
|
|
"logps/chosen": -467.2818938193344,
|
|
"logps/rejected": -359.4695952472952,
|
|
"loss": 0.5184,
|
|
"loss/base_kto": 3.4629993438720703,
|
|
"loss/total_with_kl": 4.147392749786377,
|
|
"metrics/advantage_var": 411.0470886230469,
|
|
"regularization/advantage_var": 411.0470886230469,
|
|
"regularization/kl": 0.6843933463096619,
|
|
"rewards/chosen": 0.6384369718669523,
|
|
"rewards/margins": 0.519010156132372,
|
|
"rewards/rejected": 0.11942681573458028,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 19.082189559936523,
|
|
"kl": 25.172948837280273,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -37199617.19440124,
|
|
"logits/rejected": -38692976.527472526,
|
|
"logps/chosen": -480.5903965785381,
|
|
"logps/rejected": -378.8040374803768,
|
|
"loss": 0.5202,
|
|
"loss/base_kto": 3.122431993484497,
|
|
"loss/total_with_kl": 4.161515712738037,
|
|
"metrics/advantage_var": 624.0739135742188,
|
|
"regularization/advantage_var": 624.0739135742188,
|
|
"regularization/kl": 1.0390831232070923,
|
|
"rewards/chosen": 0.7134314458440659,
|
|
"rewards/margins": 0.9238890964523861,
|
|
"rewards/rejected": -0.21045765060832025,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 29.694541931152344,
|
|
"kl": 23.97586441040039,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36667727.01234568,
|
|
"logits/rejected": -38678712.70886076,
|
|
"logps/chosen": -491.08825231481484,
|
|
"logps/rejected": -355.8345282832278,
|
|
"loss": 0.5137,
|
|
"loss/base_kto": 3.161720037460327,
|
|
"loss/total_with_kl": 4.109841346740723,
|
|
"metrics/advantage_var": 569.44189453125,
|
|
"regularization/advantage_var": 569.44189453125,
|
|
"regularization/kl": 0.9481207728385925,
|
|
"rewards/chosen": 0.7007974695276331,
|
|
"rewards/margins": 0.8591527961198009,
|
|
"rewards/rejected": -0.15835532659216772,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 37.35356521606445,
|
|
"kl": 13.508429527282715,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36980185.6,
|
|
"logits/rejected": -37730924.8,
|
|
"logps/chosen": -471.428515625,
|
|
"logps/rejected": -397.226708984375,
|
|
"loss": 0.5269,
|
|
"loss/base_kto": 3.168551206588745,
|
|
"loss/total_with_kl": 4.21511697769165,
|
|
"metrics/advantage_var": 628.5682983398438,
|
|
"regularization/advantage_var": 628.5682983398438,
|
|
"regularization/kl": 1.046566128730774,
|
|
"rewards/chosen": 0.5748261928558349,
|
|
"rewards/margins": 0.9435202836990356,
|
|
"rewards/rejected": -0.3686940908432007,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 15.61967658996582,
|
|
"kl": 15.977557182312012,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35587951.094339624,
|
|
"logits/rejected": -37552681.34161491,
|
|
"logps/chosen": -497.69845715408803,
|
|
"logps/rejected": -377.67396156832297,
|
|
"loss": 0.5383,
|
|
"loss/base_kto": 3.2179372310638428,
|
|
"loss/total_with_kl": 4.306275844573975,
|
|
"metrics/advantage_var": 653.6566772460938,
|
|
"regularization/advantage_var": 653.6566772460938,
|
|
"regularization/kl": 1.0883382558822632,
|
|
"rewards/chosen": 0.589692697585004,
|
|
"rewards/margins": 0.8593480555618819,
|
|
"rewards/rejected": -0.2696553579768779,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 14.306334495544434,
|
|
"kl": 22.224899291992188,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -34981326.603369065,
|
|
"logits/rejected": -36668825.926634766,
|
|
"logps/chosen": -444.37418644716695,
|
|
"logps/rejected": -381.15854266347685,
|
|
"loss": 0.5307,
|
|
"loss/base_kto": 3.1858081817626953,
|
|
"loss/total_with_kl": 4.245278358459473,
|
|
"metrics/advantage_var": 636.3184204101562,
|
|
"regularization/advantage_var": 636.3184204101562,
|
|
"regularization/kl": 1.0594701766967773,
|
|
"rewards/chosen": 0.6363208531238036,
|
|
"rewards/margins": 0.8607075276792886,
|
|
"rewards/rejected": -0.22438667455548494,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 20.933927536010742,
|
|
"kl": 18.01068115234375,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35257615.67346939,
|
|
"logits/rejected": -36309441.09486781,
|
|
"logps/chosen": -470.021340266876,
|
|
"logps/rejected": -370.5529500388802,
|
|
"loss": 0.5114,
|
|
"loss/base_kto": 3.1673390865325928,
|
|
"loss/total_with_kl": 4.09150505065918,
|
|
"metrics/advantage_var": 555.0546264648438,
|
|
"regularization/advantage_var": 555.0546264648438,
|
|
"regularization/kl": 0.9241658449172974,
|
|
"rewards/chosen": 0.6581263280176854,
|
|
"rewards/margins": 0.9154616052368971,
|
|
"rewards/rejected": -0.2573352772192117,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 18.352964401245117,
|
|
"kl": 17.618865966796875,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -33307081.82414307,
|
|
"logits/rejected": -35403944.14449918,
|
|
"logps/chosen": -474.8101248137109,
|
|
"logps/rejected": -367.839131773399,
|
|
"loss": 0.5158,
|
|
"loss/base_kto": 3.2249488830566406,
|
|
"loss/total_with_kl": 4.126284599304199,
|
|
"metrics/advantage_var": 541.3427124023438,
|
|
"regularization/advantage_var": 541.3427124023438,
|
|
"regularization/kl": 0.9013355374336243,
|
|
"rewards/chosen": 0.6527705867670688,
|
|
"rewards/margins": 0.8131337198890541,
|
|
"rewards/rejected": -0.16036313312198533,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 18.690711975097656,
|
|
"kl": 28.0926513671875,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -35240297.59050445,
|
|
"logits/rejected": -36293794.217821784,
|
|
"logps/chosen": -491.0958827893175,
|
|
"logps/rejected": -364.5375412541254,
|
|
"loss": 0.5354,
|
|
"loss/base_kto": 3.255042314529419,
|
|
"loss/total_with_kl": 4.283196926116943,
|
|
"metrics/advantage_var": 617.5104370117188,
|
|
"regularization/advantage_var": 617.5104370117188,
|
|
"regularization/kl": 1.028154969215393,
|
|
"rewards/chosen": 0.7327139200725612,
|
|
"rewards/margins": 0.7675044784626592,
|
|
"rewards/rejected": -0.03479055839009804,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 12.565786361694336,
|
|
"kl": 25.228748321533203,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34886545.12101911,
|
|
"logits/rejected": -35423938.74846626,
|
|
"logps/chosen": -492.3561902866242,
|
|
"logps/rejected": -387.10527703220856,
|
|
"loss": 0.4978,
|
|
"loss/base_kto": 3.202287435531616,
|
|
"loss/total_with_kl": 3.982314348220825,
|
|
"metrics/advantage_var": 468.484375,
|
|
"regularization/advantage_var": 468.484375,
|
|
"regularization/kl": 0.7800265550613403,
|
|
"rewards/chosen": 0.7081752704207305,
|
|
"rewards/margins": 0.8083388998706802,
|
|
"rewards/rejected": -0.10016362944994968,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 11.628968238830566,
|
|
"kl": 17.416303634643555,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34922655.4251497,
|
|
"logits/rejected": -37737050.35294118,
|
|
"logps/chosen": -486.67870508982037,
|
|
"logps/rejected": -356.3923100490196,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.192002534866333,
|
|
"loss/total_with_kl": 4.143036842346191,
|
|
"metrics/advantage_var": 571.1918334960938,
|
|
"regularization/advantage_var": 571.1918334960938,
|
|
"regularization/kl": 0.9510343670845032,
|
|
"rewards/chosen": 0.6224495893466973,
|
|
"rewards/margins": 0.8962472915126873,
|
|
"rewards/rejected": -0.2737977021659901,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 10.224623680114746,
|
|
"kl": 31.75507926940918,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34713961.59512938,
|
|
"logits/rejected": -36450215.242375605,
|
|
"logps/chosen": -488.0044710806697,
|
|
"logps/rejected": -363.065459470305,
|
|
"loss": 0.4992,
|
|
"loss/base_kto": 3.1356523036956787,
|
|
"loss/total_with_kl": 3.9932098388671875,
|
|
"metrics/advantage_var": 515.0497436523438,
|
|
"regularization/advantage_var": 515.0497436523438,
|
|
"regularization/kl": 0.8575578927993774,
|
|
"rewards/chosen": 0.8930493126902588,
|
|
"rewards/margins": 0.8721640395696909,
|
|
"rewards/rejected": 0.020885273120567847,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 25.271808624267578,
|
|
"kl": 25.127153396606445,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35238961.83359498,
|
|
"logits/rejected": -35146329.978227064,
|
|
"logps/chosen": -472.45496467817895,
|
|
"logps/rejected": -354.9506706842924,
|
|
"loss": 0.5133,
|
|
"loss/base_kto": 3.2005226612091064,
|
|
"loss/total_with_kl": 4.106726169586182,
|
|
"metrics/advantage_var": 544.2662963867188,
|
|
"regularization/advantage_var": 544.2662963867188,
|
|
"regularization/kl": 0.9062032699584961,
|
|
"rewards/chosen": 0.6507727414884223,
|
|
"rewards/margins": 0.8073777753036075,
|
|
"rewards/rejected": -0.15660503381518517,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 9.767934799194336,
|
|
"kl": 22.692121505737305,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35365387.37777778,
|
|
"logits/rejected": -36197316.13538461,
|
|
"logps/chosen": -478.4297123015873,
|
|
"logps/rejected": -372.9775480769231,
|
|
"loss": 0.5071,
|
|
"loss/base_kto": 3.130155086517334,
|
|
"loss/total_with_kl": 4.057077407836914,
|
|
"metrics/advantage_var": 556.7100830078125,
|
|
"regularization/advantage_var": 556.7100830078125,
|
|
"regularization/kl": 0.9269222617149353,
|
|
"rewards/chosen": 0.7219639369419643,
|
|
"rewards/margins": 0.9413163069840316,
|
|
"rewards/rejected": -0.2193523700420673,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 15.217293739318848,
|
|
"kl": 14.585739135742188,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -37254442.46054751,
|
|
"logits/rejected": -37072221.62063733,
|
|
"logps/chosen": -519.4237117552335,
|
|
"logps/rejected": -377.64406297420334,
|
|
"loss": 0.5299,
|
|
"loss/base_kto": 3.207568883895874,
|
|
"loss/total_with_kl": 4.23884916305542,
|
|
"metrics/advantage_var": 619.3873901367188,
|
|
"regularization/advantage_var": 619.3873901367188,
|
|
"regularization/kl": 1.0312799215316772,
|
|
"rewards/chosen": 0.5522806901670693,
|
|
"rewards/margins": 0.8528270150171025,
|
|
"rewards/rejected": -0.3005463248500332,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 20.180313110351562,
|
|
"kl": 8.756698608398438,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35250730.59706362,
|
|
"logits/rejected": -36567509.781109445,
|
|
"logps/chosen": -461.12280791190864,
|
|
"logps/rejected": -379.3897816716642,
|
|
"loss": 0.5176,
|
|
"loss/base_kto": 3.2456843852996826,
|
|
"loss/total_with_kl": 4.140535831451416,
|
|
"metrics/advantage_var": 537.4483642578125,
|
|
"regularization/advantage_var": 537.4483642578125,
|
|
"regularization/kl": 0.8948515057563782,
|
|
"rewards/chosen": 0.3932627542567292,
|
|
"rewards/margins": 0.8137823528558672,
|
|
"rewards/rejected": -0.42051959859913796,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 20.089553833007812,
|
|
"kl": 8.730230331420898,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35317209.11392405,
|
|
"logits/rejected": -37072889.67901234,
|
|
"logps/chosen": -490.8776206487342,
|
|
"logps/rejected": -387.20023148148147,
|
|
"loss": 0.528,
|
|
"loss/base_kto": 3.160203695297241,
|
|
"loss/total_with_kl": 4.224068641662598,
|
|
"metrics/advantage_var": 638.9579467773438,
|
|
"regularization/advantage_var": 638.9579467773438,
|
|
"regularization/kl": 1.0638649463653564,
|
|
"rewards/chosen": 0.4458843665786936,
|
|
"rewards/margins": 0.9192902752339309,
|
|
"rewards/rejected": -0.4734059086552373,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 13.642004013061523,
|
|
"kl": 11.72966480255127,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34649173.46771654,
|
|
"logits/rejected": -35987163.88217054,
|
|
"logps/chosen": -453.5484251968504,
|
|
"logps/rejected": -370.1801356589147,
|
|
"loss": 0.4946,
|
|
"loss/base_kto": 3.227121114730835,
|
|
"loss/total_with_kl": 3.956449270248413,
|
|
"metrics/advantage_var": 438.03466796875,
|
|
"regularization/advantage_var": 438.03466796875,
|
|
"regularization/kl": 0.7293277978897095,
|
|
"rewards/chosen": 0.38410324937715307,
|
|
"rewards/margins": 0.8051071026353895,
|
|
"rewards/rejected": -0.4210038532582364,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 18.430641174316406,
|
|
"kl": 15.853436470031738,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34262446.74268105,
|
|
"logits/rejected": -35507886.453248814,
|
|
"logps/chosen": -471.2519260400616,
|
|
"logps/rejected": -360.37608954041207,
|
|
"loss": 0.5065,
|
|
"loss/base_kto": 3.2107174396514893,
|
|
"loss/total_with_kl": 4.052042007446289,
|
|
"metrics/advantage_var": 505.2998962402344,
|
|
"regularization/advantage_var": 505.2998962402344,
|
|
"regularization/kl": 0.8413244485855103,
|
|
"rewards/chosen": 0.5920350437722698,
|
|
"rewards/margins": 0.8279987956603938,
|
|
"rewards/rejected": -0.23596375188812402,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 11.596901893615723,
|
|
"kl": 19.499155044555664,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35666099.24149109,
|
|
"logits/rejected": -36556787.64404223,
|
|
"logps/chosen": -479.5274513776337,
|
|
"logps/rejected": -368.0727281297134,
|
|
"loss": 0.5073,
|
|
"loss/base_kto": 3.272367238998413,
|
|
"loss/total_with_kl": 4.058253765106201,
|
|
"metrics/advantage_var": 472.0035095214844,
|
|
"regularization/advantage_var": 472.0035095214844,
|
|
"regularization/kl": 0.7858858704566956,
|
|
"rewards/chosen": 0.5839098863895361,
|
|
"rewards/margins": 0.7571505365032475,
|
|
"rewards/rejected": -0.17324065011371134,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 21.2474365234375,
|
|
"kl": 12.786124229431152,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35003061.2179289,
|
|
"logits/rejected": -36446149.763033174,
|
|
"logps/chosen": -484.0413929675425,
|
|
"logps/rejected": -366.73514020537124,
|
|
"loss": 0.5473,
|
|
"loss/base_kto": 3.2516980171203613,
|
|
"loss/total_with_kl": 4.378077983856201,
|
|
"metrics/advantage_var": 676.5042114257812,
|
|
"regularization/advantage_var": 676.5042114257812,
|
|
"regularization/kl": 1.1263796091079712,
|
|
"rewards/chosen": 0.5038429036206772,
|
|
"rewards/margins": 0.813237486938583,
|
|
"rewards/rejected": -0.3093945833179058,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 18.45342445373535,
|
|
"kl": 11.658247947692871,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35261300.07453416,
|
|
"logits/rejected": -37697027.22012579,
|
|
"logps/chosen": -474.02406832298135,
|
|
"logps/rejected": -391.57159001572325,
|
|
"loss": 0.4958,
|
|
"loss/base_kto": 3.1406071186065674,
|
|
"loss/total_with_kl": 3.9661178588867188,
|
|
"metrics/advantage_var": 495.80230712890625,
|
|
"regularization/advantage_var": 495.80230712890625,
|
|
"regularization/kl": 0.8255107998847961,
|
|
"rewards/chosen": 0.5352091344987383,
|
|
"rewards/margins": 0.9082779859936774,
|
|
"rewards/rejected": -0.37306885149493907,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 15.466266632080078,
|
|
"kl": 15.288904190063477,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35485876.117263846,
|
|
"logits/rejected": -36272663.06306306,
|
|
"logps/chosen": -485.76348737785014,
|
|
"logps/rejected": -372.39954016516515,
|
|
"loss": 0.5452,
|
|
"loss/base_kto": 3.2038826942443848,
|
|
"loss/total_with_kl": 4.361847877502441,
|
|
"metrics/advantage_var": 695.4745483398438,
|
|
"regularization/advantage_var": 695.4745483398438,
|
|
"regularization/kl": 1.157965064048767,
|
|
"rewards/chosen": 0.5022166497544279,
|
|
"rewards/margins": 0.8467681256688189,
|
|
"rewards/rejected": -0.34455147591439095,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 18.962560653686523,
|
|
"kl": 14.576865196228027,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34796382.48085758,
|
|
"logits/rejected": -36728903.85964912,
|
|
"logps/chosen": -472.33700229709035,
|
|
"logps/rejected": -378.79022129186603,
|
|
"loss": 0.5334,
|
|
"loss/base_kto": 3.1811161041259766,
|
|
"loss/total_with_kl": 4.267554759979248,
|
|
"metrics/advantage_var": 652.5154418945312,
|
|
"regularization/advantage_var": 652.5154418945312,
|
|
"regularization/kl": 1.0864381790161133,
|
|
"rewards/chosen": 0.5820119486833605,
|
|
"rewards/margins": 0.8358198074805221,
|
|
"rewards/rejected": -0.25380785879716156,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 10.036248207092285,
|
|
"kl": 14.154253959655762,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35233821.07255521,
|
|
"logits/rejected": -37613000.52012384,
|
|
"logps/chosen": -504.1319006309148,
|
|
"logps/rejected": -370.36348684210526,
|
|
"loss": 0.5104,
|
|
"loss/base_kto": 3.1802279949188232,
|
|
"loss/total_with_kl": 4.08314847946167,
|
|
"metrics/advantage_var": 542.2943725585938,
|
|
"regularization/advantage_var": 542.2943725585938,
|
|
"regularization/kl": 0.9029200673103333,
|
|
"rewards/chosen": 0.5362653762384167,
|
|
"rewards/margins": 0.8688510682750484,
|
|
"rewards/rejected": -0.3325856920366317,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 17.11452865600586,
|
|
"kl": 14.6259183883667,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35942132.18461538,
|
|
"logits/rejected": -35888654.62857143,
|
|
"logps/chosen": -471.0679326923077,
|
|
"logps/rejected": -390.24900793650795,
|
|
"loss": 0.5128,
|
|
"loss/base_kto": 3.2217698097229004,
|
|
"loss/total_with_kl": 4.102136135101318,
|
|
"metrics/advantage_var": 528.7486572265625,
|
|
"regularization/advantage_var": 528.7486572265625,
|
|
"regularization/kl": 0.8803665041923523,
|
|
"rewards/chosen": 0.4991877159705529,
|
|
"rewards/margins": 0.8237822197062921,
|
|
"rewards/rejected": -0.3245945037357391,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 13.426315307617188,
|
|
"kl": 21.187501907348633,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -35899219.313609466,
|
|
"logits/rejected": -36108952.58278146,
|
|
"logps/chosen": -478.40939349112426,
|
|
"logps/rejected": -376.22198365066225,
|
|
"loss": 0.5256,
|
|
"loss/base_kto": 3.0841805934906006,
|
|
"loss/total_with_kl": 4.204774856567383,
|
|
"metrics/advantage_var": 673.0297241210938,
|
|
"regularization/advantage_var": 673.0297241210938,
|
|
"regularization/kl": 1.1205943822860718,
|
|
"rewards/chosen": 0.8278614473060744,
|
|
"rewards/margins": 0.9817192026512989,
|
|
"rewards/rejected": -0.15385775534522456,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 18.595876693725586,
|
|
"kl": 16.4093074798584,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -36212147.993474714,
|
|
"logits/rejected": -36312851.574212894,
|
|
"logps/chosen": -480.4773654159869,
|
|
"logps/rejected": -384.2383573838081,
|
|
"loss": 0.5144,
|
|
"loss/base_kto": 3.121016025543213,
|
|
"loss/total_with_kl": 4.115411281585693,
|
|
"metrics/advantage_var": 597.2344970703125,
|
|
"regularization/advantage_var": 597.2344970703125,
|
|
"regularization/kl": 0.9943954348564148,
|
|
"rewards/chosen": 0.6110618802890497,
|
|
"rewards/margins": 0.9445265413911159,
|
|
"rewards/rejected": -0.33346466110206613,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 15.014665603637695,
|
|
"kl": 12.478160858154297,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34328689.41772152,
|
|
"logits/rejected": -36343002.074074075,
|
|
"logps/chosen": -483.02066851265823,
|
|
"logps/rejected": -361.3313319830247,
|
|
"loss": 0.5019,
|
|
"loss/base_kto": 3.1962921619415283,
|
|
"loss/total_with_kl": 4.014953136444092,
|
|
"metrics/advantage_var": 491.6882019042969,
|
|
"regularization/advantage_var": 491.6882019042969,
|
|
"regularization/kl": 0.8186608552932739,
|
|
"rewards/chosen": 0.4755639184879351,
|
|
"rewards/margins": 0.8383861097027909,
|
|
"rewards/rejected": -0.3628221912148558,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 20.065237045288086,
|
|
"kl": 17.541168212890625,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35264845.91304348,
|
|
"logits/rejected": -36458199.748427674,
|
|
"logps/chosen": -471.6372767857143,
|
|
"logps/rejected": -382.64450668238993,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.1861112117767334,
|
|
"loss/total_with_kl": 4.144787788391113,
|
|
"metrics/advantage_var": 575.7818603515625,
|
|
"regularization/advantage_var": 575.7818603515625,
|
|
"regularization/kl": 0.9586767554283142,
|
|
"rewards/chosen": 0.6077950045188761,
|
|
"rewards/margins": 0.8923384557920926,
|
|
"rewards/rejected": -0.2845434512732164,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 7.35832405090332,
|
|
"kl": 19.599578857421875,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34626447.629057184,
|
|
"logits/rejected": -36631433.90837283,
|
|
"logps/chosen": -498.6435954404946,
|
|
"logps/rejected": -385.8411581753555,
|
|
"loss": 0.489,
|
|
"loss/base_kto": 3.1960391998291016,
|
|
"loss/total_with_kl": 3.9119606018066406,
|
|
"metrics/advantage_var": 429.9827575683594,
|
|
"regularization/advantage_var": 429.9827575683594,
|
|
"regularization/kl": 0.7159212827682495,
|
|
"rewards/chosen": 0.6508269273147701,
|
|
"rewards/margins": 0.8370466594697489,
|
|
"rewards/rejected": -0.18621973215497878,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 12.111591339111328,
|
|
"kl": 23.701385498046875,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34556157.123595506,
|
|
"logits/rejected": -35505739.592085235,
|
|
"logps/chosen": -490.5776484751204,
|
|
"logps/rejected": -352.76812214611874,
|
|
"loss": 0.4942,
|
|
"loss/base_kto": 3.158454418182373,
|
|
"loss/total_with_kl": 3.953991651535034,
|
|
"metrics/advantage_var": 477.8002014160156,
|
|
"regularization/advantage_var": 477.8002014160156,
|
|
"regularization/kl": 0.7955374121665955,
|
|
"rewards/chosen": 0.6967105283783106,
|
|
"rewards/margins": 0.8574777454018611,
|
|
"rewards/rejected": -0.16076721702355046,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 27.519393920898438,
|
|
"kl": 12.274657249450684,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34909792.55784469,
|
|
"logits/rejected": -36648925.288135596,
|
|
"logps/chosen": -481.07052297939777,
|
|
"logps/rejected": -384.71275520030815,
|
|
"loss": 0.5165,
|
|
"loss/base_kto": 3.194960355758667,
|
|
"loss/total_with_kl": 4.131756782531738,
|
|
"metrics/advantage_var": 562.6405639648438,
|
|
"regularization/advantage_var": 562.6405639648438,
|
|
"regularization/kl": 0.9367966055870056,
|
|
"rewards/chosen": 0.5420264290932052,
|
|
"rewards/margins": 0.8606375388458896,
|
|
"rewards/rejected": -0.3186111097526844,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 12.714757919311523,
|
|
"kl": 11.496892929077148,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -33980278.81339713,
|
|
"logits/rejected": -35730028.98621746,
|
|
"logps/chosen": -479.1045653907496,
|
|
"logps/rejected": -383.17237748851454,
|
|
"loss": 0.5197,
|
|
"loss/base_kto": 3.217435598373413,
|
|
"loss/total_with_kl": 4.157291412353516,
|
|
"metrics/advantage_var": 564.4779663085938,
|
|
"regularization/advantage_var": 564.4779663085938,
|
|
"regularization/kl": 0.9398557543754578,
|
|
"rewards/chosen": 0.5248745031524122,
|
|
"rewards/margins": 0.8473018526227056,
|
|
"rewards/rejected": -0.32242734947029333,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 18.978567123413086,
|
|
"kl": 9.191932678222656,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35137578.06003159,
|
|
"logits/rejected": -35601643.82071097,
|
|
"logps/chosen": -455.7292654028436,
|
|
"logps/rejected": -402.95532264296753,
|
|
"loss": 0.5785,
|
|
"loss/base_kto": 3.204576253890991,
|
|
"loss/total_with_kl": 4.6282830238342285,
|
|
"metrics/advantage_var": 855.0794067382812,
|
|
"regularization/advantage_var": 855.0794067382812,
|
|
"regularization/kl": 1.4237072467803955,
|
|
"rewards/chosen": 0.39985444609776116,
|
|
"rewards/margins": 0.8191401687389175,
|
|
"rewards/rejected": -0.4192857226411563,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 16.078025817871094,
|
|
"kl": 8.582796096801758,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -34820460.89245586,
|
|
"logits/rejected": -35349994.95890411,
|
|
"logps/chosen": -497.6932182985554,
|
|
"logps/rejected": -374.38256278538813,
|
|
"loss": 0.4972,
|
|
"loss/base_kto": 3.1804006099700928,
|
|
"loss/total_with_kl": 3.9778900146484375,
|
|
"metrics/advantage_var": 478.97271728515625,
|
|
"regularization/advantage_var": 478.97271728515625,
|
|
"regularization/kl": 0.797489583492279,
|
|
"rewards/chosen": 0.4434997166714737,
|
|
"rewards/margins": 0.8916125876760399,
|
|
"rewards/rejected": -0.4481128710045662,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 13.291605949401855,
|
|
"kl": 9.347896575927734,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -34984493.671361506,
|
|
"logits/rejected": -35968515.99375975,
|
|
"logps/chosen": -480.32619327073553,
|
|
"logps/rejected": -391.31167121684865,
|
|
"loss": 0.5093,
|
|
"loss/base_kto": 3.2492711544036865,
|
|
"loss/total_with_kl": 4.074707508087158,
|
|
"metrics/advantage_var": 495.75732421875,
|
|
"regularization/advantage_var": 495.75732421875,
|
|
"regularization/kl": 0.8254359364509583,
|
|
"rewards/chosen": 0.40781231851831473,
|
|
"rewards/margins": 0.794915751874399,
|
|
"rewards/rejected": -0.38710343335608427,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 18.700685501098633,
|
|
"kl": 16.065826416015625,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35745333.72839506,
|
|
"logits/rejected": -37859891.848101266,
|
|
"logps/chosen": -490.93663194444446,
|
|
"logps/rejected": -366.12776898734177,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.213034152984619,
|
|
"loss/total_with_kl": 4.099498271942139,
|
|
"metrics/advantage_var": 532.4105834960938,
|
|
"regularization/advantage_var": 532.4105834960938,
|
|
"regularization/kl": 0.8864635825157166,
|
|
"rewards/chosen": 0.541022218303916,
|
|
"rewards/margins": 0.8151354025930031,
|
|
"rewards/rejected": -0.2741131842890872,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 13.629622459411621,
|
|
"kl": 13.1502046585083,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33882845.03610675,
|
|
"logits/rejected": -35588783.97511664,
|
|
"logps/chosen": -498.35341444270017,
|
|
"logps/rejected": -381.63617807153963,
|
|
"loss": 0.5212,
|
|
"loss/base_kto": 3.176405668258667,
|
|
"loss/total_with_kl": 4.169734477996826,
|
|
"metrics/advantage_var": 596.593505859375,
|
|
"regularization/advantage_var": 596.593505859375,
|
|
"regularization/kl": 0.9933280944824219,
|
|
"rewards/chosen": 0.5328028033629317,
|
|
"rewards/margins": 0.8442190259189436,
|
|
"rewards/rejected": -0.31141622255601187,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 11.953184127807617,
|
|
"kl": 12.940622329711914,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34304535.343465045,
|
|
"logits/rejected": -36408683.35483871,
|
|
"logps/chosen": -488.5472074468085,
|
|
"logps/rejected": -350.0005796370968,
|
|
"loss": 0.4734,
|
|
"loss/base_kto": 3.148651599884033,
|
|
"loss/total_with_kl": 3.7874207496643066,
|
|
"metrics/advantage_var": 383.6451721191406,
|
|
"regularization/advantage_var": 383.6451721191406,
|
|
"regularization/kl": 0.6387692093849182,
|
|
"rewards/chosen": 0.5164814401180187,
|
|
"rewards/margins": 0.8778054304784018,
|
|
"rewards/rejected": -0.36132399036038304,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 11.690995216369629,
|
|
"kl": 16.725244522094727,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -33632077.489698894,
|
|
"logits/rejected": -35001544.382126346,
|
|
"logps/chosen": -463.4868264659271,
|
|
"logps/rejected": -391.66393008474574,
|
|
"loss": 0.4609,
|
|
"loss/base_kto": 3.1087894439697266,
|
|
"loss/total_with_kl": 3.6873364448547363,
|
|
"metrics/advantage_var": 347.4755859375,
|
|
"regularization/advantage_var": 347.4755859375,
|
|
"regularization/kl": 0.578546941280365,
|
|
"rewards/chosen": 0.6607287810457112,
|
|
"rewards/margins": 0.9061654861780061,
|
|
"rewards/rejected": -0.24543670513229487,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 13.24403190612793,
|
|
"kl": 13.528584480285645,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -34882627.864506625,
|
|
"logits/rejected": -36557372.485856906,
|
|
"logps/chosen": -488.2106038291605,
|
|
"logps/rejected": -363.5569883527454,
|
|
"loss": 0.4661,
|
|
"loss/base_kto": 3.1356799602508545,
|
|
"loss/total_with_kl": 3.728633165359497,
|
|
"metrics/advantage_var": 356.1280822753906,
|
|
"regularization/advantage_var": 356.1280822753906,
|
|
"regularization/kl": 0.5929532051086426,
|
|
"rewards/chosen": 0.5750067237550626,
|
|
"rewards/margins": 0.8577097362310141,
|
|
"rewards/rejected": -0.28270301247595153,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 11.460430145263672,
|
|
"kl": 16.43263816833496,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -36409579.61061947,
|
|
"logits/rejected": -36912757.36877076,
|
|
"logps/chosen": -495.63901179941,
|
|
"logps/rejected": -395.6131384966777,
|
|
"loss": 0.4628,
|
|
"loss/base_kto": 3.0920205116271973,
|
|
"loss/total_with_kl": 3.702526807785034,
|
|
"metrics/advantage_var": 366.6706237792969,
|
|
"regularization/advantage_var": 366.6706237792969,
|
|
"regularization/kl": 0.6105066537857056,
|
|
"rewards/chosen": 0.6819064610123986,
|
|
"rewards/margins": 0.9130183106736839,
|
|
"rewards/rejected": -0.2311118496612853,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 10.828267097473145,
|
|
"kl": 25.990583419799805,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35216284.7030303,
|
|
"logits/rejected": -36907652.12903226,
|
|
"logps/chosen": -482.9846590909091,
|
|
"logps/rejected": -384.07714213709676,
|
|
"loss": 0.4861,
|
|
"loss/base_kto": 3.124025583267212,
|
|
"loss/total_with_kl": 3.888789415359497,
|
|
"metrics/advantage_var": 459.31768798828125,
|
|
"regularization/advantage_var": 459.31768798828125,
|
|
"regularization/kl": 0.7647639513015747,
|
|
"rewards/chosen": 0.7531587080522018,
|
|
"rewards/margins": 0.8602593240150609,
|
|
"rewards/rejected": -0.10710061596285912,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 8.08063793182373,
|
|
"kl": 25.383153915405273,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35300228.15339233,
|
|
"logits/rejected": -36579253.156146176,
|
|
"logps/chosen": -493.70174225663715,
|
|
"logps/rejected": -387.1957537375415,
|
|
"loss": 0.4721,
|
|
"loss/base_kto": 3.09136700630188,
|
|
"loss/total_with_kl": 3.776615619659424,
|
|
"metrics/advantage_var": 411.5606384277344,
|
|
"regularization/advantage_var": 411.5606384277344,
|
|
"regularization/kl": 0.6852484941482544,
|
|
"rewards/chosen": 0.7989593775926438,
|
|
"rewards/margins": 0.8929000237420351,
|
|
"rewards/rejected": -0.09394064614939135,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 12.848079681396484,
|
|
"kl": 27.20682716369629,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35068282.04173355,
|
|
"logits/rejected": -36154971.17808219,
|
|
"logps/chosen": -473.7116272070626,
|
|
"logps/rejected": -374.97814402587517,
|
|
"loss": 0.4716,
|
|
"loss/base_kto": 3.1573140621185303,
|
|
"loss/total_with_kl": 3.7725541591644287,
|
|
"metrics/advantage_var": 369.5137023925781,
|
|
"regularization/advantage_var": 369.5137023925781,
|
|
"regularization/kl": 0.6152403354644775,
|
|
"rewards/chosen": 0.7845997818017657,
|
|
"rewards/margins": 0.8580667519348577,
|
|
"rewards/rejected": -0.07346697013309195,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.91923999786377,
|
|
"kl": 19.138383865356445,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34709870.28304821,
|
|
"logits/rejected": -35458146.05965463,
|
|
"logps/chosen": -486.7598172628305,
|
|
"logps/rejected": -348.4806220565149,
|
|
"loss": 0.4607,
|
|
"loss/base_kto": 3.083167314529419,
|
|
"loss/total_with_kl": 3.6852643489837646,
|
|
"metrics/advantage_var": 361.61968994140625,
|
|
"regularization/advantage_var": 361.61968994140625,
|
|
"regularization/kl": 0.6020968556404114,
|
|
"rewards/chosen": 0.6897910490377138,
|
|
"rewards/margins": 0.9076763763394579,
|
|
"rewards/rejected": -0.217885327301744,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 9.267581939697266,
|
|
"kl": 21.036916732788086,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34937308.12738854,
|
|
"logits/rejected": -35924099.926380366,
|
|
"logps/chosen": -441.3529060509554,
|
|
"logps/rejected": -390.4515912576687,
|
|
"loss": 0.4626,
|
|
"loss/base_kto": 3.1236610412597656,
|
|
"loss/total_with_kl": 3.701164960861206,
|
|
"metrics/advantage_var": 346.8492736816406,
|
|
"regularization/advantage_var": 346.8492736816406,
|
|
"regularization/kl": 0.5775040984153748,
|
|
"rewards/chosen": 0.6530756373314341,
|
|
"rewards/margins": 0.8518194926264399,
|
|
"rewards/rejected": -0.19874385529500574,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 7.988361835479736,
|
|
"kl": 16.773530960083008,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35164786.67507886,
|
|
"logits/rejected": -36039099.839009285,
|
|
"logps/chosen": -467.0486001577287,
|
|
"logps/rejected": -374.7779605263158,
|
|
"loss": 0.472,
|
|
"loss/base_kto": 3.072448492050171,
|
|
"loss/total_with_kl": 3.7760262489318848,
|
|
"metrics/advantage_var": 422.5692443847656,
|
|
"regularization/advantage_var": 422.5692443847656,
|
|
"regularization/kl": 0.7035778760910034,
|
|
"rewards/chosen": 0.6349966729100947,
|
|
"rewards/margins": 0.9603737208946873,
|
|
"rewards/rejected": -0.32537704798459266,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 9.556100845336914,
|
|
"kl": 21.25367546081543,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35548117.19614148,
|
|
"logits/rejected": -37375533.13069909,
|
|
"logps/chosen": -493.2891881028939,
|
|
"logps/rejected": -356.5102108662614,
|
|
"loss": 0.4614,
|
|
"loss/base_kto": 3.1235148906707764,
|
|
"loss/total_with_kl": 3.691087484359741,
|
|
"metrics/advantage_var": 340.8845520019531,
|
|
"regularization/advantage_var": 340.8845520019531,
|
|
"regularization/kl": 0.5675727725028992,
|
|
"rewards/chosen": 0.6359744056628065,
|
|
"rewards/margins": 0.8620812290593443,
|
|
"rewards/rejected": -0.22610682339653781,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 13.81753921508789,
|
|
"kl": 17.34554672241211,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -36035330.75268817,
|
|
"logits/rejected": -37101060.0699523,
|
|
"logps/chosen": -458.11578341013825,
|
|
"logps/rejected": -369.71094992050877,
|
|
"loss": 0.4713,
|
|
"loss/base_kto": 3.1017704010009766,
|
|
"loss/total_with_kl": 3.7706711292266846,
|
|
"metrics/advantage_var": 401.74224853515625,
|
|
"regularization/advantage_var": 401.74224853515625,
|
|
"regularization/kl": 0.6689009070396423,
|
|
"rewards/chosen": 0.6609592701432891,
|
|
"rewards/margins": 0.9111267529408845,
|
|
"rewards/rejected": -0.2501674827975954,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 10.761502265930176,
|
|
"kl": 18.38096046447754,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34642998.57053292,
|
|
"logits/rejected": -35020155.61370716,
|
|
"logps/chosen": -479.7155172413793,
|
|
"logps/rejected": -355.86898851246104,
|
|
"loss": 0.4729,
|
|
"loss/base_kto": 3.0909461975097656,
|
|
"loss/total_with_kl": 3.78312087059021,
|
|
"metrics/advantage_var": 415.7206115722656,
|
|
"regularization/advantage_var": 415.7206115722656,
|
|
"regularization/kl": 0.6921747326850891,
|
|
"rewards/chosen": 0.6938356979513617,
|
|
"rewards/margins": 0.9060003183240982,
|
|
"rewards/rejected": -0.21216462037273656,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 11.66818618774414,
|
|
"kl": 17.343162536621094,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34119430.705701075,
|
|
"logits/rejected": -37607920.58320127,
|
|
"logps/chosen": -511.9595531587057,
|
|
"logps/rejected": -379.2196414421553,
|
|
"loss": 0.4747,
|
|
"loss/base_kto": 3.0926403999328613,
|
|
"loss/total_with_kl": 3.7974541187286377,
|
|
"metrics/advantage_var": 423.31134033203125,
|
|
"regularization/advantage_var": 423.31134033203125,
|
|
"regularization/kl": 0.7048134207725525,
|
|
"rewards/chosen": 0.6449178216637375,
|
|
"rewards/margins": 0.887506548015758,
|
|
"rewards/rejected": -0.2425887263520206,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 11.241945266723633,
|
|
"kl": 15.661109924316406,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35104141.06122449,
|
|
"logits/rejected": -36752129.47976878,
|
|
"logps/chosen": -486.69254889455783,
|
|
"logps/rejected": -391.0997561416185,
|
|
"loss": 0.4644,
|
|
"loss/base_kto": 3.06414794921875,
|
|
"loss/total_with_kl": 3.71484375,
|
|
"metrics/advantage_var": 390.80810546875,
|
|
"regularization/advantage_var": 390.80810546875,
|
|
"regularization/kl": 0.6506955027580261,
|
|
"rewards/chosen": 0.6317532078749468,
|
|
"rewards/margins": 0.9362236890859972,
|
|
"rewards/rejected": -0.3044704812110504,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 13.073074340820312,
|
|
"kl": 10.715472221374512,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35160555.52,
|
|
"logits/rejected": -36325210.28396947,
|
|
"logps/chosen": -500.19295,
|
|
"logps/rejected": -380.86836832061067,
|
|
"loss": 0.4788,
|
|
"loss/base_kto": 3.1275978088378906,
|
|
"loss/total_with_kl": 3.830390691757202,
|
|
"metrics/advantage_var": 422.0977478027344,
|
|
"regularization/advantage_var": 422.0977478027344,
|
|
"regularization/kl": 0.702792763710022,
|
|
"rewards/chosen": 0.49115185546875,
|
|
"rewards/margins": 0.910570400092438,
|
|
"rewards/rejected": -0.419418544623688,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 9.682475090026855,
|
|
"kl": 13.271239280700684,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -34782306.23739837,
|
|
"logits/rejected": -36713338.03308271,
|
|
"logps/chosen": -492.6305386178862,
|
|
"logps/rejected": -348.612265037594,
|
|
"loss": 0.4603,
|
|
"loss/base_kto": 3.121549606323242,
|
|
"loss/total_with_kl": 3.6824288368225098,
|
|
"metrics/advantage_var": 336.8644714355469,
|
|
"regularization/advantage_var": 336.8644714355469,
|
|
"regularization/kl": 0.5608793497085571,
|
|
"rewards/chosen": 0.4957244624936484,
|
|
"rewards/margins": 0.8713876576519544,
|
|
"rewards/rejected": -0.37566319515830593,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 10.05482292175293,
|
|
"kl": 13.020792961120605,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34836158.12634823,
|
|
"logits/rejected": -37248641.01426307,
|
|
"logps/chosen": -474.77860169491527,
|
|
"logps/rejected": -363.87762480190173,
|
|
"loss": 0.4668,
|
|
"loss/base_kto": 3.0941388607025146,
|
|
"loss/total_with_kl": 3.7342286109924316,
|
|
"metrics/advantage_var": 384.4383239746094,
|
|
"regularization/advantage_var": 384.4383239746094,
|
|
"regularization/kl": 0.6400898694992065,
|
|
"rewards/chosen": 0.5670388902096977,
|
|
"rewards/margins": 0.9124092236802552,
|
|
"rewards/rejected": -0.3453703334705576,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 12.21088981628418,
|
|
"kl": 15.068017959594727,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34262025.00879765,
|
|
"logits/rejected": -35611195.93311037,
|
|
"logps/chosen": -478.4992668621701,
|
|
"logps/rejected": -355.3208612040134,
|
|
"loss": 0.4541,
|
|
"loss/base_kto": 3.116960287094116,
|
|
"loss/total_with_kl": 3.632725477218628,
|
|
"metrics/advantage_var": 309.7688903808594,
|
|
"regularization/advantage_var": 309.7688903808594,
|
|
"regularization/kl": 0.5157651901245117,
|
|
"rewards/chosen": 0.5903962434561721,
|
|
"rewards/margins": 0.8882983028517486,
|
|
"rewards/rejected": -0.2979020593955764,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 29.03020668029785,
|
|
"kl": 19.16185760498047,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -33774752.38554217,
|
|
"logits/rejected": -35445021.922077924,
|
|
"logps/chosen": -486.2610598644578,
|
|
"logps/rejected": -378.3712713068182,
|
|
"loss": 0.4639,
|
|
"loss/base_kto": 3.033926486968994,
|
|
"loss/total_with_kl": 3.7110137939453125,
|
|
"metrics/advantage_var": 406.65899658203125,
|
|
"regularization/advantage_var": 406.65899658203125,
|
|
"regularization/kl": 0.6770871877670288,
|
|
"rewards/chosen": 0.7380514029996941,
|
|
"rewards/margins": 0.9736781191180484,
|
|
"rewards/rejected": -0.2356267161183543,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.977595329284668,
|
|
"kl": 18.823400497436523,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35876762.94736842,
|
|
"logits/rejected": -36554453.333333336,
|
|
"logps/chosen": -472.7612047697368,
|
|
"logps/rejected": -371.73511904761904,
|
|
"loss": 0.4547,
|
|
"loss/base_kto": 3.0908591747283936,
|
|
"loss/total_with_kl": 3.637716770172119,
|
|
"metrics/advantage_var": 328.4429016113281,
|
|
"regularization/advantage_var": 328.4429016113281,
|
|
"regularization/kl": 0.5468574166297913,
|
|
"rewards/chosen": 0.6265672382555509,
|
|
"rewards/margins": 0.8871684636089736,
|
|
"rewards/rejected": -0.2606012253534226,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 10.993363380432129,
|
|
"kl": 16.923913955688477,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34935068.88408037,
|
|
"logits/rejected": -36321050.287519746,
|
|
"logps/chosen": -472.142436244204,
|
|
"logps/rejected": -389.6239139020537,
|
|
"loss": 0.4542,
|
|
"loss/base_kto": 3.091399908065796,
|
|
"loss/total_with_kl": 3.6336147785186768,
|
|
"metrics/advantage_var": 325.6545104980469,
|
|
"regularization/advantage_var": 325.6545104980469,
|
|
"regularization/kl": 0.5422146916389465,
|
|
"rewards/chosen": 0.6149391044604907,
|
|
"rewards/margins": 0.8938580532441844,
|
|
"rewards/rejected": -0.2789189487836937,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 12.73780345916748,
|
|
"kl": 16.31883430480957,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -35137060.9153605,
|
|
"logits/rejected": -36178711.127725855,
|
|
"logps/chosen": -483.6540948275862,
|
|
"logps/rejected": -366.36499707943926,
|
|
"loss": 0.4553,
|
|
"loss/base_kto": 3.1180169582366943,
|
|
"loss/total_with_kl": 3.642152786254883,
|
|
"metrics/advantage_var": 314.79638671875,
|
|
"regularization/advantage_var": 314.79638671875,
|
|
"regularization/kl": 0.5241360068321228,
|
|
"rewards/chosen": 0.628901849346101,
|
|
"rewards/margins": 0.8812360162561478,
|
|
"rewards/rejected": -0.25233416691004673,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 18.77777099609375,
|
|
"kl": 17.32157325744629,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35022478.58227848,
|
|
"logits/rejected": -36610834.96296296,
|
|
"logps/chosen": -488.8885482594937,
|
|
"logps/rejected": -372.50441261574076,
|
|
"loss": 0.466,
|
|
"loss/base_kto": 3.1631367206573486,
|
|
"loss/total_with_kl": 3.7282981872558594,
|
|
"metrics/advantage_var": 339.436279296875,
|
|
"regularization/advantage_var": 339.436279296875,
|
|
"regularization/kl": 0.565161406993866,
|
|
"rewards/chosen": 0.6058276212668117,
|
|
"rewards/margins": 0.8357870385393088,
|
|
"rewards/rejected": -0.2299594172724971,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 10.02481746673584,
|
|
"kl": 16.70195198059082,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35962632.33488372,
|
|
"logits/rejected": -36337080.23937008,
|
|
"logps/chosen": -488.5249031007752,
|
|
"logps/rejected": -367.4638779527559,
|
|
"loss": 0.4603,
|
|
"loss/base_kto": 3.1158604621887207,
|
|
"loss/total_with_kl": 3.6822052001953125,
|
|
"metrics/advantage_var": 340.14703369140625,
|
|
"regularization/advantage_var": 340.14703369140625,
|
|
"regularization/kl": 0.5663448572158813,
|
|
"rewards/chosen": 0.6587526779766231,
|
|
"rewards/margins": 0.8999231977077993,
|
|
"rewards/rejected": -0.24117051973117618,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 9.507659912109375,
|
|
"kl": 17.795928955078125,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -33117129.9328,
|
|
"logits/rejected": -35644486.35114504,
|
|
"logps/chosen": -449.96215,
|
|
"logps/rejected": -387.6421040076336,
|
|
"loss": 0.4522,
|
|
"loss/base_kto": 3.115571975708008,
|
|
"loss/total_with_kl": 3.617614507675171,
|
|
"metrics/advantage_var": 301.527099609375,
|
|
"regularization/advantage_var": 301.527099609375,
|
|
"regularization/kl": 0.5020425915718079,
|
|
"rewards/chosen": 0.61205830078125,
|
|
"rewards/margins": 0.8676137113847806,
|
|
"rewards/rejected": -0.25555541060353054,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 24.53262710571289,
|
|
"kl": 18.590635299682617,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35027106.025316454,
|
|
"logits/rejected": -36655957.333333336,
|
|
"logps/chosen": -457.34622231012656,
|
|
"logps/rejected": -363.8410011574074,
|
|
"loss": 0.47,
|
|
"loss/base_kto": 3.106231689453125,
|
|
"loss/total_with_kl": 3.75982666015625,
|
|
"metrics/advantage_var": 392.54962158203125,
|
|
"regularization/advantage_var": 392.54962158203125,
|
|
"regularization/kl": 0.6535950899124146,
|
|
"rewards/chosen": 0.6397764471512807,
|
|
"rewards/margins": 0.8830742124356894,
|
|
"rewards/rejected": -0.24329776528440875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 10.593400001525879,
|
|
"kl": 15.272354125976562,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35067172.132132135,
|
|
"logits/rejected": -36814694.56677525,
|
|
"logps/chosen": -501.37687687687685,
|
|
"logps/rejected": -373.24572475570034,
|
|
"loss": 0.4679,
|
|
"loss/base_kto": 3.116925001144409,
|
|
"loss/total_with_kl": 3.743285894393921,
|
|
"metrics/advantage_var": 376.1924743652344,
|
|
"regularization/advantage_var": 376.1924743652344,
|
|
"regularization/kl": 0.6263605356216431,
|
|
"rewards/chosen": 0.6096706447658596,
|
|
"rewards/margins": 0.8753517096325127,
|
|
"rewards/rejected": -0.2656810648666531,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 16.926231384277344,
|
|
"kl": 15.618414878845215,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34082707.875776395,
|
|
"logits/rejected": -35245461.73584906,
|
|
"logps/chosen": -473.8452057453416,
|
|
"logps/rejected": -383.31247543238993,
|
|
"loss": 0.4601,
|
|
"loss/base_kto": 3.1131198406219482,
|
|
"loss/total_with_kl": 3.680828094482422,
|
|
"metrics/advantage_var": 340.9659118652344,
|
|
"regularization/advantage_var": 340.9659118652344,
|
|
"regularization/kl": 0.5677081942558289,
|
|
"rewards/chosen": 0.6263199326414499,
|
|
"rewards/margins": 0.8792453975834704,
|
|
"rewards/rejected": -0.25292546494202045,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 13.325568199157715,
|
|
"kl": 18.657567977905273,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34629402.64984227,
|
|
"logits/rejected": -34480888.86687306,
|
|
"logps/chosen": -476.8927937697161,
|
|
"logps/rejected": -373.66021671826627,
|
|
"loss": 0.4595,
|
|
"loss/base_kto": 3.1403558254241943,
|
|
"loss/total_with_kl": 3.6762948036193848,
|
|
"metrics/advantage_var": 321.8852233886719,
|
|
"regularization/advantage_var": 321.8852233886719,
|
|
"regularization/kl": 0.5359388589859009,
|
|
"rewards/chosen": 0.6203554556572851,
|
|
"rewards/margins": 0.854642729430384,
|
|
"rewards/rejected": -0.23428727377309888,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 13.129006385803223,
|
|
"kl": 16.0499210357666,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34988103.29113924,
|
|
"logits/rejected": -37126311.506172836,
|
|
"logps/chosen": -477.37366495253167,
|
|
"logps/rejected": -385.6603732638889,
|
|
"loss": 0.4656,
|
|
"loss/base_kto": 3.1309595108032227,
|
|
"loss/total_with_kl": 3.724468946456909,
|
|
"metrics/advantage_var": 356.4622497558594,
|
|
"regularization/advantage_var": 356.4622497558594,
|
|
"regularization/kl": 0.5935096144676208,
|
|
"rewards/chosen": 0.6041377586654469,
|
|
"rewards/margins": 0.8480117551645909,
|
|
"rewards/rejected": -0.243873996499144,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 19.92145347595215,
|
|
"kl": 18.164865493774414,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35185378.19457735,
|
|
"logits/rejected": -36843996.71669219,
|
|
"logps/chosen": -475.40455542264755,
|
|
"logps/rejected": -377.80383805513014,
|
|
"loss": 0.4729,
|
|
"loss/base_kto": 3.0925214290618896,
|
|
"loss/total_with_kl": 3.7832858562469482,
|
|
"metrics/advantage_var": 414.87353515625,
|
|
"regularization/advantage_var": 414.87353515625,
|
|
"regularization/kl": 0.6907644271850586,
|
|
"rewards/chosen": 0.6183269202614134,
|
|
"rewards/margins": 0.8891867364343652,
|
|
"rewards/rejected": -0.27085981617295174,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 12.983906745910645,
|
|
"kl": 15.957466125488281,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -36596476.4608295,
|
|
"logits/rejected": -36720672.55961844,
|
|
"logps/chosen": -481.4262192780338,
|
|
"logps/rejected": -390.82802563593003,
|
|
"loss": 0.4621,
|
|
"loss/base_kto": 3.1289126873016357,
|
|
"loss/total_with_kl": 3.697082281112671,
|
|
"metrics/advantage_var": 341.2428894042969,
|
|
"regularization/advantage_var": 341.2428894042969,
|
|
"regularization/kl": 0.5681694149971008,
|
|
"rewards/chosen": 0.6093246530277938,
|
|
"rewards/margins": 0.8547525473168439,
|
|
"rewards/rejected": -0.24542789428905007,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 38.83980941772461,
|
|
"kl": 14.251669883728027,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -36566716.30395137,
|
|
"logits/rejected": -35861217.54340836,
|
|
"logps/chosen": -501.7251614741641,
|
|
"logps/rejected": -382.47008139067526,
|
|
"loss": 0.4769,
|
|
"loss/base_kto": 3.0682308673858643,
|
|
"loss/total_with_kl": 3.814955949783325,
|
|
"metrics/advantage_var": 448.48370361328125,
|
|
"regularization/advantage_var": 448.48370361328125,
|
|
"regularization/kl": 0.74672532081604,
|
|
"rewards/chosen": 0.6730838300246961,
|
|
"rewards/margins": 0.9568195897966012,
|
|
"rewards/rejected": -0.2837357597719051,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 6.46622896194458,
|
|
"kl": 16.409635543823242,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35249085.88346457,
|
|
"logits/rejected": -35636279.565891474,
|
|
"logps/chosen": -461.93272637795275,
|
|
"logps/rejected": -387.534011627907,
|
|
"loss": 0.4576,
|
|
"loss/base_kto": 3.0873970985412598,
|
|
"loss/total_with_kl": 3.661062717437744,
|
|
"metrics/advantage_var": 344.54400634765625,
|
|
"regularization/advantage_var": 344.54400634765625,
|
|
"regularization/kl": 0.5736657381057739,
|
|
"rewards/chosen": 0.6180735190083662,
|
|
"rewards/margins": 0.8716921467487974,
|
|
"rewards/rejected": -0.2536186277404312,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 17.902297973632812,
|
|
"kl": 18.096309661865234,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -35743181.216260165,
|
|
"logits/rejected": -37807970.93533835,
|
|
"logps/chosen": -488.88252032520325,
|
|
"logps/rejected": -375.5296992481203,
|
|
"loss": 0.4579,
|
|
"loss/base_kto": 3.0990636348724365,
|
|
"loss/total_with_kl": 3.6634557247161865,
|
|
"metrics/advantage_var": 338.9741516113281,
|
|
"regularization/advantage_var": 338.9741516113281,
|
|
"regularization/kl": 0.5643919706344604,
|
|
"rewards/chosen": 0.6239154598577236,
|
|
"rewards/margins": 0.8892775038838043,
|
|
"rewards/rejected": -0.2653620440260808,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 9.670804023742676,
|
|
"kl": 16.14820671081543,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -33755014.58970359,
|
|
"logits/rejected": -36095772.44444445,
|
|
"logps/chosen": -479.80952613104523,
|
|
"logps/rejected": -364.2539368153365,
|
|
"loss": 0.4625,
|
|
"loss/base_kto": 3.1111419200897217,
|
|
"loss/total_with_kl": 3.700047254562378,
|
|
"metrics/advantage_var": 353.6966857910156,
|
|
"regularization/advantage_var": 353.6966857910156,
|
|
"regularization/kl": 0.5889049768447876,
|
|
"rewards/chosen": 0.632166441442814,
|
|
"rewards/margins": 0.8874523820757609,
|
|
"rewards/rejected": -0.25528594063294696,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 9.49049186706543,
|
|
"kl": 17.153213500976562,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34444536.29160064,
|
|
"logits/rejected": -35969904.41910632,
|
|
"logps/chosen": -479.5860736925515,
|
|
"logps/rejected": -369.3864599383667,
|
|
"loss": 0.4625,
|
|
"loss/base_kto": 3.0792396068573,
|
|
"loss/total_with_kl": 3.700125217437744,
|
|
"metrics/advantage_var": 372.904296875,
|
|
"regularization/advantage_var": 372.904296875,
|
|
"regularization/kl": 0.6208856701850891,
|
|
"rewards/chosen": 0.6217606872461866,
|
|
"rewards/margins": 0.8986652474469763,
|
|
"rewards/rejected": -0.2769045602007897,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_loss": 0.5004740403723841,
|
|
"train_runtime": 11074.4958,
|
|
"train_samples_per_second": 13.384,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|