60927b710d
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed5 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 12.543169975280762,
|
|
"kl": 4.120953559875488,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -36844670.29333334,
|
|
"logits/rejected": -38798299.85882353,
|
|
"logps/chosen": -485.039375,
|
|
"logps/rejected": -356.75268841911765,
|
|
"loss": 0.5235,
|
|
"loss/base_kto": 3.844444751739502,
|
|
"loss/total_with_kl": 4.188304901123047,
|
|
"metrics/advantage_var": 206.522705078125,
|
|
"regularization/advantage_var": 206.522705078125,
|
|
"regularization/kl": 0.34386029839515686,
|
|
"rewards/chosen": -0.14080973307291667,
|
|
"rewards/margins": 0.12140165141984527,
|
|
"rewards/rejected": -0.26221138449276193,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 12.811156272888184,
|
|
"kl": 3.7785446643829346,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -35289351.63579278,
|
|
"logits/rejected": -36426619.81959564,
|
|
"logps/chosen": -465.6356946624804,
|
|
"logps/rejected": -380.57100505443236,
|
|
"loss": 0.5192,
|
|
"loss/base_kto": 3.862229585647583,
|
|
"loss/total_with_kl": 4.1537041664123535,
|
|
"metrics/advantage_var": 175.059814453125,
|
|
"regularization/advantage_var": 175.059814453125,
|
|
"regularization/kl": 0.2914746105670929,
|
|
"rewards/chosen": -0.040490984355261786,
|
|
"rewards/margins": 0.1250243754313221,
|
|
"rewards/rejected": -0.16551535978658388,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 15.698122024536133,
|
|
"kl": 12.603705406188965,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -34628379.719745226,
|
|
"logits/rejected": -37336217.91411043,
|
|
"logps/chosen": -480.4050059713376,
|
|
"logps/rejected": -377.2071271088957,
|
|
"loss": 0.5197,
|
|
"loss/base_kto": 3.7998244762420654,
|
|
"loss/total_with_kl": 4.15751314163208,
|
|
"metrics/advantage_var": 214.82791137695312,
|
|
"regularization/advantage_var": 214.82791137695312,
|
|
"regularization/kl": 0.35768845677375793,
|
|
"rewards/chosen": 0.17579958241456634,
|
|
"rewards/margins": 0.1800836991683376,
|
|
"rewards/rejected": -0.004284116753771261,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 19.648683547973633,
|
|
"kl": 3.3418984413146973,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36155277.312,
|
|
"logits/rejected": -37515326.53435115,
|
|
"logps/chosen": -479.7178,
|
|
"logps/rejected": -372.2624284351145,
|
|
"loss": 0.5243,
|
|
"loss/base_kto": 3.84568452835083,
|
|
"loss/total_with_kl": 4.1944899559021,
|
|
"metrics/advantage_var": 209.49282836914062,
|
|
"regularization/advantage_var": 209.49282836914062,
|
|
"regularization/kl": 0.3488055169582367,
|
|
"rewards/chosen": 0.002515827178955078,
|
|
"rewards/margins": 0.14408785052699896,
|
|
"rewards/rejected": -0.1415720233480439,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 20.74751853942871,
|
|
"kl": 4.541070461273193,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35040742.27552674,
|
|
"logits/rejected": -36573913.001508296,
|
|
"logps/chosen": -447.9112135332253,
|
|
"logps/rejected": -362.3477564102564,
|
|
"loss": 0.5183,
|
|
"loss/base_kto": 3.8213508129119873,
|
|
"loss/total_with_kl": 4.146453380584717,
|
|
"metrics/advantage_var": 195.25657653808594,
|
|
"regularization/advantage_var": 195.25657653808594,
|
|
"regularization/kl": 0.32510218024253845,
|
|
"rewards/chosen": 0.0025399392002791988,
|
|
"rewards/margins": 0.17132490783391655,
|
|
"rewards/rejected": -0.16878496863363734,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 16.51796531677246,
|
|
"kl": 14.160174369812012,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35284802.25882353,
|
|
"logits/rejected": -36192822.61333334,
|
|
"logps/chosen": -481.9678308823529,
|
|
"logps/rejected": -367.79505208333336,
|
|
"loss": 0.5241,
|
|
"loss/base_kto": 3.804245710372925,
|
|
"loss/total_with_kl": 4.193059921264648,
|
|
"metrics/advantage_var": 233.52197265625,
|
|
"regularization/advantage_var": 233.52197265625,
|
|
"regularization/kl": 0.3888140618801117,
|
|
"rewards/chosen": 0.2053975273581112,
|
|
"rewards/margins": 0.19320797695833092,
|
|
"rewards/rejected": 0.012189550399780274,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 17.835784912109375,
|
|
"kl": 26.637731552124023,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -34895098.65074627,
|
|
"logits/rejected": -36363267.35737705,
|
|
"logps/chosen": -472.32835820895525,
|
|
"logps/rejected": -354.3390881147541,
|
|
"loss": 0.5104,
|
|
"loss/base_kto": 3.6856160163879395,
|
|
"loss/total_with_kl": 4.083138465881348,
|
|
"metrics/advantage_var": 238.752197265625,
|
|
"regularization/advantage_var": 238.752197265625,
|
|
"regularization/kl": 0.3975224196910858,
|
|
"rewards/chosen": 0.5182701452454525,
|
|
"rewards/margins": 0.2600720731559289,
|
|
"rewards/rejected": 0.25819807208952356,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 12.635039329528809,
|
|
"kl": 22.309574127197266,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36942477.68553459,
|
|
"logits/rejected": -36688085.06832298,
|
|
"logps/chosen": -472.00181800314465,
|
|
"logps/rejected": -363.2448078416149,
|
|
"loss": 0.5068,
|
|
"loss/base_kto": 3.64274525642395,
|
|
"loss/total_with_kl": 4.054568290710449,
|
|
"metrics/advantage_var": 247.34121704101562,
|
|
"regularization/advantage_var": 247.34121704101562,
|
|
"regularization/kl": 0.4118230938911438,
|
|
"rewards/chosen": 0.4226521426026926,
|
|
"rewards/margins": 0.32631989464945543,
|
|
"rewards/rejected": 0.09633224795323721,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 16.836219787597656,
|
|
"kl": 13.491203308105469,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -35903285.45054945,
|
|
"logits/rejected": -37087630.133748055,
|
|
"logps/chosen": -470.5286008634223,
|
|
"logps/rejected": -348.26875972006223,
|
|
"loss": 0.5112,
|
|
"loss/base_kto": 3.6826083660125732,
|
|
"loss/total_with_kl": 4.089618682861328,
|
|
"metrics/advantage_var": 244.4507598876953,
|
|
"regularization/advantage_var": 244.4507598876953,
|
|
"regularization/kl": 0.4070105254650116,
|
|
"rewards/chosen": 0.2844440214675113,
|
|
"rewards/margins": 0.3015338118236149,
|
|
"rewards/rejected": -0.01708979035610359,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 13.599421501159668,
|
|
"kl": 16.24431610107422,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37906976.880733944,
|
|
"logits/rejected": -37935149.801916935,
|
|
"logps/chosen": -508.9643539755352,
|
|
"logps/rejected": -361.9020567092652,
|
|
"loss": 0.5253,
|
|
"loss/base_kto": 3.7127296924591064,
|
|
"loss/total_with_kl": 4.202786445617676,
|
|
"metrics/advantage_var": 294.3282165527344,
|
|
"regularization/advantage_var": 294.3282165527344,
|
|
"regularization/kl": 0.49005642533302307,
|
|
"rewards/chosen": 0.3227073832753966,
|
|
"rewards/margins": 0.2382622342833152,
|
|
"rewards/rejected": 0.08444514899208141,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 14.82326602935791,
|
|
"kl": 22.02607536315918,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36723064.03641882,
|
|
"logits/rejected": -38682254.634460546,
|
|
"logps/chosen": -496.69783763277695,
|
|
"logps/rejected": -375.44781602254426,
|
|
"loss": 0.522,
|
|
"loss/base_kto": 3.7374000549316406,
|
|
"loss/total_with_kl": 4.175693035125732,
|
|
"metrics/advantage_var": 263.2390441894531,
|
|
"regularization/advantage_var": 263.2390441894531,
|
|
"regularization/kl": 0.4382929801940918,
|
|
"rewards/chosen": 0.40924155621680575,
|
|
"rewards/margins": 0.25828569259588186,
|
|
"rewards/rejected": 0.15095586362092392,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 14.854036331176758,
|
|
"kl": 18.0318660736084,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35792166.61897356,
|
|
"logits/rejected": -36740894.94505495,
|
|
"logps/chosen": -473.66475505443236,
|
|
"logps/rejected": -365.25201138147565,
|
|
"loss": 0.5187,
|
|
"loss/base_kto": 3.751659393310547,
|
|
"loss/total_with_kl": 4.149405479431152,
|
|
"metrics/advantage_var": 238.88633728027344,
|
|
"regularization/advantage_var": 238.88633728027344,
|
|
"regularization/kl": 0.3977457582950592,
|
|
"rewards/chosen": 0.3046710546799062,
|
|
"rewards/margins": 0.21607648928087997,
|
|
"rewards/rejected": 0.0885945653990262,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 13.262442588806152,
|
|
"kl": 11.475546836853027,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36391940.83779527,
|
|
"logits/rejected": -37548813.09767442,
|
|
"logps/chosen": -486.6638779527559,
|
|
"logps/rejected": -380.0341569767442,
|
|
"loss": 0.5205,
|
|
"loss/base_kto": 3.744659423828125,
|
|
"loss/total_with_kl": 4.164302349090576,
|
|
"metrics/advantage_var": 252.03750610351562,
|
|
"regularization/advantage_var": 252.03750610351562,
|
|
"regularization/kl": 0.41964250802993774,
|
|
"rewards/chosen": 0.1893415195735421,
|
|
"rewards/margins": 0.24742802720814117,
|
|
"rewards/rejected": -0.05808650763459908,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 14.660490036010742,
|
|
"kl": 9.907742500305176,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -34189718.3232,
|
|
"logits/rejected": -36631816.207633585,
|
|
"logps/chosen": -478.3143,
|
|
"logps/rejected": -383.78740458015267,
|
|
"loss": 0.5213,
|
|
"loss/base_kto": 3.738574266433716,
|
|
"loss/total_with_kl": 4.170376777648926,
|
|
"metrics/advantage_var": 259.3409118652344,
|
|
"regularization/advantage_var": 259.3409118652344,
|
|
"regularization/kl": 0.43180257081985474,
|
|
"rewards/chosen": 0.1726495849609375,
|
|
"rewards/margins": 0.21923724798537392,
|
|
"rewards/rejected": -0.046587663024436426,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 17.37724494934082,
|
|
"kl": 12.741228103637695,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -35913408.790123455,
|
|
"logits/rejected": -37017386.12658228,
|
|
"logps/chosen": -481.0546875,
|
|
"logps/rejected": -390.03822191455697,
|
|
"loss": 0.5244,
|
|
"loss/base_kto": 3.753157377243042,
|
|
"loss/total_with_kl": 4.194825649261475,
|
|
"metrics/advantage_var": 265.2659606933594,
|
|
"regularization/advantage_var": 265.2659606933594,
|
|
"regularization/kl": 0.4416678547859192,
|
|
"rewards/chosen": 0.2545143880961854,
|
|
"rewards/margins": 0.23339311553828695,
|
|
"rewards/rejected": 0.02112127255789841,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 13.298713684082031,
|
|
"kl": 25.486494064331055,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37696522.77894737,
|
|
"logits/rejected": -37902026.30243903,
|
|
"logps/chosen": -467.2192669172932,
|
|
"logps/rejected": -386.305462398374,
|
|
"loss": 0.5081,
|
|
"loss/base_kto": 3.634143829345703,
|
|
"loss/total_with_kl": 4.064520359039307,
|
|
"metrics/advantage_var": 258.484375,
|
|
"regularization/advantage_var": 258.484375,
|
|
"regularization/kl": 0.43037644028663635,
|
|
"rewards/chosen": 0.5204786257636278,
|
|
"rewards/margins": 0.3019425713540115,
|
|
"rewards/rejected": 0.21853605440961635,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 12.883255004882812,
|
|
"kl": 26.868762969970703,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -37809326.12792512,
|
|
"logits/rejected": -37679905.25195618,
|
|
"logps/chosen": -506.40039976599064,
|
|
"logps/rejected": -363.810348200313,
|
|
"loss": 0.5101,
|
|
"loss/base_kto": 3.6604385375976562,
|
|
"loss/total_with_kl": 4.0811991691589355,
|
|
"metrics/advantage_var": 252.7090301513672,
|
|
"regularization/advantage_var": 252.7090301513672,
|
|
"regularization/kl": 0.42076048254966736,
|
|
"rewards/chosen": 0.4910184529939791,
|
|
"rewards/margins": 0.2956581946557553,
|
|
"rewards/rejected": 0.1953602583382238,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 14.052704811096191,
|
|
"kl": 16.62283706665039,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34937062.63063063,
|
|
"logits/rejected": -35884028.66449511,
|
|
"logps/chosen": -488.95978791291293,
|
|
"logps/rejected": -387.6422536644951,
|
|
"loss": 0.516,
|
|
"loss/base_kto": 3.669461965560913,
|
|
"loss/total_with_kl": 4.127758026123047,
|
|
"metrics/advantage_var": 275.25274658203125,
|
|
"regularization/advantage_var": 275.25274658203125,
|
|
"regularization/kl": 0.4582958221435547,
|
|
"rewards/chosen": 0.34843284446555933,
|
|
"rewards/margins": 0.30156181448437486,
|
|
"rewards/rejected": 0.04687102998118447,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 20.06379508972168,
|
|
"kl": 19.192947387695312,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -34175236.770186335,
|
|
"logits/rejected": -37451206.03773585,
|
|
"logps/chosen": -494.44691381987576,
|
|
"logps/rejected": -355.5341489779874,
|
|
"loss": 0.5154,
|
|
"loss/base_kto": 3.7321929931640625,
|
|
"loss/total_with_kl": 4.122966289520264,
|
|
"metrics/advantage_var": 234.69839477539062,
|
|
"regularization/advantage_var": 234.69839477539062,
|
|
"regularization/kl": 0.39077281951904297,
|
|
"rewards/chosen": 0.32846332336804884,
|
|
"rewards/margins": 0.20946806472485652,
|
|
"rewards/rejected": 0.11899525864319231,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 13.178326606750488,
|
|
"kl": 19.160587310791016,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35872586.1682243,
|
|
"logits/rejected": -37736929.504702196,
|
|
"logps/chosen": -477.0406931464174,
|
|
"logps/rejected": -379.07494122257054,
|
|
"loss": 0.5182,
|
|
"loss/base_kto": 3.6772613525390625,
|
|
"loss/total_with_kl": 4.145966053009033,
|
|
"metrics/advantage_var": 281.50408935546875,
|
|
"regularization/advantage_var": 281.50408935546875,
|
|
"regularization/kl": 0.46870431303977966,
|
|
"rewards/chosen": 0.3526080835645444,
|
|
"rewards/margins": 0.29702494885057923,
|
|
"rewards/rejected": 0.05558313471396515,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 15.641792297363281,
|
|
"kl": 25.410354614257812,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36205324.641975306,
|
|
"logits/rejected": -36690198.6835443,
|
|
"logps/chosen": -482.1506558641975,
|
|
"logps/rejected": -380.53772745253167,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.672055959701538,
|
|
"loss/total_with_kl": 4.143238544464111,
|
|
"metrics/advantage_var": 282.9927673339844,
|
|
"regularization/advantage_var": 282.9927673339844,
|
|
"regularization/kl": 0.47118303179740906,
|
|
"rewards/chosen": 0.4387426494080343,
|
|
"rewards/margins": 0.24976545617922818,
|
|
"rewards/rejected": 0.18897719322880613,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 16.935712814331055,
|
|
"kl": 17.426271438598633,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -37085468.26498423,
|
|
"logits/rejected": -38433402.05572756,
|
|
"logps/chosen": -483.97550276025237,
|
|
"logps/rejected": -412.6327883126935,
|
|
"loss": 0.5227,
|
|
"loss/base_kto": 3.6580123901367188,
|
|
"loss/total_with_kl": 4.1814093589782715,
|
|
"metrics/advantage_var": 314.3523864746094,
|
|
"regularization/advantage_var": 314.3523864746094,
|
|
"regularization/kl": 0.5233966708183289,
|
|
"rewards/chosen": 0.3245206044675424,
|
|
"rewards/margins": 0.2917093894182003,
|
|
"rewards/rejected": 0.032811215049342105,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 12.15158748626709,
|
|
"kl": 14.632761001586914,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -34396176.22820919,
|
|
"logits/rejected": -36999442.53929122,
|
|
"logps/chosen": -500.4534469096672,
|
|
"logps/rejected": -367.6519164098613,
|
|
"loss": 0.5088,
|
|
"loss/base_kto": 3.6314399242401123,
|
|
"loss/total_with_kl": 4.070034027099609,
|
|
"metrics/advantage_var": 263.419677734375,
|
|
"regularization/advantage_var": 263.419677734375,
|
|
"regularization/kl": 0.4385937750339508,
|
|
"rewards/chosen": 0.3130121729831121,
|
|
"rewards/margins": 0.3304324994158677,
|
|
"rewards/rejected": -0.017420326432755623,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 16.809125900268555,
|
|
"kl": 26.657337188720703,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37657088.0,
|
|
"logits/rejected": -38298680.51948052,
|
|
"logps/chosen": -483.0152484939759,
|
|
"logps/rejected": -374.9105113636364,
|
|
"loss": 0.5256,
|
|
"loss/base_kto": 3.6715781688690186,
|
|
"loss/total_with_kl": 4.204737186431885,
|
|
"metrics/advantage_var": 320.2157287597656,
|
|
"regularization/advantage_var": 320.2157287597656,
|
|
"regularization/kl": 0.5331591963768005,
|
|
"rewards/chosen": 0.45624385971620857,
|
|
"rewards/margins": 0.2573161290072813,
|
|
"rewards/rejected": 0.1989277307089273,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 15.978750228881836,
|
|
"kl": 25.155349731445312,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -37110075.5624037,
|
|
"logits/rejected": -39355640.29160064,
|
|
"logps/chosen": -505.7794202619414,
|
|
"logps/rejected": -369.95671553090335,
|
|
"loss": 0.5209,
|
|
"loss/base_kto": 3.7089500427246094,
|
|
"loss/total_with_kl": 4.167476654052734,
|
|
"metrics/advantage_var": 275.3914489746094,
|
|
"regularization/advantage_var": 275.3914489746094,
|
|
"regularization/kl": 0.45852676033973694,
|
|
"rewards/chosen": 0.45925052213742296,
|
|
"rewards/margins": 0.23406726190440988,
|
|
"rewards/rejected": 0.22518326023301308,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 16.09609603881836,
|
|
"kl": 24.378955841064453,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36918953.001626015,
|
|
"logits/rejected": -39944783.30225564,
|
|
"logps/chosen": -502.0104674796748,
|
|
"logps/rejected": -344.4579652255639,
|
|
"loss": 0.5044,
|
|
"loss/base_kto": 3.5950279235839844,
|
|
"loss/total_with_kl": 4.035430431365967,
|
|
"metrics/advantage_var": 264.5058898925781,
|
|
"regularization/advantage_var": 264.5058898925781,
|
|
"regularization/kl": 0.4404023289680481,
|
|
"rewards/chosen": 0.4226708388910061,
|
|
"rewards/margins": 0.34771922531421745,
|
|
"rewards/rejected": 0.07495161357678864,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 15.821316719055176,
|
|
"kl": 18.246322631835938,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -36656654.8519084,
|
|
"logits/rejected": -37230896.7424,
|
|
"logps/chosen": -468.0425095419847,
|
|
"logps/rejected": -393.6592,
|
|
"loss": 0.5114,
|
|
"loss/base_kto": 3.6468300819396973,
|
|
"loss/total_with_kl": 4.09090518951416,
|
|
"metrics/advantage_var": 266.7118835449219,
|
|
"regularization/advantage_var": 266.7118835449219,
|
|
"regularization/kl": 0.44407525658607483,
|
|
"rewards/chosen": 0.30877767082389074,
|
|
"rewards/margins": 0.3212894612901993,
|
|
"rewards/rejected": -0.012511790466308594,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 13.75169563293457,
|
|
"kl": 23.669757843017578,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35771069.41518579,
|
|
"logits/rejected": -37964163.291981846,
|
|
"logps/chosen": -481.0651252019386,
|
|
"logps/rejected": -360.60795669440245,
|
|
"loss": 0.5131,
|
|
"loss/base_kto": 3.6904549598693848,
|
|
"loss/total_with_kl": 4.104562759399414,
|
|
"metrics/advantage_var": 248.71353149414062,
|
|
"regularization/advantage_var": 248.71353149414062,
|
|
"regularization/kl": 0.414108008146286,
|
|
"rewards/chosen": 0.405015591080624,
|
|
"rewards/margins": 0.26723809811305593,
|
|
"rewards/rejected": 0.13777749296756808,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 13.048786163330078,
|
|
"kl": 16.03829574584961,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36146570.35598706,
|
|
"logits/rejected": -37301346.99697885,
|
|
"logps/chosen": -480.2477245145631,
|
|
"logps/rejected": -357.41906627643505,
|
|
"loss": 0.5083,
|
|
"loss/base_kto": 3.60404372215271,
|
|
"loss/total_with_kl": 4.066466808319092,
|
|
"metrics/advantage_var": 277.7317810058594,
|
|
"regularization/advantage_var": 277.7317810058594,
|
|
"regularization/kl": 0.4624234139919281,
|
|
"rewards/chosen": 0.2579197065729925,
|
|
"rewards/margins": 0.3390387656487543,
|
|
"rewards/rejected": -0.08111905907576178,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 13.477405548095703,
|
|
"kl": 18.233827590942383,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36576879.16819572,
|
|
"logits/rejected": -37704373.57188498,
|
|
"logps/chosen": -485.9089258409786,
|
|
"logps/rejected": -382.1067292332268,
|
|
"loss": 0.5179,
|
|
"loss/base_kto": 3.648549795150757,
|
|
"loss/total_with_kl": 4.143303871154785,
|
|
"metrics/advantage_var": 297.1498107910156,
|
|
"regularization/advantage_var": 297.1498107910156,
|
|
"regularization/kl": 0.4947544038295746,
|
|
"rewards/chosen": 0.3176503079382287,
|
|
"rewards/margins": 0.3248079200898408,
|
|
"rewards/rejected": -0.007157612151612108,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 15.82155990600586,
|
|
"kl": 16.25971031188965,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -36148114.6407767,
|
|
"logits/rejected": -37517466.68277945,
|
|
"logps/chosen": -494.61392597087377,
|
|
"logps/rejected": -384.3298243957704,
|
|
"loss": 0.5124,
|
|
"loss/base_kto": 3.6538937091827393,
|
|
"loss/total_with_kl": 4.099232196807861,
|
|
"metrics/advantage_var": 267.470458984375,
|
|
"regularization/advantage_var": 267.470458984375,
|
|
"regularization/kl": 0.44533830881118774,
|
|
"rewards/chosen": 0.3224094230380259,
|
|
"rewards/margins": 0.31607459241104546,
|
|
"rewards/rejected": 0.006334830626980416,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 15.940324783325195,
|
|
"kl": 13.046548843383789,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -36451385.668188736,
|
|
"logits/rejected": -38100400.282504015,
|
|
"logps/chosen": -463.5775780060883,
|
|
"logps/rejected": -372.9582162921348,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.7482807636260986,
|
|
"loss/total_with_kl": 4.200253486633301,
|
|
"metrics/advantage_var": 271.4549560546875,
|
|
"regularization/advantage_var": 271.4549560546875,
|
|
"regularization/kl": 0.4519725441932678,
|
|
"rewards/chosen": 0.17628667125963185,
|
|
"rewards/margins": 0.23125859582253,
|
|
"rewards/rejected": -0.05497192456289815,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 15.46878433227539,
|
|
"kl": 20.6762752532959,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36713859.332312405,
|
|
"logits/rejected": -37496995.31738437,
|
|
"logps/chosen": -494.5665199081164,
|
|
"logps/rejected": -363.8888058213716,
|
|
"loss": 0.5074,
|
|
"loss/base_kto": 3.6548168659210205,
|
|
"loss/total_with_kl": 4.0588860511779785,
|
|
"metrics/advantage_var": 242.6844940185547,
|
|
"regularization/advantage_var": 242.6844940185547,
|
|
"regularization/kl": 0.4040696620941162,
|
|
"rewards/chosen": 0.38200334020267035,
|
|
"rewards/margins": 0.2799816528638732,
|
|
"rewards/rejected": 0.1020216873387971,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 12.483373641967773,
|
|
"kl": 20.401687622070312,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36751360.0,
|
|
"logits/rejected": -36997946.83229814,
|
|
"logps/chosen": -483.50756682389937,
|
|
"logps/rejected": -364.694439052795,
|
|
"loss": 0.5117,
|
|
"loss/base_kto": 3.648693084716797,
|
|
"loss/total_with_kl": 4.0936126708984375,
|
|
"metrics/advantage_var": 267.2190856933594,
|
|
"regularization/advantage_var": 267.2190856933594,
|
|
"regularization/kl": 0.44491979479789734,
|
|
"rewards/chosen": 0.34530965937008645,
|
|
"rewards/margins": 0.3172409569566578,
|
|
"rewards/rejected": 0.028068702413428643,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 15.437085151672363,
|
|
"kl": 13.675241470336914,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37608755.03403565,
|
|
"logits/rejected": -38460568.904977374,
|
|
"logps/chosen": -489.1448541329011,
|
|
"logps/rejected": -388.56605863499243,
|
|
"loss": 0.5254,
|
|
"loss/base_kto": 3.689847946166992,
|
|
"loss/total_with_kl": 4.2030487060546875,
|
|
"metrics/advantage_var": 308.2286682128906,
|
|
"regularization/advantage_var": 308.2286682128906,
|
|
"regularization/kl": 0.5132007002830505,
|
|
"rewards/chosen": 0.20648723831053103,
|
|
"rewards/margins": 0.2850018667712152,
|
|
"rewards/rejected": -0.07851462846068415,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 12.771230697631836,
|
|
"kl": 13.965453147888184,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -34826274.65846154,
|
|
"logits/rejected": -38143960.99047619,
|
|
"logps/chosen": -461.40567307692305,
|
|
"logps/rejected": -380.9243303571429,
|
|
"loss": 0.5205,
|
|
"loss/base_kto": 3.686608076095581,
|
|
"loss/total_with_kl": 4.164395809173584,
|
|
"metrics/advantage_var": 286.95953369140625,
|
|
"regularization/advantage_var": 286.95953369140625,
|
|
"regularization/kl": 0.477787584066391,
|
|
"rewards/chosen": 0.2752271916316106,
|
|
"rewards/margins": 0.27373041795723607,
|
|
"rewards/rejected": 0.001496773674374535,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 13.932594299316406,
|
|
"kl": 18.443395614624023,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -35767063.55828221,
|
|
"logits/rejected": -37548113.52866242,
|
|
"logps/chosen": -495.91583588957053,
|
|
"logps/rejected": -361.82633359872614,
|
|
"loss": 0.5335,
|
|
"loss/base_kto": 3.6956048011779785,
|
|
"loss/total_with_kl": 4.2682905197143555,
|
|
"metrics/advantage_var": 343.9552307128906,
|
|
"regularization/advantage_var": 343.9552307128906,
|
|
"regularization/kl": 0.5726854205131531,
|
|
"rewards/chosen": 0.278569203944294,
|
|
"rewards/margins": 0.2421474736841132,
|
|
"rewards/rejected": 0.03642173026018082,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 12.6790771484375,
|
|
"kl": 22.159164428710938,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -34496757.62722853,
|
|
"logits/rejected": -36220798.26244344,
|
|
"logps/chosen": -498.22346029173417,
|
|
"logps/rejected": -372.95364347662144,
|
|
"loss": 0.5189,
|
|
"loss/base_kto": 3.713252305984497,
|
|
"loss/total_with_kl": 4.150824546813965,
|
|
"metrics/advantage_var": 262.8059997558594,
|
|
"regularization/advantage_var": 262.8059997558594,
|
|
"regularization/kl": 0.43757200241088867,
|
|
"rewards/chosen": 0.32671659151197835,
|
|
"rewards/margins": 0.24346621489836717,
|
|
"rewards/rejected": 0.08325037661361119,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 14.06401538848877,
|
|
"kl": 13.91886043548584,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -35481282.89032258,
|
|
"logits/rejected": -36621333.787234046,
|
|
"logps/chosen": -516.0988911290323,
|
|
"logps/rejected": -376.4156060030395,
|
|
"loss": 0.5219,
|
|
"loss/base_kto": 3.4332756996154785,
|
|
"loss/total_with_kl": 4.175444602966309,
|
|
"metrics/advantage_var": 445.7469787597656,
|
|
"regularization/advantage_var": 445.7469787597656,
|
|
"regularization/kl": 0.7421687245368958,
|
|
"rewards/chosen": 0.32127651091544857,
|
|
"rewards/margins": 0.6127056305473231,
|
|
"rewards/rejected": -0.2914291196318745,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 31.391620635986328,
|
|
"kl": 9.995851516723633,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35555537.63779528,
|
|
"logits/rejected": -36252800.595348835,
|
|
"logps/chosen": -480.66633858267716,
|
|
"logps/rejected": -378.59905523255816,
|
|
"loss": 0.5267,
|
|
"loss/base_kto": 3.1529393196105957,
|
|
"loss/total_with_kl": 4.213776588439941,
|
|
"metrics/advantage_var": 637.1392822265625,
|
|
"regularization/advantage_var": 637.1392822265625,
|
|
"regularization/kl": 1.0608367919921875,
|
|
"rewards/chosen": 0.43852298766609255,
|
|
"rewards/margins": 0.9586412633898086,
|
|
"rewards/rejected": -0.520118275723716,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 10.978106498718262,
|
|
"kl": 9.668081283569336,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -34221189.43030303,
|
|
"logits/rejected": -35982834.787096776,
|
|
"logps/chosen": -480.0691287878788,
|
|
"logps/rejected": -369.32998991935483,
|
|
"loss": 0.5207,
|
|
"loss/base_kto": 3.168668508529663,
|
|
"loss/total_with_kl": 4.165344715118408,
|
|
"metrics/advantage_var": 598.6043701171875,
|
|
"regularization/advantage_var": 598.6043701171875,
|
|
"regularization/kl": 0.9966762661933899,
|
|
"rewards/chosen": 0.5383653120561079,
|
|
"rewards/margins": 0.8908628312723378,
|
|
"rewards/rejected": -0.35249751921622985,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 15.422128677368164,
|
|
"kl": 16.428346633911133,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -34393119.03030303,
|
|
"logits/rejected": -35830791.84073507,
|
|
"logps/chosen": -477.65166467304624,
|
|
"logps/rejected": -420.2646439509954,
|
|
"loss": 0.5309,
|
|
"loss/base_kto": 3.1263301372528076,
|
|
"loss/total_with_kl": 4.246942043304443,
|
|
"metrics/advantage_var": 673.0401611328125,
|
|
"regularization/advantage_var": 673.0401611328125,
|
|
"regularization/kl": 1.1206120252609253,
|
|
"rewards/chosen": 0.5925096332361444,
|
|
"rewards/margins": 0.9288811324539971,
|
|
"rewards/rejected": -0.3363714992178527,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 24.02975845336914,
|
|
"kl": 11.534314155578613,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35253049.11717496,
|
|
"logits/rejected": -36137595.9086758,
|
|
"logps/chosen": -459.44858547351527,
|
|
"logps/rejected": -382.470605022831,
|
|
"loss": 0.5164,
|
|
"loss/base_kto": 3.155416250228882,
|
|
"loss/total_with_kl": 4.131539821624756,
|
|
"metrics/advantage_var": 586.2604370117188,
|
|
"regularization/advantage_var": 586.2604370117188,
|
|
"regularization/kl": 0.9761235117912292,
|
|
"rewards/chosen": 0.5091067784096609,
|
|
"rewards/margins": 0.9455399173223321,
|
|
"rewards/rejected": -0.4364331389126712,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 12.157356262207031,
|
|
"kl": 9.995984077453613,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35482280.60422961,
|
|
"logits/rejected": -36495823.94822007,
|
|
"logps/chosen": -475.25368202416917,
|
|
"logps/rejected": -379.808909789644,
|
|
"loss": 0.5235,
|
|
"loss/base_kto": 3.122894763946533,
|
|
"loss/total_with_kl": 4.1876912117004395,
|
|
"metrics/advantage_var": 639.517578125,
|
|
"regularization/advantage_var": 639.517578125,
|
|
"regularization/kl": 1.064796805381775,
|
|
"rewards/chosen": 0.4884158592569864,
|
|
"rewards/margins": 0.991856444594011,
|
|
"rewards/rejected": -0.5034405853370246,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 36.18532180786133,
|
|
"kl": 23.652647018432617,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35271855.13713405,
|
|
"logits/rejected": -36747095.22662441,
|
|
"logps/chosen": -478.3598805855162,
|
|
"logps/rejected": -366.32740689381933,
|
|
"loss": 0.5085,
|
|
"loss/base_kto": 3.146047592163086,
|
|
"loss/total_with_kl": 4.067641735076904,
|
|
"metrics/advantage_var": 553.5099487304688,
|
|
"regularization/advantage_var": 553.5099487304688,
|
|
"regularization/kl": 0.9215942621231079,
|
|
"rewards/chosen": 0.7144313630043817,
|
|
"rewards/margins": 0.8990142017369243,
|
|
"rewards/rejected": -0.18458283873254258,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 21.897268295288086,
|
|
"kl": 17.896835327148438,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -36141747.24018838,
|
|
"logits/rejected": -38051234.83670296,
|
|
"logps/chosen": -484.2305239403454,
|
|
"logps/rejected": -371.8453052099533,
|
|
"loss": 0.5288,
|
|
"loss/base_kto": 3.164278745651245,
|
|
"loss/total_with_kl": 4.2304887771606445,
|
|
"metrics/advantage_var": 640.3662719726562,
|
|
"regularization/advantage_var": 640.3662719726562,
|
|
"regularization/kl": 1.0662097930908203,
|
|
"rewards/chosen": 0.5933866635596546,
|
|
"rewards/margins": 0.8884789244157492,
|
|
"rewards/rejected": -0.29509226085609447,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 26.35476303100586,
|
|
"kl": 21.59039878845215,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34994205.07255521,
|
|
"logits/rejected": -35554253.2755418,
|
|
"logps/chosen": -488.9683063880126,
|
|
"logps/rejected": -374.1844040247678,
|
|
"loss": 0.5335,
|
|
"loss/base_kto": 3.176971912384033,
|
|
"loss/total_with_kl": 4.267880916595459,
|
|
"metrics/advantage_var": 655.200439453125,
|
|
"regularization/advantage_var": 655.200439453125,
|
|
"regularization/kl": 1.0909088850021362,
|
|
"rewards/chosen": 0.6922250260313979,
|
|
"rewards/margins": 0.912030339000027,
|
|
"rewards/rejected": -0.21980531296862907,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 13.80142593383789,
|
|
"kl": 19.11087417602539,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35398798.6228482,
|
|
"logits/rejected": -36274508.280811235,
|
|
"logps/chosen": -446.09086463223787,
|
|
"logps/rejected": -370.3956952028081,
|
|
"loss": 0.5157,
|
|
"loss/base_kto": 3.1798524856567383,
|
|
"loss/total_with_kl": 4.125638008117676,
|
|
"metrics/advantage_var": 568.0391235351562,
|
|
"regularization/advantage_var": 568.0391235351562,
|
|
"regularization/kl": 0.9457851648330688,
|
|
"rewards/chosen": 0.6188518303287608,
|
|
"rewards/margins": 0.8779430454177941,
|
|
"rewards/rejected": -0.25909121508903327,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 18.03839683532715,
|
|
"kl": 11.0951566696167,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -34223405.94871795,
|
|
"logits/rejected": -35617698.34146342,
|
|
"logps/chosen": -497.5053084935897,
|
|
"logps/rejected": -369.2091749237805,
|
|
"loss": 0.4938,
|
|
"loss/base_kto": 3.15341854095459,
|
|
"loss/total_with_kl": 3.95076060295105,
|
|
"metrics/advantage_var": 478.8841247558594,
|
|
"regularization/advantage_var": 478.8841247558594,
|
|
"regularization/kl": 0.79734206199646,
|
|
"rewards/chosen": 0.518190432817508,
|
|
"rewards/margins": 0.9124734483710522,
|
|
"rewards/rejected": -0.3942830155535442,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 23.129491806030273,
|
|
"kl": 18.38189125061035,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35689517.85074627,
|
|
"logits/rejected": -37217471.3704918,
|
|
"logps/chosen": -495.32621268656715,
|
|
"logps/rejected": -389.0386782786885,
|
|
"loss": 0.5284,
|
|
"loss/base_kto": 3.210973024368286,
|
|
"loss/total_with_kl": 4.227263450622559,
|
|
"metrics/advantage_var": 610.3844604492188,
|
|
"regularization/advantage_var": 610.3844604492188,
|
|
"regularization/kl": 1.0162901878356934,
|
|
"rewards/chosen": 0.6094223705690298,
|
|
"rewards/margins": 0.8007466608602376,
|
|
"rewards/rejected": -0.19132429029120773,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 17.110923767089844,
|
|
"kl": 11.127530097961426,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34592222.7120743,
|
|
"logits/rejected": -36780035.23028391,
|
|
"logps/chosen": -468.953076625387,
|
|
"logps/rejected": -366.334976340694,
|
|
"loss": 0.5164,
|
|
"loss/base_kto": 3.234569549560547,
|
|
"loss/total_with_kl": 4.1309356689453125,
|
|
"metrics/advantage_var": 538.3580322265625,
|
|
"regularization/advantage_var": 538.3580322265625,
|
|
"regularization/kl": 0.8963660597801208,
|
|
"rewards/chosen": 0.48085187315571787,
|
|
"rewards/margins": 0.8364997334210225,
|
|
"rewards/rejected": -0.3556478602653046,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 41.96747970581055,
|
|
"kl": 17.045631408691406,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35331633.64848485,
|
|
"logits/rejected": -35348565.88387097,
|
|
"logps/chosen": -501.39375,
|
|
"logps/rejected": -365.53077116935486,
|
|
"loss": 0.511,
|
|
"loss/base_kto": 3.1432340145111084,
|
|
"loss/total_with_kl": 4.087615489959717,
|
|
"metrics/advantage_var": 567.1957397460938,
|
|
"regularization/advantage_var": 567.1957397460938,
|
|
"regularization/kl": 0.9443808794021606,
|
|
"rewards/chosen": 0.6437827370383523,
|
|
"rewards/margins": 0.9052675353466941,
|
|
"rewards/rejected": -0.26148479830834176,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.453969955444336,
|
|
"kl": 22.517139434814453,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34838537.64521193,
|
|
"logits/rejected": -36997868.491446346,
|
|
"logps/chosen": -505.01285321821035,
|
|
"logps/rejected": -372.1735031104199,
|
|
"loss": 0.5373,
|
|
"loss/base_kto": 3.1928532123565674,
|
|
"loss/total_with_kl": 4.298562526702881,
|
|
"metrics/advantage_var": 664.0895385742188,
|
|
"regularization/advantage_var": 664.0895385742188,
|
|
"regularization/kl": 1.1057090759277344,
|
|
"rewards/chosen": 0.6341987741611067,
|
|
"rewards/margins": 0.8541001601432583,
|
|
"rewards/rejected": -0.21990138598215153,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 15.8773775100708,
|
|
"kl": 18.72047233581543,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34075852.16692427,
|
|
"logits/rejected": -35870826.76777251,
|
|
"logps/chosen": -469.90098531684697,
|
|
"logps/rejected": -373.26209518167457,
|
|
"loss": 0.519,
|
|
"loss/base_kto": 3.2500557899475098,
|
|
"loss/total_with_kl": 4.152146339416504,
|
|
"metrics/advantage_var": 541.7962036132812,
|
|
"regularization/advantage_var": 541.7962036132812,
|
|
"regularization/kl": 0.9020906686782837,
|
|
"rewards/chosen": 0.5916375892792456,
|
|
"rewards/margins": 0.7798564397510194,
|
|
"rewards/rejected": -0.1882188504717738,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 16.176589965820312,
|
|
"kl": 12.963971138000488,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -33798691.42138365,
|
|
"logits/rejected": -35253715.47826087,
|
|
"logps/chosen": -466.51346305031444,
|
|
"logps/rejected": -373.9714188664596,
|
|
"loss": 0.5093,
|
|
"loss/base_kto": 3.253509521484375,
|
|
"loss/total_with_kl": 4.0741472244262695,
|
|
"metrics/advantage_var": 492.8753967285156,
|
|
"regularization/advantage_var": 492.8753967285156,
|
|
"regularization/kl": 0.8206375241279602,
|
|
"rewards/chosen": 0.4933123318654186,
|
|
"rewards/margins": 0.7834087806808059,
|
|
"rewards/rejected": -0.2900964488153872,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 9.588994979858398,
|
|
"kl": 14.815385818481445,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -33478852.737007875,
|
|
"logits/rejected": -35595449.74883721,
|
|
"logps/chosen": -474.4527559055118,
|
|
"logps/rejected": -379.58488372093024,
|
|
"loss": 0.53,
|
|
"loss/base_kto": 3.1816983222961426,
|
|
"loss/total_with_kl": 4.240319728851318,
|
|
"metrics/advantage_var": 635.8090209960938,
|
|
"regularization/advantage_var": 635.8090209960938,
|
|
"regularization/kl": 1.0586220026016235,
|
|
"rewards/chosen": 0.5786730668676181,
|
|
"rewards/margins": 0.8986347803184418,
|
|
"rewards/rejected": -0.31996171345082364,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 28.82682991027832,
|
|
"kl": 12.66755199432373,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -33937022.96272285,
|
|
"logits/rejected": -37700251.22171946,
|
|
"logps/chosen": -478.9775628038898,
|
|
"logps/rejected": -364.1681278280543,
|
|
"loss": 0.5093,
|
|
"loss/base_kto": 3.163775682449341,
|
|
"loss/total_with_kl": 4.074222564697266,
|
|
"metrics/advantage_var": 546.8151245117188,
|
|
"regularization/advantage_var": 546.8151245117188,
|
|
"regularization/kl": 0.9104471206665039,
|
|
"rewards/chosen": 0.5768320834810575,
|
|
"rewards/margins": 0.9143526770926007,
|
|
"rewards/rejected": -0.3375205936115432,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.246854782104492,
|
|
"kl": 11.055014610290527,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -34606991.144246355,
|
|
"logits/rejected": -35550660.53695324,
|
|
"logps/chosen": -489.21545786061586,
|
|
"logps/rejected": -372.8019183634992,
|
|
"loss": 0.5018,
|
|
"loss/base_kto": 3.223292589187622,
|
|
"loss/total_with_kl": 4.014786243438721,
|
|
"metrics/advantage_var": 475.37164306640625,
|
|
"regularization/advantage_var": 475.37164306640625,
|
|
"regularization/kl": 0.791493833065033,
|
|
"rewards/chosen": 0.47648707162619025,
|
|
"rewards/margins": 0.8354702452554676,
|
|
"rewards/rejected": -0.35898317362927745,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 15.94455623626709,
|
|
"kl": 13.05618953704834,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -34256073.030674845,
|
|
"logits/rejected": -36733035.6178344,
|
|
"logps/chosen": -488.2274252300613,
|
|
"logps/rejected": -383.648636544586,
|
|
"loss": 0.5286,
|
|
"loss/base_kto": 3.19297194480896,
|
|
"loss/total_with_kl": 4.229190349578857,
|
|
"metrics/advantage_var": 622.3533325195312,
|
|
"regularization/advantage_var": 622.3533325195312,
|
|
"regularization/kl": 1.0362184047698975,
|
|
"rewards/chosen": 0.5630929396927722,
|
|
"rewards/margins": 0.8562681206989426,
|
|
"rewards/rejected": -0.2931751810061704,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 13.716262817382812,
|
|
"kl": 8.40222454071045,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -34286942.94835681,
|
|
"logits/rejected": -35281736.287051484,
|
|
"logps/chosen": -472.13703051643193,
|
|
"logps/rejected": -386.0454368174727,
|
|
"loss": 0.5424,
|
|
"loss/base_kto": 3.1698734760284424,
|
|
"loss/total_with_kl": 4.3393144607543945,
|
|
"metrics/advantage_var": 702.3667602539062,
|
|
"regularization/advantage_var": 702.3667602539062,
|
|
"regularization/kl": 1.169440746307373,
|
|
"rewards/chosen": 0.44656825774525627,
|
|
"rewards/margins": 0.9094737368550456,
|
|
"rewards/rejected": -0.4629054791097894,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 10.339768409729004,
|
|
"kl": 11.972759246826172,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34969125.14826498,
|
|
"logits/rejected": -35130276.0619195,
|
|
"logps/chosen": -472.61070583596216,
|
|
"logps/rejected": -411.7797503869969,
|
|
"loss": 0.5372,
|
|
"loss/base_kto": 3.1880974769592285,
|
|
"loss/total_with_kl": 4.297727584838867,
|
|
"metrics/advantage_var": 666.4446411132812,
|
|
"regularization/advantage_var": 666.4446411132812,
|
|
"regularization/kl": 1.1096304655075073,
|
|
"rewards/chosen": 0.47758507954209384,
|
|
"rewards/margins": 0.8865025331780507,
|
|
"rewards/rejected": -0.40891745363595683,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 11.059826850891113,
|
|
"kl": 10.49127197265625,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -34979327.150912106,
|
|
"logits/rejected": -36496079.97636632,
|
|
"logps/chosen": -495.18055555555554,
|
|
"logps/rejected": -351.90615768094534,
|
|
"loss": 0.541,
|
|
"loss/base_kto": 3.2105355262756348,
|
|
"loss/total_with_kl": 4.328180313110352,
|
|
"metrics/advantage_var": 671.25830078125,
|
|
"regularization/advantage_var": 671.25830078125,
|
|
"regularization/kl": 1.117645025253296,
|
|
"rewards/chosen": 0.443090688727586,
|
|
"rewards/margins": 0.857144714126866,
|
|
"rewards/rejected": -0.41405402539927993,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 36.40850830078125,
|
|
"kl": 13.112838745117188,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34023483.04704097,
|
|
"logits/rejected": -33377875.272141706,
|
|
"logps/chosen": -506.0541540212443,
|
|
"logps/rejected": -412.02136171497585,
|
|
"loss": 0.5097,
|
|
"loss/base_kto": 3.1219232082366943,
|
|
"loss/total_with_kl": 4.077780723571777,
|
|
"metrics/advantage_var": 574.0885009765625,
|
|
"regularization/advantage_var": 574.0885009765625,
|
|
"regularization/kl": 0.9558573961257935,
|
|
"rewards/chosen": 0.4977967142154306,
|
|
"rewards/margins": 0.9400747054688273,
|
|
"rewards/rejected": -0.44227799125339673,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 11.101387023925781,
|
|
"kl": 10.55687427520752,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34665638.62243286,
|
|
"logits/rejected": -35676875.375579596,
|
|
"logps/chosen": -494.42229462875196,
|
|
"logps/rejected": -385.0909486089644,
|
|
"loss": 0.5129,
|
|
"loss/base_kto": 3.198608875274658,
|
|
"loss/total_with_kl": 4.103453159332275,
|
|
"metrics/advantage_var": 543.4500122070312,
|
|
"regularization/advantage_var": 543.4500122070312,
|
|
"regularization/kl": 0.9048442840576172,
|
|
"rewards/chosen": 0.4689873910816548,
|
|
"rewards/margins": 0.851609663272415,
|
|
"rewards/rejected": -0.3826222721907602,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 19.276321411132812,
|
|
"kl": 14.524676322937012,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34034861.76969697,
|
|
"logits/rejected": -36807617.23870968,
|
|
"logps/chosen": -497.4969696969697,
|
|
"logps/rejected": -359.0599798387097,
|
|
"loss": 0.5002,
|
|
"loss/base_kto": 3.159895896911621,
|
|
"loss/total_with_kl": 4.001439094543457,
|
|
"metrics/advantage_var": 505.43121337890625,
|
|
"regularization/advantage_var": 505.43121337890625,
|
|
"regularization/kl": 0.8415428996086121,
|
|
"rewards/chosen": 0.630764493075284,
|
|
"rewards/margins": 0.8790538597666273,
|
|
"rewards/rejected": -0.24828936669134324,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 18.631711959838867,
|
|
"kl": 16.653379440307617,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -31887962.352941178,
|
|
"logits/rejected": -35704069.652996846,
|
|
"logps/chosen": -502.30674342105266,
|
|
"logps/rejected": -358.23821963722395,
|
|
"loss": 0.6035,
|
|
"loss/base_kto": 3.2512428760528564,
|
|
"loss/total_with_kl": 4.827615261077881,
|
|
"metrics/advantage_var": 946.7701416015625,
|
|
"regularization/advantage_var": 946.7701416015625,
|
|
"regularization/kl": 1.5763721466064453,
|
|
"rewards/chosen": 0.5880422680739648,
|
|
"rewards/margins": 0.7506182154474121,
|
|
"rewards/rejected": -0.16257594737344735,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 15.079712867736816,
|
|
"kl": 12.630504608154297,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33385437.341538463,
|
|
"logits/rejected": -36008300.08888889,
|
|
"logps/chosen": -487.2771153846154,
|
|
"logps/rejected": -366.0266121031746,
|
|
"loss": 0.507,
|
|
"loss/base_kto": 3.1973884105682373,
|
|
"loss/total_with_kl": 4.05588960647583,
|
|
"metrics/advantage_var": 515.6163940429688,
|
|
"regularization/advantage_var": 515.6163940429688,
|
|
"regularization/kl": 0.8585012555122375,
|
|
"rewards/chosen": 0.5683293269230769,
|
|
"rewards/margins": 0.8383688641409588,
|
|
"rewards/rejected": -0.27003953721788193,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.889738082885742,
|
|
"kl": 17.27866554260254,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35216221.12755906,
|
|
"logits/rejected": -36704541.76744186,
|
|
"logps/chosen": -474.2298720472441,
|
|
"logps/rejected": -373.54161821705424,
|
|
"loss": 0.5008,
|
|
"loss/base_kto": 3.242872714996338,
|
|
"loss/total_with_kl": 4.006385326385498,
|
|
"metrics/advantage_var": 458.565673828125,
|
|
"regularization/advantage_var": 458.565673828125,
|
|
"regularization/kl": 0.7635118365287781,
|
|
"rewards/chosen": 0.5789116326279528,
|
|
"rewards/margins": 0.8074839635463757,
|
|
"rewards/rejected": -0.22857233091842297,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 17.977779388427734,
|
|
"kl": 16.265213012695312,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -34899636.236760125,
|
|
"logits/rejected": -35481513.329153605,
|
|
"logps/chosen": -484.6946066978193,
|
|
"logps/rejected": -359.53198471786834,
|
|
"loss": 0.5002,
|
|
"loss/base_kto": 3.170069694519043,
|
|
"loss/total_with_kl": 4.0019989013671875,
|
|
"metrics/advantage_var": 499.6572265625,
|
|
"regularization/advantage_var": 499.6572265625,
|
|
"regularization/kl": 0.8319292068481445,
|
|
"rewards/chosen": 0.5630078657393887,
|
|
"rewards/margins": 0.8864267490600686,
|
|
"rewards/rejected": -0.32341888332067986,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 12.042512893676758,
|
|
"kl": 14.595972061157227,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -35094110.44660194,
|
|
"logits/rejected": -35696708.060422964,
|
|
"logps/chosen": -483.64917071197414,
|
|
"logps/rejected": -388.2859469410876,
|
|
"loss": 0.5008,
|
|
"loss/base_kto": 3.1869797706604004,
|
|
"loss/total_with_kl": 4.00601863861084,
|
|
"metrics/advantage_var": 491.9150390625,
|
|
"regularization/advantage_var": 491.9150390625,
|
|
"regularization/kl": 0.8190385699272156,
|
|
"rewards/chosen": 0.5053484771629754,
|
|
"rewards/margins": 0.8603355380786428,
|
|
"rewards/rejected": -0.3549870609156675,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 19.413347244262695,
|
|
"kl": 9.449725151062012,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -33266667.42503864,
|
|
"logits/rejected": -33330983.22906793,
|
|
"logps/chosen": -456.65948608964453,
|
|
"logps/rejected": -382.70156990521326,
|
|
"loss": 0.4896,
|
|
"loss/base_kto": 3.1942191123962402,
|
|
"loss/total_with_kl": 3.9166457653045654,
|
|
"metrics/advantage_var": 433.8897399902344,
|
|
"regularization/advantage_var": 433.8897399902344,
|
|
"regularization/kl": 0.7224263548851013,
|
|
"rewards/chosen": 0.3941137204767195,
|
|
"rewards/margins": 0.843319501711292,
|
|
"rewards/rejected": -0.4492057812345725,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 75.15998077392578,
|
|
"kl": 14.229202270507812,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34895341.513343796,
|
|
"logits/rejected": -36524114.8118196,
|
|
"logps/chosen": -492.09379905808476,
|
|
"logps/rejected": -378.3467875194401,
|
|
"loss": 0.5099,
|
|
"loss/base_kto": 3.2120933532714844,
|
|
"loss/total_with_kl": 4.079436779022217,
|
|
"metrics/advantage_var": 520.9268188476562,
|
|
"regularization/advantage_var": 520.9268188476562,
|
|
"regularization/kl": 0.8673431277275085,
|
|
"rewards/chosen": 0.5472287547831632,
|
|
"rewards/margins": 0.8261867090593831,
|
|
"rewards/rejected": -0.2789579542762199,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 15.21430778503418,
|
|
"kl": 14.696313858032227,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34389920.0,
|
|
"logits/rejected": -35830291.2,
|
|
"logps/chosen": -478.40322265625,
|
|
"logps/rejected": -374.1216552734375,
|
|
"loss": 0.5258,
|
|
"loss/base_kto": 3.215595006942749,
|
|
"loss/total_with_kl": 4.20621919631958,
|
|
"metrics/advantage_var": 594.9694213867188,
|
|
"regularization/advantage_var": 594.9694213867188,
|
|
"regularization/kl": 0.9906242489814758,
|
|
"rewards/chosen": 0.5080112934112548,
|
|
"rewards/margins": 0.8089938640594482,
|
|
"rewards/rejected": -0.30098257064819334,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 15.669301986694336,
|
|
"kl": 16.72923469543457,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35849677.43209877,
|
|
"logits/rejected": -35671811.24050633,
|
|
"logps/chosen": -462.44092399691357,
|
|
"logps/rejected": -370.6542474287975,
|
|
"loss": 0.5038,
|
|
"loss/base_kto": 3.188962697982788,
|
|
"loss/total_with_kl": 4.0304388999938965,
|
|
"metrics/advantage_var": 505.3913269042969,
|
|
"regularization/advantage_var": 505.3913269042969,
|
|
"regularization/kl": 0.8414764404296875,
|
|
"rewards/chosen": 0.5550163645803192,
|
|
"rewards/margins": 0.8617818122693273,
|
|
"rewards/rejected": -0.3067654476890081,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 12.49700927734375,
|
|
"kl": 18.269855499267578,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -33705777.951219514,
|
|
"logits/rejected": -35187472.41025641,
|
|
"logps/chosen": -472.6994092987805,
|
|
"logps/rejected": -375.2556590544872,
|
|
"loss": 0.5022,
|
|
"loss/base_kto": 3.181504964828491,
|
|
"loss/total_with_kl": 4.017747402191162,
|
|
"metrics/advantage_var": 502.24786376953125,
|
|
"regularization/advantage_var": 502.24786376953125,
|
|
"regularization/kl": 0.8362426161766052,
|
|
"rewards/chosen": 0.6359493906905012,
|
|
"rewards/margins": 0.8660423930098371,
|
|
"rewards/rejected": -0.23009300231933594,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 12.752656936645508,
|
|
"kl": 20.18071746826172,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34223343.09822866,
|
|
"logits/rejected": -37117616.364188164,
|
|
"logps/chosen": -496.8047504025765,
|
|
"logps/rejected": -355.42398520485585,
|
|
"loss": 0.4887,
|
|
"loss/base_kto": 3.239885091781616,
|
|
"loss/total_with_kl": 3.9096763134002686,
|
|
"metrics/advantage_var": 402.2770690917969,
|
|
"regularization/advantage_var": 402.2770690917969,
|
|
"regularization/kl": 0.6697913408279419,
|
|
"rewards/chosen": 0.5882177183977456,
|
|
"rewards/margins": 0.78272604085157,
|
|
"rewards/rejected": -0.19450832245382446,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 18.113637924194336,
|
|
"kl": 16.881772994995117,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -33521202.567901235,
|
|
"logits/rejected": -35090931.037974685,
|
|
"logps/chosen": -440.4192708333333,
|
|
"logps/rejected": -374.43030557753167,
|
|
"loss": 0.5304,
|
|
"loss/base_kto": 3.154949188232422,
|
|
"loss/total_with_kl": 4.243502140045166,
|
|
"metrics/advantage_var": 653.7854614257812,
|
|
"regularization/advantage_var": 653.7854614257812,
|
|
"regularization/kl": 1.0885528326034546,
|
|
"rewards/chosen": 0.6528695659872926,
|
|
"rewards/margins": 0.9338312424762414,
|
|
"rewards/rejected": -0.28096167648894876,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 11.993721961975098,
|
|
"kl": 18.08906364440918,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34671162.37133551,
|
|
"logits/rejected": -35041301.590361446,
|
|
"logps/chosen": -450.2229234527687,
|
|
"logps/rejected": -353.789109563253,
|
|
"loss": 0.5308,
|
|
"loss/base_kto": 3.213454484939575,
|
|
"loss/total_with_kl": 4.246249675750732,
|
|
"metrics/advantage_var": 620.2974243164062,
|
|
"regularization/advantage_var": 620.2974243164062,
|
|
"regularization/kl": 1.0327951908111572,
|
|
"rewards/chosen": 0.5425039782197933,
|
|
"rewards/margins": 0.7466175133539824,
|
|
"rewards/rejected": -0.2041135351341891,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 12.469103813171387,
|
|
"kl": 13.226333618164062,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34055782.4,
|
|
"logits/rejected": -35245022.13543307,
|
|
"logps/chosen": -481.1217054263566,
|
|
"logps/rejected": -385.636343503937,
|
|
"loss": 0.4687,
|
|
"loss/base_kto": 3.1283042430877686,
|
|
"loss/total_with_kl": 3.749284505844116,
|
|
"metrics/advantage_var": 372.9611511230469,
|
|
"regularization/advantage_var": 372.9611511230469,
|
|
"regularization/kl": 0.6209803223609924,
|
|
"rewards/chosen": 0.5851207644440407,
|
|
"rewards/margins": 0.9028250928458617,
|
|
"rewards/rejected": -0.3177043284018209,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 10.519600868225098,
|
|
"kl": 11.149703025817871,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -33789724.44444445,
|
|
"logits/rejected": -36133481.55076923,
|
|
"logps/chosen": -487.6768353174603,
|
|
"logps/rejected": -357.6958173076923,
|
|
"loss": 0.4581,
|
|
"loss/base_kto": 3.121828556060791,
|
|
"loss/total_with_kl": 3.665121078491211,
|
|
"metrics/advantage_var": 326.3017272949219,
|
|
"regularization/advantage_var": 326.3017272949219,
|
|
"regularization/kl": 0.5432922840118408,
|
|
"rewards/chosen": 0.49341018918960816,
|
|
"rewards/margins": 0.8931621986547524,
|
|
"rewards/rejected": -0.3997520094651442,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 12.13588809967041,
|
|
"kl": 11.816821098327637,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34190863.03554869,
|
|
"logits/rejected": -35444425.4028436,
|
|
"logps/chosen": -464.54849304482224,
|
|
"logps/rejected": -384.7115422590837,
|
|
"loss": 0.4741,
|
|
"loss/base_kto": 3.143925905227661,
|
|
"loss/total_with_kl": 3.793105363845825,
|
|
"metrics/advantage_var": 389.8977355957031,
|
|
"regularization/advantage_var": 389.8977355957031,
|
|
"regularization/kl": 0.6491796970367432,
|
|
"rewards/chosen": 0.5393455540746958,
|
|
"rewards/margins": 0.870848314830768,
|
|
"rewards/rejected": -0.33150276075607227,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 9.222172737121582,
|
|
"kl": 14.158614158630371,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -34220178.72238806,
|
|
"logits/rejected": -35187272.18360656,
|
|
"logps/chosen": -485.69538246268655,
|
|
"logps/rejected": -386.8644467213115,
|
|
"loss": 0.4604,
|
|
"loss/base_kto": 3.12493896484375,
|
|
"loss/total_with_kl": 3.6833016872406006,
|
|
"metrics/advantage_var": 335.3531494140625,
|
|
"regularization/advantage_var": 335.3531494140625,
|
|
"regularization/kl": 0.5583630204200745,
|
|
"rewards/chosen": 0.5551715907765858,
|
|
"rewards/margins": 0.8964328072372671,
|
|
"rewards/rejected": -0.34126121646068136,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 7.134238243103027,
|
|
"kl": 21.103641510009766,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -32563412.65175719,
|
|
"logits/rejected": -35934458.51987767,
|
|
"logps/chosen": -465.9977535942492,
|
|
"logps/rejected": -364.90720565749234,
|
|
"loss": 0.4641,
|
|
"loss/base_kto": 3.120934009552002,
|
|
"loss/total_with_kl": 3.7129979133605957,
|
|
"metrics/advantage_var": 355.593994140625,
|
|
"regularization/advantage_var": 355.593994140625,
|
|
"regularization/kl": 0.5920640230178833,
|
|
"rewards/chosen": 0.69232908986247,
|
|
"rewards/margins": 0.850305316202481,
|
|
"rewards/rejected": -0.157976226340011,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 15.896869659423828,
|
|
"kl": 14.406694412231445,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33975640.367407404,
|
|
"logits/rejected": -34251909.712396696,
|
|
"logps/chosen": -487.14523148148146,
|
|
"logps/rejected": -378.45958161157023,
|
|
"loss": 0.4635,
|
|
"loss/base_kto": 3.096287250518799,
|
|
"loss/total_with_kl": 3.708249807357788,
|
|
"metrics/advantage_var": 367.5450134277344,
|
|
"regularization/advantage_var": 367.5450134277344,
|
|
"regularization/kl": 0.6119624376296997,
|
|
"rewards/chosen": 0.6030554651331018,
|
|
"rewards/margins": 0.8958246654817588,
|
|
"rewards/rejected": -0.292769200348657,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 13.382315635681152,
|
|
"kl": 20.42544174194336,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34242197.06329114,
|
|
"logits/rejected": -35010718.02469136,
|
|
"logps/chosen": -484.60091969936707,
|
|
"logps/rejected": -369.2682291666667,
|
|
"loss": 0.4697,
|
|
"loss/base_kto": 3.1071879863739014,
|
|
"loss/total_with_kl": 3.7578208446502686,
|
|
"metrics/advantage_var": 390.7705993652344,
|
|
"regularization/advantage_var": 390.7705993652344,
|
|
"regularization/kl": 0.6506330370903015,
|
|
"rewards/chosen": 0.6587396452698526,
|
|
"rewards/margins": 0.8763825824473459,
|
|
"rewards/rejected": -0.21764293717749325,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 16.108022689819336,
|
|
"kl": 19.32630729675293,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -33720700.94753577,
|
|
"logits/rejected": -36886896.07373272,
|
|
"logps/chosen": -495.4686009538951,
|
|
"logps/rejected": -350.099846390169,
|
|
"loss": 0.4616,
|
|
"loss/base_kto": 3.1080052852630615,
|
|
"loss/total_with_kl": 3.6930859088897705,
|
|
"metrics/advantage_var": 351.3998718261719,
|
|
"regularization/advantage_var": 351.3998718261719,
|
|
"regularization/kl": 0.5850808024406433,
|
|
"rewards/chosen": 0.6325717556078597,
|
|
"rewards/margins": 0.8702764530834981,
|
|
"rewards/rejected": -0.23770469747563844,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 12.411762237548828,
|
|
"kl": 13.545578002929688,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -33760238.75038285,
|
|
"logits/rejected": -34265813.1291866,
|
|
"logps/chosen": -480.08250382848394,
|
|
"logps/rejected": -387.300139553429,
|
|
"loss": 0.4612,
|
|
"loss/base_kto": 3.0721538066864014,
|
|
"loss/total_with_kl": 3.6892669200897217,
|
|
"metrics/advantage_var": 370.63873291015625,
|
|
"regularization/advantage_var": 370.63873291015625,
|
|
"regularization/kl": 0.617113471031189,
|
|
"rewards/chosen": 0.6181547156154288,
|
|
"rewards/margins": 0.9403173012009302,
|
|
"rewards/rejected": -0.32216258558550137,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 14.86486530303955,
|
|
"kl": 18.197635650634766,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -33103795.2,
|
|
"logits/rejected": -35078176.0,
|
|
"logps/chosen": -502.35322265625,
|
|
"logps/rejected": -399.6092529296875,
|
|
"loss": 0.4644,
|
|
"loss/base_kto": 3.097761631011963,
|
|
"loss/total_with_kl": 3.7150864601135254,
|
|
"metrics/advantage_var": 370.7657775878906,
|
|
"regularization/advantage_var": 370.7657775878906,
|
|
"regularization/kl": 0.6173250079154968,
|
|
"rewards/chosen": 0.6517374038696289,
|
|
"rewards/margins": 0.8863547801971435,
|
|
"rewards/rejected": -0.23461737632751464,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 10.893120765686035,
|
|
"kl": 19.56777572631836,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -34881686.01273885,
|
|
"logits/rejected": -35484128.58895706,
|
|
"logps/chosen": -495.26502786624206,
|
|
"logps/rejected": -368.6687116564417,
|
|
"loss": 0.4654,
|
|
"loss/base_kto": 3.087596893310547,
|
|
"loss/total_with_kl": 3.7230966091156006,
|
|
"metrics/advantage_var": 381.6813659667969,
|
|
"regularization/advantage_var": 381.6813659667969,
|
|
"regularization/kl": 0.6354994177818298,
|
|
"rewards/chosen": 0.6605051125690435,
|
|
"rewards/margins": 0.898327920208698,
|
|
"rewards/rejected": -0.23782280763965444,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 10.211127281188965,
|
|
"kl": 18.727100372314453,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -33754411.47169811,
|
|
"logits/rejected": -36048046.9068323,
|
|
"logps/chosen": -479.561320754717,
|
|
"logps/rejected": -371.8881259704969,
|
|
"loss": 0.4568,
|
|
"loss/base_kto": 3.1105823516845703,
|
|
"loss/total_with_kl": 3.654071569442749,
|
|
"metrics/advantage_var": 326.4197692871094,
|
|
"regularization/advantage_var": 326.4197692871094,
|
|
"regularization/kl": 0.5434889197349548,
|
|
"rewards/chosen": 0.629716213394261,
|
|
"rewards/margins": 0.8747631962483104,
|
|
"rewards/rejected": -0.2450469828540494,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 9.33775806427002,
|
|
"kl": 17.56424903869629,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33742450.4274062,
|
|
"logits/rejected": -36088784.4077961,
|
|
"logps/chosen": -461.97695758564436,
|
|
"logps/rejected": -378.94284107946027,
|
|
"loss": 0.4673,
|
|
"loss/base_kto": 3.0938880443573,
|
|
"loss/total_with_kl": 3.73811411857605,
|
|
"metrics/advantage_var": 386.9224548339844,
|
|
"regularization/advantage_var": 386.9224548339844,
|
|
"regularization/kl": 0.6442259550094604,
|
|
"rewards/chosen": 0.6264443327513255,
|
|
"rewards/margins": 0.9080303235083884,
|
|
"rewards/rejected": -0.28158599075706287,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 9.468308448791504,
|
|
"kl": 15.773188591003418,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -33561020.258461535,
|
|
"logits/rejected": -35213975.16190476,
|
|
"logps/chosen": -476.34865384615387,
|
|
"logps/rejected": -368.75644841269843,
|
|
"loss": 0.46,
|
|
"loss/base_kto": 3.101701498031616,
|
|
"loss/total_with_kl": 3.680065870285034,
|
|
"metrics/advantage_var": 347.36602783203125,
|
|
"regularization/advantage_var": 347.36602783203125,
|
|
"regularization/kl": 0.578364372253418,
|
|
"rewards/chosen": 0.5799172738882211,
|
|
"rewards/margins": 0.9101130853264032,
|
|
"rewards/rejected": -0.33019581143818205,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 13.366183280944824,
|
|
"kl": 16.339414596557617,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -33723471.47462687,
|
|
"logits/rejected": -36503273.33770492,
|
|
"logps/chosen": -480.34757462686565,
|
|
"logps/rejected": -391.7977715163934,
|
|
"loss": 0.4703,
|
|
"loss/base_kto": 3.1189987659454346,
|
|
"loss/total_with_kl": 3.762113332748413,
|
|
"metrics/advantage_var": 386.2548828125,
|
|
"regularization/advantage_var": 386.2548828125,
|
|
"regularization/kl": 0.6431143283843994,
|
|
"rewards/chosen": 0.6421425890566698,
|
|
"rewards/margins": 0.8741011972149689,
|
|
"rewards/rejected": -0.2319586081582992,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 12.77984619140625,
|
|
"kl": 20.899572372436523,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -33416177.440758295,
|
|
"logits/rejected": -34146674.34930448,
|
|
"logps/chosen": -497.25557859399686,
|
|
"logps/rejected": -371.7069889876352,
|
|
"loss": 0.4536,
|
|
"loss/base_kto": 3.0386874675750732,
|
|
"loss/total_with_kl": 3.628633499145508,
|
|
"metrics/advantage_var": 354.32208251953125,
|
|
"regularization/advantage_var": 354.32208251953125,
|
|
"regularization/kl": 0.5899462103843689,
|
|
"rewards/chosen": 0.6879725644562105,
|
|
"rewards/margins": 0.9389171471757333,
|
|
"rewards/rejected": -0.2509445827195228,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 10.726768493652344,
|
|
"kl": 16.755084991455078,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34819377.51124438,
|
|
"logits/rejected": -34807413.76835237,
|
|
"logps/chosen": -485.03513868065966,
|
|
"logps/rejected": -387.9455546492659,
|
|
"loss": 0.4663,
|
|
"loss/base_kto": 3.120441198348999,
|
|
"loss/total_with_kl": 3.7302353382110596,
|
|
"metrics/advantage_var": 366.2425842285156,
|
|
"regularization/advantage_var": 366.2425842285156,
|
|
"regularization/kl": 0.6097939610481262,
|
|
"rewards/chosen": 0.6576518869471514,
|
|
"rewards/margins": 0.904373363117752,
|
|
"rewards/rejected": -0.24672147617060053,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 10.555054664611816,
|
|
"kl": 18.0380802154541,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -33776249.198067635,
|
|
"logits/rejected": -35350058.731411226,
|
|
"logps/chosen": -464.94187801932367,
|
|
"logps/rejected": -373.35541540212444,
|
|
"loss": 0.4604,
|
|
"loss/base_kto": 3.0973966121673584,
|
|
"loss/total_with_kl": 3.683020830154419,
|
|
"metrics/advantage_var": 351.7262878417969,
|
|
"regularization/advantage_var": 351.7262878417969,
|
|
"regularization/kl": 0.5856242179870605,
|
|
"rewards/chosen": 0.6328215422453703,
|
|
"rewards/margins": 0.9020929992054898,
|
|
"rewards/rejected": -0.2692714569601195,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 11.60404109954834,
|
|
"kl": 15.615480422973633,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -35260598.18404908,
|
|
"logits/rejected": -35910375.541401275,
|
|
"logps/chosen": -466.3318155674847,
|
|
"logps/rejected": -394.04160031847135,
|
|
"loss": 0.4613,
|
|
"loss/base_kto": 3.1451575756073,
|
|
"loss/total_with_kl": 3.690443515777588,
|
|
"metrics/advantage_var": 327.49920654296875,
|
|
"regularization/advantage_var": 327.49920654296875,
|
|
"regularization/kl": 0.5452861785888672,
|
|
"rewards/chosen": 0.5481845411054928,
|
|
"rewards/margins": 0.8254203133051843,
|
|
"rewards/rejected": -0.2772357721996915,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 10.604949951171875,
|
|
"kl": 15.750246047973633,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35190154.598726116,
|
|
"logits/rejected": -36175922.25766871,
|
|
"logps/chosen": -486.2613953025478,
|
|
"logps/rejected": -375.42479869631904,
|
|
"loss": 0.4666,
|
|
"loss/base_kto": 3.1520683765411377,
|
|
"loss/total_with_kl": 3.7330169677734375,
|
|
"metrics/advantage_var": 348.9179382324219,
|
|
"regularization/advantage_var": 348.9179382324219,
|
|
"regularization/kl": 0.5809482932090759,
|
|
"rewards/chosen": 0.5965832752786624,
|
|
"rewards/margins": 0.8676013210902278,
|
|
"rewards/rejected": -0.27101804581156536,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 11.35732650756836,
|
|
"kl": 15.332303047180176,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34536307.93920973,
|
|
"logits/rejected": -36262184.33440515,
|
|
"logps/chosen": -486.8729578267477,
|
|
"logps/rejected": -381.66268086816723,
|
|
"loss": 0.4665,
|
|
"loss/base_kto": 3.0221824645996094,
|
|
"loss/total_with_kl": 3.7316601276397705,
|
|
"metrics/advantage_var": 426.1127624511719,
|
|
"regularization/advantage_var": 426.1127624511719,
|
|
"regularization/kl": 0.7094777226448059,
|
|
"rewards/chosen": 0.6987152563402356,
|
|
"rewards/margins": 0.9797875969344629,
|
|
"rewards/rejected": -0.2810723405942273,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.866669654846191,
|
|
"kl": 15.0243501663208,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -33443058.93093093,
|
|
"logits/rejected": -35621791.27035831,
|
|
"logps/chosen": -484.6421734234234,
|
|
"logps/rejected": -369.73427320846906,
|
|
"loss": 0.4629,
|
|
"loss/base_kto": 3.0962038040161133,
|
|
"loss/total_with_kl": 3.703148365020752,
|
|
"metrics/advantage_var": 364.53125,
|
|
"regularization/advantage_var": 364.53125,
|
|
"regularization/kl": 0.6069445013999939,
|
|
"rewards/chosen": 0.6645719511014921,
|
|
"rewards/margins": 0.8922062443501688,
|
|
"rewards/rejected": -0.2276342932486767,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 11.085612297058105,
|
|
"kl": 16.760417938232422,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34392750.84839204,
|
|
"logits/rejected": -35218632.880382776,
|
|
"logps/chosen": -488.3175727411945,
|
|
"logps/rejected": -368.4882376395534,
|
|
"loss": 0.4771,
|
|
"loss/base_kto": 3.1475400924682617,
|
|
"loss/total_with_kl": 3.816805601119995,
|
|
"metrics/advantage_var": 401.9615173339844,
|
|
"regularization/advantage_var": 401.9615173339844,
|
|
"regularization/kl": 0.669265866279602,
|
|
"rewards/chosen": 0.6359957684784169,
|
|
"rewards/margins": 0.8359292722818709,
|
|
"rewards/rejected": -0.19993350380345395,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.890966415405273,
|
|
"kl": 16.326244354248047,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -33934182.319115326,
|
|
"logits/rejected": -35683687.270479135,
|
|
"logps/chosen": -453.66706161137444,
|
|
"logps/rejected": -376.3212905718702,
|
|
"loss": 0.4589,
|
|
"loss/base_kto": 3.11423921585083,
|
|
"loss/total_with_kl": 3.6709487438201904,
|
|
"metrics/advantage_var": 334.3602294921875,
|
|
"regularization/advantage_var": 334.3602294921875,
|
|
"regularization/kl": 0.5567097663879395,
|
|
"rewards/chosen": 0.5950622076483512,
|
|
"rewards/margins": 0.8764101987898576,
|
|
"rewards/rejected": -0.2813479911415065,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 10.93013858795166,
|
|
"kl": 17.41450309753418,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34556746.34850863,
|
|
"logits/rejected": -36726424.08709176,
|
|
"logps/chosen": -481.96031200941917,
|
|
"logps/rejected": -383.06653382581646,
|
|
"loss": 0.4609,
|
|
"loss/base_kto": 3.1031627655029297,
|
|
"loss/total_with_kl": 3.6871914863586426,
|
|
"metrics/advantage_var": 350.7679138183594,
|
|
"regularization/advantage_var": 350.7679138183594,
|
|
"regularization/kl": 0.5840285420417786,
|
|
"rewards/chosen": 0.6313654302240728,
|
|
"rewards/margins": 0.8693554343171302,
|
|
"rewards/rejected": -0.23799000409305746,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 19.807714462280273,
|
|
"kl": 15.906045913696289,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -32977395.828843106,
|
|
"logits/rejected": -36120291.99383667,
|
|
"logps/chosen": -491.4402733755943,
|
|
"logps/rejected": -364.68160150231125,
|
|
"loss": 0.469,
|
|
"loss/base_kto": 3.1000936031341553,
|
|
"loss/total_with_kl": 3.7519099712371826,
|
|
"metrics/advantage_var": 391.4812927246094,
|
|
"regularization/advantage_var": 391.4812927246094,
|
|
"regularization/kl": 0.6518163681030273,
|
|
"rewards/chosen": 0.6381141432869454,
|
|
"rewards/margins": 0.9079072378690549,
|
|
"rewards/rejected": -0.2697930945821095,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 13.495177268981934,
|
|
"kl": 19.9287109375,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -34606156.322981365,
|
|
"logits/rejected": -35822624.20125786,
|
|
"logps/chosen": -481.2013295807453,
|
|
"logps/rejected": -383.01299626572325,
|
|
"loss": 0.4718,
|
|
"loss/base_kto": 3.075888156890869,
|
|
"loss/total_with_kl": 3.774658203125,
|
|
"metrics/advantage_var": 419.6817321777344,
|
|
"regularization/advantage_var": 419.6817321777344,
|
|
"regularization/kl": 0.6987700462341309,
|
|
"rewards/chosen": 0.6444550034422312,
|
|
"rewards/margins": 0.9164425586041931,
|
|
"rewards/rejected": -0.271987555161962,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 7.06175422668457,
|
|
"kl": 18.30661964416504,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34172116.18018018,
|
|
"logits/rejected": -36401799.08794788,
|
|
"logps/chosen": -487.959740990991,
|
|
"logps/rejected": -366.29690044788276,
|
|
"loss": 0.4621,
|
|
"loss/base_kto": 3.1006484031677246,
|
|
"loss/total_with_kl": 3.69689679145813,
|
|
"metrics/advantage_var": 358.107177734375,
|
|
"regularization/advantage_var": 358.107177734375,
|
|
"regularization/kl": 0.59624844789505,
|
|
"rewards/chosen": 0.6270985188068928,
|
|
"rewards/margins": 0.9056602879755105,
|
|
"rewards/rejected": -0.2785617691686177,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 18.84697151184082,
|
|
"kl": 18.24949073791504,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -33402331.66451613,
|
|
"logits/rejected": -35917532.31515151,
|
|
"logps/chosen": -486.2614919354839,
|
|
"logps/rejected": -381.3780539772727,
|
|
"loss": 0.4683,
|
|
"loss/base_kto": 3.128180742263794,
|
|
"loss/total_with_kl": 3.7462964057922363,
|
|
"metrics/advantage_var": 371.2406921386719,
|
|
"regularization/advantage_var": 371.2406921386719,
|
|
"regularization/kl": 0.6181157231330872,
|
|
"rewards/chosen": 0.6342254146452873,
|
|
"rewards/margins": 0.8826653652992882,
|
|
"rewards/rejected": -0.24843995065400096,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 7.345629692077637,
|
|
"kl": 17.771638870239258,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34244737.73755656,
|
|
"logits/rejected": -36181315.630470015,
|
|
"logps/chosen": -490.74622926093514,
|
|
"logps/rejected": -387.5709076175041,
|
|
"loss": 0.4641,
|
|
"loss/base_kto": 3.0852608680725098,
|
|
"loss/total_with_kl": 3.712777853012085,
|
|
"metrics/advantage_var": 376.8869934082031,
|
|
"regularization/advantage_var": 376.8869934082031,
|
|
"regularization/kl": 0.6275168061256409,
|
|
"rewards/chosen": 0.6421334797440611,
|
|
"rewards/margins": 0.898888053522804,
|
|
"rewards/rejected": -0.2567545737787429,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 20.201440811157227,
|
|
"kl": 19.56184959411621,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35519863.031847134,
|
|
"logits/rejected": -36617341.64417178,
|
|
"logps/chosen": -490.218949044586,
|
|
"logps/rejected": -369.78192101226995,
|
|
"loss": 0.4643,
|
|
"loss/base_kto": 3.0941224098205566,
|
|
"loss/total_with_kl": 3.714582920074463,
|
|
"metrics/advantage_var": 372.6491394042969,
|
|
"regularization/advantage_var": 372.6491394042969,
|
|
"regularization/kl": 0.6204608082771301,
|
|
"rewards/chosen": 0.6528654159254329,
|
|
"rewards/margins": 0.9218227763159381,
|
|
"rewards/rejected": -0.26895736039050516,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 12.100944519042969,
|
|
"kl": 17.12972068786621,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35650007.87460815,
|
|
"logits/rejected": -36424097.894081,
|
|
"logps/chosen": -483.08297413793105,
|
|
"logps/rejected": -365.57919587227417,
|
|
"loss": 0.4544,
|
|
"loss/base_kto": 3.0918233394622803,
|
|
"loss/total_with_kl": 3.6350114345550537,
|
|
"metrics/advantage_var": 326.2388610839844,
|
|
"regularization/advantage_var": 326.2388610839844,
|
|
"regularization/kl": 0.5431877374649048,
|
|
"rewards/chosen": 0.6503526454435247,
|
|
"rewards/margins": 0.8949310218094952,
|
|
"rewards/rejected": -0.2445783763659706,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 9.107111930847168,
|
|
"kl": 19.432432174682617,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33601410.67954911,
|
|
"logits/rejected": -34967527.13808801,
|
|
"logps/chosen": -488.8157206119163,
|
|
"logps/rejected": -382.0994641502276,
|
|
"loss": 0.4683,
|
|
"loss/base_kto": 3.1135101318359375,
|
|
"loss/total_with_kl": 3.746328830718994,
|
|
"metrics/advantage_var": 380.0712585449219,
|
|
"regularization/advantage_var": 380.0712585449219,
|
|
"regularization/kl": 0.6328186392784119,
|
|
"rewards/chosen": 0.6293476491734602,
|
|
"rewards/margins": 0.865551180024833,
|
|
"rewards/rejected": -0.23620353085137283,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 14.301103591918945,
|
|
"kl": 18.90772819519043,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34614046.65372168,
|
|
"logits/rejected": -34828009.57099698,
|
|
"logps/chosen": -453.7316949838188,
|
|
"logps/rejected": -389.2943023036254,
|
|
"loss": 0.4701,
|
|
"loss/base_kto": 3.0868051052093506,
|
|
"loss/total_with_kl": 3.7606430053710938,
|
|
"metrics/advantage_var": 404.7074890136719,
|
|
"regularization/advantage_var": 404.7074890136719,
|
|
"regularization/kl": 0.6738380193710327,
|
|
"rewards/chosen": 0.6064947924567657,
|
|
"rewards/margins": 0.9340541339335289,
|
|
"rewards/rejected": -0.32755934147676313,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.741808891296387,
|
|
"kl": 19.727737426757812,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34053906.79282219,
|
|
"logits/rejected": -36005737.547226384,
|
|
"logps/chosen": -469.11776101141925,
|
|
"logps/rejected": -350.3216829085457,
|
|
"loss": 0.4565,
|
|
"loss/base_kto": 3.1069390773773193,
|
|
"loss/total_with_kl": 3.6520142555236816,
|
|
"metrics/advantage_var": 327.37249755859375,
|
|
"regularization/advantage_var": 327.37249755859375,
|
|
"regularization/kl": 0.5450751781463623,
|
|
"rewards/chosen": 0.6125880081151865,
|
|
"rewards/margins": 0.8740746964116632,
|
|
"rewards/rejected": -0.26148668829647675,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 9.603240966796875,
|
|
"kl": 16.149274826049805,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -35375289.32515337,
|
|
"logits/rejected": -34030203.92356688,
|
|
"logps/chosen": -442.3598542944785,
|
|
"logps/rejected": -356.9175955414013,
|
|
"loss": 0.4606,
|
|
"loss/base_kto": 3.1312148571014404,
|
|
"loss/total_with_kl": 3.684755802154541,
|
|
"metrics/advantage_var": 332.45709228515625,
|
|
"regularization/advantage_var": 332.45709228515625,
|
|
"regularization/kl": 0.5535410046577454,
|
|
"rewards/chosen": 0.5940659411845763,
|
|
"rewards/margins": 0.8722834269404481,
|
|
"rewards/rejected": -0.2782174857558718,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 11.697710990905762,
|
|
"kl": 16.6291446685791,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34522619.96850394,
|
|
"logits/rejected": -36080825.74883721,
|
|
"logps/chosen": -467.4814468503937,
|
|
"logps/rejected": -375.39093992248064,
|
|
"loss": 0.4535,
|
|
"loss/base_kto": 3.1075026988983154,
|
|
"loss/total_with_kl": 3.628373861312866,
|
|
"metrics/advantage_var": 312.8353576660156,
|
|
"regularization/advantage_var": 312.8353576660156,
|
|
"regularization/kl": 0.5208708643913269,
|
|
"rewards/chosen": 0.6153376061146654,
|
|
"rewards/margins": 0.909383792221739,
|
|
"rewards/rejected": -0.29404618610707367,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_loss": 0.5004168742678944,
|
|
"train_runtime": 11053.4701,
|
|
"train_samples_per_second": 13.409,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|