2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 20.017595291137695,
|
||
|
|
"kl": 10.869776725769043,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -37405406.679365076,
|
||
|
|
"logits/rejected": -37543794.21538462,
|
||
|
|
"logps/chosen": -492.95694444444445,
|
||
|
|
"logps/rejected": -370.13004807692306,
|
||
|
|
"loss": 0.6113,
|
||
|
|
"loss/base_kto": 3.935161590576172,
|
||
|
|
"metrics/advantage_var": 221.8739776611328,
|
||
|
|
"regularization/lipschitz_norm": 986.458984375,
|
||
|
|
"regularization/w1": 0.9548923373222351,
|
||
|
|
"rewards/chosen": 0.08865559895833333,
|
||
|
|
"rewards/margins": 0.057333157857259114,
|
||
|
|
"rewards/rejected": 0.03132244110107422,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 21.927038192749023,
|
||
|
|
"kl": 6.196732997894287,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -36026778.874028,
|
||
|
|
"logits/rejected": -37112846.467817895,
|
||
|
|
"logps/chosen": -500.83461314152413,
|
||
|
|
"logps/rejected": -377.2483810832025,
|
||
|
|
"loss": 0.6022,
|
||
|
|
"loss/base_kto": 3.907296895980835,
|
||
|
|
"metrics/advantage_var": 200.6800994873047,
|
||
|
|
"regularization/lipschitz_norm": 939.9888916015625,
|
||
|
|
"regularization/w1": 0.909909188747406,
|
||
|
|
"rewards/chosen": 0.020851707755128852,
|
||
|
|
"rewards/margins": 0.07231350693355923,
|
||
|
|
"rewards/rejected": -0.051461799178430384,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 30.5780086517334,
|
||
|
|
"kl": 12.766398429870605,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -35985940.35474006,
|
||
|
|
"logits/rejected": -36749187.680511184,
|
||
|
|
"logps/chosen": -473.48427943425077,
|
||
|
|
"logps/rejected": -395.1400259584665,
|
||
|
|
"loss": 0.5988,
|
||
|
|
"loss/base_kto": 3.871070146560669,
|
||
|
|
"metrics/advantage_var": 196.3928985595703,
|
||
|
|
"regularization/lipschitz_norm": 949.5232543945312,
|
||
|
|
"regularization/w1": 0.9191384315490723,
|
||
|
|
"rewards/chosen": 0.23350781548642965,
|
||
|
|
"rewards/margins": 0.09413125743244102,
|
||
|
|
"rewards/rejected": 0.13937655805398863,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 17.839956283569336,
|
||
|
|
"kl": 6.810754299163818,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -36709510.2685624,
|
||
|
|
"logits/rejected": -39179683.41267388,
|
||
|
|
"logps/chosen": -479.3947966034755,
|
||
|
|
"logps/rejected": -389.4709717928903,
|
||
|
|
"loss": 0.5976,
|
||
|
|
"loss/base_kto": 3.8942711353302,
|
||
|
|
"metrics/advantage_var": 187.96533203125,
|
||
|
|
"regularization/lipschitz_norm": 916.04296875,
|
||
|
|
"regularization/w1": 0.8867295384407043,
|
||
|
|
"rewards/chosen": 0.09592528742448225,
|
||
|
|
"rewards/margins": 0.09882228099376347,
|
||
|
|
"rewards/rejected": -0.002896993569281223,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 17.139768600463867,
|
||
|
|
"kl": 11.984885215759277,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -37125205.737756714,
|
||
|
|
"logits/rejected": -37139505.0633694,
|
||
|
|
"logps/chosen": -470.8042061611374,
|
||
|
|
"logps/rejected": -378.0897894126739,
|
||
|
|
"loss": 0.5941,
|
||
|
|
"loss/base_kto": 3.876530408859253,
|
||
|
|
"metrics/advantage_var": 166.6145477294922,
|
||
|
|
"regularization/lipschitz_norm": 904.8302001953125,
|
||
|
|
"regularization/w1": 0.8758754730224609,
|
||
|
|
"rewards/chosen": 0.20821557790747186,
|
||
|
|
"rewards/margins": 0.10907115967156333,
|
||
|
|
"rewards/rejected": 0.09914441823590853,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 22.828672409057617,
|
||
|
|
"kl": 9.92587947845459,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -37828626.56193353,
|
||
|
|
"logits/rejected": -38083183.01618123,
|
||
|
|
"logps/chosen": -477.83100453172204,
|
||
|
|
"logps/rejected": -366.81697512135923,
|
||
|
|
"loss": 0.5944,
|
||
|
|
"loss/base_kto": 3.863205671310425,
|
||
|
|
"metrics/advantage_var": 179.6526641845703,
|
||
|
|
"regularization/lipschitz_norm": 921.1351928710938,
|
||
|
|
"regularization/w1": 0.8916589021682739,
|
||
|
|
"rewards/chosen": 0.14145618911236074,
|
||
|
|
"rewards/margins": 0.12850694462241316,
|
||
|
|
"rewards/rejected": 0.012949244489947569,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 24.155696868896484,
|
||
|
|
"kl": 13.541302680969238,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -36501507.2,
|
||
|
|
"logits/rejected": -38708019.2,
|
||
|
|
"logps/chosen": -496.6033203125,
|
||
|
|
"logps/rejected": -365.200244140625,
|
||
|
|
"loss": 0.588,
|
||
|
|
"loss/base_kto": 3.8093273639678955,
|
||
|
|
"metrics/advantage_var": 222.50718688964844,
|
||
|
|
"regularization/lipschitz_norm": 923.9501953125,
|
||
|
|
"regularization/w1": 0.8943838477134705,
|
||
|
|
"rewards/chosen": 0.23306601047515868,
|
||
|
|
"rewards/margins": 0.1751942753791809,
|
||
|
|
"rewards/rejected": 0.05787173509597778,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 21.56728172302246,
|
||
|
|
"kl": 5.2361602783203125,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36570532.90797546,
|
||
|
|
"logits/rejected": -37827029.60509554,
|
||
|
|
"logps/chosen": -480.0208013803681,
|
||
|
|
"logps/rejected": -366.4399134156051,
|
||
|
|
"loss": 0.5907,
|
||
|
|
"loss/base_kto": 3.812206268310547,
|
||
|
|
"metrics/advantage_var": 198.59609985351562,
|
||
|
|
"regularization/lipschitz_norm": 943.3603515625,
|
||
|
|
"regularization/w1": 0.913172721862793,
|
||
|
|
"rewards/chosen": 0.07574753088453796,
|
||
|
|
"rewards/margins": 0.1770774376154328,
|
||
|
|
"rewards/rejected": -0.10132990673089483,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 20.541982650756836,
|
||
|
|
"kl": 6.803701877593994,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -37411657.73462783,
|
||
|
|
"logits/rejected": -37252049.59516616,
|
||
|
|
"logps/chosen": -470.3621561488673,
|
||
|
|
"logps/rejected": -370.6001935422961,
|
||
|
|
"loss": 0.5872,
|
||
|
|
"loss/base_kto": 3.8091697692871094,
|
||
|
|
"metrics/advantage_var": 228.62413024902344,
|
||
|
|
"regularization/lipschitz_norm": 917.7543334960938,
|
||
|
|
"regularization/w1": 0.8883861899375916,
|
||
|
|
"rewards/chosen": 0.08275365983784005,
|
||
|
|
"rewards/margins": 0.1808527950379026,
|
||
|
|
"rewards/rejected": -0.09809913520006255,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 25.392168045043945,
|
||
|
|
"kl": 8.498290061950684,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -36947681.1465798,
|
||
|
|
"logits/rejected": -38496046.8948949,
|
||
|
|
"logps/chosen": -474.0551201140065,
|
||
|
|
"logps/rejected": -368.8321368243243,
|
||
|
|
"loss": 0.5953,
|
||
|
|
"loss/base_kto": 3.8641929626464844,
|
||
|
|
"metrics/advantage_var": 185.65187072753906,
|
||
|
|
"regularization/lipschitz_norm": 928.2777709960938,
|
||
|
|
"regularization/w1": 0.8985727429389954,
|
||
|
|
"rewards/chosen": 0.0563995348902401,
|
||
|
|
"rewards/margins": 0.12253679145527682,
|
||
|
|
"rewards/rejected": -0.06613725656503672,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 17.997512817382812,
|
||
|
|
"kl": 7.209378719329834,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -37346435.204712816,
|
||
|
|
"logits/rejected": -37922608.133111484,
|
||
|
|
"logps/chosen": -495.16168078055966,
|
||
|
|
"logps/rejected": -380.97626351913476,
|
||
|
|
"loss": 0.5939,
|
||
|
|
"loss/base_kto": 3.7523562908172607,
|
||
|
|
"metrics/advantage_var": 245.42568969726562,
|
||
|
|
"regularization/lipschitz_norm": 1032.0877685546875,
|
||
|
|
"regularization/w1": 0.9990608096122742,
|
||
|
|
"rewards/chosen": 0.11389820846085696,
|
||
|
|
"rewards/margins": 0.250538354116535,
|
||
|
|
"rewards/rejected": -0.13664014565567803,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 22.708410263061523,
|
||
|
|
"kl": 13.33154296875,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -36773372.75949367,
|
||
|
|
"logits/rejected": -38125264.59259259,
|
||
|
|
"logps/chosen": -496.90481606012656,
|
||
|
|
"logps/rejected": -376.5931712962963,
|
||
|
|
"loss": 0.5987,
|
||
|
|
"loss/base_kto": 3.77742075920105,
|
||
|
|
"metrics/advantage_var": 245.73403930664062,
|
||
|
|
"regularization/lipschitz_norm": 1045.599853515625,
|
||
|
|
"regularization/w1": 1.0121406316757202,
|
||
|
|
"rewards/chosen": 0.28304865390439576,
|
||
|
|
"rewards/margins": 0.22781992018232122,
|
||
|
|
"rewards/rejected": 0.055228733722074534,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 21.75596809387207,
|
||
|
|
"kl": 4.437197685241699,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -37280626.12048193,
|
||
|
|
"logits/rejected": -37577601.66233766,
|
||
|
|
"logps/chosen": -481.93260542168673,
|
||
|
|
"logps/rejected": -365.2028713474026,
|
||
|
|
"loss": 0.5822,
|
||
|
|
"loss/base_kto": 3.75117564201355,
|
||
|
|
"metrics/advantage_var": 213.0432586669922,
|
||
|
|
"regularization/lipschitz_norm": 936.2691650390625,
|
||
|
|
"regularization/w1": 0.9063085913658142,
|
||
|
|
"rewards/chosen": 0.07063835787485881,
|
||
|
|
"rewards/margins": 0.26035296303288435,
|
||
|
|
"rewards/rejected": -0.18971460515802557,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 22.17413330078125,
|
||
|
|
"kl": 14.323687553405762,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -36430384.5382263,
|
||
|
|
"logits/rejected": -38332144.4600639,
|
||
|
|
"logps/chosen": -527.8588016055046,
|
||
|
|
"logps/rejected": -367.6101238019169,
|
||
|
|
"loss": 0.5948,
|
||
|
|
"loss/base_kto": 3.7395730018615723,
|
||
|
|
"metrics/advantage_var": 253.6971893310547,
|
||
|
|
"regularization/lipschitz_norm": 1052.7191162109375,
|
||
|
|
"regularization/w1": 1.0190320014953613,
|
||
|
|
"rewards/chosen": 0.3271077007328698,
|
||
|
|
"rewards/margins": 0.2468259708366413,
|
||
|
|
"rewards/rejected": 0.08028172989622853,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 18.199485778808594,
|
||
|
|
"kl": 16.366008758544922,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -35703982.9068323,
|
||
|
|
"logits/rejected": -37515186.716981135,
|
||
|
|
"logps/chosen": -478.0859860248447,
|
||
|
|
"logps/rejected": -355.0528203616352,
|
||
|
|
"loss": 0.5848,
|
||
|
|
"loss/base_kto": 3.7101800441741943,
|
||
|
|
"metrics/advantage_var": 211.41197204589844,
|
||
|
|
"regularization/lipschitz_norm": 1000.478515625,
|
||
|
|
"regularization/w1": 0.9684632420539856,
|
||
|
|
"rewards/chosen": 0.3133388306043163,
|
||
|
|
"rewards/margins": 0.24097956558275818,
|
||
|
|
"rewards/rejected": 0.07235926502155808,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 20.52756118774414,
|
||
|
|
"kl": 8.322763442993164,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -36121608.16586922,
|
||
|
|
"logits/rejected": -37411954.474732004,
|
||
|
|
"logps/chosen": -465.50897129186603,
|
||
|
|
"logps/rejected": -383.57831642419603,
|
||
|
|
"loss": 0.5926,
|
||
|
|
"loss/base_kto": 3.800311326980591,
|
||
|
|
"metrics/advantage_var": 228.5869140625,
|
||
|
|
"regularization/lipschitz_norm": 971.8804931640625,
|
||
|
|
"regularization/w1": 0.9407802820205688,
|
||
|
|
"rewards/chosen": 0.1454428105453175,
|
||
|
|
"rewards/margins": 0.19636511671710277,
|
||
|
|
"rewards/rejected": -0.05092230617178527,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 21.62049674987793,
|
||
|
|
"kl": 7.547786235809326,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -35656501.68996961,
|
||
|
|
"logits/rejected": -37903672.797427654,
|
||
|
|
"logps/chosen": -470.7935505319149,
|
||
|
|
"logps/rejected": -369.6180667202572,
|
||
|
|
"loss": 0.599,
|
||
|
|
"loss/base_kto": 3.807238817214966,
|
||
|
|
"metrics/advantage_var": 236.3555145263672,
|
||
|
|
"regularization/lipschitz_norm": 1017.7091064453125,
|
||
|
|
"regularization/w1": 0.9851425290107727,
|
||
|
|
"rewards/chosen": 0.09910588134023557,
|
||
|
|
"rewards/margins": 0.20289699862507782,
|
||
|
|
"rewards/rejected": -0.10379111728484225,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 18.61902618408203,
|
||
|
|
"kl": 5.990298748016357,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -35846357.26280992,
|
||
|
|
"logits/rejected": -37901671.53777778,
|
||
|
|
"logps/chosen": -483.161673553719,
|
||
|
|
"logps/rejected": -374.85791666666665,
|
||
|
|
"loss": 0.5923,
|
||
|
|
"loss/base_kto": 3.7925915718078613,
|
||
|
|
"metrics/advantage_var": 223.52725219726562,
|
||
|
|
"regularization/lipschitz_norm": 977.4733276367188,
|
||
|
|
"regularization/w1": 0.9461941719055176,
|
||
|
|
"rewards/chosen": -0.004993493103783978,
|
||
|
|
"rewards/margins": 0.1786275192660077,
|
||
|
|
"rewards/rejected": -0.18362101236979167,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 25.671829223632812,
|
||
|
|
"kl": 7.032774448394775,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -35546496.40378549,
|
||
|
|
"logits/rejected": -37972183.578947365,
|
||
|
|
"logps/chosen": -494.2342271293375,
|
||
|
|
"logps/rejected": -359.9824400154799,
|
||
|
|
"loss": 0.5932,
|
||
|
|
"loss/base_kto": 3.762411594390869,
|
||
|
|
"metrics/advantage_var": 246.97886657714844,
|
||
|
|
"regularization/lipschitz_norm": 1015.7227783203125,
|
||
|
|
"regularization/w1": 0.983219563961029,
|
||
|
|
"rewards/chosen": 0.05271367695805402,
|
||
|
|
"rewards/margins": 0.23495319761854888,
|
||
|
|
"rewards/rejected": -0.18223952066049487,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 24.56637191772461,
|
||
|
|
"kl": 7.599175453186035,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -34486925.95638629,
|
||
|
|
"logits/rejected": -35730056.42633229,
|
||
|
|
"logps/chosen": -509.3734423676012,
|
||
|
|
"logps/rejected": -383.3430887539185,
|
||
|
|
"loss": 0.5985,
|
||
|
|
"loss/base_kto": 3.7800540924072266,
|
||
|
|
"metrics/advantage_var": 239.8534698486328,
|
||
|
|
"regularization/lipschitz_norm": 1040.890869140625,
|
||
|
|
"regularization/w1": 1.0075823068618774,
|
||
|
|
"rewards/chosen": 0.10800875607309311,
|
||
|
|
"rewards/margins": 0.21338056014603224,
|
||
|
|
"rewards/rejected": -0.10537180407293911,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 22.619426727294922,
|
||
|
|
"kl": 4.125832557678223,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36172878.003241494,
|
||
|
|
"logits/rejected": -37643858.63046757,
|
||
|
|
"logps/chosen": -467.92038087520257,
|
||
|
|
"logps/rejected": -380.50228601055807,
|
||
|
|
"loss": 0.5943,
|
||
|
|
"loss/base_kto": 3.7586922645568848,
|
||
|
|
"metrics/advantage_var": 258.6198425292969,
|
||
|
|
"regularization/lipschitz_norm": 1028.2740478515625,
|
||
|
|
"regularization/w1": 0.9953693747520447,
|
||
|
|
"rewards/chosen": -0.0877177085443679,
|
||
|
|
"rewards/margins": 0.20979935736402672,
|
||
|
|
"rewards/rejected": -0.2975170659083946,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 22.970863342285156,
|
||
|
|
"kl": 11.10529613494873,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -37031548.121212125,
|
||
|
|
"logits/rejected": -37050519.948387094,
|
||
|
|
"logps/chosen": -480.84966856060606,
|
||
|
|
"logps/rejected": -400.41219758064517,
|
||
|
|
"loss": 0.5994,
|
||
|
|
"loss/base_kto": 3.72294020652771,
|
||
|
|
"metrics/advantage_var": 293.3447265625,
|
||
|
|
"regularization/lipschitz_norm": 1107.8349609375,
|
||
|
|
"regularization/w1": 1.072384238243103,
|
||
|
|
"rewards/chosen": 0.1748742999452533,
|
||
|
|
"rewards/margins": 0.2780853398268057,
|
||
|
|
"rewards/rejected": -0.10321103988155242,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 17.754562377929688,
|
||
|
|
"kl": 8.446227073669434,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -36642416.5553719,
|
||
|
|
"logits/rejected": -37213680.07111111,
|
||
|
|
"logps/chosen": -482.2271694214876,
|
||
|
|
"logps/rejected": -382.54599537037035,
|
||
|
|
"loss": 0.5964,
|
||
|
|
"loss/base_kto": 3.7078018188476562,
|
||
|
|
"metrics/advantage_var": 289.71759033203125,
|
||
|
|
"regularization/lipschitz_norm": 1098.6951904296875,
|
||
|
|
"regularization/w1": 1.0635368824005127,
|
||
|
|
"rewards/chosen": 0.049709506074259106,
|
||
|
|
"rewards/margins": 0.27176474063010403,
|
||
|
|
"rewards/rejected": -0.22205523455584492,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 27.884546279907227,
|
||
|
|
"kl": 2.1241166591644287,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -36130041.6,
|
||
|
|
"logits/rejected": -37182835.2,
|
||
|
|
"logps/chosen": -513.2166015625,
|
||
|
|
"logps/rejected": -378.71630859375,
|
||
|
|
"loss": 0.6128,
|
||
|
|
"loss/base_kto": 3.7935006618499756,
|
||
|
|
"metrics/advantage_var": 307.8874206542969,
|
||
|
|
"regularization/lipschitz_norm": 1145.83984375,
|
||
|
|
"regularization/w1": 1.109173059463501,
|
||
|
|
"rewards/chosen": -0.16095976829528807,
|
||
|
|
"rewards/margins": 0.21801607608795165,
|
||
|
|
"rewards/rejected": -0.3789758443832397,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 24.344051361083984,
|
||
|
|
"kl": 5.434615135192871,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36868043.82165605,
|
||
|
|
"logits/rejected": -36816107.582822084,
|
||
|
|
"logps/chosen": -516.2912022292994,
|
||
|
|
"logps/rejected": -397.26303680981596,
|
||
|
|
"loss": 0.5893,
|
||
|
|
"loss/base_kto": 3.6992299556732178,
|
||
|
|
"metrics/advantage_var": 280.0695495605469,
|
||
|
|
"regularization/lipschitz_norm": 1048.950439453125,
|
||
|
|
"regularization/w1": 1.0153840780258179,
|
||
|
|
"rewards/chosen": 0.09616808678693832,
|
||
|
|
"rewards/margins": 0.2888349889194992,
|
||
|
|
"rewards/rejected": -0.19266690213256088,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 21.51359748840332,
|
||
|
|
"kl": 10.364459991455078,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -36672948.61349693,
|
||
|
|
"logits/rejected": -38122861.24840765,
|
||
|
|
"logps/chosen": -490.30032592024537,
|
||
|
|
"logps/rejected": -368.547173566879,
|
||
|
|
"loss": 0.5921,
|
||
|
|
"loss/base_kto": 3.679784059524536,
|
||
|
|
"metrics/advantage_var": 260.45025634765625,
|
||
|
|
"regularization/lipschitz_norm": 1091.844482421875,
|
||
|
|
"regularization/w1": 1.0569053888320923,
|
||
|
|
"rewards/chosen": 0.19242699745973926,
|
||
|
|
"rewards/margins": 0.306936713615162,
|
||
|
|
"rewards/rejected": -0.11450971615542273,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 20.921213150024414,
|
||
|
|
"kl": 13.729880332946777,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -34964199.07964602,
|
||
|
|
"logits/rejected": -37574656.0,
|
||
|
|
"logps/chosen": -510.9582872418879,
|
||
|
|
"logps/rejected": -350.2104962624585,
|
||
|
|
"loss": 0.5895,
|
||
|
|
"loss/base_kto": 3.6536691188812256,
|
||
|
|
"metrics/advantage_var": 272.725341796875,
|
||
|
|
"regularization/lipschitz_norm": 1097.171142578125,
|
||
|
|
"regularization/w1": 1.0620616674423218,
|
||
|
|
"rewards/chosen": 0.3045457147918971,
|
||
|
|
"rewards/margins": 0.3334412164704653,
|
||
|
|
"rewards/rejected": -0.028895501678568185,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 29.47673797607422,
|
||
|
|
"kl": 20.595972061157227,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -34999925.28278041,
|
||
|
|
"logits/rejected": -37622620.98299845,
|
||
|
|
"logps/chosen": -483.7810525276461,
|
||
|
|
"logps/rejected": -345.6037963678516,
|
||
|
|
"loss": 0.5887,
|
||
|
|
"loss/base_kto": 3.6905770301818848,
|
||
|
|
"metrics/advantage_var": 254.7733612060547,
|
||
|
|
"regularization/lipschitz_norm": 1052.6541748046875,
|
||
|
|
"regularization/w1": 1.018969178199768,
|
||
|
|
"rewards/chosen": 0.36258464891575337,
|
||
|
|
"rewards/margins": 0.2585487864691851,
|
||
|
|
"rewards/rejected": 0.10403586244656829,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 24.759416580200195,
|
||
|
|
"kl": 18.578369140625,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -36682243.22012579,
|
||
|
|
"logits/rejected": -38303826.68322981,
|
||
|
|
"logps/chosen": -479.5780267295597,
|
||
|
|
"logps/rejected": -387.1138392857143,
|
||
|
|
"loss": 0.5889,
|
||
|
|
"loss/base_kto": 3.7273178100585938,
|
||
|
|
"metrics/advantage_var": 239.63267517089844,
|
||
|
|
"regularization/lipschitz_norm": 1016.1982421875,
|
||
|
|
"regularization/w1": 0.9836799502372742,
|
||
|
|
"rewards/chosen": 0.25962666445558175,
|
||
|
|
"rewards/margins": 0.2517788414843562,
|
||
|
|
"rewards/rejected": 0.007847822971225524,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 18.48613739013672,
|
||
|
|
"kl": 8.944536209106445,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -36596557.35825545,
|
||
|
|
"logits/rejected": -37825038.44514106,
|
||
|
|
"logps/chosen": -481.3895054517134,
|
||
|
|
"logps/rejected": -366.57014106583074,
|
||
|
|
"loss": 0.5883,
|
||
|
|
"loss/base_kto": 3.755094289779663,
|
||
|
|
"metrics/advantage_var": 241.492919921875,
|
||
|
|
"regularization/lipschitz_norm": 982.9472045898438,
|
||
|
|
"regularization/w1": 0.9514929056167603,
|
||
|
|
"rewards/chosen": 0.10775050938686478,
|
||
|
|
"rewards/margins": 0.22339647706769722,
|
||
|
|
"rewards/rejected": -0.11564596768083243,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 24.27670669555664,
|
||
|
|
"kl": 6.778843879699707,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -36233756.672,
|
||
|
|
"logits/rejected": -36525079.45038168,
|
||
|
|
"logps/chosen": -496.5119,
|
||
|
|
"logps/rejected": -366.8752385496183,
|
||
|
|
"loss": 0.5981,
|
||
|
|
"loss/base_kto": 3.714728593826294,
|
||
|
|
"metrics/advantage_var": 260.7563171386719,
|
||
|
|
"regularization/lipschitz_norm": 1105.4476318359375,
|
||
|
|
"regularization/w1": 1.0700734853744507,
|
||
|
|
"rewards/chosen": 0.05918572998046875,
|
||
|
|
"rewards/margins": 0.2862464119714635,
|
||
|
|
"rewards/rejected": -0.22706068199099474,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 29.453887939453125,
|
||
|
|
"kl": 3.2028260231018066,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -37143531.2324493,
|
||
|
|
"logits/rejected": -36793922.50391236,
|
||
|
|
"logps/chosen": -466.21294851794073,
|
||
|
|
"logps/rejected": -389.4480389280125,
|
||
|
|
"loss": 0.6028,
|
||
|
|
"loss/base_kto": 3.8090901374816895,
|
||
|
|
"metrics/advantage_var": 238.0695343017578,
|
||
|
|
"regularization/lipschitz_norm": 1047.0986328125,
|
||
|
|
"regularization/w1": 1.0135915279388428,
|
||
|
|
"rewards/chosen": -0.17376682799245563,
|
||
|
|
"rewards/margins": 0.18634799642496908,
|
||
|
|
"rewards/rejected": -0.3601148244174247,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 25.4942626953125,
|
||
|
|
"kl": 13.659431457519531,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36567567.900621116,
|
||
|
|
"logits/rejected": -36893097.056603774,
|
||
|
|
"logps/chosen": -476.733501552795,
|
||
|
|
"logps/rejected": -381.26049036949684,
|
||
|
|
"loss": 0.5951,
|
||
|
|
"loss/base_kto": 3.7146377563476562,
|
||
|
|
"metrics/advantage_var": 265.953125,
|
||
|
|
"regularization/lipschitz_norm": 1080.6595458984375,
|
||
|
|
"regularization/w1": 1.0460784435272217,
|
||
|
|
"rewards/chosen": 0.15029237877508128,
|
||
|
|
"rewards/margins": 0.2527857608751206,
|
||
|
|
"rewards/rejected": -0.1024933821000393,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 16.184741973876953,
|
||
|
|
"kl": 12.682997703552246,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -37725187.2926045,
|
||
|
|
"logits/rejected": -38742644.717325225,
|
||
|
|
"logps/chosen": -486.11284163987136,
|
||
|
|
"logps/rejected": -367.1682655775076,
|
||
|
|
"loss": 0.598,
|
||
|
|
"loss/base_kto": 3.700289249420166,
|
||
|
|
"metrics/advantage_var": 298.099609375,
|
||
|
|
"regularization/lipschitz_norm": 1119.477294921875,
|
||
|
|
"regularization/w1": 1.0836539268493652,
|
||
|
|
"rewards/chosen": 0.15572850267220156,
|
||
|
|
"rewards/margins": 0.2574530519396313,
|
||
|
|
"rewards/rejected": -0.10172454926742971,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 18.903465270996094,
|
||
|
|
"kl": 11.158483505249023,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -37589995.61408882,
|
||
|
|
"logits/rejected": -38158531.98086125,
|
||
|
|
"logps/chosen": -481.54857388973966,
|
||
|
|
"logps/rejected": -372.3416068580542,
|
||
|
|
"loss": 0.5953,
|
||
|
|
"loss/base_kto": 3.7120754718780518,
|
||
|
|
"metrics/advantage_var": 256.5372009277344,
|
||
|
|
"regularization/lipschitz_norm": 1085.4078369140625,
|
||
|
|
"regularization/w1": 1.0506747961044312,
|
||
|
|
"rewards/chosen": 0.09413982934645088,
|
||
|
|
"rewards/margins": 0.2781820952673789,
|
||
|
|
"rewards/rejected": -0.18404226592092804,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 17.967939376831055,
|
||
|
|
"kl": 16.590667724609375,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -36044548.711656444,
|
||
|
|
"logits/rejected": -36358597.299363054,
|
||
|
|
"logps/chosen": -468.0887653374233,
|
||
|
|
"logps/rejected": -381.5858628582803,
|
||
|
|
"loss": 0.5952,
|
||
|
|
"loss/base_kto": 3.7376930713653564,
|
||
|
|
"metrics/advantage_var": 244.9986114501953,
|
||
|
|
"regularization/lipschitz_norm": 1058.009033203125,
|
||
|
|
"regularization/w1": 1.0241526365280151,
|
||
|
|
"rewards/chosen": 0.24092291615491995,
|
||
|
|
"rewards/margins": 0.24871887263899173,
|
||
|
|
"rewards/rejected": -0.007795956484071768,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 19.054676055908203,
|
||
|
|
"kl": 25.2733211517334,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -36187785.28706624,
|
||
|
|
"logits/rejected": -38553244.928792566,
|
||
|
|
"logps/chosen": -498.87849960567826,
|
||
|
|
"logps/rejected": -344.6284104102167,
|
||
|
|
"loss": 0.5969,
|
||
|
|
"loss/base_kto": 3.720266103744507,
|
||
|
|
"metrics/advantage_var": 270.4605407714844,
|
||
|
|
"regularization/lipschitz_norm": 1089.65625,
|
||
|
|
"regularization/w1": 1.054787278175354,
|
||
|
|
"rewards/chosen": 0.4106831355049783,
|
||
|
|
"rewards/margins": 0.22542031963387246,
|
||
|
|
"rewards/rejected": 0.18526281587110585,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 27.553903579711914,
|
||
|
|
"kl": 10.561861038208008,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -34855298.35582822,
|
||
|
|
"logits/rejected": -36138877.55414013,
|
||
|
|
"logps/chosen": -467.36881710122697,
|
||
|
|
"logps/rejected": -373.711236066879,
|
||
|
|
"loss": 0.5905,
|
||
|
|
"loss/base_kto": 3.72161865234375,
|
||
|
|
"metrics/advantage_var": 237.09024047851562,
|
||
|
|
"regularization/lipschitz_norm": 1035.80078125,
|
||
|
|
"regularization/w1": 1.002655029296875,
|
||
|
|
"rewards/chosen": 0.1594490800167154,
|
||
|
|
"rewards/margins": 0.2713456920990328,
|
||
|
|
"rewards/rejected": -0.11189661208231738,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 23.264455795288086,
|
||
|
|
"kl": 8.589877128601074,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -34713675.85185185,
|
||
|
|
"logits/rejected": -36206457.96042618,
|
||
|
|
"logps/chosen": -443.14457528180355,
|
||
|
|
"logps/rejected": -350.6369387366819,
|
||
|
|
"loss": 0.6027,
|
||
|
|
"loss/base_kto": 3.5902230739593506,
|
||
|
|
"metrics/advantage_var": 664.9645385742188,
|
||
|
|
"regularization/lipschitz_norm": 1271.913330078125,
|
||
|
|
"regularization/w1": 1.2312120199203491,
|
||
|
|
"rewards/chosen": 0.15100901138379377,
|
||
|
|
"rewards/margins": 0.37334378169094273,
|
||
|
|
"rewards/rejected": -0.22233477030714896,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 16.499601364135742,
|
||
|
|
"kl": 9.61130428314209,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -36948227.69823435,
|
||
|
|
"logits/rejected": -36996618.13089802,
|
||
|
|
"logps/chosen": -479.370636035313,
|
||
|
|
"logps/rejected": -375.1408152587519,
|
||
|
|
"loss": 0.619,
|
||
|
|
"loss/base_kto": 3.298659086227417,
|
||
|
|
"metrics/advantage_var": 788.0341186523438,
|
||
|
|
"regularization/lipschitz_norm": 1708.1888427734375,
|
||
|
|
"regularization/w1": 1.6535266637802124,
|
||
|
|
"rewards/chosen": 0.3780508270998445,
|
||
|
|
"rewards/margins": 0.8341672000545629,
|
||
|
|
"rewards/rejected": -0.4561163729547184,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 15.769074440002441,
|
||
|
|
"kl": 7.957960605621338,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -35042104.35220126,
|
||
|
|
"logits/rejected": -36433633.78881988,
|
||
|
|
"logps/chosen": -482.4151926100629,
|
||
|
|
"logps/rejected": -374.84474475931677,
|
||
|
|
"loss": 0.6042,
|
||
|
|
"loss/base_kto": 3.308922529220581,
|
||
|
|
"metrics/advantage_var": 586.6065673828125,
|
||
|
|
"regularization/lipschitz_norm": 1575.0994873046875,
|
||
|
|
"regularization/w1": 1.5246961116790771,
|
||
|
|
"rewards/chosen": 0.30838679667538815,
|
||
|
|
"rewards/margins": 0.7849945517795953,
|
||
|
|
"rewards/rejected": -0.4766077551042071,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 22.673343658447266,
|
||
|
|
"kl": 10.246726036071777,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -35047955.66205837,
|
||
|
|
"logits/rejected": -36608283.268680446,
|
||
|
|
"logps/chosen": -483.68764400921657,
|
||
|
|
"logps/rejected": -370.5650586248013,
|
||
|
|
"loss": 0.6116,
|
||
|
|
"loss/base_kto": 3.275951385498047,
|
||
|
|
"metrics/advantage_var": 692.44091796875,
|
||
|
|
"regularization/lipschitz_norm": 1669.9676513671875,
|
||
|
|
"regularization/w1": 1.6165287494659424,
|
||
|
|
"rewards/chosen": 0.5250440559445805,
|
||
|
|
"rewards/margins": 0.8289676946568221,
|
||
|
|
"rewards/rejected": -0.30392363871224165,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 22.069089889526367,
|
||
|
|
"kl": 9.033541679382324,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -35870101.13395639,
|
||
|
|
"logits/rejected": -36145094.21943574,
|
||
|
|
"logps/chosen": -479.1555685358255,
|
||
|
|
"logps/rejected": -388.91322981974923,
|
||
|
|
"loss": 0.6235,
|
||
|
|
"loss/base_kto": 3.292295455932617,
|
||
|
|
"metrics/advantage_var": 665.8187866210938,
|
||
|
|
"regularization/lipschitz_norm": 1751.5699462890625,
|
||
|
|
"regularization/w1": 1.6955196857452393,
|
||
|
|
"rewards/chosen": 0.41297247105298385,
|
||
|
|
"rewards/margins": 0.8165806437178056,
|
||
|
|
"rewards/rejected": -0.4036081726648217,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 22.259113311767578,
|
||
|
|
"kl": 16.263540267944336,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -34954428.21183801,
|
||
|
|
"logits/rejected": -37720333.64263323,
|
||
|
|
"logps/chosen": -464.213152258567,
|
||
|
|
"logps/rejected": -376.9896404780564,
|
||
|
|
"loss": 0.616,
|
||
|
|
"loss/base_kto": 3.227264404296875,
|
||
|
|
"metrics/advantage_var": 705.037109375,
|
||
|
|
"regularization/lipschitz_norm": 1756.6126708984375,
|
||
|
|
"regularization/w1": 1.7004011869430542,
|
||
|
|
"rewards/chosen": 0.5953559281298676,
|
||
|
|
"rewards/margins": 0.8454651313130328,
|
||
|
|
"rewards/rejected": -0.25010920318316515,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 19.32278060913086,
|
||
|
|
"kl": 7.220941066741943,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -34986736.89561271,
|
||
|
|
"logits/rejected": -35957831.134087235,
|
||
|
|
"logps/chosen": -487.258793494705,
|
||
|
|
"logps/rejected": -369.0515195880452,
|
||
|
|
"loss": 0.6056,
|
||
|
|
"loss/base_kto": 3.3698794841766357,
|
||
|
|
"metrics/advantage_var": 557.4229736328125,
|
||
|
|
"regularization/lipschitz_norm": 1523.3997802734375,
|
||
|
|
"regularization/w1": 1.4746509790420532,
|
||
|
|
"rewards/chosen": 0.27471369802500945,
|
||
|
|
"rewards/margins": 0.6957687661319611,
|
||
|
|
"rewards/rejected": -0.42105506810695176,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 16.878324508666992,
|
||
|
|
"kl": 17.644540786743164,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -35317784.86191199,
|
||
|
|
"logits/rejected": -36752972.6763285,
|
||
|
|
"logps/chosen": -483.5268873292868,
|
||
|
|
"logps/rejected": -372.4219504830918,
|
||
|
|
"loss": 0.6082,
|
||
|
|
"loss/base_kto": 3.299276113510132,
|
||
|
|
"metrics/advantage_var": 595.0243530273438,
|
||
|
|
"regularization/lipschitz_norm": 1617.9088134765625,
|
||
|
|
"regularization/w1": 1.5661356449127197,
|
||
|
|
"rewards/chosen": 0.518430255432592,
|
||
|
|
"rewards/margins": 0.7378777614633766,
|
||
|
|
"rewards/rejected": -0.21944750603078453,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 20.07608413696289,
|
||
|
|
"kl": 13.129855155944824,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35702567.54471545,
|
||
|
|
"logits/rejected": -37748951.578947365,
|
||
|
|
"logps/chosen": -504.4263211382114,
|
||
|
|
"logps/rejected": -369.752772556391,
|
||
|
|
"loss": 0.5953,
|
||
|
|
"loss/base_kto": 3.360877275466919,
|
||
|
|
"metrics/advantage_var": 504.4501953125,
|
||
|
|
"regularization/lipschitz_norm": 1448.0484619140625,
|
||
|
|
"regularization/w1": 1.401710867881775,
|
||
|
|
"rewards/chosen": 0.43839076592670223,
|
||
|
|
"rewards/margins": 0.6803039748523366,
|
||
|
|
"rewards/rejected": -0.2419132089256344,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 16.529178619384766,
|
||
|
|
"kl": 12.449128150939941,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -35306447.387442574,
|
||
|
|
"logits/rejected": -35949135.208931416,
|
||
|
|
"logps/chosen": -466.2633996937213,
|
||
|
|
"logps/rejected": -362.49855462519935,
|
||
|
|
"loss": 0.6025,
|
||
|
|
"loss/base_kto": 3.272773027420044,
|
||
|
|
"metrics/advantage_var": 615.2431030273438,
|
||
|
|
"regularization/lipschitz_norm": 1598.7708740234375,
|
||
|
|
"regularization/w1": 1.5476102828979492,
|
||
|
|
"rewards/chosen": 0.5132591165773354,
|
||
|
|
"rewards/margins": 0.815122329101944,
|
||
|
|
"rewards/rejected": -0.30186321252460874,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 24.012788772583008,
|
||
|
|
"kl": 16.31725311279297,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -34741235.435582824,
|
||
|
|
"logits/rejected": -35833347.26114649,
|
||
|
|
"logps/chosen": -495.16478144171776,
|
||
|
|
"logps/rejected": -385.30269705414014,
|
||
|
|
"loss": 0.5986,
|
||
|
|
"loss/base_kto": 3.215549945831299,
|
||
|
|
"metrics/advantage_var": 641.9974975585938,
|
||
|
|
"regularization/lipschitz_norm": 1625.2222900390625,
|
||
|
|
"regularization/w1": 1.573215126991272,
|
||
|
|
"rewards/chosen": 0.5474949936193922,
|
||
|
|
"rewards/margins": 0.8771340396748883,
|
||
|
|
"rewards/rejected": -0.3296390460554961,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 20.914777755737305,
|
||
|
|
"kl": 21.8082218170166,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -34782408.94191523,
|
||
|
|
"logits/rejected": -36444150.44479004,
|
||
|
|
"logps/chosen": -490.38412480376763,
|
||
|
|
"logps/rejected": -391.69717146189737,
|
||
|
|
"loss": 0.6032,
|
||
|
|
"loss/base_kto": 3.2490127086639404,
|
||
|
|
"metrics/advantage_var": 631.494384765625,
|
||
|
|
"regularization/lipschitz_norm": 1628.42919921875,
|
||
|
|
"regularization/w1": 1.5763194561004639,
|
||
|
|
"rewards/chosen": 0.5423311865311028,
|
||
|
|
"rewards/margins": 0.7854773798737714,
|
||
|
|
"rewards/rejected": -0.24314619334266865,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 20.474929809570312,
|
||
|
|
"kl": 12.542684555053711,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -35852532.19713832,
|
||
|
|
"logits/rejected": -36600629.08755761,
|
||
|
|
"logps/chosen": -502.88384340222575,
|
||
|
|
"logps/rejected": -370.43073156682027,
|
||
|
|
"loss": 0.6004,
|
||
|
|
"loss/base_kto": 3.3460891246795654,
|
||
|
|
"metrics/advantage_var": 558.3849487304688,
|
||
|
|
"regularization/lipschitz_norm": 1505.5638427734375,
|
||
|
|
"regularization/w1": 1.4573858976364136,
|
||
|
|
"rewards/chosen": 0.46179650432544217,
|
||
|
|
"rewards/margins": 0.7181949528886502,
|
||
|
|
"rewards/rejected": -0.25639844856320804,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 18.697038650512695,
|
||
|
|
"kl": 13.531590461730957,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -34661934.83911672,
|
||
|
|
"logits/rejected": -37135775.306501545,
|
||
|
|
"logps/chosen": -474.7382689274448,
|
||
|
|
"logps/rejected": -384.8535216718266,
|
||
|
|
"loss": 0.6086,
|
||
|
|
"loss/base_kto": 3.375030517578125,
|
||
|
|
"metrics/advantage_var": 682.4904174804688,
|
||
|
|
"regularization/lipschitz_norm": 1543.2581787109375,
|
||
|
|
"regularization/w1": 1.493873953819275,
|
||
|
|
"rewards/chosen": 0.37385640911499407,
|
||
|
|
"rewards/margins": 0.6713754906322638,
|
||
|
|
"rewards/rejected": -0.29751908151726975,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 19.288637161254883,
|
||
|
|
"kl": 9.594658851623535,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -34427238.72611465,
|
||
|
|
"logits/rejected": -36342859.38650307,
|
||
|
|
"logps/chosen": -474.343252388535,
|
||
|
|
"logps/rejected": -365.4795580904908,
|
||
|
|
"loss": 0.5989,
|
||
|
|
"loss/base_kto": 3.3052353858947754,
|
||
|
|
"metrics/advantage_var": 535.04150390625,
|
||
|
|
"regularization/lipschitz_norm": 1535.0205078125,
|
||
|
|
"regularization/w1": 1.4858999252319336,
|
||
|
|
"rewards/chosen": 0.31807237369999003,
|
||
|
|
"rewards/margins": 0.7722628677926856,
|
||
|
|
"rewards/rejected": -0.4541904940926956,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 20.058246612548828,
|
||
|
|
"kl": 14.58768367767334,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34454455.84407796,
|
||
|
|
"logits/rejected": -36876521.86623165,
|
||
|
|
"logps/chosen": -458.3414074212894,
|
||
|
|
"logps/rejected": -374.77928731647637,
|
||
|
|
"loss": 0.599,
|
||
|
|
"loss/base_kto": 3.2784602642059326,
|
||
|
|
"metrics/advantage_var": 587.8887939453125,
|
||
|
|
"regularization/lipschitz_norm": 1563.96435546875,
|
||
|
|
"regularization/w1": 1.5139175653457642,
|
||
|
|
"rewards/chosen": 0.5640859074857103,
|
||
|
|
"rewards/margins": 0.7959707378298808,
|
||
|
|
"rewards/rejected": -0.23188483034417057,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 21.521879196166992,
|
||
|
|
"kl": 32.227752685546875,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34971616.34621329,
|
||
|
|
"logits/rejected": -36296021.333333336,
|
||
|
|
"logps/chosen": -499.96232612055644,
|
||
|
|
"logps/rejected": -392.9808698657188,
|
||
|
|
"loss": 0.6138,
|
||
|
|
"loss/base_kto": 3.271024703979492,
|
||
|
|
"metrics/advantage_var": 653.11767578125,
|
||
|
|
"regularization/lipschitz_norm": 1693.3758544921875,
|
||
|
|
"regularization/w1": 1.6391880512237549,
|
||
|
|
"rewards/chosen": 0.7821665650357419,
|
||
|
|
"rewards/margins": 0.7898490775691618,
|
||
|
|
"rewards/rejected": -0.007682512533419882,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 24.230466842651367,
|
||
|
|
"kl": 14.0050048828125,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -35453792.0,
|
||
|
|
"logits/rejected": -34380684.8,
|
||
|
|
"logps/chosen": -508.399609375,
|
||
|
|
"logps/rejected": -382.64130859375,
|
||
|
|
"loss": 0.6045,
|
||
|
|
"loss/base_kto": 3.273062229156494,
|
||
|
|
"metrics/advantage_var": 611.5560913085938,
|
||
|
|
"regularization/lipschitz_norm": 1614.3800048828125,
|
||
|
|
"regularization/w1": 1.5627198219299316,
|
||
|
|
"rewards/chosen": 0.5066928863525391,
|
||
|
|
"rewards/margins": 0.7878694534301758,
|
||
|
|
"rewards/rejected": -0.2811765670776367,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 21.83265495300293,
|
||
|
|
"kl": 17.23431968688965,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -35426775.13615023,
|
||
|
|
"logits/rejected": -36576267.1825273,
|
||
|
|
"logps/chosen": -466.0667546948357,
|
||
|
|
"logps/rejected": -374.26372367394697,
|
||
|
|
"loss": 0.6095,
|
||
|
|
"loss/base_kto": 3.3570492267608643,
|
||
|
|
"metrics/advantage_var": 571.9691772460938,
|
||
|
|
"regularization/lipschitz_norm": 1569.212158203125,
|
||
|
|
"regularization/w1": 1.5189971923828125,
|
||
|
|
"rewards/chosen": 0.49200439453125,
|
||
|
|
"rewards/margins": 0.6891589945824396,
|
||
|
|
"rewards/rejected": -0.19715460005118954,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 18.959505081176758,
|
||
|
|
"kl": 11.298334121704102,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -35625665.98757764,
|
||
|
|
"logits/rejected": -36577070.6918239,
|
||
|
|
"logps/chosen": -463.794448757764,
|
||
|
|
"logps/rejected": -383.4762185534591,
|
||
|
|
"loss": 0.6036,
|
||
|
|
"loss/base_kto": 3.3050782680511475,
|
||
|
|
"metrics/advantage_var": 597.6561889648438,
|
||
|
|
"regularization/lipschitz_norm": 1574.0662841796875,
|
||
|
|
"regularization/w1": 1.5236961841583252,
|
||
|
|
"rewards/chosen": 0.44647396869540956,
|
||
|
|
"rewards/margins": 0.7729159674470678,
|
||
|
|
"rewards/rejected": -0.3264419987516583,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 21.067062377929688,
|
||
|
|
"kl": 14.25390625,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -35856666.040688574,
|
||
|
|
"logits/rejected": -37136176.324492976,
|
||
|
|
"logps/chosen": -491.2589495305164,
|
||
|
|
"logps/rejected": -375.8061134945398,
|
||
|
|
"loss": 0.6123,
|
||
|
|
"loss/base_kto": 3.3427810668945312,
|
||
|
|
"metrics/advantage_var": 645.2509765625,
|
||
|
|
"regularization/lipschitz_norm": 1606.885986328125,
|
||
|
|
"regularization/w1": 1.555465579032898,
|
||
|
|
"rewards/chosen": 0.4693153632078932,
|
||
|
|
"rewards/margins": 0.693963658832101,
|
||
|
|
"rewards/rejected": -0.2246482956242078,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 20.4710750579834,
|
||
|
|
"kl": 11.749445915222168,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -36544750.72392638,
|
||
|
|
"logits/rejected": -37266813.55414013,
|
||
|
|
"logps/chosen": -488.17628450920245,
|
||
|
|
"logps/rejected": -390.2607484076433,
|
||
|
|
"loss": 0.6129,
|
||
|
|
"loss/base_kto": 3.323289155960083,
|
||
|
|
"metrics/advantage_var": 666.1123046875,
|
||
|
|
"regularization/lipschitz_norm": 1631.8944091796875,
|
||
|
|
"regularization/w1": 1.5796737670898438,
|
||
|
|
"rewards/chosen": 0.40534388068263516,
|
||
|
|
"rewards/margins": 0.7730458059724938,
|
||
|
|
"rewards/rejected": -0.3677019252898587,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 16.940189361572266,
|
||
|
|
"kl": 19.096708297729492,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -35006235.386046514,
|
||
|
|
"logits/rejected": -36909391.42047244,
|
||
|
|
"logps/chosen": -477.328003875969,
|
||
|
|
"logps/rejected": -369.3166338582677,
|
||
|
|
"loss": 0.5981,
|
||
|
|
"loss/base_kto": 3.3004188537597656,
|
||
|
|
"metrics/advantage_var": 536.4371948242188,
|
||
|
|
"regularization/lipschitz_norm": 1533.2161865234375,
|
||
|
|
"regularization/w1": 1.484153151512146,
|
||
|
|
"rewards/chosen": 0.5849333060804264,
|
||
|
|
"rewards/margins": 0.7282379296617194,
|
||
|
|
"rewards/rejected": -0.14330462358129306,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 16.300283432006836,
|
||
|
|
"kl": 17.500856399536133,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -35477111.69834088,
|
||
|
|
"logits/rejected": -36938391.85737439,
|
||
|
|
"logps/chosen": -489.6779788838612,
|
||
|
|
"logps/rejected": -361.63416734197733,
|
||
|
|
"loss": 0.6121,
|
||
|
|
"loss/base_kto": 3.352329969406128,
|
||
|
|
"metrics/advantage_var": 603.9823608398438,
|
||
|
|
"regularization/lipschitz_norm": 1595.181884765625,
|
||
|
|
"regularization/w1": 1.5441360473632812,
|
||
|
|
"rewards/chosen": 0.5508544829815941,
|
||
|
|
"rewards/margins": 0.7190373011795277,
|
||
|
|
"rewards/rejected": -0.16818281819793354,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 18.363264083862305,
|
||
|
|
"kl": 29.446781158447266,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -34925766.3496063,
|
||
|
|
"logits/rejected": -35609017.35193799,
|
||
|
|
"logps/chosen": -464.2686023622047,
|
||
|
|
"logps/rejected": -376.27398255813955,
|
||
|
|
"loss": 0.5887,
|
||
|
|
"loss/base_kto": 3.2562196254730225,
|
||
|
|
"metrics/advantage_var": 536.0402221679688,
|
||
|
|
"regularization/lipschitz_norm": 1501.6697998046875,
|
||
|
|
"regularization/w1": 1.4536161422729492,
|
||
|
|
"rewards/chosen": 0.6845081239234744,
|
||
|
|
"rewards/margins": 0.7690583811700981,
|
||
|
|
"rewards/rejected": -0.08455025724662367,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 25.284282684326172,
|
||
|
|
"kl": 19.516387939453125,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -35681520.27716535,
|
||
|
|
"logits/rejected": -37468080.62015504,
|
||
|
|
"logps/chosen": -463.6035433070866,
|
||
|
|
"logps/rejected": -383.9078972868217,
|
||
|
|
"loss": 0.6071,
|
||
|
|
"loss/base_kto": 3.3693978786468506,
|
||
|
|
"metrics/advantage_var": 537.1165161132812,
|
||
|
|
"regularization/lipschitz_norm": 1536.220947265625,
|
||
|
|
"regularization/w1": 1.4870620965957642,
|
||
|
|
"rewards/chosen": 0.5293718473179134,
|
||
|
|
"rewards/margins": 0.6587132212050864,
|
||
|
|
"rewards/rejected": -0.12934137388717296,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 18.165491104125977,
|
||
|
|
"kl": 15.14261531829834,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -34315581.90327613,
|
||
|
|
"logits/rejected": -37249107.330203444,
|
||
|
|
"logps/chosen": -486.13377535101404,
|
||
|
|
"logps/rejected": -377.13715277777777,
|
||
|
|
"loss": 0.5993,
|
||
|
|
"loss/base_kto": 3.3054962158203125,
|
||
|
|
"metrics/advantage_var": 600.05224609375,
|
||
|
|
"regularization/lipschitz_norm": 1537.8310546875,
|
||
|
|
"regularization/w1": 1.488620400428772,
|
||
|
|
"rewards/chosen": 0.5017440248391186,
|
||
|
|
"rewards/margins": 0.7489487531820495,
|
||
|
|
"rewards/rejected": -0.24720472834293086,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 18.07354164123535,
|
||
|
|
"kl": 17.446069717407227,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -34741082.838709675,
|
||
|
|
"logits/rejected": -37393233.80604134,
|
||
|
|
"logps/chosen": -499.9599174347158,
|
||
|
|
"logps/rejected": -350.125,
|
||
|
|
"loss": 0.6022,
|
||
|
|
"loss/base_kto": 3.2626919746398926,
|
||
|
|
"metrics/advantage_var": 661.6778564453125,
|
||
|
|
"regularization/lipschitz_norm": 1606.317138671875,
|
||
|
|
"regularization/w1": 1.554914951324463,
|
||
|
|
"rewards/chosen": 0.5640367546022945,
|
||
|
|
"rewards/margins": 0.8372905669221972,
|
||
|
|
"rewards/rejected": -0.27325381231990264,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 18.38632583618164,
|
||
|
|
"kl": 15.903594970703125,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -35562097.5131783,
|
||
|
|
"logits/rejected": -37191428.43464567,
|
||
|
|
"logps/chosen": -496.19951550387594,
|
||
|
|
"logps/rejected": -394.4829724409449,
|
||
|
|
"loss": 0.6118,
|
||
|
|
"loss/base_kto": 3.290264844894409,
|
||
|
|
"metrics/advantage_var": 667.70263671875,
|
||
|
|
"regularization/lipschitz_norm": 1656.7857666015625,
|
||
|
|
"regularization/w1": 1.6037687063217163,
|
||
|
|
"rewards/chosen": 0.5458641821099807,
|
||
|
|
"rewards/margins": 0.7589617259015038,
|
||
|
|
"rewards/rejected": -0.21309754379152313,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 18.004058837890625,
|
||
|
|
"kl": 10.730722427368164,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -34024478.44757433,
|
||
|
|
"logits/rejected": -35676543.40093604,
|
||
|
|
"logps/chosen": -482.30604460093895,
|
||
|
|
"logps/rejected": -364.5335413416537,
|
||
|
|
"loss": 0.5917,
|
||
|
|
"loss/base_kto": 3.3022239208221436,
|
||
|
|
"metrics/advantage_var": 518.4223022460938,
|
||
|
|
"regularization/lipschitz_norm": 1478.7027587890625,
|
||
|
|
"regularization/w1": 1.4313843250274658,
|
||
|
|
"rewards/chosen": 0.42893554496466646,
|
||
|
|
"rewards/margins": 0.7466070366143716,
|
||
|
|
"rewards/rejected": -0.31767149164970504,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 16.194839477539062,
|
||
|
|
"kl": 13.344321250915527,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -35144017.68549618,
|
||
|
|
"logits/rejected": -37702552.7808,
|
||
|
|
"logps/chosen": -504.9664599236641,
|
||
|
|
"logps/rejected": -385.011,
|
||
|
|
"loss": 0.6025,
|
||
|
|
"loss/base_kto": 3.37534761428833,
|
||
|
|
"metrics/advantage_var": 533.4691162109375,
|
||
|
|
"regularization/lipschitz_norm": 1492.0777587890625,
|
||
|
|
"regularization/w1": 1.444331169128418,
|
||
|
|
"rewards/chosen": 0.4436519710162214,
|
||
|
|
"rewards/margins": 0.6707798518755964,
|
||
|
|
"rewards/rejected": -0.227127880859375,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 17.043498992919922,
|
||
|
|
"kl": 19.01972770690918,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -36703572.24801272,
|
||
|
|
"logits/rejected": -37849591.34869432,
|
||
|
|
"logps/chosen": -471.9105226550079,
|
||
|
|
"logps/rejected": -382.0459389400922,
|
||
|
|
"loss": 0.6176,
|
||
|
|
"loss/base_kto": 3.325809955596924,
|
||
|
|
"metrics/advantage_var": 736.2930297851562,
|
||
|
|
"regularization/lipschitz_norm": 1668.2176513671875,
|
||
|
|
"regularization/w1": 1.6148347854614258,
|
||
|
|
"rewards/chosen": 0.5692052310525139,
|
||
|
|
"rewards/margins": 0.7573018004829575,
|
||
|
|
"rewards/rejected": -0.18809656943044356,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 17.405284881591797,
|
||
|
|
"kl": 14.538073539733887,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -36876243.12989046,
|
||
|
|
"logits/rejected": -37138048.59906396,
|
||
|
|
"logps/chosen": -478.1436815336463,
|
||
|
|
"logps/rejected": -375.7367638455538,
|
||
|
|
"loss": 0.5932,
|
||
|
|
"loss/base_kto": 3.2952370643615723,
|
||
|
|
"metrics/advantage_var": 489.7381896972656,
|
||
|
|
"regularization/lipschitz_norm": 1498.0330810546875,
|
||
|
|
"regularization/w1": 1.4500960111618042,
|
||
|
|
"rewards/chosen": 0.45022635541984546,
|
||
|
|
"rewards/margins": 0.7333660171879568,
|
||
|
|
"rewards/rejected": -0.28313966176811134,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 18.89828109741211,
|
||
|
|
"kl": 12.498826026916504,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -35515587.2,
|
||
|
|
"logits/rejected": -36558387.2,
|
||
|
|
"logps/chosen": -515.36123046875,
|
||
|
|
"logps/rejected": -359.762841796875,
|
||
|
|
"loss": 0.6076,
|
||
|
|
"loss/base_kto": 3.353375196456909,
|
||
|
|
"metrics/advantage_var": 578.0367431640625,
|
||
|
|
"regularization/lipschitz_norm": 1557.6429443359375,
|
||
|
|
"regularization/w1": 1.5077983140945435,
|
||
|
|
"rewards/chosen": 0.40036706924438475,
|
||
|
|
"rewards/margins": 0.7235270500183105,
|
||
|
|
"rewards/rejected": -0.32315998077392577,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 16.251623153686523,
|
||
|
|
"kl": 11.878005981445312,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34584073.157723576,
|
||
|
|
"logits/rejected": -35917657.696240604,
|
||
|
|
"logps/chosen": -446.88099593495934,
|
||
|
|
"logps/rejected": -391.1749530075188,
|
||
|
|
"loss": 0.5884,
|
||
|
|
"loss/base_kto": 3.2654101848602295,
|
||
|
|
"metrics/advantage_var": 531.6594848632812,
|
||
|
|
"regularization/lipschitz_norm": 1489.2799072265625,
|
||
|
|
"regularization/w1": 1.4416228532791138,
|
||
|
|
"rewards/chosen": 0.35830549534743394,
|
||
|
|
"rewards/margins": 0.7987000302687215,
|
||
|
|
"rewards/rejected": -0.4403945349212876,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 15.030388832092285,
|
||
|
|
"kl": 9.553278923034668,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -35934383.83838384,
|
||
|
|
"logits/rejected": -36504301.341107875,
|
||
|
|
"logps/chosen": -475.02251683501686,
|
||
|
|
"logps/rejected": -368.0303389212828,
|
||
|
|
"loss": 0.6037,
|
||
|
|
"loss/base_kto": 3.316877841949463,
|
||
|
|
"metrics/advantage_var": 537.7207641601562,
|
||
|
|
"regularization/lipschitz_norm": 1562.980712890625,
|
||
|
|
"regularization/w1": 1.5129653215408325,
|
||
|
|
"rewards/chosen": 0.2434273665200179,
|
||
|
|
"rewards/margins": 0.7115952192498375,
|
||
|
|
"rewards/rejected": -0.4681678527298196,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 19.4332218170166,
|
||
|
|
"kl": 6.0943708419799805,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -35515151.15022762,
|
||
|
|
"logits/rejected": -35172688.38647343,
|
||
|
|
"logps/chosen": -454.55187784522,
|
||
|
|
"logps/rejected": -363.44930052334945,
|
||
|
|
"loss": 0.6159,
|
||
|
|
"loss/base_kto": 3.403013229370117,
|
||
|
|
"metrics/advantage_var": 592.5694580078125,
|
||
|
|
"regularization/lipschitz_norm": 1574.67529296875,
|
||
|
|
"regularization/w1": 1.5242856740951538,
|
||
|
|
"rewards/chosen": 0.20428188943356412,
|
||
|
|
"rewards/margins": 0.6780351013963761,
|
||
|
|
"rewards/rejected": -0.473753211962812,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 22.03345489501953,
|
||
|
|
"kl": 10.232871055603027,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -35184254.00623053,
|
||
|
|
"logits/rejected": -36078678.670846395,
|
||
|
|
"logps/chosen": -465.66968457943926,
|
||
|
|
"logps/rejected": -364.7988832288401,
|
||
|
|
"loss": 0.5986,
|
||
|
|
"loss/base_kto": 3.317852020263672,
|
||
|
|
"metrics/advantage_var": 528.4635620117188,
|
||
|
|
"regularization/lipschitz_norm": 1519.39501953125,
|
||
|
|
"regularization/w1": 1.4707742929458618,
|
||
|
|
"rewards/chosen": 0.38940341747438423,
|
||
|
|
"rewards/margins": 0.7393428176002046,
|
||
|
|
"rewards/rejected": -0.3499394001258204,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 21.75213050842285,
|
||
|
|
"kl": 10.051443099975586,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -35845955.057660624,
|
||
|
|
"logits/rejected": -37086398.19316493,
|
||
|
|
"logps/chosen": -502.7420716639209,
|
||
|
|
"logps/rejected": -375.42143387815753,
|
||
|
|
"loss": 0.594,
|
||
|
|
"loss/base_kto": 3.3372910022735596,
|
||
|
|
"metrics/advantage_var": 543.7970581054688,
|
||
|
|
"regularization/lipschitz_norm": 1461.751953125,
|
||
|
|
"regularization/w1": 1.4149760007858276,
|
||
|
|
"rewards/chosen": 0.29541211701696357,
|
||
|
|
"rewards/margins": 0.6923868138182498,
|
||
|
|
"rewards/rejected": -0.3969746968012862,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 18.577890396118164,
|
||
|
|
"kl": 6.203124523162842,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34675876.102564104,
|
||
|
|
"logits/rejected": -36907634.29969419,
|
||
|
|
"logps/chosen": -494.60902443910254,
|
||
|
|
"logps/rejected": -376.49820814220186,
|
||
|
|
"loss": 0.579,
|
||
|
|
"loss/base_kto": 3.252350330352783,
|
||
|
|
"metrics/advantage_var": 481.51007080078125,
|
||
|
|
"regularization/lipschitz_norm": 1425.4237060546875,
|
||
|
|
"regularization/w1": 1.3798102140426636,
|
||
|
|
"rewards/chosen": 0.3323974609375,
|
||
|
|
"rewards/margins": 0.8132524869493023,
|
||
|
|
"rewards/rejected": -0.48085502601180236,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 18.51157569885254,
|
||
|
|
"kl": 6.404454708099365,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34582219.04821151,
|
||
|
|
"logits/rejected": -36116576.452119306,
|
||
|
|
"logps/chosen": -474.65814541213064,
|
||
|
|
"logps/rejected": -364.9083594976452,
|
||
|
|
"loss": 0.5561,
|
||
|
|
"loss/base_kto": 3.221468687057495,
|
||
|
|
"metrics/advantage_var": 404.1481628417969,
|
||
|
|
"regularization/lipschitz_norm": 1267.6856689453125,
|
||
|
|
"regularization/w1": 1.2271196842193604,
|
||
|
|
"rewards/chosen": 0.34772355004009525,
|
||
|
|
"rewards/margins": 0.8442625596493476,
|
||
|
|
"rewards/rejected": -0.49653900960925235,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 17.475242614746094,
|
||
|
|
"kl": 9.11551570892334,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -36855277.53442623,
|
||
|
|
"logits/rejected": -37261107.2,
|
||
|
|
"logps/chosen": -488.87254098360654,
|
||
|
|
"logps/rejected": -377.4211287313433,
|
||
|
|
"loss": 0.5633,
|
||
|
|
"loss/base_kto": 3.271669864654541,
|
||
|
|
"metrics/advantage_var": 416.1767272949219,
|
||
|
|
"regularization/lipschitz_norm": 1275.496337890625,
|
||
|
|
"regularization/w1": 1.234680414199829,
|
||
|
|
"rewards/chosen": 0.3925642670178022,
|
||
|
|
"rewards/margins": 0.7673343375635111,
|
||
|
|
"rewards/rejected": -0.37477007054570893,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 18.36026954650879,
|
||
|
|
"kl": 8.405257225036621,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -35935978.732307695,
|
||
|
|
"logits/rejected": -36367967.8984127,
|
||
|
|
"logps/chosen": -466.0245192307692,
|
||
|
|
"logps/rejected": -376.2544890873016,
|
||
|
|
"loss": 0.5641,
|
||
|
|
"loss/base_kto": 3.1944825649261475,
|
||
|
|
"metrics/advantage_var": 483.5879821777344,
|
||
|
|
"regularization/lipschitz_norm": 1362.2386474609375,
|
||
|
|
"regularization/w1": 1.318647027015686,
|
||
|
|
"rewards/chosen": 0.43166912372295674,
|
||
|
|
"rewards/margins": 0.8600282371437157,
|
||
|
|
"rewards/rejected": -0.42835911342075894,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 15.171882629394531,
|
||
|
|
"kl": 8.67348861694336,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -33117539.07120743,
|
||
|
|
"logits/rejected": -34332077.627760254,
|
||
|
|
"logps/chosen": -481.45133513931887,
|
||
|
|
"logps/rejected": -358.57314668769715,
|
||
|
|
"loss": 0.5556,
|
||
|
|
"loss/base_kto": 3.215223789215088,
|
||
|
|
"metrics/advantage_var": 417.1731872558594,
|
||
|
|
"regularization/lipschitz_norm": 1270.332275390625,
|
||
|
|
"regularization/w1": 1.2296816110610962,
|
||
|
|
"rewards/chosen": 0.44595573118227555,
|
||
|
|
"rewards/margins": 0.840025559251824,
|
||
|
|
"rewards/rejected": -0.3940698280695485,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 20.370195388793945,
|
||
|
|
"kl": 14.783319473266602,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -35879426.426540285,
|
||
|
|
"logits/rejected": -37747735.74034003,
|
||
|
|
"logps/chosen": -473.7768562401264,
|
||
|
|
"logps/rejected": -362.18407071097374,
|
||
|
|
"loss": 0.5624,
|
||
|
|
"loss/base_kto": 3.1979527473449707,
|
||
|
|
"metrics/advantage_var": 464.1930847167969,
|
||
|
|
"regularization/lipschitz_norm": 1344.4075927734375,
|
||
|
|
"regularization/w1": 1.3013864755630493,
|
||
|
|
"rewards/chosen": 0.4723948498284459,
|
||
|
|
"rewards/margins": 0.856767755577335,
|
||
|
|
"rewards/rejected": -0.3843729057488891,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 16.537639617919922,
|
||
|
|
"kl": 12.888556480407715,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -34248097.64350453,
|
||
|
|
"logits/rejected": -35691367.55987055,
|
||
|
|
"logps/chosen": -448.9484516616314,
|
||
|
|
"logps/rejected": -382.2485335760518,
|
||
|
|
"loss": 0.5549,
|
||
|
|
"loss/base_kto": 3.1708996295928955,
|
||
|
|
"metrics/advantage_var": 426.1979675292969,
|
||
|
|
"regularization/lipschitz_norm": 1310.096923828125,
|
||
|
|
"regularization/w1": 1.2681736946105957,
|
||
|
|
"rewards/chosen": 0.5071284821386424,
|
||
|
|
"rewards/margins": 0.8765312718995899,
|
||
|
|
"rewards/rejected": -0.3694027897609476,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 18.989715576171875,
|
||
|
|
"kl": 17.40022087097168,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -35120997.84946237,
|
||
|
|
"logits/rejected": -36973699.86645469,
|
||
|
|
"logps/chosen": -482.90264976958525,
|
||
|
|
"logps/rejected": -384.970786963434,
|
||
|
|
"loss": 0.5473,
|
||
|
|
"loss/base_kto": 3.1376843452453613,
|
||
|
|
"metrics/advantage_var": 430.1627502441406,
|
||
|
|
"regularization/lipschitz_norm": 1281.6112060546875,
|
||
|
|
"regularization/w1": 1.2405997514724731,
|
||
|
|
"rewards/chosen": 0.6347455612159179,
|
||
|
|
"rewards/margins": 0.8940889374362329,
|
||
|
|
"rewards/rejected": -0.259343376220315,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 19.608339309692383,
|
||
|
|
"kl": 18.455400466918945,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -34972480.3072,
|
||
|
|
"logits/rejected": -37169592.86717557,
|
||
|
|
"logps/chosen": -469.4927,
|
||
|
|
"logps/rejected": -378.4319179389313,
|
||
|
|
"loss": 0.5594,
|
||
|
|
"loss/base_kto": 3.1552579402923584,
|
||
|
|
"metrics/advantage_var": 453.8045959472656,
|
||
|
|
"regularization/lipschitz_norm": 1363.4388427734375,
|
||
|
|
"regularization/w1": 1.319808840751648,
|
||
|
|
"rewards/chosen": 0.58350087890625,
|
||
|
|
"rewards/margins": 0.8539570677779103,
|
||
|
|
"rewards/rejected": -0.2704561888716603,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 23.085952758789062,
|
||
|
|
"kl": 16.185239791870117,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -35424293.867488444,
|
||
|
|
"logits/rejected": -36946043.33438986,
|
||
|
|
"logps/chosen": -508.084312403698,
|
||
|
|
"logps/rejected": -369.8927297939778,
|
||
|
|
"loss": 0.5555,
|
||
|
|
"loss/base_kto": 3.178971290588379,
|
||
|
|
"metrics/advantage_var": 425.2232971191406,
|
||
|
|
"regularization/lipschitz_norm": 1306.8421630859375,
|
||
|
|
"regularization/w1": 1.2650232315063477,
|
||
|
|
"rewards/chosen": 0.5859161518020513,
|
||
|
|
"rewards/margins": 0.8434531955956948,
|
||
|
|
"rewards/rejected": -0.2575370437936435,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 15.182164192199707,
|
||
|
|
"kl": 16.984102249145508,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -35689651.15890851,
|
||
|
|
"logits/rejected": -36672535.37899543,
|
||
|
|
"logps/chosen": -448.7127307383628,
|
||
|
|
"logps/rejected": -389.4425656392694,
|
||
|
|
"loss": 0.5554,
|
||
|
|
"loss/base_kto": 3.1280744075775146,
|
||
|
|
"metrics/advantage_var": 495.2984313964844,
|
||
|
|
"regularization/lipschitz_norm": 1358.5955810546875,
|
||
|
|
"regularization/w1": 1.3151204586029053,
|
||
|
|
"rewards/chosen": 0.5651231401422051,
|
||
|
|
"rewards/margins": 0.8971262182835555,
|
||
|
|
"rewards/rejected": -0.3320030781413504,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 21.04039192199707,
|
||
|
|
"kl": 11.082907676696777,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35638392.565149136,
|
||
|
|
"logits/rejected": -35987035.57076205,
|
||
|
|
"logps/chosen": -478.27158555729983,
|
||
|
|
"logps/rejected": -380.9886518273717,
|
||
|
|
"loss": 0.5676,
|
||
|
|
"loss/base_kto": 3.171964645385742,
|
||
|
|
"metrics/advantage_var": 796.0897827148438,
|
||
|
|
"regularization/lipschitz_norm": 1414.4652099609375,
|
||
|
|
"regularization/w1": 1.3692022562026978,
|
||
|
|
"rewards/chosen": 0.4200789183612147,
|
||
|
|
"rewards/margins": 0.8561153086034872,
|
||
|
|
"rewards/rejected": -0.43603639024227253,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 18.560272216796875,
|
||
|
|
"kl": 11.838644027709961,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -35915638.24215247,
|
||
|
|
"logits/rejected": -37247797.21112929,
|
||
|
|
"logps/chosen": -480.7824177877429,
|
||
|
|
"logps/rejected": -383.7034574468085,
|
||
|
|
"loss": 0.5652,
|
||
|
|
"loss/base_kto": 3.1558837890625,
|
||
|
|
"metrics/advantage_var": 484.69781494140625,
|
||
|
|
"regularization/lipschitz_norm": 1410.4854736328125,
|
||
|
|
"regularization/w1": 1.3653497695922852,
|
||
|
|
"rewards/chosen": 0.5410139827984866,
|
||
|
|
"rewards/margins": 0.8891102653079486,
|
||
|
|
"rewards/rejected": -0.34809628250946195,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 20.190584182739258,
|
||
|
|
"kl": 16.06940460205078,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -34849311.079872206,
|
||
|
|
"logits/rejected": -36955771.6941896,
|
||
|
|
"logps/chosen": -477.25279552715654,
|
||
|
|
"logps/rejected": -343.90586773700306,
|
||
|
|
"loss": 0.5584,
|
||
|
|
"loss/base_kto": 3.134000301361084,
|
||
|
|
"metrics/advantage_var": 466.61962890625,
|
||
|
|
"regularization/lipschitz_norm": 1377.008056640625,
|
||
|
|
"regularization/w1": 1.3329437971115112,
|
||
|
|
"rewards/chosen": 0.6188795193315695,
|
||
|
|
"rewards/margins": 0.9074625573816698,
|
||
|
|
"rewards/rejected": -0.2885830380501003,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 18.073322296142578,
|
||
|
|
"kl": 17.676794052124023,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -35403344.15335463,
|
||
|
|
"logits/rejected": -36786003.7675841,
|
||
|
|
"logps/chosen": -502.3608226837061,
|
||
|
|
"logps/rejected": -349.6362528669725,
|
||
|
|
"loss": 0.5579,
|
||
|
|
"loss/base_kto": 3.1910133361816406,
|
||
|
|
"metrics/advantage_var": 391.9412841796875,
|
||
|
|
"regularization/lipschitz_norm": 1314.455078125,
|
||
|
|
"regularization/w1": 1.2723925113677979,
|
||
|
|
"rewards/chosen": 0.5328241415298023,
|
||
|
|
"rewards/margins": 0.8114488365406729,
|
||
|
|
"rewards/rejected": -0.2786246950108706,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 16.1851749420166,
|
||
|
|
"kl": 15.725171089172363,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -34462659.00940439,
|
||
|
|
"logits/rejected": -36155548.31152648,
|
||
|
|
"logps/chosen": -484.4577292319749,
|
||
|
|
"logps/rejected": -371.28185845015577,
|
||
|
|
"loss": 0.5524,
|
||
|
|
"loss/base_kto": 3.1397719383239746,
|
||
|
|
"metrics/advantage_var": 444.7248229980469,
|
||
|
|
"regularization/lipschitz_norm": 1321.617919921875,
|
||
|
|
"regularization/w1": 1.27932608127594,
|
||
|
|
"rewards/chosen": 0.5605026292950382,
|
||
|
|
"rewards/margins": 0.8856563485226716,
|
||
|
|
"rewards/rejected": -0.32515371922763336,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 17.90858268737793,
|
||
|
|
"kl": 12.214306831359863,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -35744870.23771791,
|
||
|
|
"logits/rejected": -37153225.565485366,
|
||
|
|
"logps/chosen": -488.4059033280507,
|
||
|
|
"logps/rejected": -403.81151290446843,
|
||
|
|
"loss": 0.5675,
|
||
|
|
"loss/base_kto": 3.2203705310821533,
|
||
|
|
"metrics/advantage_var": 443.23126220703125,
|
||
|
|
"regularization/lipschitz_norm": 1363.0374755859375,
|
||
|
|
"regularization/w1": 1.3194202184677124,
|
||
|
|
"rewards/chosen": 0.4475267896939382,
|
||
|
|
"rewards/margins": 0.8189232703071412,
|
||
|
|
"rewards/rejected": -0.371396480613203,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 21.16765594482422,
|
||
|
|
"kl": 12.547263145446777,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -35205254.78009631,
|
||
|
|
"logits/rejected": -36215410.557077624,
|
||
|
|
"logps/chosen": -489.56530898876406,
|
||
|
|
"logps/rejected": -378.8430127473364,
|
||
|
|
"loss": 0.5505,
|
||
|
|
"loss/base_kto": 3.197359800338745,
|
||
|
|
"metrics/advantage_var": 401.6699523925781,
|
||
|
|
"regularization/lipschitz_norm": 1246.5228271484375,
|
||
|
|
"regularization/w1": 1.2066341638565063,
|
||
|
|
"rewards/chosen": 0.4505366391201846,
|
||
|
|
"rewards/margins": 0.8340050115754738,
|
||
|
|
"rewards/rejected": -0.3834683724552892,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 21.010990142822266,
|
||
|
|
"kl": 10.815085411071777,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -35025544.37747336,
|
||
|
|
"logits/rejected": -36055533.097913325,
|
||
|
|
"logps/chosen": -475.53695776255705,
|
||
|
|
"logps/rejected": -392.6695676163724,
|
||
|
|
"loss": 0.5621,
|
||
|
|
"loss/base_kto": 3.2230801582336426,
|
||
|
|
"metrics/advantage_var": 426.4375915527344,
|
||
|
|
"regularization/lipschitz_norm": 1316.1917724609375,
|
||
|
|
"regularization/w1": 1.274073600769043,
|
||
|
|
"rewards/chosen": 0.471170317999667,
|
||
|
|
"rewards/margins": 0.8352990553498125,
|
||
|
|
"rewards/rejected": -0.36412873735014545,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 15.986784934997559,
|
||
|
|
"kl": 12.288924217224121,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -36367155.2,
|
||
|
|
"logits/rejected": -37064815.70909091,
|
||
|
|
"logps/chosen": -484.96446572580646,
|
||
|
|
"logps/rejected": -372.9198390151515,
|
||
|
|
"loss": 0.5555,
|
||
|
|
"loss/base_kto": 3.165621519088745,
|
||
|
|
"metrics/advantage_var": 570.4794311523438,
|
||
|
|
"regularization/lipschitz_norm": 1320.803955078125,
|
||
|
|
"regularization/w1": 1.2785381078720093,
|
||
|
|
"rewards/chosen": 0.5384136569115423,
|
||
|
|
"rewards/margins": 0.8526477026916086,
|
||
|
|
"rewards/rejected": -0.3142340457800663,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 21.033382415771484,
|
||
|
|
"kl": 11.921859741210938,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35882482.79210926,
|
||
|
|
"logits/rejected": -37186660.58615137,
|
||
|
|
"logps/chosen": -492.34408194233686,
|
||
|
|
"logps/rejected": -382.951690821256,
|
||
|
|
"loss": 0.5608,
|
||
|
|
"loss/base_kto": 3.1837589740753174,
|
||
|
|
"metrics/advantage_var": 444.9703674316406,
|
||
|
|
"regularization/lipschitz_norm": 1345.833984375,
|
||
|
|
"regularization/w1": 1.302767276763916,
|
||
|
|
"rewards/chosen": 0.5376987949307663,
|
||
|
|
"rewards/margins": 0.8656762933031621,
|
||
|
|
"rewards/rejected": -0.3279774983723958,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 15.930805206298828,
|
||
|
|
"kl": 13.818572998046875,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -35410512.58505564,
|
||
|
|
"logits/rejected": -36944862.96774194,
|
||
|
|
"logps/chosen": -483.8710254372019,
|
||
|
|
"logps/rejected": -351.28338613671275,
|
||
|
|
"loss": 0.5445,
|
||
|
|
"loss/base_kto": 3.2034249305725098,
|
||
|
|
"metrics/advantage_var": 382.9445495605469,
|
||
|
|
"regularization/lipschitz_norm": 1190.4652099609375,
|
||
|
|
"regularization/w1": 1.1523703336715698,
|
||
|
|
"rewards/chosen": 0.48938628713732113,
|
||
|
|
"rewards/margins": 0.8164141075428271,
|
||
|
|
"rewards/rejected": -0.32702782040550593,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 16.114788055419922,
|
||
|
|
"kl": 11.364229202270508,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -35163680.28828829,
|
||
|
|
"logits/rejected": -38131788.716612376,
|
||
|
|
"logps/chosen": -484.7685810810811,
|
||
|
|
"logps/rejected": -365.84512418566777,
|
||
|
|
"loss": 0.559,
|
||
|
|
"loss/base_kto": 3.2034904956817627,
|
||
|
|
"metrics/advantage_var": 469.1863708496094,
|
||
|
|
"regularization/lipschitz_norm": 1310.663330078125,
|
||
|
|
"regularization/w1": 1.2687219381332397,
|
||
|
|
"rewards/chosen": 0.5054551316453172,
|
||
|
|
"rewards/margins": 0.8541521105840133,
|
||
|
|
"rewards/rejected": -0.34869697893869606,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 16.752727508544922,
|
||
|
|
"kl": 12.859704971313477,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -35552411.343653254,
|
||
|
|
"logits/rejected": -35836808.479495265,
|
||
|
|
"logps/chosen": -475.7380030959752,
|
||
|
|
"logps/rejected": -401.7384660883281,
|
||
|
|
"loss": 0.5614,
|
||
|
|
"loss/base_kto": 3.1534054279327393,
|
||
|
|
"metrics/advantage_var": 482.09307861328125,
|
||
|
|
"regularization/lipschitz_norm": 1381.8914794921875,
|
||
|
|
"regularization/w1": 1.3376706838607788,
|
||
|
|
"rewards/chosen": 0.5472315738075658,
|
||
|
|
"rewards/margins": 0.9019963256390056,
|
||
|
|
"rewards/rejected": -0.35476475183143974,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 14.49431324005127,
|
||
|
|
"kl": 14.295180320739746,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -35231242.76276276,
|
||
|
|
"logits/rejected": -35790407.71335505,
|
||
|
|
"logps/chosen": -478.3442192192192,
|
||
|
|
"logps/rejected": -377.14851384364823,
|
||
|
|
"loss": 0.5595,
|
||
|
|
"loss/base_kto": 3.1992764472961426,
|
||
|
|
"metrics/advantage_var": 423.2524108886719,
|
||
|
|
"regularization/lipschitz_norm": 1319.3045654296875,
|
||
|
|
"regularization/w1": 1.277086615562439,
|
||
|
|
"rewards/chosen": 0.5252729994398696,
|
||
|
|
"rewards/margins": 0.8552534085463944,
|
||
|
|
"rewards/rejected": -0.3299804091065248,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 14.851458549499512,
|
||
|
|
"kl": 15.1482572555542,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -34456233.5744,
|
||
|
|
"logits/rejected": -36623664.854961835,
|
||
|
|
"logps/chosen": -501.77555,
|
||
|
|
"logps/rejected": -373.3756202290076,
|
||
|
|
"loss": 0.5556,
|
||
|
|
"loss/base_kto": 3.139052152633667,
|
||
|
|
"metrics/advantage_var": 441.7076110839844,
|
||
|
|
"regularization/lipschitz_norm": 1348.7933349609375,
|
||
|
|
"regularization/w1": 1.3056318759918213,
|
||
|
|
"rewards/chosen": 0.54389921875,
|
||
|
|
"rewards/margins": 0.8732354637553674,
|
||
|
|
"rewards/rejected": -0.32933624500536735,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 21.785266876220703,
|
||
|
|
"kl": 13.95178508758545,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -35514544.20779221,
|
||
|
|
"logits/rejected": -36804163.855421685,
|
||
|
|
"logps/chosen": -505.4075689935065,
|
||
|
|
"logps/rejected": -370.59779743975906,
|
||
|
|
"loss": 0.569,
|
||
|
|
"loss/base_kto": 3.2066452503204346,
|
||
|
|
"metrics/advantage_var": 438.0576171875,
|
||
|
|
"regularization/lipschitz_norm": 1389.8353271484375,
|
||
|
|
"regularization/w1": 1.3453606367111206,
|
||
|
|
"rewards/chosen": 0.5691466393408837,
|
||
|
|
"rewards/margins": 0.8382363266423356,
|
||
|
|
"rewards/rejected": -0.2690896873014519,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 15.563207626342773,
|
||
|
|
"kl": 13.7863187789917,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -35387645.99058085,
|
||
|
|
"logits/rejected": -37348127.452566095,
|
||
|
|
"logps/chosen": -479.9228806907378,
|
||
|
|
"logps/rejected": -379.8424620917574,
|
||
|
|
"loss": 0.5596,
|
||
|
|
"loss/base_kto": 3.234290361404419,
|
||
|
|
"metrics/advantage_var": 432.3598327636719,
|
||
|
|
"regularization/lipschitz_norm": 1283.27392578125,
|
||
|
|
"regularization/w1": 1.2422091960906982,
|
||
|
|
"rewards/chosen": 0.4893600487821331,
|
||
|
|
"rewards/margins": 0.7867642890833335,
|
||
|
|
"rewards/rejected": -0.29740424030120044,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 20.491729736328125,
|
||
|
|
"kl": 13.381392478942871,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -34800635.0371567,
|
||
|
|
"logits/rejected": -34597505.35552194,
|
||
|
|
"logps/chosen": -464.2064317447496,
|
||
|
|
"logps/rejected": -386.40369704992435,
|
||
|
|
"loss": 0.5493,
|
||
|
|
"loss/base_kto": 3.139193058013916,
|
||
|
|
"metrics/advantage_var": 452.55224609375,
|
||
|
|
"regularization/lipschitz_norm": 1297.0628662109375,
|
||
|
|
"regularization/w1": 1.2555568218231201,
|
||
|
|
"rewards/chosen": 0.5594025397724405,
|
||
|
|
"rewards/margins": 0.9078356260349689,
|
||
|
|
"rewards/rejected": -0.3484330862625284,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 18.10733413696289,
|
||
|
|
"kl": 12.210433006286621,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -34073104.91291291,
|
||
|
|
"logits/rejected": -36275066.57980456,
|
||
|
|
"logps/chosen": -485.66516516516515,
|
||
|
|
"logps/rejected": -353.6661746742671,
|
||
|
|
"loss": 0.5535,
|
||
|
|
"loss/base_kto": 3.1714961528778076,
|
||
|
|
"metrics/advantage_var": 417.6553649902344,
|
||
|
|
"regularization/lipschitz_norm": 1297.9132080078125,
|
||
|
|
"regularization/w1": 1.2563799619674683,
|
||
|
|
"rewards/chosen": 0.5340808032153247,
|
||
|
|
"rewards/margins": 0.8667566252216994,
|
||
|
|
"rewards/rejected": -0.3326758220063747,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 17.645549774169922,
|
||
|
|
"kl": 13.995226860046387,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -35230927.9751938,
|
||
|
|
"logits/rejected": -36144800.453543305,
|
||
|
|
"logps/chosen": -495.7886143410853,
|
||
|
|
"logps/rejected": -365.29559547244094,
|
||
|
|
"loss": 0.5629,
|
||
|
|
"loss/base_kto": 3.245443105697632,
|
||
|
|
"metrics/advantage_var": 407.492431640625,
|
||
|
|
"regularization/lipschitz_norm": 1299.3297119140625,
|
||
|
|
"regularization/w1": 1.2577511072158813,
|
||
|
|
"rewards/chosen": 0.45776499666908915,
|
||
|
|
"rewards/margins": 0.7973788315608215,
|
||
|
|
"rewards/rejected": -0.3396138348917323,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 17.04086685180664,
|
||
|
|
"kl": 14.748274803161621,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -35406739.27099842,
|
||
|
|
"logits/rejected": -35839195.31587057,
|
||
|
|
"logps/chosen": -476.064926703645,
|
||
|
|
"logps/rejected": -381.61782550077044,
|
||
|
|
"loss": 0.5546,
|
||
|
|
"loss/base_kto": 3.1653707027435303,
|
||
|
|
"metrics/advantage_var": 434.7702331542969,
|
||
|
|
"regularization/lipschitz_norm": 1313.435546875,
|
||
|
|
"regularization/w1": 1.271405577659607,
|
||
|
|
"rewards/chosen": 0.5823736659320523,
|
||
|
|
"rewards/margins": 0.8787721167864316,
|
||
|
|
"rewards/rejected": -0.2963984508543793,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 20.287521362304688,
|
||
|
|
"kl": 13.361541748046875,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -35161287.72755418,
|
||
|
|
"logits/rejected": -36467007.79810726,
|
||
|
|
"logps/chosen": -491.35714976780184,
|
||
|
|
"logps/rejected": -371.70539235015775,
|
||
|
|
"loss": 0.5524,
|
||
|
|
"loss/base_kto": 3.1137187480926514,
|
||
|
|
"metrics/advantage_var": 448.5995178222656,
|
||
|
|
"regularization/lipschitz_norm": 1348.6898193359375,
|
||
|
|
"regularization/w1": 1.3055317401885986,
|
||
|
|
"rewards/chosen": 0.5504875065003386,
|
||
|
|
"rewards/margins": 0.9152884741289862,
|
||
|
|
"rewards/rejected": -0.3648009676286475,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 17.823556900024414,
|
||
|
|
"kl": 13.623834609985352,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -36393625.91411043,
|
||
|
|
"logits/rejected": -37944633.070063695,
|
||
|
|
"logps/chosen": -498.00393021472394,
|
||
|
|
"logps/rejected": -398.20989251592357,
|
||
|
|
"loss": 0.5678,
|
||
|
|
"loss/base_kto": 3.221618413925171,
|
||
|
|
"metrics/advantage_var": 467.2951354980469,
|
||
|
|
"regularization/lipschitz_norm": 1364.040283203125,
|
||
|
|
"regularization/w1": 1.3203909397125244,
|
||
|
|
"rewards/chosen": 0.49074713584104196,
|
||
|
|
"rewards/margins": 0.8423864886273551,
|
||
|
|
"rewards/rejected": -0.3516393527863132,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 15.583972930908203,
|
||
|
|
"kl": 11.398027420043945,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -34613680.11887073,
|
||
|
|
"logits/rejected": -36446707.347611204,
|
||
|
|
"logps/chosen": -452.58014487369985,
|
||
|
|
"logps/rejected": -372.89026462108734,
|
||
|
|
"loss": 0.5615,
|
||
|
|
"loss/base_kto": 3.2216451168060303,
|
||
|
|
"metrics/advantage_var": 434.85235595703125,
|
||
|
|
"regularization/lipschitz_norm": 1312.203125,
|
||
|
|
"regularization/w1": 1.2702126502990723,
|
||
|
|
"rewards/chosen": 0.5213648440402118,
|
||
|
|
"rewards/margins": 0.8280147701142697,
|
||
|
|
"rewards/rejected": -0.3066499260740579,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 21.09757423400879,
|
||
|
|
"kl": 13.159136772155762,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -34305653.78403756,
|
||
|
|
"logits/rejected": -36084855.81279251,
|
||
|
|
"logps/chosen": -486.7087245696401,
|
||
|
|
"logps/rejected": -369.51564937597504,
|
||
|
|
"loss": 0.5627,
|
||
|
|
"loss/base_kto": 3.169769763946533,
|
||
|
|
"metrics/advantage_var": 475.5580749511719,
|
||
|
|
"regularization/lipschitz_norm": 1375.818359375,
|
||
|
|
"regularization/w1": 1.3317921161651611,
|
||
|
|
"rewards/chosen": 0.5698012947476526,
|
||
|
|
"rewards/margins": 0.8729822452278593,
|
||
|
|
"rewards/rejected": -0.3031809504802067,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 15.512831687927246,
|
||
|
|
"kl": 12.718172073364258,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -34757713.32252836,
|
||
|
|
"logits/rejected": -35187043.23378582,
|
||
|
|
"logps/chosen": -496.6736223662885,
|
||
|
|
"logps/rejected": -380.04128959276017,
|
||
|
|
"loss": 0.5592,
|
||
|
|
"loss/base_kto": 3.1521763801574707,
|
||
|
|
"metrics/advantage_var": 437.49273681640625,
|
||
|
|
"regularization/lipschitz_norm": 1365.0662841796875,
|
||
|
|
"regularization/w1": 1.3213841915130615,
|
||
|
|
"rewards/chosen": 0.5392303219490732,
|
||
|
|
"rewards/margins": 0.8790295613939496,
|
||
|
|
"rewards/rejected": -0.3397992394448765,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 14.884849548339844,
|
||
|
|
"kl": 12.334362983703613,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -36344090.6993865,
|
||
|
|
"logits/rejected": -36805273.27388535,
|
||
|
|
"logps/chosen": -468.74693251533745,
|
||
|
|
"logps/rejected": -391.46616242038215,
|
||
|
|
"loss": 0.5497,
|
||
|
|
"loss/base_kto": 3.1781673431396484,
|
||
|
|
"metrics/advantage_var": 424.24005126953125,
|
||
|
|
"regularization/lipschitz_norm": 1259.74560546875,
|
||
|
|
"regularization/w1": 1.2194337844848633,
|
||
|
|
"rewards/chosen": 0.5212264265750814,
|
||
|
|
"rewards/margins": 0.864103565526863,
|
||
|
|
"rewards/rejected": -0.34287713895178146,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.98294195064275e+17,
|
||
|
|
"train_loss": 0.5861379849066595,
|
||
|
|
"train_runtime": 11032.7786,
|
||
|
|
"train_samples_per_second": 13.434,
|
||
|
|
"train_steps_per_second": 0.21
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.98294195064275e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|