e90962e0d2
Model: Gueule-d-ange/aup-fullft-kto_w1_mmd-w1lam8.4e-4_mmdrho8.4e-4_kr0.1-seed3 Source: Original Platform
2459 lines
92 KiB
JSON
2459 lines
92 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 20.5758113861084,
|
|
"kl": 16.81874656677246,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37939062.65275708,
|
|
"logits/rejected": -37362454.27914614,
|
|
"logps/chosen": -463.85492734724295,
|
|
"logps/rejected": -394.7774014778325,
|
|
"loss": 0.6076,
|
|
"loss/base_kto": 3.887288808822632,
|
|
"metrics/advantage_var": 215.61167907714844,
|
|
"regularization/lipschitz_norm": 992.4693603515625,
|
|
"regularization/mmd": 0.1399136632680893,
|
|
"regularization/rkhs_norm": 166.5638885498047,
|
|
"regularization/w1": 0.8336742520332336,
|
|
"rewards/chosen": 0.28945038251123556,
|
|
"rewards/margins": 0.07722112203684106,
|
|
"rewards/rejected": 0.2122292604743945,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 32.53988265991211,
|
|
"kl": 8.703550338745117,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36340779.64590164,
|
|
"logits/rejected": -37285878.82985075,
|
|
"logps/chosen": -483.72079918032784,
|
|
"logps/rejected": -371.6361473880597,
|
|
"loss": 0.5888,
|
|
"loss/base_kto": 3.9180238246917725,
|
|
"metrics/advantage_var": 138.6885528564453,
|
|
"regularization/lipschitz_norm": 807.34423828125,
|
|
"regularization/mmd": 0.11442476511001587,
|
|
"regularization/rkhs_norm": 136.21995544433594,
|
|
"regularization/w1": 0.6781691908836365,
|
|
"rewards/chosen": 0.08447965403072169,
|
|
"rewards/margins": 0.06347960324400585,
|
|
"rewards/rejected": 0.021000050786715834,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 19.574037551879883,
|
|
"kl": 3.263112783432007,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36193813.13015873,
|
|
"logits/rejected": -37613397.85846154,
|
|
"logps/chosen": -482.15208333333334,
|
|
"logps/rejected": -347.1869711538462,
|
|
"loss": 0.598,
|
|
"loss/base_kto": 3.870199680328369,
|
|
"metrics/advantage_var": 195.90281677246094,
|
|
"regularization/lipschitz_norm": 928.9747314453125,
|
|
"regularization/mmd": 0.13382986187934875,
|
|
"regularization/rkhs_norm": 159.32127380371094,
|
|
"regularization/w1": 0.780338704586029,
|
|
"rewards/chosen": -0.051958592732747395,
|
|
"rewards/margins": 0.12822778750688602,
|
|
"rewards/rejected": -0.1801863802396334,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 22.571435928344727,
|
|
"kl": 4.356871128082275,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35967044.266666666,
|
|
"logits/rejected": -38276518.20307692,
|
|
"logps/chosen": -492.0336805555556,
|
|
"logps/rejected": -381.4961057692308,
|
|
"loss": 0.6131,
|
|
"loss/base_kto": 3.9751222133636475,
|
|
"metrics/advantage_var": 193.7662353515625,
|
|
"regularization/lipschitz_norm": 949.2344970703125,
|
|
"regularization/mmd": 0.13238470256328583,
|
|
"regularization/rkhs_norm": 157.60084533691406,
|
|
"regularization/w1": 0.797356903553009,
|
|
"rewards/chosen": -0.11964053199404762,
|
|
"rewards/margins": -0.01796555124359689,
|
|
"rewards/rejected": -0.10167498075045073,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 22.95536994934082,
|
|
"kl": 11.476832389831543,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -35546065.08688784,
|
|
"logits/rejected": -35757168.370942816,
|
|
"logps/chosen": -496.32237361769353,
|
|
"logps/rejected": -362.64678323029364,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.8341598510742188,
|
|
"metrics/advantage_var": 185.092041015625,
|
|
"regularization/lipschitz_norm": 947.2268676757812,
|
|
"regularization/mmd": 0.13265080749988556,
|
|
"regularization/rkhs_norm": 157.91763305664062,
|
|
"regularization/w1": 0.7956705093383789,
|
|
"rewards/chosen": 0.16115551989225416,
|
|
"rewards/margins": 0.16172028228484361,
|
|
"rewards/rejected": -0.0005647623925894452,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 21.858009338378906,
|
|
"kl": 17.05402183532715,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -35402981.75334324,
|
|
"logits/rejected": -37303574.35255354,
|
|
"logps/chosen": -482.651838781575,
|
|
"logps/rejected": -376.8109297775947,
|
|
"loss": 0.5927,
|
|
"loss/base_kto": 3.8168811798095703,
|
|
"metrics/advantage_var": 193.15184020996094,
|
|
"regularization/lipschitz_norm": 941.7293090820312,
|
|
"regularization/mmd": 0.13376301527023315,
|
|
"regularization/rkhs_norm": 159.24168395996094,
|
|
"regularization/w1": 0.791052520275116,
|
|
"rewards/chosen": 0.2940350287382174,
|
|
"rewards/margins": 0.17179536127625025,
|
|
"rewards/rejected": 0.12223966746196716,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 17.485946655273438,
|
|
"kl": 9.870765686035156,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -34899143.64779874,
|
|
"logits/rejected": -36140066.98136646,
|
|
"logps/chosen": -489.23265526729557,
|
|
"logps/rejected": -406.5649505046584,
|
|
"loss": 0.5965,
|
|
"loss/base_kto": 3.8522095680236816,
|
|
"metrics/advantage_var": 195.4907684326172,
|
|
"regularization/lipschitz_norm": 934.1529541015625,
|
|
"regularization/mmd": 0.1350112408399582,
|
|
"regularization/rkhs_norm": 160.72767639160156,
|
|
"regularization/w1": 0.7846884727478027,
|
|
"rewards/chosen": 0.1335341735455975,
|
|
"rewards/margins": 0.13950597845468166,
|
|
"rewards/rejected": -0.005971804909084154,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 17.309173583984375,
|
|
"kl": 9.891693115234375,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -35866083.73006135,
|
|
"logits/rejected": -37282020.280254774,
|
|
"logps/chosen": -460.4609375,
|
|
"logps/rejected": -385.0896198248408,
|
|
"loss": 0.6043,
|
|
"loss/base_kto": 3.861051321029663,
|
|
"metrics/advantage_var": 212.87026977539062,
|
|
"regularization/lipschitz_norm": 990.7973022460938,
|
|
"regularization/mmd": 0.1408347487449646,
|
|
"regularization/rkhs_norm": 167.66041564941406,
|
|
"regularization/w1": 0.8322697877883911,
|
|
"rewards/chosen": 0.12143621854255536,
|
|
"rewards/margins": 0.11573324613044599,
|
|
"rewards/rejected": 0.005702972412109375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 18.654821395874023,
|
|
"kl": 5.233686923980713,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36207757.58199357,
|
|
"logits/rejected": -37505129.82370821,
|
|
"logps/chosen": -499.97221663987136,
|
|
"logps/rejected": -372.81532579787233,
|
|
"loss": 0.5988,
|
|
"loss/base_kto": 3.820173740386963,
|
|
"metrics/advantage_var": 226.052001953125,
|
|
"regularization/lipschitz_norm": 985.3980712890625,
|
|
"regularization/mmd": 0.14236484467983246,
|
|
"regularization/rkhs_norm": 169.48196411132812,
|
|
"regularization/w1": 0.8277344107627869,
|
|
"rewards/chosen": -0.0008185262465400328,
|
|
"rewards/margins": 0.17960524511019915,
|
|
"rewards/rejected": -0.18042377135673918,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 18.692941665649414,
|
|
"kl": 12.598770141601562,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -37093495.36196319,
|
|
"logits/rejected": -36910389.808917195,
|
|
"logps/chosen": -475.8856403374233,
|
|
"logps/rejected": -383.1358479299363,
|
|
"loss": 0.5957,
|
|
"loss/base_kto": 3.8510231971740723,
|
|
"metrics/advantage_var": 197.63787841796875,
|
|
"regularization/lipschitz_norm": 929.4817504882812,
|
|
"regularization/mmd": 0.13404138386249542,
|
|
"regularization/rkhs_norm": 159.5730743408203,
|
|
"regularization/w1": 0.7807646989822388,
|
|
"rewards/chosen": 0.18085748871411284,
|
|
"rewards/margins": 0.13592879828364782,
|
|
"rewards/rejected": 0.04492869043046502,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 24.100107192993164,
|
|
"kl": 11.595545768737793,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36806138.2656,
|
|
"logits/rejected": -36215240.50076336,
|
|
"logps/chosen": -488.6162,
|
|
"logps/rejected": -355.44444179389313,
|
|
"loss": 0.5923,
|
|
"loss/base_kto": 3.8025033473968506,
|
|
"metrics/advantage_var": 202.3291778564453,
|
|
"regularization/lipschitz_norm": 952.2705078125,
|
|
"regularization/mmd": 0.13572661578655243,
|
|
"regularization/rkhs_norm": 161.5792999267578,
|
|
"regularization/w1": 0.7999071478843689,
|
|
"rewards/chosen": 0.18568939208984375,
|
|
"rewards/margins": 0.18039366991465328,
|
|
"rewards/rejected": 0.005295722175190467,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 26.79574966430664,
|
|
"kl": 22.761743545532227,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35564447.45595054,
|
|
"logits/rejected": -37149658.79304897,
|
|
"logps/chosen": -464.9503477588872,
|
|
"logps/rejected": -362.37638230647707,
|
|
"loss": 0.5982,
|
|
"loss/base_kto": 3.7932441234588623,
|
|
"metrics/advantage_var": 246.3934783935547,
|
|
"regularization/lipschitz_norm": 1006.77734375,
|
|
"regularization/mmd": 0.14705203473567963,
|
|
"regularization/rkhs_norm": 175.06195068359375,
|
|
"regularization/w1": 0.845693051815033,
|
|
"rewards/chosen": 0.3306272527349787,
|
|
"rewards/margins": 0.15932515638466893,
|
|
"rewards/rejected": 0.1713020963503098,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 17.627756118774414,
|
|
"kl": 6.831817150115967,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36917455.72107766,
|
|
"logits/rejected": -38925057.97226502,
|
|
"logps/chosen": -495.70483359746436,
|
|
"logps/rejected": -385.986999229584,
|
|
"loss": 0.6027,
|
|
"loss/base_kto": 3.8173015117645264,
|
|
"metrics/advantage_var": 229.39768981933594,
|
|
"regularization/lipschitz_norm": 1021.6549072265625,
|
|
"regularization/mmd": 0.14589278399944305,
|
|
"regularization/rkhs_norm": 173.681884765625,
|
|
"regularization/w1": 0.8581901788711548,
|
|
"rewards/chosen": 0.061865067519778875,
|
|
"rewards/margins": 0.16022221671151918,
|
|
"rewards/rejected": -0.0983571491917403,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 19.737218856811523,
|
|
"kl": 9.316094398498535,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36195308.43312102,
|
|
"logits/rejected": -37284034.74846626,
|
|
"logps/chosen": -483.2488554936306,
|
|
"logps/rejected": -369.80840203220856,
|
|
"loss": 0.5917,
|
|
"loss/base_kto": 3.7503457069396973,
|
|
"metrics/advantage_var": 227.41404724121094,
|
|
"regularization/lipschitz_norm": 999.7609252929688,
|
|
"regularization/mmd": 0.14342042803764343,
|
|
"regularization/rkhs_norm": 170.7386016845703,
|
|
"regularization/w1": 0.8397992253303528,
|
|
"rewards/chosen": 0.1580808724567389,
|
|
"rewards/margins": 0.22238207070229643,
|
|
"rewards/rejected": -0.06430119824555754,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 23.97921371459961,
|
|
"kl": 8.586662292480469,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -37549160.162754305,
|
|
"logits/rejected": -38236319.750390016,
|
|
"logps/chosen": -485.34477699530515,
|
|
"logps/rejected": -376.4766965678627,
|
|
"loss": 0.594,
|
|
"loss/base_kto": 3.733426809310913,
|
|
"metrics/advantage_var": 241.8022003173828,
|
|
"regularization/lipschitz_norm": 1037.521240234375,
|
|
"regularization/mmd": 0.14734356105327606,
|
|
"regularization/rkhs_norm": 175.40899658203125,
|
|
"regularization/w1": 0.8715177774429321,
|
|
"rewards/chosen": 0.14147711621017336,
|
|
"rewards/margins": 0.24885428605282645,
|
|
"rewards/rejected": -0.10737716984265308,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 18.06004524230957,
|
|
"kl": 6.466609954833984,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -35774181.09309309,
|
|
"logits/rejected": -37414938.684039086,
|
|
"logps/chosen": -485.11294106606607,
|
|
"logps/rejected": -348.24541938110747,
|
|
"loss": 0.5904,
|
|
"loss/base_kto": 3.8044323921203613,
|
|
"metrics/advantage_var": 211.8214874267578,
|
|
"regularization/lipschitz_norm": 930.0750122070312,
|
|
"regularization/mmd": 0.1376616209745407,
|
|
"regularization/rkhs_norm": 163.88287353515625,
|
|
"regularization/w1": 0.7812630534172058,
|
|
"rewards/chosen": 0.13392112204978415,
|
|
"rewards/margins": 0.1955093907647138,
|
|
"rewards/rejected": -0.06158826871492964,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.632667541503906,
|
|
"kl": 14.662436485290527,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -35479061.590361446,
|
|
"logits/rejected": -37082886.64935065,
|
|
"logps/chosen": -476.8266189759036,
|
|
"logps/rejected": -385.76212459415586,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.810995578765869,
|
|
"metrics/advantage_var": 233.14382934570312,
|
|
"regularization/lipschitz_norm": 1004.1724853515625,
|
|
"regularization/mmd": 0.14769704639911652,
|
|
"regularization/rkhs_norm": 175.82981872558594,
|
|
"regularization/w1": 0.8435049057006836,
|
|
"rewards/chosen": 0.28432138282132435,
|
|
"rewards/margins": 0.15912314241731917,
|
|
"rewards/rejected": 0.12519824040400518,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 21.117231369018555,
|
|
"kl": 16.584177017211914,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -34552640.91866029,
|
|
"logits/rejected": -35992721.83767228,
|
|
"logps/chosen": -498.9676036682616,
|
|
"logps/rejected": -353.48281967840734,
|
|
"loss": 0.5942,
|
|
"loss/base_kto": 3.7767086029052734,
|
|
"metrics/advantage_var": 222.5546417236328,
|
|
"regularization/lipschitz_norm": 991.2771606445312,
|
|
"regularization/mmd": 0.1440141201019287,
|
|
"regularization/rkhs_norm": 171.44537353515625,
|
|
"regularization/w1": 0.8326728940010071,
|
|
"rewards/chosen": 0.2840682277649023,
|
|
"rewards/margins": 0.2045750483267772,
|
|
"rewards/rejected": 0.07949317943812512,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 22.647109985351562,
|
|
"kl": 6.5619940757751465,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35596677.9392,
|
|
"logits/rejected": -37279328.146564886,
|
|
"logps/chosen": -496.00245,
|
|
"logps/rejected": -375.54489503816797,
|
|
"loss": 0.594,
|
|
"loss/base_kto": 3.7610390186309814,
|
|
"metrics/advantage_var": 233.2661590576172,
|
|
"regularization/lipschitz_norm": 1008.4299926757812,
|
|
"regularization/mmd": 0.14352834224700928,
|
|
"regularization/rkhs_norm": 170.86709594726562,
|
|
"regularization/w1": 0.847081184387207,
|
|
"rewards/chosen": 0.05087820129394531,
|
|
"rewards/margins": 0.22610913316566528,
|
|
"rewards/rejected": -0.17523093187171995,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 19.77211570739746,
|
|
"kl": 7.502200603485107,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -35290638.17846154,
|
|
"logits/rejected": -37069814.24761905,
|
|
"logps/chosen": -491.0845673076923,
|
|
"logps/rejected": -384.30562996031745,
|
|
"loss": 0.6055,
|
|
"loss/base_kto": 3.79258131980896,
|
|
"metrics/advantage_var": 269.2097473144531,
|
|
"regularization/lipschitz_norm": 1070.2557373046875,
|
|
"regularization/mmd": 0.15260519087314606,
|
|
"regularization/rkhs_norm": 181.6728515625,
|
|
"regularization/w1": 0.8990148901939392,
|
|
"rewards/chosen": 0.07853038494403546,
|
|
"rewards/margins": 0.1819285172682542,
|
|
"rewards/rejected": -0.10339813232421875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 22.727684020996094,
|
|
"kl": 16.226346969604492,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -35434669.82716049,
|
|
"logits/rejected": -39028742.48101266,
|
|
"logps/chosen": -492.70076195987656,
|
|
"logps/rejected": -355.4692444620253,
|
|
"loss": 0.593,
|
|
"loss/base_kto": 3.71561598777771,
|
|
"metrics/advantage_var": 290.4650573730469,
|
|
"regularization/lipschitz_norm": 1039.91552734375,
|
|
"regularization/mmd": 0.15495604276657104,
|
|
"regularization/rkhs_norm": 184.47149658203125,
|
|
"regularization/w1": 0.8735290765762329,
|
|
"rewards/chosen": 0.21321501555266203,
|
|
"rewards/margins": 0.25089171510652597,
|
|
"rewards/rejected": -0.037676699553863914,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 27.383163452148438,
|
|
"kl": 11.542716026306152,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -34382502.27652733,
|
|
"logits/rejected": -36393355.282674775,
|
|
"logps/chosen": -478.00753617363347,
|
|
"logps/rejected": -374.1326225303951,
|
|
"loss": 0.5948,
|
|
"loss/base_kto": 3.7610371112823486,
|
|
"metrics/advantage_var": 238.17247009277344,
|
|
"regularization/lipschitz_norm": 1013.21728515625,
|
|
"regularization/mmd": 0.14663951098918915,
|
|
"regularization/rkhs_norm": 174.5708465576172,
|
|
"regularization/w1": 0.8511025309562683,
|
|
"rewards/chosen": 0.11473729909424613,
|
|
"rewards/margins": 0.21073798787320164,
|
|
"rewards/rejected": -0.09600068877895553,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 17.471017837524414,
|
|
"kl": 12.239995002746582,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35423626.084848486,
|
|
"logits/rejected": -36434494.76129032,
|
|
"logps/chosen": -460.1627840909091,
|
|
"logps/rejected": -381.32910786290324,
|
|
"loss": 0.5962,
|
|
"loss/base_kto": 3.7733826637268066,
|
|
"metrics/advantage_var": 226.7340545654297,
|
|
"regularization/lipschitz_norm": 1011.71875,
|
|
"regularization/mmd": 0.14616459608078003,
|
|
"regularization/rkhs_norm": 174.0054931640625,
|
|
"regularization/w1": 0.8498438000679016,
|
|
"rewards/chosen": 0.19644125736120976,
|
|
"rewards/margins": 0.21708291343696423,
|
|
"rewards/rejected": -0.020641656075754473,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 19.190166473388672,
|
|
"kl": 15.33696460723877,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -37426184.29821718,
|
|
"logits/rejected": -37660202.47360483,
|
|
"logps/chosen": -499.82860615883305,
|
|
"logps/rejected": -364.14755373303166,
|
|
"loss": 0.597,
|
|
"loss/base_kto": 3.758046865463257,
|
|
"metrics/advantage_var": 251.15982055664062,
|
|
"regularization/lipschitz_norm": 1030.2054443359375,
|
|
"regularization/mmd": 0.1522947996854782,
|
|
"regularization/rkhs_norm": 181.30332946777344,
|
|
"regularization/w1": 0.8653726577758789,
|
|
"rewards/chosen": 0.2285088488116643,
|
|
"rewards/margins": 0.22294865928116367,
|
|
"rewards/rejected": 0.005560189530500639,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 16.460947036743164,
|
|
"kl": 7.460095405578613,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -34947084.54517611,
|
|
"logits/rejected": -35680053.48644338,
|
|
"logps/chosen": -496.16993683001533,
|
|
"logps/rejected": -382.27262759170657,
|
|
"loss": 0.5912,
|
|
"loss/base_kto": 3.7523276805877686,
|
|
"metrics/advantage_var": 233.4416046142578,
|
|
"regularization/lipschitz_norm": 989.814453125,
|
|
"regularization/mmd": 0.14545856416225433,
|
|
"regularization/rkhs_norm": 173.1649627685547,
|
|
"regularization/w1": 0.8314442038536072,
|
|
"rewards/chosen": 0.07514801726406968,
|
|
"rewards/margins": 0.2212005466310329,
|
|
"rewards/rejected": -0.14605252936696322,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 23.77032470703125,
|
|
"kl": 11.39418888092041,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36298183.80487805,
|
|
"logits/rejected": -37196773.743589744,
|
|
"logps/chosen": -477.8599466463415,
|
|
"logps/rejected": -366.67387820512823,
|
|
"loss": 0.5898,
|
|
"loss/base_kto": 3.6974334716796875,
|
|
"metrics/advantage_var": 272.9799499511719,
|
|
"regularization/lipschitz_norm": 1035.4735107421875,
|
|
"regularization/mmd": 0.15107448399066925,
|
|
"regularization/rkhs_norm": 179.85057067871094,
|
|
"regularization/w1": 0.8697977066040039,
|
|
"rewards/chosen": 0.20378950165539253,
|
|
"rewards/margins": 0.2523843554722808,
|
|
"rewards/rejected": -0.04859485381688827,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 23.84042739868164,
|
|
"kl": 15.18149471282959,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37342823.04402516,
|
|
"logits/rejected": -38569392.49689441,
|
|
"logps/chosen": -483.0530169025157,
|
|
"logps/rejected": -378.34714673913044,
|
|
"loss": 0.5946,
|
|
"loss/base_kto": 3.7625184059143066,
|
|
"metrics/advantage_var": 227.36949157714844,
|
|
"regularization/lipschitz_norm": 1011.3478393554688,
|
|
"regularization/mmd": 0.14458440244197845,
|
|
"regularization/rkhs_norm": 172.12429809570312,
|
|
"regularization/w1": 0.8495321273803711,
|
|
"rewards/chosen": 0.24147647881657822,
|
|
"rewards/margins": 0.19763926026541634,
|
|
"rewards/rejected": 0.04383721855116186,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 19.289283752441406,
|
|
"kl": 12.326022148132324,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35266791.8490566,
|
|
"logits/rejected": -37670123.32919255,
|
|
"logps/chosen": -474.1544811320755,
|
|
"logps/rejected": -359.5655570652174,
|
|
"loss": 0.5863,
|
|
"loss/base_kto": 3.7010605335235596,
|
|
"metrics/advantage_var": 240.5254669189453,
|
|
"regularization/lipschitz_norm": 1004.1748046875,
|
|
"regularization/mmd": 0.14562593400478363,
|
|
"regularization/rkhs_norm": 173.36419677734375,
|
|
"regularization/w1": 0.8435068130493164,
|
|
"rewards/chosen": 0.20909660867175217,
|
|
"rewards/margins": 0.25939465973581105,
|
|
"rewards/rejected": -0.05029805106405886,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 19.455881118774414,
|
|
"kl": 14.824966430664062,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -35932914.85196374,
|
|
"logits/rejected": -36527412.19417476,
|
|
"logps/chosen": -482.98319486404836,
|
|
"logps/rejected": -364.8072411003236,
|
|
"loss": 0.5934,
|
|
"loss/base_kto": 3.7563083171844482,
|
|
"metrics/advantage_var": 229.74356079101562,
|
|
"regularization/lipschitz_norm": 1006.9215087890625,
|
|
"regularization/mmd": 0.1447356939315796,
|
|
"regularization/rkhs_norm": 172.30441284179688,
|
|
"regularization/w1": 0.845814049243927,
|
|
"rewards/chosen": 0.22214694757836104,
|
|
"rewards/margins": 0.22541619251619605,
|
|
"rewards/rejected": -0.003269244937835002,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 25.28292465209961,
|
|
"kl": 10.836579322814941,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36878989.26953748,
|
|
"logits/rejected": -37204030.72588055,
|
|
"logps/chosen": -482.2801036682616,
|
|
"logps/rejected": -372.3872511485452,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.7240941524505615,
|
|
"metrics/advantage_var": 252.6636199951172,
|
|
"regularization/lipschitz_norm": 1069.439697265625,
|
|
"regularization/mmd": 0.15283682942390442,
|
|
"regularization/rkhs_norm": 181.9486083984375,
|
|
"regularization/w1": 0.8983292579650879,
|
|
"rewards/chosen": 0.17491405432304127,
|
|
"rewards/margins": 0.2543630580479617,
|
|
"rewards/rejected": -0.07944900372492043,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 19.931053161621094,
|
|
"kl": 14.318656921386719,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37684136.60547504,
|
|
"logits/rejected": -37465844.73444613,
|
|
"logps/chosen": -504.9434380032206,
|
|
"logps/rejected": -389.2411086874052,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.7321908473968506,
|
|
"metrics/advantage_var": 237.4651336669922,
|
|
"regularization/lipschitz_norm": 1049.4945068359375,
|
|
"regularization/mmd": 0.1490609049797058,
|
|
"regularization/rkhs_norm": 177.45347595214844,
|
|
"regularization/w1": 0.8815754055976868,
|
|
"rewards/chosen": 0.1969018466230752,
|
|
"rewards/margins": 0.25105918474376293,
|
|
"rewards/rejected": -0.054157338120687716,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 20.67220115661621,
|
|
"kl": 6.832446575164795,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -35766081.028938904,
|
|
"logits/rejected": -36465218.91793313,
|
|
"logps/chosen": -480.4846262057878,
|
|
"logps/rejected": -383.38150645896656,
|
|
"loss": 0.6006,
|
|
"loss/base_kto": 3.8007652759552,
|
|
"metrics/advantage_var": 238.26353454589844,
|
|
"regularization/lipschitz_norm": 1017.7281494140625,
|
|
"regularization/mmd": 0.148813396692276,
|
|
"regularization/rkhs_norm": 177.15879821777344,
|
|
"regularization/w1": 0.8548915982246399,
|
|
"rewards/chosen": 0.048960977045286125,
|
|
"rewards/margins": 0.18022028073247742,
|
|
"rewards/rejected": -0.1312593036871913,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 19.877376556396484,
|
|
"kl": 10.88232135772705,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36232104.585365854,
|
|
"logits/rejected": -36577145.43589743,
|
|
"logps/chosen": -504.7980182926829,
|
|
"logps/rejected": -382.63426482371796,
|
|
"loss": 0.5954,
|
|
"loss/base_kto": 3.724571943283081,
|
|
"metrics/advantage_var": 240.63926696777344,
|
|
"regularization/lipschitz_norm": 1058.7271728515625,
|
|
"regularization/mmd": 0.14946918189525604,
|
|
"regularization/rkhs_norm": 177.9394989013672,
|
|
"regularization/w1": 0.8893308639526367,
|
|
"rewards/chosen": 0.2028577851086128,
|
|
"rewards/margins": 0.2583897881689781,
|
|
"rewards/rejected": -0.055532003060365334,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 17.519556045532227,
|
|
"kl": 10.411486625671387,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -35376066.03821656,
|
|
"logits/rejected": -34851588.711656444,
|
|
"logps/chosen": -471.26313694267515,
|
|
"logps/rejected": -366.3571223159509,
|
|
"loss": 0.5967,
|
|
"loss/base_kto": 3.739750623703003,
|
|
"metrics/advantage_var": 257.7501220703125,
|
|
"regularization/lipschitz_norm": 1049.66015625,
|
|
"regularization/mmd": 0.1522214263677597,
|
|
"regularization/rkhs_norm": 181.2159881591797,
|
|
"regularization/w1": 0.8817145228385925,
|
|
"rewards/chosen": 0.09120152710349697,
|
|
"rewards/margins": 0.22122545999927437,
|
|
"rewards/rejected": -0.1300239328957774,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 37.141357421875,
|
|
"kl": 11.514135360717773,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37315386.006042294,
|
|
"logits/rejected": -37213283.41747573,
|
|
"logps/chosen": -477.4136612537764,
|
|
"logps/rejected": -371.179636933657,
|
|
"loss": 0.6061,
|
|
"loss/base_kto": 3.7803478240966797,
|
|
"metrics/advantage_var": 266.41778564453125,
|
|
"regularization/lipschitz_norm": 1088.9759521484375,
|
|
"regularization/mmd": 0.1539926379919052,
|
|
"regularization/rkhs_norm": 183.32456970214844,
|
|
"regularization/w1": 0.9147397875785828,
|
|
"rewards/chosen": 0.11777314246601209,
|
|
"rewards/margins": 0.16899296537122233,
|
|
"rewards/rejected": -0.05121982290521023,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 25.138517379760742,
|
|
"kl": 20.77939796447754,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35795514.4913928,
|
|
"logits/rejected": -35336252.705148205,
|
|
"logps/chosen": -492.52963615023475,
|
|
"logps/rejected": -373.00672776911074,
|
|
"loss": 0.5921,
|
|
"loss/base_kto": 3.708894729614258,
|
|
"metrics/advantage_var": 245.8461151123047,
|
|
"regularization/lipschitz_norm": 1045.6116943359375,
|
|
"regularization/mmd": 0.1495220810174942,
|
|
"regularization/rkhs_norm": 178.00247192382812,
|
|
"regularization/w1": 0.8783137202262878,
|
|
"rewards/chosen": 0.3159759473726037,
|
|
"rewards/margins": 0.2526352321338867,
|
|
"rewards/rejected": 0.06334071523871697,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 50.69983673095703,
|
|
"kl": 17.803647994995117,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36249531.51592357,
|
|
"logits/rejected": -37110237.44785276,
|
|
"logps/chosen": -474.6914808917197,
|
|
"logps/rejected": -358.6874520705521,
|
|
"loss": 0.5949,
|
|
"loss/base_kto": 3.720240831375122,
|
|
"metrics/advantage_var": 270.0563659667969,
|
|
"regularization/lipschitz_norm": 1052.604736328125,
|
|
"regularization/mmd": 0.15498439967632294,
|
|
"regularization/rkhs_norm": 184.50523376464844,
|
|
"regularization/w1": 0.8841879963874817,
|
|
"rewards/chosen": 0.30120555610413763,
|
|
"rewards/margins": 0.23346862034232901,
|
|
"rewards/rejected": 0.06773693576180861,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 16.54012107849121,
|
|
"kl": 11.75640869140625,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35117618.32137285,
|
|
"logits/rejected": -36250923.668231614,
|
|
"logps/chosen": -508.0092628705148,
|
|
"logps/rejected": -371.91378129890455,
|
|
"loss": 0.5896,
|
|
"loss/base_kto": 3.7095627784729004,
|
|
"metrics/advantage_var": 245.6681365966797,
|
|
"regularization/lipschitz_norm": 1019.3429565429688,
|
|
"regularization/mmd": 0.15087106823921204,
|
|
"regularization/rkhs_norm": 179.6083984375,
|
|
"regularization/w1": 0.8562480807304382,
|
|
"rewards/chosen": 0.1789109918888943,
|
|
"rewards/margins": 0.2717039147908896,
|
|
"rewards/rejected": -0.09279292290199531,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 21.221527099609375,
|
|
"kl": 12.573948860168457,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36170574.677067086,
|
|
"logits/rejected": -37090292.74725275,
|
|
"logps/chosen": -488.9374024960998,
|
|
"logps/rejected": -362.7842425431711,
|
|
"loss": 0.6028,
|
|
"loss/base_kto": 3.579540967941284,
|
|
"metrics/advantage_var": 391.9313049316406,
|
|
"regularization/lipschitz_norm": 1264.6966552734375,
|
|
"regularization/mmd": 0.1801941841840744,
|
|
"regularization/rkhs_norm": 214.5168914794922,
|
|
"regularization/w1": 1.0623451471328735,
|
|
"rewards/chosen": 0.24875042181863055,
|
|
"rewards/margins": 0.4382941653414303,
|
|
"rewards/rejected": -0.18954374352279973,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 18.517812728881836,
|
|
"kl": 22.912307739257812,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -35347484.35692307,
|
|
"logits/rejected": -35902610.28571428,
|
|
"logps/chosen": -484.1655769230769,
|
|
"logps/rejected": -381.25637400793653,
|
|
"loss": 0.6263,
|
|
"loss/base_kto": 3.3201873302459717,
|
|
"metrics/advantage_var": 688.6425170898438,
|
|
"regularization/lipschitz_norm": 1724.913330078125,
|
|
"regularization/mmd": 0.24138009548187256,
|
|
"regularization/rkhs_norm": 287.35723876953125,
|
|
"regularization/w1": 1.4489272832870483,
|
|
"rewards/chosen": 0.6670431753305288,
|
|
"rewards/margins": 0.795169204693283,
|
|
"rewards/rejected": -0.12812602936275422,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 15.87170124053955,
|
|
"kl": 16.535335540771484,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35519548.04938272,
|
|
"logits/rejected": -36609273.51898734,
|
|
"logps/chosen": -495.86299189814815,
|
|
"logps/rejected": -398.7484671677215,
|
|
"loss": 0.6077,
|
|
"loss/base_kto": 3.2903213500976562,
|
|
"metrics/advantage_var": 619.0986938476562,
|
|
"regularization/lipschitz_norm": 1596.5906982421875,
|
|
"regularization/mmd": 0.22998562455177307,
|
|
"regularization/rkhs_norm": 273.79241943359375,
|
|
"regularization/w1": 1.3411362171173096,
|
|
"rewards/chosen": 0.5365299648708768,
|
|
"rewards/margins": 0.7547019175168667,
|
|
"rewards/rejected": -0.2181719526459899,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 23.145084381103516,
|
|
"kl": 17.9096736907959,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35584755.220611915,
|
|
"logits/rejected": -36302705.04400607,
|
|
"logps/chosen": -490.21864935587763,
|
|
"logps/rejected": -393.79267830045524,
|
|
"loss": 0.6058,
|
|
"loss/base_kto": 3.2760345935821533,
|
|
"metrics/advantage_var": 598.9376831054688,
|
|
"regularization/lipschitz_norm": 1604.2181396484375,
|
|
"regularization/mmd": 0.22320659458637238,
|
|
"regularization/rkhs_norm": 265.72216796875,
|
|
"regularization/w1": 1.3475432395935059,
|
|
"rewards/chosen": 0.5825202192469303,
|
|
"rewards/margins": 0.7996018709490239,
|
|
"rewards/rejected": -0.21708165170209362,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 16.735595703125,
|
|
"kl": 5.670738697052002,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35206040.7808,
|
|
"logits/rejected": -36138748.48244275,
|
|
"logps/chosen": -488.7985,
|
|
"logps/rejected": -390.2844704198473,
|
|
"loss": 0.6209,
|
|
"loss/base_kto": 3.335326910018921,
|
|
"metrics/advantage_var": 616.02587890625,
|
|
"regularization/lipschitz_norm": 1667.9228515625,
|
|
"regularization/mmd": 0.23069031536579132,
|
|
"regularization/rkhs_norm": 274.6313171386719,
|
|
"regularization/w1": 1.4010552167892456,
|
|
"rewards/chosen": 0.1739558837890625,
|
|
"rewards/margins": 0.7510106719271827,
|
|
"rewards/rejected": -0.5770547881381203,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 19.573148727416992,
|
|
"kl": 11.090420722961426,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35530803.360501565,
|
|
"logits/rejected": -36086717.00934579,
|
|
"logps/chosen": -470.6704055642633,
|
|
"logps/rejected": -362.2284852024922,
|
|
"loss": 0.6109,
|
|
"loss/base_kto": 3.280815839767456,
|
|
"metrics/advantage_var": 622.8731689453125,
|
|
"regularization/lipschitz_norm": 1638.9910888671875,
|
|
"regularization/mmd": 0.2300119698047638,
|
|
"regularization/rkhs_norm": 273.8237609863281,
|
|
"regularization/w1": 1.376752495765686,
|
|
"rewards/chosen": 0.47481845389339244,
|
|
"rewards/margins": 0.7902909230449695,
|
|
"rewards/rejected": -0.3154724691515771,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 18.900663375854492,
|
|
"kl": 10.095346450805664,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -34952144.73846154,
|
|
"logits/rejected": -35983451.02222222,
|
|
"logps/chosen": -475.2823076923077,
|
|
"logps/rejected": -384.0702876984127,
|
|
"loss": 0.6132,
|
|
"loss/base_kto": 3.3758606910705566,
|
|
"metrics/advantage_var": 604.9810791015625,
|
|
"regularization/lipschitz_norm": 1555.710205078125,
|
|
"regularization/mmd": 0.2226063311100006,
|
|
"regularization/rkhs_norm": 265.0075378417969,
|
|
"regularization/w1": 1.3067964315414429,
|
|
"rewards/chosen": 0.2885575045072115,
|
|
"rewards/margins": 0.6789511037600637,
|
|
"rewards/rejected": -0.39039359925285216,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 14.313115119934082,
|
|
"kl": 19.707983016967773,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35337982.033794165,
|
|
"logits/rejected": -36343599.61844197,
|
|
"logps/chosen": -487.78619431643625,
|
|
"logps/rejected": -354.9862380763116,
|
|
"loss": 0.5923,
|
|
"loss/base_kto": 3.183997631072998,
|
|
"metrics/advantage_var": 639.8392944335938,
|
|
"regularization/lipschitz_norm": 1578.1241455078125,
|
|
"regularization/mmd": 0.2291080504655838,
|
|
"regularization/rkhs_norm": 272.7476501464844,
|
|
"regularization/w1": 1.3256242275238037,
|
|
"rewards/chosen": 0.6527138049335157,
|
|
"rewards/margins": 0.8894780062327009,
|
|
"rewards/rejected": -0.2367642012991852,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 18.604690551757812,
|
|
"kl": 12.943008422851562,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -34232329.767885536,
|
|
"logits/rejected": -34887053.960061446,
|
|
"logps/chosen": -473.987480127186,
|
|
"logps/rejected": -348.35186251920123,
|
|
"loss": 0.599,
|
|
"loss/base_kto": 3.257160186767578,
|
|
"metrics/advantage_var": 590.5481567382812,
|
|
"regularization/lipschitz_norm": 1565.31982421875,
|
|
"regularization/mmd": 0.2202320098876953,
|
|
"regularization/rkhs_norm": 262.1809997558594,
|
|
"regularization/w1": 1.3148685693740845,
|
|
"rewards/chosen": 0.4355853009489269,
|
|
"rewards/margins": 0.8222324213646335,
|
|
"rewards/rejected": -0.3866471204157066,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 14.078204154968262,
|
|
"kl": 12.49372386932373,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -34542604.8,
|
|
"logits/rejected": -35873574.4,
|
|
"logps/chosen": -435.915283203125,
|
|
"logps/rejected": -376.5451171875,
|
|
"loss": 0.591,
|
|
"loss/base_kto": 3.2920472621917725,
|
|
"metrics/advantage_var": 531.306884765625,
|
|
"regularization/lipschitz_norm": 1460.0257568359375,
|
|
"regularization/mmd": 0.20950733125209808,
|
|
"regularization/rkhs_norm": 249.41348266601562,
|
|
"regularization/w1": 1.226421594619751,
|
|
"rewards/chosen": 0.36907153129577636,
|
|
"rewards/margins": 0.7647289276123046,
|
|
"rewards/rejected": -0.3956573963165283,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 19.92791748046875,
|
|
"kl": 14.54132080078125,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -33633025.23274478,
|
|
"logits/rejected": -36365460.06697108,
|
|
"logps/chosen": -512.2743780096308,
|
|
"logps/rejected": -389.1756326103501,
|
|
"loss": 0.6156,
|
|
"loss/base_kto": 3.275949478149414,
|
|
"metrics/advantage_var": 649.2162475585938,
|
|
"regularization/lipschitz_norm": 1685.2899169921875,
|
|
"regularization/mmd": 0.232918381690979,
|
|
"regularization/rkhs_norm": 277.2838134765625,
|
|
"regularization/w1": 1.415643572807312,
|
|
"rewards/chosen": 0.47823205154933285,
|
|
"rewards/margins": 0.7808198306945955,
|
|
"rewards/rejected": -0.3025877791452626,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 34.92301940917969,
|
|
"kl": 12.782320976257324,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35761696.405063294,
|
|
"logits/rejected": -34188525.03703704,
|
|
"logps/chosen": -468.26399327531647,
|
|
"logps/rejected": -374.8835358796296,
|
|
"loss": 0.6063,
|
|
"loss/base_kto": 3.275601625442505,
|
|
"metrics/advantage_var": 801.8453979492188,
|
|
"regularization/lipschitz_norm": 1597.693115234375,
|
|
"regularization/mmd": 0.23303942382335663,
|
|
"regularization/rkhs_norm": 277.4279479980469,
|
|
"regularization/w1": 1.34206223487854,
|
|
"rewards/chosen": 0.5217841908901553,
|
|
"rewards/margins": 0.8165568688415441,
|
|
"rewards/rejected": -0.2947726779513889,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 20.192045211791992,
|
|
"kl": 15.831377029418945,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -34121421.26475038,
|
|
"logits/rejected": -36387265.9644588,
|
|
"logps/chosen": -472.9985344175492,
|
|
"logps/rejected": -377.53367326332796,
|
|
"loss": 0.612,
|
|
"loss/base_kto": 3.369748830795288,
|
|
"metrics/advantage_var": 577.3466186523438,
|
|
"regularization/lipschitz_norm": 1555.6634521484375,
|
|
"regularization/mmd": 0.21918420493602753,
|
|
"regularization/rkhs_norm": 260.9335632324219,
|
|
"regularization/w1": 1.3067572116851807,
|
|
"rewards/chosen": 0.438801683203716,
|
|
"rewards/margins": 0.6660324885838023,
|
|
"rewards/rejected": -0.22723080538008633,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 20.490093231201172,
|
|
"kl": 24.298540115356445,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34470267.25925926,
|
|
"logits/rejected": -36366078.73057851,
|
|
"logps/chosen": -482.6902777777778,
|
|
"logps/rejected": -360.40996900826445,
|
|
"loss": 0.5991,
|
|
"loss/base_kto": 3.32615065574646,
|
|
"metrics/advantage_var": 559.3719482421875,
|
|
"regularization/lipschitz_norm": 1490.1746826171875,
|
|
"regularization/mmd": 0.21459384262561798,
|
|
"regularization/rkhs_norm": 255.46885681152344,
|
|
"regularization/w1": 1.251746654510498,
|
|
"rewards/chosen": 0.6812146448206019,
|
|
"rewards/margins": 0.7386676300954922,
|
|
"rewards/rejected": -0.05745298527489024,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 13.66446590423584,
|
|
"kl": 22.269344329833984,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -35219875.544764794,
|
|
"logits/rejected": -36463080.09017713,
|
|
"logps/chosen": -484.26242412746586,
|
|
"logps/rejected": -381.37097423510465,
|
|
"loss": 0.6116,
|
|
"loss/base_kto": 3.2787845134735107,
|
|
"metrics/advantage_var": 724.5116577148438,
|
|
"regularization/lipschitz_norm": 1644.1292724609375,
|
|
"regularization/mmd": 0.2325955480337143,
|
|
"regularization/rkhs_norm": 276.8994445800781,
|
|
"regularization/w1": 1.3810687065124512,
|
|
"rewards/chosen": 0.6422472940772003,
|
|
"rewards/margins": 0.7504795311758818,
|
|
"rewards/rejected": -0.10823223709868156,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 17.01795196533203,
|
|
"kl": 13.111950874328613,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34974592.58986175,
|
|
"logits/rejected": -36928982.48648649,
|
|
"logps/chosen": -498.9107142857143,
|
|
"logps/rejected": -368.5732064785374,
|
|
"loss": 0.6137,
|
|
"loss/base_kto": 3.329371690750122,
|
|
"metrics/advantage_var": 623.7325439453125,
|
|
"regularization/lipschitz_norm": 1612.164306640625,
|
|
"regularization/mmd": 0.22597865760326385,
|
|
"regularization/rkhs_norm": 269.022216796875,
|
|
"regularization/w1": 1.3542180061340332,
|
|
"rewards/chosen": 0.4364518547937068,
|
|
"rewards/margins": 0.7611252633383958,
|
|
"rewards/rejected": -0.324673408544689,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 19.452707290649414,
|
|
"kl": 14.179718017578125,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -34936724.54320987,
|
|
"logits/rejected": -35825404.75949367,
|
|
"logps/chosen": -478.56968557098764,
|
|
"logps/rejected": -357.82345233386076,
|
|
"loss": 0.5943,
|
|
"loss/base_kto": 3.280343770980835,
|
|
"metrics/advantage_var": 581.9765014648438,
|
|
"regularization/lipschitz_norm": 1497.4761962890625,
|
|
"regularization/mmd": 0.21621882915496826,
|
|
"regularization/rkhs_norm": 257.40338134765625,
|
|
"regularization/w1": 1.2578800916671753,
|
|
"rewards/chosen": 0.5376853942871094,
|
|
"rewards/margins": 0.8094757176652739,
|
|
"rewards/rejected": -0.27179032337816456,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 20.629621505737305,
|
|
"kl": 12.300616264343262,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35334603.75510204,
|
|
"logits/rejected": -36255396.827371694,
|
|
"logps/chosen": -471.71085164835165,
|
|
"logps/rejected": -370.98532270606535,
|
|
"loss": 0.6058,
|
|
"loss/base_kto": 3.260944366455078,
|
|
"metrics/advantage_var": 635.8053588867188,
|
|
"regularization/lipschitz_norm": 1616.2861328125,
|
|
"regularization/mmd": 0.22804021835327148,
|
|
"regularization/rkhs_norm": 271.4764709472656,
|
|
"regularization/w1": 1.357680320739746,
|
|
"rewards/chosen": 0.4329929321863962,
|
|
"rewards/margins": 0.792609865863678,
|
|
"rewards/rejected": -0.35961693367728176,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 17.78253746032715,
|
|
"kl": 16.477476119995117,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34614334.49765258,
|
|
"logits/rejected": -36624674.74570983,
|
|
"logps/chosen": -498.6641236306729,
|
|
"logps/rejected": -393.81108619344775,
|
|
"loss": 0.6232,
|
|
"loss/base_kto": 3.346104145050049,
|
|
"metrics/advantage_var": 653.1791381835938,
|
|
"regularization/lipschitz_norm": 1673.0789794921875,
|
|
"regularization/mmd": 0.23400922119617462,
|
|
"regularization/rkhs_norm": 278.5823974609375,
|
|
"regularization/w1": 1.4053863286972046,
|
|
"rewards/chosen": 0.4271581228909918,
|
|
"rewards/margins": 0.6889516433378644,
|
|
"rewards/rejected": -0.26179352044687254,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 22.57742691040039,
|
|
"kl": 22.35113525390625,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -32957121.56097561,
|
|
"logits/rejected": -36542132.51282051,
|
|
"logps/chosen": -473.2158441310976,
|
|
"logps/rejected": -367.58969350961536,
|
|
"loss": 0.5908,
|
|
"loss/base_kto": 3.2424709796905518,
|
|
"metrics/advantage_var": 522.5794067382812,
|
|
"regularization/lipschitz_norm": 1508.7386474609375,
|
|
"regularization/mmd": 0.216816708445549,
|
|
"regularization/rkhs_norm": 258.11517333984375,
|
|
"regularization/w1": 1.2673405408859253,
|
|
"rewards/chosen": 0.7005406356439358,
|
|
"rewards/margins": 0.8212893633934913,
|
|
"rewards/rejected": -0.12074872774955554,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 24.7183895111084,
|
|
"kl": 19.532506942749023,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35172556.16199377,
|
|
"logits/rejected": -35466846.69592477,
|
|
"logps/chosen": -463.80909267912773,
|
|
"logps/rejected": -370.18005485893417,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.305455446243286,
|
|
"metrics/advantage_var": 577.9130859375,
|
|
"regularization/lipschitz_norm": 1493.23974609375,
|
|
"regularization/mmd": 0.22011370956897736,
|
|
"regularization/rkhs_norm": 262.0401306152344,
|
|
"regularization/w1": 1.2543214559555054,
|
|
"rewards/chosen": 0.5439251100534219,
|
|
"rewards/margins": 0.7324913837098066,
|
|
"rewards/rejected": -0.1885662736563847,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 14.92409896850586,
|
|
"kl": 13.285479545593262,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35531352.79092383,
|
|
"logits/rejected": -36195709.49019608,
|
|
"logps/chosen": -472.19403363047,
|
|
"logps/rejected": -351.37101715686276,
|
|
"loss": 0.5986,
|
|
"loss/base_kto": 3.337872266769409,
|
|
"metrics/advantage_var": 564.6458129882812,
|
|
"regularization/lipschitz_norm": 1476.8944091796875,
|
|
"regularization/mmd": 0.21070383489131927,
|
|
"regularization/rkhs_norm": 250.837890625,
|
|
"regularization/w1": 1.240591287612915,
|
|
"rewards/chosen": 0.44879150390625,
|
|
"rewards/margins": 0.717865450529671,
|
|
"rewards/rejected": -0.269073946623421,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 14.848271369934082,
|
|
"kl": 13.913154602050781,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35476883.0984127,
|
|
"logits/rejected": -36927629.78461538,
|
|
"logps/chosen": -499.249503968254,
|
|
"logps/rejected": -385.41865384615386,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.3059029579162598,
|
|
"metrics/advantage_var": 570.443603515625,
|
|
"regularization/lipschitz_norm": 1541.2706298828125,
|
|
"regularization/mmd": 0.22412748634815216,
|
|
"regularization/rkhs_norm": 266.81842041015625,
|
|
"regularization/w1": 1.2946674823760986,
|
|
"rewards/chosen": 0.44281790597098214,
|
|
"rewards/margins": 0.758338113302713,
|
|
"rewards/rejected": -0.31552020733173075,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 18.774168014526367,
|
|
"kl": 13.343116760253906,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35339089.986928105,
|
|
"logits/rejected": -36073269.65269461,
|
|
"logps/chosen": -486.8254187091503,
|
|
"logps/rejected": -396.15269461077844,
|
|
"loss": 0.6162,
|
|
"loss/base_kto": 3.307917356491089,
|
|
"metrics/advantage_var": 598.0717163085938,
|
|
"regularization/lipschitz_norm": 1659.9559326171875,
|
|
"regularization/mmd": 0.22696201503276825,
|
|
"regularization/rkhs_norm": 270.19287109375,
|
|
"regularization/w1": 1.3943630456924438,
|
|
"rewards/chosen": 0.36707116419972935,
|
|
"rewards/margins": 0.7442218311275923,
|
|
"rewards/rejected": -0.37715066692786303,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 16.591535568237305,
|
|
"kl": 15.358591079711914,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34459275.63636363,
|
|
"logits/rejected": -35436834.29283489,
|
|
"logps/chosen": -496.42158111285266,
|
|
"logps/rejected": -374.64193925233644,
|
|
"loss": 0.6077,
|
|
"loss/base_kto": 3.287248373031616,
|
|
"metrics/advantage_var": 660.1201171875,
|
|
"regularization/lipschitz_norm": 1603.1197509765625,
|
|
"regularization/mmd": 0.22733287513256073,
|
|
"regularization/rkhs_norm": 270.6343688964844,
|
|
"regularization/w1": 1.3466205596923828,
|
|
"rewards/chosen": 0.5073473700161638,
|
|
"rewards/margins": 0.7654089780662636,
|
|
"rewards/rejected": -0.2580616080500998,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 21.79800033569336,
|
|
"kl": 8.014674186706543,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -34720075.15227629,
|
|
"logits/rejected": -35828568.783825815,
|
|
"logps/chosen": -499.52266483516485,
|
|
"logps/rejected": -373.08040921461895,
|
|
"loss": 0.6157,
|
|
"loss/base_kto": 3.312753677368164,
|
|
"metrics/advantage_var": 660.47265625,
|
|
"regularization/lipschitz_norm": 1642.948486328125,
|
|
"regularization/mmd": 0.23258309066295624,
|
|
"regularization/rkhs_norm": 276.8846130371094,
|
|
"regularization/w1": 1.3800767660140991,
|
|
"rewards/chosen": 0.37633331155103267,
|
|
"rewards/margins": 0.7964948183581138,
|
|
"rewards/rejected": -0.4201615068070811,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 21.787792205810547,
|
|
"kl": 11.092989921569824,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -36956741.530864194,
|
|
"logits/rejected": -36305531.1392405,
|
|
"logps/chosen": -481.8137538580247,
|
|
"logps/rejected": -380.1670292721519,
|
|
"loss": 0.6044,
|
|
"loss/base_kto": 3.354888916015625,
|
|
"metrics/advantage_var": 610.9271850585938,
|
|
"regularization/lipschitz_norm": 1505.3267822265625,
|
|
"regularization/mmd": 0.21620440483093262,
|
|
"regularization/rkhs_norm": 257.3861999511719,
|
|
"regularization/w1": 1.2644745111465454,
|
|
"rewards/chosen": 0.40942858472282506,
|
|
"rewards/margins": 0.673559261571804,
|
|
"rewards/rejected": -0.26413067684897895,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 15.7304105758667,
|
|
"kl": 11.27443790435791,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -35081312.0,
|
|
"logits/rejected": -35930259.2,
|
|
"logps/chosen": -509.440673828125,
|
|
"logps/rejected": -362.72119140625,
|
|
"loss": 0.6024,
|
|
"loss/base_kto": 3.330542802810669,
|
|
"metrics/advantage_var": 518.4404296875,
|
|
"regularization/lipschitz_norm": 1518.8363037109375,
|
|
"regularization/mmd": 0.2130681574344635,
|
|
"regularization/rkhs_norm": 253.652587890625,
|
|
"regularization/w1": 1.2758225202560425,
|
|
"rewards/chosen": 0.40545225143432617,
|
|
"rewards/margins": 0.6953860759735108,
|
|
"rewards/rejected": -0.2899338245391846,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 21.205015182495117,
|
|
"kl": 8.930121421813965,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -33595198.22512235,
|
|
"logits/rejected": -34796834.15892054,
|
|
"logps/chosen": -465.2855831973899,
|
|
"logps/rejected": -372.65877998500747,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.3770835399627686,
|
|
"metrics/advantage_var": 492.9637145996094,
|
|
"regularization/lipschitz_norm": 1480.6346435546875,
|
|
"regularization/mmd": 0.2036602795124054,
|
|
"regularization/rkhs_norm": 242.4527130126953,
|
|
"regularization/w1": 1.2437330484390259,
|
|
"rewards/chosen": 0.241196973094349,
|
|
"rewards/margins": 0.6486426948135675,
|
|
"rewards/rejected": -0.4074457217192185,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 17.333303451538086,
|
|
"kl": 10.10449504852295,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34304521.54037267,
|
|
"logits/rejected": -35787589.2327044,
|
|
"logps/chosen": -462.81317934782606,
|
|
"logps/rejected": -408.51533018867923,
|
|
"loss": 0.6066,
|
|
"loss/base_kto": 3.3171639442443848,
|
|
"metrics/advantage_var": 561.0217895507812,
|
|
"regularization/lipschitz_norm": 1567.8363037109375,
|
|
"regularization/mmd": 0.21870306134223938,
|
|
"regularization/rkhs_norm": 260.3608093261719,
|
|
"regularization/w1": 1.316982626914978,
|
|
"rewards/chosen": 0.3505211587277999,
|
|
"rewards/margins": 0.7447365805843742,
|
|
"rewards/rejected": -0.39421542185657427,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 19.61787986755371,
|
|
"kl": 11.242029190063477,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35242940.39291465,
|
|
"logits/rejected": -36643089.481031865,
|
|
"logps/chosen": -485.9791163446055,
|
|
"logps/rejected": -394.5627371016692,
|
|
"loss": 0.6038,
|
|
"loss/base_kto": 3.283898115158081,
|
|
"metrics/advantage_var": 582.4346923828125,
|
|
"regularization/lipschitz_norm": 1574.2464599609375,
|
|
"regularization/mmd": 0.22452466189861298,
|
|
"regularization/rkhs_norm": 267.291259765625,
|
|
"regularization/w1": 1.3223670721054077,
|
|
"rewards/chosen": 0.35564372236218045,
|
|
"rewards/margins": 0.7960194136617347,
|
|
"rewards/rejected": -0.44037569129955423,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 15.509449005126953,
|
|
"kl": 5.774592876434326,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34188235.11737089,
|
|
"logits/rejected": -34408949.61622465,
|
|
"logps/chosen": -467.61551251956183,
|
|
"logps/rejected": -364.19283833853353,
|
|
"loss": 0.6066,
|
|
"loss/base_kto": 3.3596699237823486,
|
|
"metrics/advantage_var": 553.5308227539062,
|
|
"regularization/lipschitz_norm": 1517.386474609375,
|
|
"regularization/mmd": 0.21879544854164124,
|
|
"regularization/rkhs_norm": 260.47076416015625,
|
|
"regularization/w1": 1.2746046781539917,
|
|
"rewards/chosen": 0.2672982611380086,
|
|
"rewards/margins": 0.7456527761779852,
|
|
"rewards/rejected": -0.4783545150399766,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 19.400392532348633,
|
|
"kl": 8.308951377868652,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -34242840.77419355,
|
|
"logits/rejected": -35609572.07272727,
|
|
"logps/chosen": -487.59637096774196,
|
|
"logps/rejected": -374.04543087121215,
|
|
"loss": 0.606,
|
|
"loss/base_kto": 3.2787225246429443,
|
|
"metrics/advantage_var": 566.2044677734375,
|
|
"regularization/lipschitz_norm": 1603.628173828125,
|
|
"regularization/mmd": 0.22200237214565277,
|
|
"regularization/rkhs_norm": 264.2885437011719,
|
|
"regularization/w1": 1.3470476865768433,
|
|
"rewards/chosen": 0.362433107437626,
|
|
"rewards/margins": 0.7876839989208173,
|
|
"rewards/rejected": -0.4252508914831913,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 17.38187026977539,
|
|
"kl": 7.223329067230225,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35521924.62650602,
|
|
"logits/rejected": -34677154.90909091,
|
|
"logps/chosen": -489.94126506024094,
|
|
"logps/rejected": -386.2927150974026,
|
|
"loss": 0.6039,
|
|
"loss/base_kto": 3.33835768699646,
|
|
"metrics/advantage_var": 510.293212890625,
|
|
"regularization/lipschitz_norm": 1523.18359375,
|
|
"regularization/mmd": 0.21299903094768524,
|
|
"regularization/rkhs_norm": 253.5702667236328,
|
|
"regularization/w1": 1.279474139213562,
|
|
"rewards/chosen": 0.3077183459178511,
|
|
"rewards/margins": 0.7415810568458391,
|
|
"rewards/rejected": -0.433862710927988,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 13.490575790405273,
|
|
"kl": 13.166955947875977,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34149653.23317684,
|
|
"logits/rejected": -35491149.878315136,
|
|
"logps/chosen": -485.71273474178406,
|
|
"logps/rejected": -367.64489079563185,
|
|
"loss": 0.5847,
|
|
"loss/base_kto": 3.3421249389648438,
|
|
"metrics/advantage_var": 476.4816589355469,
|
|
"regularization/lipschitz_norm": 1352.8961181640625,
|
|
"regularization/mmd": 0.1990450620651245,
|
|
"regularization/rkhs_norm": 236.95840454101562,
|
|
"regularization/w1": 1.1364328861236572,
|
|
"rewards/chosen": 0.4030161874021909,
|
|
"rewards/margins": 0.730635266896816,
|
|
"rewards/rejected": -0.3276190794946251,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 20.736454010009766,
|
|
"kl": 10.202261924743652,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35675823.85410334,
|
|
"logits/rejected": -35379628.03858521,
|
|
"logps/chosen": -480.85020896656533,
|
|
"logps/rejected": -378.16780546623795,
|
|
"loss": 0.6031,
|
|
"loss/base_kto": 3.305757522583008,
|
|
"metrics/advantage_var": 567.9496459960938,
|
|
"regularization/lipschitz_norm": 1546.3131103515625,
|
|
"regularization/mmd": 0.22012697160243988,
|
|
"regularization/rkhs_norm": 262.055908203125,
|
|
"regularization/w1": 1.298902988433838,
|
|
"rewards/chosen": 0.39345200373409006,
|
|
"rewards/margins": 0.7709371256787997,
|
|
"rewards/rejected": -0.3774851219447096,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 16.839126586914062,
|
|
"kl": 11.110552787780762,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35211466.013605446,
|
|
"logits/rejected": -35807820.94797688,
|
|
"logps/chosen": -465.5361394557823,
|
|
"logps/rejected": -378.2607026734104,
|
|
"loss": 0.5933,
|
|
"loss/base_kto": 3.285698413848877,
|
|
"metrics/advantage_var": 516.9336547851562,
|
|
"regularization/lipschitz_norm": 1489.0986328125,
|
|
"regularization/mmd": 0.2096785306930542,
|
|
"regularization/rkhs_norm": 249.6172637939453,
|
|
"regularization/w1": 1.2508429288864136,
|
|
"rewards/chosen": 0.3581001904545998,
|
|
"rewards/margins": 0.7581950771902843,
|
|
"rewards/rejected": -0.4000948867356846,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 16.851472854614258,
|
|
"kl": 6.301633358001709,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -34655394.34146342,
|
|
"logits/rejected": -35765563.07692308,
|
|
"logps/chosen": -497.6768769054878,
|
|
"logps/rejected": -359.32166466346155,
|
|
"loss": 0.6056,
|
|
"loss/base_kto": 3.331233263015747,
|
|
"metrics/advantage_var": 577.916259765625,
|
|
"regularization/lipschitz_norm": 1545.646728515625,
|
|
"regularization/mmd": 0.21512162685394287,
|
|
"regularization/rkhs_norm": 256.09716796875,
|
|
"regularization/w1": 1.298343300819397,
|
|
"rewards/chosen": 0.2812550707561214,
|
|
"rewards/margins": 0.7595724293110593,
|
|
"rewards/rejected": -0.4783173585549379,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 18.61896514892578,
|
|
"kl": 8.949078559875488,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34263785.27810651,
|
|
"logits/rejected": -35954813.456953645,
|
|
"logps/chosen": -486.1563424556213,
|
|
"logps/rejected": -371.0991825331126,
|
|
"loss": 0.6065,
|
|
"loss/base_kto": 3.3602044582366943,
|
|
"metrics/advantage_var": 523.9320068359375,
|
|
"regularization/lipschitz_norm": 1520.69921875,
|
|
"regularization/mmd": 0.21460656821727753,
|
|
"regularization/rkhs_norm": 255.48399353027344,
|
|
"regularization/w1": 1.2773874998092651,
|
|
"rewards/chosen": 0.3820487930929872,
|
|
"rewards/margins": 0.7061543400569126,
|
|
"rewards/rejected": -0.3241055469639254,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 15.663430213928223,
|
|
"kl": 13.345422744750977,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34523730.36152571,
|
|
"logits/rejected": -36343477.285925925,
|
|
"logps/chosen": -481.4479166666667,
|
|
"logps/rejected": -361.3463888888889,
|
|
"loss": 0.5615,
|
|
"loss/base_kto": 3.151230573654175,
|
|
"metrics/advantage_var": 496.30792236328125,
|
|
"regularization/lipschitz_norm": 1353.939697265625,
|
|
"regularization/mmd": 0.2035330832004547,
|
|
"regularization/rkhs_norm": 242.30126953125,
|
|
"regularization/w1": 1.1373093128204346,
|
|
"rewards/chosen": 0.5420287276184184,
|
|
"rewards/margins": 0.9054012138742054,
|
|
"rewards/rejected": -0.36337248625578705,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 16.60082244873047,
|
|
"kl": 4.14560604095459,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -34222845.11433172,
|
|
"logits/rejected": -35006299.289833084,
|
|
"logps/chosen": -467.01056763285027,
|
|
"logps/rejected": -376.38766122913506,
|
|
"loss": 0.5649,
|
|
"loss/base_kto": 3.2409398555755615,
|
|
"metrics/advantage_var": 410.4333190917969,
|
|
"regularization/lipschitz_norm": 1293.7279052734375,
|
|
"regularization/mmd": 0.19129277765750885,
|
|
"regularization/rkhs_norm": 227.7294921875,
|
|
"regularization/w1": 1.0867313146591187,
|
|
"rewards/chosen": 0.2668771052706069,
|
|
"rewards/margins": 0.8168642499144803,
|
|
"rewards/rejected": -0.5499871446438733,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 20.203248977661133,
|
|
"kl": 7.194047451019287,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -32956076.840764333,
|
|
"logits/rejected": -35525902.134969324,
|
|
"logps/chosen": -459.06797372611464,
|
|
"logps/rejected": -380.4706192484663,
|
|
"loss": 0.5599,
|
|
"loss/base_kto": 3.150766372680664,
|
|
"metrics/advantage_var": 444.0229187011719,
|
|
"regularization/lipschitz_norm": 1342.6407470703125,
|
|
"regularization/mmd": 0.20039306581020355,
|
|
"regularization/rkhs_norm": 238.5631866455078,
|
|
"regularization/w1": 1.1278181076049805,
|
|
"rewards/chosen": 0.3972078797164237,
|
|
"rewards/margins": 0.9025857135450279,
|
|
"rewards/rejected": -0.5053778338286042,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 19.274572372436523,
|
|
"kl": 9.700411796569824,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -34070280.718266256,
|
|
"logits/rejected": -36092401.4637224,
|
|
"logps/chosen": -484.202883126935,
|
|
"logps/rejected": -378.37867212145113,
|
|
"loss": 0.5628,
|
|
"loss/base_kto": 3.1901321411132812,
|
|
"metrics/advantage_var": 441.826171875,
|
|
"regularization/lipschitz_norm": 1326.1185302734375,
|
|
"regularization/mmd": 0.19808267056941986,
|
|
"regularization/rkhs_norm": 235.81272888183594,
|
|
"regularization/w1": 1.1139395236968994,
|
|
"rewards/chosen": 0.41013607816430436,
|
|
"rewards/margins": 0.8686482489980485,
|
|
"rewards/rejected": -0.45851217083374407,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 17.713275909423828,
|
|
"kl": 16.2282657623291,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33278884.450331125,
|
|
"logits/rejected": -34799952.28402367,
|
|
"logps/chosen": -450.5597061258278,
|
|
"logps/rejected": -373.2557784763314,
|
|
"loss": 0.5504,
|
|
"loss/base_kto": 3.0897510051727295,
|
|
"metrics/advantage_var": 430.9189147949219,
|
|
"regularization/lipschitz_norm": 1330.6058349609375,
|
|
"regularization/mmd": 0.1960301399230957,
|
|
"regularization/rkhs_norm": 233.36924743652344,
|
|
"regularization/w1": 1.1177090406417847,
|
|
"rewards/chosen": 0.5879785398773799,
|
|
"rewards/margins": 0.9322853051007758,
|
|
"rewards/rejected": -0.34430676522339587,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 16.43911361694336,
|
|
"kl": 10.072003364562988,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -34154459.537091985,
|
|
"logits/rejected": -35742905.87458746,
|
|
"logps/chosen": -467.65968100890206,
|
|
"logps/rejected": -372.0141295379538,
|
|
"loss": 0.5513,
|
|
"loss/base_kto": 3.162606954574585,
|
|
"metrics/advantage_var": 420.5672912597656,
|
|
"regularization/lipschitz_norm": 1258.2938232421875,
|
|
"regularization/mmd": 0.19048896431922913,
|
|
"regularization/rkhs_norm": 226.7725830078125,
|
|
"regularization/w1": 1.056966781616211,
|
|
"rewards/chosen": 0.49096303877561664,
|
|
"rewards/margins": 0.8975468940300386,
|
|
"rewards/rejected": -0.4065838552544219,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 12.407207489013672,
|
|
"kl": 16.136207580566406,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -33702826.15568862,
|
|
"logits/rejected": -34882456.26143791,
|
|
"logps/chosen": -469.79098053892216,
|
|
"logps/rejected": -356.289164624183,
|
|
"loss": 0.5506,
|
|
"loss/base_kto": 3.1579740047454834,
|
|
"metrics/advantage_var": 393.1698913574219,
|
|
"regularization/lipschitz_norm": 1260.581298828125,
|
|
"regularization/mmd": 0.1881152242422104,
|
|
"regularization/rkhs_norm": 223.94668579101562,
|
|
"regularization/w1": 1.05888831615448,
|
|
"rewards/chosen": 0.5765886592293927,
|
|
"rewards/margins": 0.8646527058522737,
|
|
"rewards/rejected": -0.28806404662288093,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 17.052236557006836,
|
|
"kl": 20.198055267333984,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34386622.71406492,
|
|
"logits/rejected": -34062801.8957346,
|
|
"logps/chosen": -474.3678516228748,
|
|
"logps/rejected": -367.033150671406,
|
|
"loss": 0.5576,
|
|
"loss/base_kto": 3.1356256008148193,
|
|
"metrics/advantage_var": 421.7980651855469,
|
|
"regularization/lipschitz_norm": 1345.038818359375,
|
|
"regularization/mmd": 0.19535815715789795,
|
|
"regularization/rkhs_norm": 232.56924438476562,
|
|
"regularization/w1": 1.1298326253890991,
|
|
"rewards/chosen": 0.6413323285959235,
|
|
"rewards/margins": 0.8828754882615781,
|
|
"rewards/rejected": -0.24154315966565462,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 12.275116920471191,
|
|
"kl": 20.29020118713379,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34066009.396825396,
|
|
"logits/rejected": -36403187.39692308,
|
|
"logps/chosen": -470.1501488095238,
|
|
"logps/rejected": -351.7273076923077,
|
|
"loss": 0.5482,
|
|
"loss/base_kto": 3.1216049194335938,
|
|
"metrics/advantage_var": 432.7414245605469,
|
|
"regularization/lipschitz_norm": 1274.5933837890625,
|
|
"regularization/mmd": 0.19359762966632843,
|
|
"regularization/rkhs_norm": 230.473388671875,
|
|
"regularization/w1": 1.0706584453582764,
|
|
"rewards/chosen": 0.6454751635354663,
|
|
"rewards/margins": 0.9016580483386514,
|
|
"rewards/rejected": -0.2561828848031851,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 17.391368865966797,
|
|
"kl": 15.871638298034668,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35591220.9118541,
|
|
"logits/rejected": -35558969.62057878,
|
|
"logps/chosen": -510.8705357142857,
|
|
"logps/rejected": -366.8060942524116,
|
|
"loss": 0.5625,
|
|
"loss/base_kto": 3.204577684402466,
|
|
"metrics/advantage_var": 406.9927673339844,
|
|
"regularization/lipschitz_norm": 1313.569091796875,
|
|
"regularization/mmd": 0.19188784062862396,
|
|
"regularization/rkhs_norm": 228.43789672851562,
|
|
"regularization/w1": 1.1033979654312134,
|
|
"rewards/chosen": 0.5263376902664324,
|
|
"rewards/margins": 0.8345126331171157,
|
|
"rewards/rejected": -0.30817494285068325,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 15.490945816040039,
|
|
"kl": 14.441825866699219,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -34796995.17593985,
|
|
"logits/rejected": -35708375.20650406,
|
|
"logps/chosen": -478.46508458646616,
|
|
"logps/rejected": -369.3638973577236,
|
|
"loss": 0.5561,
|
|
"loss/base_kto": 3.1802051067352295,
|
|
"metrics/advantage_var": 448.74444580078125,
|
|
"regularization/lipschitz_norm": 1276.7210693359375,
|
|
"regularization/mmd": 0.19625453650951385,
|
|
"regularization/rkhs_norm": 233.63633728027344,
|
|
"regularization/w1": 1.0724457502365112,
|
|
"rewards/chosen": 0.580721426368656,
|
|
"rewards/margins": 0.8381055637781454,
|
|
"rewards/rejected": -0.25738413740948934,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 16.70163917541504,
|
|
"kl": 16.509061813354492,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -33709276.92598425,
|
|
"logits/rejected": -35695503.28062016,
|
|
"logps/chosen": -485.6650590551181,
|
|
"logps/rejected": -369.8797722868217,
|
|
"loss": 0.5535,
|
|
"loss/base_kto": 3.1416361331939697,
|
|
"metrics/advantage_var": 412.0977478027344,
|
|
"regularization/lipschitz_norm": 1298.9788818359375,
|
|
"regularization/mmd": 0.19537167251110077,
|
|
"regularization/rkhs_norm": 232.5853271484375,
|
|
"regularization/w1": 1.0911420583724976,
|
|
"rewards/chosen": 0.5842356283833662,
|
|
"rewards/margins": 0.8872957371300353,
|
|
"rewards/rejected": -0.3030601087466691,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 15.036764144897461,
|
|
"kl": 16.303043365478516,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -35427242.666666664,
|
|
"logits/rejected": -36441643.70731708,
|
|
"logps/chosen": -470.7861578525641,
|
|
"logps/rejected": -388.3401295731707,
|
|
"loss": 0.5537,
|
|
"loss/base_kto": 3.210310459136963,
|
|
"metrics/advantage_var": 389.3489074707031,
|
|
"regularization/lipschitz_norm": 1229.18896484375,
|
|
"regularization/mmd": 0.18670466542243958,
|
|
"regularization/rkhs_norm": 222.2674560546875,
|
|
"regularization/w1": 1.032518744468689,
|
|
"rewards/chosen": 0.5515383695944761,
|
|
"rewards/margins": 0.8148118645940593,
|
|
"rewards/rejected": -0.2632734949995832,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 14.74306869506836,
|
|
"kl": 14.714845657348633,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -33078640.64,
|
|
"logits/rejected": -35340725.74045801,
|
|
"logps/chosen": -491.2714,
|
|
"logps/rejected": -382.17867366412213,
|
|
"loss": 0.5506,
|
|
"loss/base_kto": 3.153940200805664,
|
|
"metrics/advantage_var": 408.9933166503906,
|
|
"regularization/lipschitz_norm": 1263.418212890625,
|
|
"regularization/mmd": 0.1893465220928192,
|
|
"regularization/rkhs_norm": 225.4125213623047,
|
|
"regularization/w1": 1.0612711906433105,
|
|
"rewards/chosen": 0.540904443359375,
|
|
"rewards/margins": 0.8782191247838145,
|
|
"rewards/rejected": -0.33731468142443943,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 17.674104690551758,
|
|
"kl": 12.485431671142578,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -34308014.331288345,
|
|
"logits/rejected": -36068224.81528662,
|
|
"logps/chosen": -481.9826495398773,
|
|
"logps/rejected": -391.43978901273886,
|
|
"loss": 0.5636,
|
|
"loss/base_kto": 3.1916422843933105,
|
|
"metrics/advantage_var": 413.0802917480469,
|
|
"regularization/lipschitz_norm": 1338.4227294921875,
|
|
"regularization/mmd": 0.1930001825094223,
|
|
"regularization/rkhs_norm": 229.76211547851562,
|
|
"regularization/w1": 1.1242752075195312,
|
|
"rewards/chosen": 0.5023735374029429,
|
|
"rewards/margins": 0.8465773162955086,
|
|
"rewards/rejected": -0.3442037788925657,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 14.2947359085083,
|
|
"kl": 12.327513694763184,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35413580.64670659,
|
|
"logits/rejected": -35673466.14379085,
|
|
"logps/chosen": -463.0991766467066,
|
|
"logps/rejected": -396.7017463235294,
|
|
"loss": 0.5658,
|
|
"loss/base_kto": 3.20099139213562,
|
|
"metrics/advantage_var": 460.7687683105469,
|
|
"regularization/lipschitz_norm": 1338.125732421875,
|
|
"regularization/mmd": 0.2013152688741684,
|
|
"regularization/rkhs_norm": 239.66104125976562,
|
|
"regularization/w1": 1.124025583267212,
|
|
"rewards/chosen": 0.5205276397887818,
|
|
"rewards/margins": 0.8575834943368185,
|
|
"rewards/rejected": -0.33705585454803666,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 31.240116119384766,
|
|
"kl": 16.550434112548828,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35559477.97829457,
|
|
"logits/rejected": -35316447.34488189,
|
|
"logps/chosen": -493.40707364341085,
|
|
"logps/rejected": -362.5683070866142,
|
|
"loss": 0.5543,
|
|
"loss/base_kto": 3.1523091793060303,
|
|
"metrics/advantage_var": 423.706298828125,
|
|
"regularization/lipschitz_norm": 1295.798828125,
|
|
"regularization/mmd": 0.19368301331996918,
|
|
"regularization/rkhs_norm": 230.5749969482422,
|
|
"regularization/w1": 1.0884710550308228,
|
|
"rewards/chosen": 0.6050730623940165,
|
|
"rewards/margins": 0.8743481810040681,
|
|
"rewards/rejected": -0.26927511861005166,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 18.28568458557129,
|
|
"kl": 12.479555130004883,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -34717428.06616541,
|
|
"logits/rejected": -34797165.060162604,
|
|
"logps/chosen": -489.3280545112782,
|
|
"logps/rejected": -395.2839430894309,
|
|
"loss": 0.5742,
|
|
"loss/base_kto": 3.2356834411621094,
|
|
"metrics/advantage_var": 445.1636657714844,
|
|
"regularization/lipschitz_norm": 1380.8460693359375,
|
|
"regularization/mmd": 0.1978769749403,
|
|
"regularization/rkhs_norm": 235.56785583496094,
|
|
"regularization/w1": 1.159910798072815,
|
|
"rewards/chosen": 0.4817421361019737,
|
|
"rewards/margins": 0.7853476266856881,
|
|
"rewards/rejected": -0.30360549058371444,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 20.037012100219727,
|
|
"kl": 16.443828582763672,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -32732970.26791277,
|
|
"logits/rejected": -36248964.4137931,
|
|
"logps/chosen": -503.50603582554515,
|
|
"logps/rejected": -360.37661637931035,
|
|
"loss": 0.5559,
|
|
"loss/base_kto": 3.1413798332214355,
|
|
"metrics/advantage_var": 422.8507995605469,
|
|
"regularization/lipschitz_norm": 1322.3681640625,
|
|
"regularization/mmd": 0.19529306888580322,
|
|
"regularization/rkhs_norm": 232.4917449951172,
|
|
"regularization/w1": 1.110789179801941,
|
|
"rewards/chosen": 0.582496619298822,
|
|
"rewards/margins": 0.8898033215329766,
|
|
"rewards/rejected": -0.30730670223415457,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 13.40541934967041,
|
|
"kl": 13.286964416503906,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34099776.19571865,
|
|
"logits/rejected": -35596026.27476038,
|
|
"logps/chosen": -460.0447247706422,
|
|
"logps/rejected": -377.27837959265173,
|
|
"loss": 0.557,
|
|
"loss/base_kto": 3.1802804470062256,
|
|
"metrics/advantage_var": 421.4122619628906,
|
|
"regularization/lipschitz_norm": 1286.3289794921875,
|
|
"regularization/mmd": 0.19494830071926117,
|
|
"regularization/rkhs_norm": 232.08132934570312,
|
|
"regularization/w1": 1.0805162191390991,
|
|
"rewards/chosen": 0.5918159134891054,
|
|
"rewards/margins": 0.8760400123527985,
|
|
"rewards/rejected": -0.2842240988636931,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 16.829252243041992,
|
|
"kl": 15.944289207458496,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -34565764.62166405,
|
|
"logits/rejected": -35543361.69206843,
|
|
"logps/chosen": -485.2910616169545,
|
|
"logps/rejected": -396.3447220062208,
|
|
"loss": 0.5557,
|
|
"loss/base_kto": 3.19449782371521,
|
|
"metrics/advantage_var": 423.94140625,
|
|
"regularization/lipschitz_norm": 1261.138427734375,
|
|
"regularization/mmd": 0.19156305491924286,
|
|
"regularization/rkhs_norm": 228.05125427246094,
|
|
"regularization/w1": 1.0593562126159668,
|
|
"rewards/chosen": 0.5014354280624509,
|
|
"rewards/margins": 0.8286953520181517,
|
|
"rewards/rejected": -0.3272599239557008,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 15.671279907226562,
|
|
"kl": 14.265724182128906,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -34766777.1572327,
|
|
"logits/rejected": -35768644.37267081,
|
|
"logps/chosen": -475.6912342767296,
|
|
"logps/rejected": -404.1916246118012,
|
|
"loss": 0.5656,
|
|
"loss/base_kto": 3.1690189838409424,
|
|
"metrics/advantage_var": 452.4483337402344,
|
|
"regularization/lipschitz_norm": 1378.263916015625,
|
|
"regularization/mmd": 0.19829636812210083,
|
|
"regularization/rkhs_norm": 236.0670928955078,
|
|
"regularization/w1": 1.1577415466308594,
|
|
"rewards/chosen": 0.5210393749692905,
|
|
"rewards/margins": 0.8297014341134943,
|
|
"rewards/rejected": -0.3086620591442037,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 15.375778198242188,
|
|
"kl": 12.648940086364746,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -32652997.17274168,
|
|
"logits/rejected": -35350613.201849,
|
|
"logps/chosen": -510.5975633914422,
|
|
"logps/rejected": -373.8830893682589,
|
|
"loss": 0.5555,
|
|
"loss/base_kto": 3.1570632457733154,
|
|
"metrics/advantage_var": 412.75006103515625,
|
|
"regularization/lipschitz_norm": 1301.3785400390625,
|
|
"regularization/mmd": 0.19356784224510193,
|
|
"regularization/rkhs_norm": 230.4379119873047,
|
|
"regularization/w1": 1.0931578874588013,
|
|
"rewards/chosen": 0.5231033059194236,
|
|
"rewards/margins": 0.8678444924052431,
|
|
"rewards/rejected": -0.34474118648581953,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 17.0692138671875,
|
|
"kl": 10.169137954711914,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -34557681.73607038,
|
|
"logits/rejected": -35296550.52842809,
|
|
"logps/chosen": -480.4759897360704,
|
|
"logps/rejected": -381.8172554347826,
|
|
"loss": 0.562,
|
|
"loss/base_kto": 3.2036292552948,
|
|
"metrics/advantage_var": 402.7635498046875,
|
|
"regularization/lipschitz_norm": 1310.4910888671875,
|
|
"regularization/mmd": 0.1911742240190506,
|
|
"regularization/rkhs_norm": 227.58837890625,
|
|
"regularization/w1": 1.100812554359436,
|
|
"rewards/chosen": 0.5259508149714763,
|
|
"rewards/margins": 0.8284874381065359,
|
|
"rewards/rejected": -0.3025366231350596,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 16.717609405517578,
|
|
"kl": 14.279626846313477,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -34697203.11949685,
|
|
"logits/rejected": -36516603.229813665,
|
|
"logps/chosen": -497.8650746855346,
|
|
"logps/rejected": -389.54755434782606,
|
|
"loss": 0.5643,
|
|
"loss/base_kto": 3.158191442489624,
|
|
"metrics/advantage_var": 485.89434814453125,
|
|
"regularization/lipschitz_norm": 1373.371337890625,
|
|
"regularization/mmd": 0.20268678665161133,
|
|
"regularization/rkhs_norm": 241.29380798339844,
|
|
"regularization/w1": 1.1536319255828857,
|
|
"rewards/chosen": 0.5690512507216735,
|
|
"rewards/margins": 0.8931313210751164,
|
|
"rewards/rejected": -0.32408007035344283,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 19.559938430786133,
|
|
"kl": 14.719937324523926,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34417054.8681672,
|
|
"logits/rejected": -37310068.717325225,
|
|
"logps/chosen": -510.5387359324759,
|
|
"logps/rejected": -335.9021181610942,
|
|
"loss": 0.5544,
|
|
"loss/base_kto": 3.178042411804199,
|
|
"metrics/advantage_var": 411.0628356933594,
|
|
"regularization/lipschitz_norm": 1269.0377197265625,
|
|
"regularization/mmd": 0.19129517674446106,
|
|
"regularization/rkhs_norm": 227.7323455810547,
|
|
"regularization/w1": 1.0659916400909424,
|
|
"rewards/chosen": 0.520851797612917,
|
|
"rewards/margins": 0.8334851380948461,
|
|
"rewards/rejected": -0.31263334048192915,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 15.9247465133667,
|
|
"kl": 15.070162773132324,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -34143676.8,
|
|
"logits/rejected": -34931616.0,
|
|
"logps/chosen": -463.45068359375,
|
|
"logps/rejected": -384.00576171875,
|
|
"loss": 0.5587,
|
|
"loss/base_kto": 3.2235469818115234,
|
|
"metrics/advantage_var": 408.8959655761719,
|
|
"regularization/lipschitz_norm": 1258.9595947265625,
|
|
"regularization/mmd": 0.18841521441936493,
|
|
"regularization/rkhs_norm": 224.3038330078125,
|
|
"regularization/w1": 1.0575259923934937,
|
|
"rewards/chosen": 0.5217224597930908,
|
|
"rewards/margins": 0.7967797040939331,
|
|
"rewards/rejected": -0.2750572443008423,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 14.346929550170898,
|
|
"kl": 13.841235160827637,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -33959118.73817035,
|
|
"logits/rejected": -35736728.173374616,
|
|
"logps/chosen": -488.43158517350156,
|
|
"logps/rejected": -372.11822755417955,
|
|
"loss": 0.5516,
|
|
"loss/base_kto": 3.17337965965271,
|
|
"metrics/advantage_var": 398.7940368652344,
|
|
"regularization/lipschitz_norm": 1251.205810546875,
|
|
"regularization/mmd": 0.18865838646888733,
|
|
"regularization/rkhs_norm": 224.5933074951172,
|
|
"regularization/w1": 1.0510128736495972,
|
|
"rewards/chosen": 0.5624405051631506,
|
|
"rewards/margins": 0.8490387579703899,
|
|
"rewards/rejected": -0.2865982528072393,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 14.097362518310547,
|
|
"kl": 12.440503120422363,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34075282.28571428,
|
|
"logits/rejected": -35199549.686746985,
|
|
"logps/chosen": -471.0784801136364,
|
|
"logps/rejected": -365.6298004518072,
|
|
"loss": 0.5446,
|
|
"loss/base_kto": 3.1548008918762207,
|
|
"metrics/advantage_var": 377.3975830078125,
|
|
"regularization/lipschitz_norm": 1210.8582763671875,
|
|
"regularization/mmd": 0.18470266461372375,
|
|
"regularization/rkhs_norm": 219.88414001464844,
|
|
"regularization/w1": 1.0171208381652832,
|
|
"rewards/chosen": 0.5069566701913809,
|
|
"rewards/margins": 0.869115985553415,
|
|
"rewards/rejected": -0.36215931536203405,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 14.731213569641113,
|
|
"kl": 11.466120719909668,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34719825.17073171,
|
|
"logits/rejected": -36402766.76923077,
|
|
"logps/chosen": -477.3217416158537,
|
|
"logps/rejected": -384.74464142628204,
|
|
"loss": 0.5598,
|
|
"loss/base_kto": 3.1886706352233887,
|
|
"metrics/advantage_var": 416.6258239746094,
|
|
"regularization/lipschitz_norm": 1307.2476806640625,
|
|
"regularization/mmd": 0.19199298322200775,
|
|
"regularization/rkhs_norm": 228.56309509277344,
|
|
"regularization/w1": 1.0980881452560425,
|
|
"rewards/chosen": 0.5373472818514196,
|
|
"rewards/margins": 0.8622382499785479,
|
|
"rewards/rejected": -0.3248909681271284,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 12.770276069641113,
|
|
"kl": 13.92181396484375,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -34218596.4244373,
|
|
"logits/rejected": -34844469.68996961,
|
|
"logps/chosen": -501.15851085209005,
|
|
"logps/rejected": -387.0451652735562,
|
|
"loss": 0.555,
|
|
"loss/base_kto": 3.1139473915100098,
|
|
"metrics/advantage_var": 455.1474609375,
|
|
"regularization/lipschitz_norm": 1338.568359375,
|
|
"regularization/mmd": 0.20162983238697052,
|
|
"regularization/rkhs_norm": 240.03555297851562,
|
|
"regularization/w1": 1.1243975162506104,
|
|
"rewards/chosen": 0.565682463323955,
|
|
"rewards/margins": 0.934562598996388,
|
|
"rewards/rejected": -0.36888013567243305,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 16.815080642700195,
|
|
"kl": 13.590758323669434,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -34037570.66255778,
|
|
"logits/rejected": -35828039.80982567,
|
|
"logps/chosen": -444.85959167950693,
|
|
"logps/rejected": -366.44552297939777,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.210402011871338,
|
|
"metrics/advantage_var": 568.79443359375,
|
|
"regularization/lipschitz_norm": 1287.3560791015625,
|
|
"regularization/mmd": 0.19464194774627686,
|
|
"regularization/rkhs_norm": 231.7165985107422,
|
|
"regularization/w1": 1.0813791751861572,
|
|
"rewards/chosen": 0.5177080950861903,
|
|
"rewards/margins": 0.7827321744709714,
|
|
"rewards/rejected": -0.2650240793847811,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 17.200023651123047,
|
|
"kl": 11.700523376464844,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35237611.41122913,
|
|
"logits/rejected": -36001030.18357488,
|
|
"logps/chosen": -505.28106031866463,
|
|
"logps/rejected": -374.33031400966183,
|
|
"loss": 0.5603,
|
|
"loss/base_kto": 3.1586668491363525,
|
|
"metrics/advantage_var": 419.7179870605469,
|
|
"regularization/lipschitz_norm": 1341.960693359375,
|
|
"regularization/mmd": 0.196184903383255,
|
|
"regularization/rkhs_norm": 233.55345153808594,
|
|
"regularization/w1": 1.1272468566894531,
|
|
"rewards/chosen": 0.5335658164595979,
|
|
"rewards/margins": 0.8919853361512495,
|
|
"rewards/rejected": -0.3584195196916516,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 19.267410278320312,
|
|
"kl": 11.981795310974121,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -33486727.145135567,
|
|
"logits/rejected": -34819092.38591118,
|
|
"logps/chosen": -476.64005183413076,
|
|
"logps/rejected": -368.72468415007654,
|
|
"loss": 0.5524,
|
|
"loss/base_kto": 3.148010730743408,
|
|
"metrics/advantage_var": 409.183349609375,
|
|
"regularization/lipschitz_norm": 1285.4906005859375,
|
|
"regularization/mmd": 0.19134068489074707,
|
|
"regularization/rkhs_norm": 227.78652954101562,
|
|
"regularization/w1": 1.0798120498657227,
|
|
"rewards/chosen": 0.5336724726967454,
|
|
"rewards/margins": 0.8878623352152897,
|
|
"rewards/rejected": -0.35418986251854423,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 17.440208435058594,
|
|
"kl": 11.75373649597168,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35822797.4522293,
|
|
"logits/rejected": -36269143.95092025,
|
|
"logps/chosen": -483.26731687898086,
|
|
"logps/rejected": -396.0221673696319,
|
|
"loss": 0.5646,
|
|
"loss/base_kto": 3.2159557342529297,
|
|
"metrics/advantage_var": 405.16900634765625,
|
|
"regularization/lipschitz_norm": 1322.2730712890625,
|
|
"regularization/mmd": 0.18996773660182953,
|
|
"regularization/rkhs_norm": 226.1520538330078,
|
|
"regularization/w1": 1.1107094287872314,
|
|
"rewards/chosen": 0.47785094437325837,
|
|
"rewards/margins": 0.8298864668341959,
|
|
"rewards/rejected": -0.3520355224609375,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 19.567821502685547,
|
|
"kl": 11.295672416687012,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -33572715.227202475,
|
|
"logits/rejected": -36172177.18799368,
|
|
"logps/chosen": -480.57959814528596,
|
|
"logps/rejected": -388.7501974723539,
|
|
"loss": 0.558,
|
|
"loss/base_kto": 3.155613422393799,
|
|
"metrics/advantage_var": 415.93408203125,
|
|
"regularization/lipschitz_norm": 1326.3670654296875,
|
|
"regularization/mmd": 0.19387857615947723,
|
|
"regularization/rkhs_norm": 230.807861328125,
|
|
"regularization/w1": 1.1141483783721924,
|
|
"rewards/chosen": 0.48880226571556223,
|
|
"rewards/margins": 0.8732924611360549,
|
|
"rewards/rejected": -0.3844901954204927,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 16.0477294921875,
|
|
"kl": 11.07884693145752,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -33802797.969040245,
|
|
"logits/rejected": -35872819.68454259,
|
|
"logps/chosen": -480.07410990712077,
|
|
"logps/rejected": -364.4790023659306,
|
|
"loss": 0.5558,
|
|
"loss/base_kto": 3.205296039581299,
|
|
"metrics/advantage_var": 400.572509765625,
|
|
"regularization/lipschitz_norm": 1255.5992431640625,
|
|
"regularization/mmd": 0.18615250289440155,
|
|
"regularization/rkhs_norm": 221.610107421875,
|
|
"regularization/w1": 1.0547033548355103,
|
|
"rewards/chosen": 0.49147085656322564,
|
|
"rewards/margins": 0.8313376391517633,
|
|
"rewards/rejected": -0.3398667825885376,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 14.80739974975586,
|
|
"kl": 13.7398042678833,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -33166031.1197411,
|
|
"logits/rejected": -34830008.07250755,
|
|
"logps/chosen": -500.65544093851133,
|
|
"logps/rejected": -377.02317787009065,
|
|
"loss": 0.5535,
|
|
"loss/base_kto": 3.211195468902588,
|
|
"metrics/advantage_var": 388.3515319824219,
|
|
"regularization/lipschitz_norm": 1225.2630615234375,
|
|
"regularization/mmd": 0.1875758022069931,
|
|
"regularization/rkhs_norm": 223.30455017089844,
|
|
"regularization/w1": 1.0292209386825562,
|
|
"rewards/chosen": 0.5005215641750101,
|
|
"rewards/margins": 0.8226207715662016,
|
|
"rewards/rejected": -0.32209920739119147,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_loss": 0.5863801720640615,
|
|
"train_runtime": 11092.9859,
|
|
"train_samples_per_second": 13.361,
|
|
"train_steps_per_second": 0.209
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|