1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 10.767630577087402,
|
|
"kl": 18.16756248474121,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -39039743.18730158,
|
|
"logits/rejected": -38866717.14461538,
|
|
"logps/chosen": -492.0530257936508,
|
|
"logps/rejected": -370.46858173076924,
|
|
"loss": 0.475,
|
|
"loss/base_kto": 3.799793004989624,
|
|
"metrics/advantage_var": 574.7128295898438,
|
|
"rewards/chosen": 0.17904757545107886,
|
|
"rewards/margins": 0.1815789432665367,
|
|
"rewards/rejected": -0.0025313678154578576,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 9.717031478881836,
|
|
"kl": 13.010165214538574,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -38004318.75583204,
|
|
"logits/rejected": -38601925.72684459,
|
|
"logps/chosen": -497.3858864696734,
|
|
"logps/rejected": -377.9446624803768,
|
|
"loss": 0.4455,
|
|
"loss/base_kto": 3.5642311573028564,
|
|
"metrics/advantage_var": 685.6561889648438,
|
|
"rewards/chosen": 0.36572222909882873,
|
|
"rewards/margins": 0.4868113319950954,
|
|
"rewards/rejected": -0.12108910289626668,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 9.341075897216797,
|
|
"kl": 41.835079193115234,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -37667752.318042815,
|
|
"logits/rejected": -38090961.38019169,
|
|
"logps/chosen": -464.83524464831805,
|
|
"logps/rejected": -390.9385982428115,
|
|
"loss": 0.4285,
|
|
"loss/base_kto": 3.4281976222991943,
|
|
"metrics/advantage_var": 1246.1971435546875,
|
|
"rewards/chosen": 1.098412003356747,
|
|
"rewards/margins": 0.5388954622444018,
|
|
"rewards/rejected": 0.5595165411123453,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 11.590300559997559,
|
|
"kl": 10.14930248260498,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37663127.658767775,
|
|
"logits/rejected": -39938677.910355486,
|
|
"logps/chosen": -475.84547788309635,
|
|
"logps/rejected": -390.74145092735705,
|
|
"loss": 0.4368,
|
|
"loss/base_kto": 3.494100332260132,
|
|
"metrics/advantage_var": 1539.3009033203125,
|
|
"rewards/chosen": 0.45085223591158174,
|
|
"rewards/margins": 0.5807951755873388,
|
|
"rewards/rejected": -0.1299429396757571,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 10.024121284484863,
|
|
"kl": 7.55079984664917,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -38242689.01105845,
|
|
"logits/rejected": -37975244.16692427,
|
|
"logps/chosen": -469.24180489731435,
|
|
"logps/rejected": -385.064142194745,
|
|
"loss": 0.4168,
|
|
"loss/base_kto": 3.334552764892578,
|
|
"metrics/advantage_var": 2498.937744140625,
|
|
"rewards/chosen": 0.3644587078365669,
|
|
"rewards/margins": 0.9627499309999652,
|
|
"rewards/rejected": -0.5982912231633983,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 7.915276050567627,
|
|
"kl": 25.576862335205078,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -39889112.55589124,
|
|
"logits/rejected": -39712290.79611651,
|
|
"logps/chosen": -466.6666351963746,
|
|
"logps/rejected": -365.86524069579286,
|
|
"loss": 0.4039,
|
|
"loss/base_kto": 3.2308411598205566,
|
|
"metrics/advantage_var": 2719.4296875,
|
|
"rewards/chosen": 1.257896584686556,
|
|
"rewards/margins": 1.1497725955196048,
|
|
"rewards/rejected": 0.1081239891669511,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 9.480732917785645,
|
|
"kl": 21.655780792236328,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -39355852.8,
|
|
"logits/rejected": -40773708.8,
|
|
"logps/chosen": -486.68837890625,
|
|
"logps/rejected": -367.1841796875,
|
|
"loss": 0.3849,
|
|
"loss/base_kto": 3.0790488719940186,
|
|
"metrics/advantage_var": 3540.862548828125,
|
|
"rewards/chosen": 1.2245597839355469,
|
|
"rewards/margins": 1.3650813817977905,
|
|
"rewards/rejected": -0.14052159786224366,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 7.545348644256592,
|
|
"kl": 17.1834659576416,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -39774104.34355828,
|
|
"logits/rejected": -40451626.39490446,
|
|
"logps/chosen": -471.5014378834356,
|
|
"logps/rejected": -368.3911723726115,
|
|
"loss": 0.3973,
|
|
"loss/base_kto": 3.1785781383514404,
|
|
"metrics/advantage_var": 3657.906982421875,
|
|
"rewards/chosen": 0.9276840116348735,
|
|
"rewards/margins": 1.2241404154279167,
|
|
"rewards/rejected": -0.2964564037930434,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 7.958818435668945,
|
|
"kl": 12.04544734954834,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -40987777.24271844,
|
|
"logits/rejected": -40042390.81570997,
|
|
"logps/chosen": -464.29571197411,
|
|
"logps/rejected": -378.16432212990935,
|
|
"loss": 0.3868,
|
|
"loss/base_kto": 3.0944411754608154,
|
|
"metrics/advantage_var": 4641.96533203125,
|
|
"rewards/chosen": 0.6893991019733515,
|
|
"rewards/margins": 1.5439098212957174,
|
|
"rewards/rejected": -0.8545107193223659,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 9.993614196777344,
|
|
"kl": 28.704259872436523,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -40937725.49837133,
|
|
"logits/rejected": -41801798.726726726,
|
|
"logps/chosen": -460.3880293159609,
|
|
"logps/rejected": -372.76360735735733,
|
|
"loss": 0.3701,
|
|
"loss/base_kto": 2.960904598236084,
|
|
"metrics/advantage_var": 5026.33154296875,
|
|
"rewards/chosen": 1.4231080294432004,
|
|
"rewards/margins": 1.8823935478717853,
|
|
"rewards/rejected": -0.45928551842858484,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 7.57697868347168,
|
|
"kl": 22.163732528686523,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -42421246.491899855,
|
|
"logits/rejected": -42000642.98169717,
|
|
"logps/chosen": -480.43611929307804,
|
|
"logps/rejected": -385.0795029118136,
|
|
"loss": 0.3502,
|
|
"loss/base_kto": 2.8012216091156006,
|
|
"metrics/advantage_var": 5297.86181640625,
|
|
"rewards/chosen": 1.5864491525911266,
|
|
"rewards/margins": 2.1334137249834937,
|
|
"rewards/rejected": -0.5469645723923668,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 10.573487281799316,
|
|
"kl": 36.15892028808594,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -42419916.151898734,
|
|
"logits/rejected": -42658585.28395062,
|
|
"logps/chosen": -481.88399920886076,
|
|
"logps/rejected": -379.80114293981484,
|
|
"loss": 0.3628,
|
|
"loss/base_kto": 2.9025771617889404,
|
|
"metrics/advantage_var": 5566.79541015625,
|
|
"rewards/chosen": 1.7851292091079904,
|
|
"rewards/margins": 2.0506995788755447,
|
|
"rewards/rejected": -0.265570369767554,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 9.307571411132812,
|
|
"kl": 11.663390159606934,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -43635637.975903615,
|
|
"logits/rejected": -42869414.233766235,
|
|
"logps/chosen": -473.2483998493976,
|
|
"logps/rejected": -375.81688818993507,
|
|
"loss": 0.3579,
|
|
"loss/base_kto": 2.8630669116973877,
|
|
"metrics/advantage_var": 5602.22607421875,
|
|
"rewards/chosen": 0.9390560288027109,
|
|
"rewards/margins": 2.1901733883108787,
|
|
"rewards/rejected": -1.2511173595081677,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 7.039910793304443,
|
|
"kl": 21.85430908203125,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -43542039.48623853,
|
|
"logits/rejected": -43862886.23642173,
|
|
"logps/chosen": -515.8261181192661,
|
|
"logps/rejected": -375.6230780750799,
|
|
"loss": 0.3502,
|
|
"loss/base_kto": 2.801739454269409,
|
|
"metrics/advantage_var": 5745.34326171875,
|
|
"rewards/chosen": 1.5303742295011469,
|
|
"rewards/margins": 2.2513882340469986,
|
|
"rewards/rejected": -0.7210140045458516,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 10.740477561950684,
|
|
"kl": 25.465660095214844,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -43092120.64596273,
|
|
"logits/rejected": -43654388.72955975,
|
|
"logps/chosen": -467.90828804347825,
|
|
"logps/rejected": -364.07173742138366,
|
|
"loss": 0.3515,
|
|
"loss/base_kto": 2.8117666244506836,
|
|
"metrics/advantage_var": 5965.7041015625,
|
|
"rewards/chosen": 1.3311123107531055,
|
|
"rewards/margins": 2.160643422560299,
|
|
"rewards/rejected": -0.8295311118071934,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 9.744688987731934,
|
|
"kl": 17.645050048828125,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -44299203.57256778,
|
|
"logits/rejected": -44484341.41500766,
|
|
"logps/chosen": -453.71630781499204,
|
|
"logps/rejected": -392.85255551301685,
|
|
"loss": 0.3614,
|
|
"loss/base_kto": 2.8914036750793457,
|
|
"metrics/advantage_var": 7013.50634765625,
|
|
"rewards/chosen": 1.324709075489683,
|
|
"rewards/margins": 2.303055521878944,
|
|
"rewards/rejected": -0.978346446389261,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 7.595713138580322,
|
|
"kl": 15.515207290649414,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -43934022.80851064,
|
|
"logits/rejected": -44995774.971061096,
|
|
"logps/chosen": -458.45117781155017,
|
|
"logps/rejected": -380.3065213022508,
|
|
"loss": 0.3471,
|
|
"loss/base_kto": 2.7768313884735107,
|
|
"metrics/advantage_var": 7399.63134765625,
|
|
"rewards/chosen": 1.3333393317225495,
|
|
"rewards/margins": 2.5059771730484135,
|
|
"rewards/rejected": -1.1726378413258642,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 9.919568061828613,
|
|
"kl": 21.106929779052734,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -43839122.40661157,
|
|
"logits/rejected": -44855490.180740744,
|
|
"logps/chosen": -468.6537190082645,
|
|
"logps/rejected": -384.3515972222222,
|
|
"loss": 0.3418,
|
|
"loss/base_kto": 2.7347142696380615,
|
|
"metrics/advantage_var": 8413.7919921875,
|
|
"rewards/chosen": 1.445809255552686,
|
|
"rewards/margins": 2.578799978208936,
|
|
"rewards/rejected": -1.13299072265625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 9.684901237487793,
|
|
"kl": 29.528310775756836,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -44314246.056782335,
|
|
"logits/rejected": -45580085.10216718,
|
|
"logps/chosen": -474.65348974763407,
|
|
"logps/rejected": -365.3409442724458,
|
|
"loss": 0.3338,
|
|
"loss/base_kto": 2.670184373855591,
|
|
"metrics/advantage_var": 8667.9609375,
|
|
"rewards/chosen": 2.0107843537435923,
|
|
"rewards/margins": 2.7288761472172522,
|
|
"rewards/rejected": -0.71809179347366,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 9.16874885559082,
|
|
"kl": 15.83648681640625,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -43639409.24610592,
|
|
"logits/rejected": -43527915.93730407,
|
|
"logps/chosen": -496.9823792834891,
|
|
"logps/rejected": -396.6072443181818,
|
|
"loss": 0.3452,
|
|
"loss/base_kto": 2.7614870071411133,
|
|
"metrics/advantage_var": 9076.080078125,
|
|
"rewards/chosen": 1.3471149195020444,
|
|
"rewards/margins": 2.7789011487462645,
|
|
"rewards/rejected": -1.4317862292442203,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 7.9212493896484375,
|
|
"kl": 20.031747817993164,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -44447096.73905997,
|
|
"logits/rejected": -45098418.00301659,
|
|
"logps/chosen": -451.9254457050243,
|
|
"logps/rejected": -391.1688584087481,
|
|
"loss": 0.334,
|
|
"loss/base_kto": 2.671661615371704,
|
|
"metrics/advantage_var": 9060.0986328125,
|
|
"rewards/chosen": 1.5117804776134522,
|
|
"rewards/margins": 2.875957186082297,
|
|
"rewards/rejected": -1.3641767084688443,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 10.213101387023926,
|
|
"kl": 25.621938705444336,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -45655890.23030303,
|
|
"logits/rejected": -44756073.70322581,
|
|
"logps/chosen": -461.4982481060606,
|
|
"logps/rejected": -408.54506048387094,
|
|
"loss": 0.3187,
|
|
"loss/base_kto": 2.5492563247680664,
|
|
"metrics/advantage_var": 8461.5302734375,
|
|
"rewards/chosen": 2.110018273555871,
|
|
"rewards/margins": 3.026513418307635,
|
|
"rewards/rejected": -0.9164951447517641,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 7.825594425201416,
|
|
"kl": 38.75930404663086,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -46050952.25123967,
|
|
"logits/rejected": -45506256.59259259,
|
|
"logps/chosen": -464.1485020661157,
|
|
"logps/rejected": -389.2406481481481,
|
|
"loss": 0.3459,
|
|
"loss/base_kto": 2.76682186126709,
|
|
"metrics/advantage_var": 9444.908203125,
|
|
"rewards/chosen": 1.8575812725981404,
|
|
"rewards/margins": 2.7491066089696683,
|
|
"rewards/rejected": -0.8915253363715278,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 7.18243932723999,
|
|
"kl": 30.665624618530273,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -46040105.6,
|
|
"logits/rejected": -45821132.8,
|
|
"logps/chosen": -493.368310546875,
|
|
"logps/rejected": -386.420751953125,
|
|
"loss": 0.3471,
|
|
"loss/base_kto": 2.7768948078155518,
|
|
"metrics/advantage_var": 10165.1181640625,
|
|
"rewards/chosen": 1.8238739013671874,
|
|
"rewards/margins": 2.9732950210571287,
|
|
"rewards/rejected": -1.1494211196899413,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 7.917231559753418,
|
|
"kl": 41.2276496887207,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -47402153.579617836,
|
|
"logits/rejected": -46038845.25153374,
|
|
"logps/chosen": -491.7269108280255,
|
|
"logps/rejected": -406.0648006134969,
|
|
"loss": 0.31,
|
|
"loss/base_kto": 2.479668140411377,
|
|
"metrics/advantage_var": 11272.2255859375,
|
|
"rewards/chosen": 2.552598698124005,
|
|
"rewards/margins": 3.6254450932304083,
|
|
"rewards/rejected": -1.0728463951064033,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 7.153599739074707,
|
|
"kl": 17.412412643432617,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -46631119.31288344,
|
|
"logits/rejected": -47223462.318471335,
|
|
"logps/chosen": -476.2435774539877,
|
|
"logps/rejected": -383.86176353503186,
|
|
"loss": 0.3234,
|
|
"loss/base_kto": 2.587484836578369,
|
|
"metrics/advantage_var": 10399.548828125,
|
|
"rewards/chosen": 1.5981022185343174,
|
|
"rewards/margins": 3.244075375505854,
|
|
"rewards/rejected": -1.6459731569715366,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 5.476096153259277,
|
|
"kl": 27.94318199157715,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -44313476.15339233,
|
|
"logits/rejected": -45888906.63122924,
|
|
"logps/chosen": -489.6357853982301,
|
|
"logps/rejected": -359.63382475083057,
|
|
"loss": 0.3029,
|
|
"loss/base_kto": 2.423074245452881,
|
|
"metrics/advantage_var": 9790.6162109375,
|
|
"rewards/chosen": 2.436796025188975,
|
|
"rewards/margins": 3.40802311976954,
|
|
"rewards/rejected": -0.9712270945805648,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 7.021985054016113,
|
|
"kl": 13.960652351379395,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -44421100.587677725,
|
|
"logits/rejected": -46085900.26584235,
|
|
"logps/chosen": -475.8004541864139,
|
|
"logps/rejected": -367.6319551777434,
|
|
"loss": 0.3452,
|
|
"loss/base_kto": 2.7619121074676514,
|
|
"metrics/advantage_var": 10958.1064453125,
|
|
"rewards/chosen": 1.160645495470478,
|
|
"rewards/margins": 3.2594266035896817,
|
|
"rewards/rejected": -2.098781108119204,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 7.780029773712158,
|
|
"kl": 52.58126449584961,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -46219425.00628931,
|
|
"logits/rejected": -47191268.9689441,
|
|
"logps/chosen": -455.8256682389937,
|
|
"logps/rejected": -392.7049689440994,
|
|
"loss": 0.319,
|
|
"loss/base_kto": 2.5518600940704346,
|
|
"metrics/advantage_var": 9555.4716796875,
|
|
"rewards/chosen": 2.634859097078911,
|
|
"rewards/margins": 3.1861237000243694,
|
|
"rewards/rejected": -0.5512646029454581,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 8.380097389221191,
|
|
"kl": 22.55097007751465,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -46950616.92211838,
|
|
"logits/rejected": -47377892.71473354,
|
|
"logps/chosen": -465.6770833333333,
|
|
"logps/rejected": -380.56700626959247,
|
|
"loss": 0.3297,
|
|
"loss/base_kto": 2.6379048824310303,
|
|
"metrics/advantage_var": 11059.1005859375,
|
|
"rewards/chosen": 1.678995007666472,
|
|
"rewards/margins": 3.1943280338469187,
|
|
"rewards/rejected": -1.5153330261804467,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 8.63371753692627,
|
|
"kl": 23.489225387573242,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -47105205.8624,
|
|
"logits/rejected": -46306636.995419845,
|
|
"logps/chosen": -475.60705,
|
|
"logps/rejected": -378.4205629770992,
|
|
"loss": 0.3171,
|
|
"loss/base_kto": 2.5367319583892822,
|
|
"metrics/advantage_var": 11255.9462890625,
|
|
"rewards/chosen": 2.1496705078125,
|
|
"rewards/margins": 3.5312632365219465,
|
|
"rewards/rejected": -1.3815927287094465,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 6.207841396331787,
|
|
"kl": 15.559234619140625,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -47738959.87519501,
|
|
"logits/rejected": -46858073.33959311,
|
|
"logps/chosen": -453.6013553042122,
|
|
"logps/rejected": -410.0959996087637,
|
|
"loss": 0.3377,
|
|
"loss/base_kto": 2.701637029647827,
|
|
"metrics/advantage_var": 13703.7685546875,
|
|
"rewards/chosen": 1.087391012730353,
|
|
"rewards/margins": 3.5123001954743867,
|
|
"rewards/rejected": -2.4249091827440337,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 8.613212585449219,
|
|
"kl": 41.37206268310547,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -46806706.08695652,
|
|
"logits/rejected": -46441462.33962264,
|
|
"logps/chosen": -452.17434976708074,
|
|
"logps/rejected": -389.0125786163522,
|
|
"loss": 0.3111,
|
|
"loss/base_kto": 2.4888522624969482,
|
|
"metrics/advantage_var": 11084.3515625,
|
|
"rewards/chosen": 2.6062068583802405,
|
|
"rewards/margins": 3.4839085277493442,
|
|
"rewards/rejected": -0.8777016693691038,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 9.352055549621582,
|
|
"kl": 18.29364585876465,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -48808864.51446945,
|
|
"logits/rejected": -48817167.56231003,
|
|
"logps/chosen": -473.30335610932474,
|
|
"logps/rejected": -386.868944718845,
|
|
"loss": 0.3207,
|
|
"loss/base_kto": 2.5655012130737305,
|
|
"metrics/advantage_var": 14518.6845703125,
|
|
"rewards/chosen": 1.4366721984274518,
|
|
"rewards/margins": 3.508463641484063,
|
|
"rewards/rejected": -2.071791443056611,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 5.842999458312988,
|
|
"kl": 26.263547897338867,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -49060517.43950995,
|
|
"logits/rejected": -48573222.78787879,
|
|
"logps/chosen": -463.4508996937213,
|
|
"logps/rejected": -388.22393341307816,
|
|
"loss": 0.3168,
|
|
"loss/base_kto": 2.5340232849121094,
|
|
"metrics/advantage_var": 13444.134765625,
|
|
"rewards/chosen": 1.9039039319726263,
|
|
"rewards/margins": 3.6761795250746996,
|
|
"rewards/rejected": -1.7722755931020733,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 8.439847946166992,
|
|
"kl": 37.183834075927734,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -46772820.80981595,
|
|
"logits/rejected": -46420320.20382166,
|
|
"logps/chosen": -445.3078029141104,
|
|
"logps/rejected": -388.06966560509557,
|
|
"loss": 0.3295,
|
|
"loss/base_kto": 2.6357457637786865,
|
|
"metrics/advantage_var": 11617.8359375,
|
|
"rewards/chosen": 2.5190156339867715,
|
|
"rewards/margins": 3.17519050631833,
|
|
"rewards/rejected": -0.6561748723315585,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 8.572729110717773,
|
|
"kl": 18.840547561645508,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -47739316.08832808,
|
|
"logits/rejected": -48534800.643962845,
|
|
"logps/chosen": -481.36883872239747,
|
|
"logps/rejected": -363.2466137770898,
|
|
"loss": 0.3124,
|
|
"loss/base_kto": 2.4990785121917725,
|
|
"metrics/advantage_var": 12671.8056640625,
|
|
"rewards/chosen": 2.161652092677938,
|
|
"rewards/margins": 3.838208848192644,
|
|
"rewards/rejected": -1.6765567555147058,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 7.6415181159973145,
|
|
"kl": 23.328641891479492,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -46437382.28220859,
|
|
"logits/rejected": -46758641.32484076,
|
|
"logps/chosen": -447.5244440184049,
|
|
"logps/rejected": -387.3877886146497,
|
|
"loss": 0.3227,
|
|
"loss/base_kto": 2.5819222927093506,
|
|
"metrics/advantage_var": 14022.7490234375,
|
|
"rewards/chosen": 2.143888134166507,
|
|
"rewards/margins": 3.623439510217711,
|
|
"rewards/rejected": -1.4795513760512042,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 6.992530822753906,
|
|
"kl": 26.622074127197266,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -46215003.104669884,
|
|
"logits/rejected": -46835125.96651446,
|
|
"logps/chosen": -422.12354066022544,
|
|
"logps/rejected": -368.4546708523592,
|
|
"loss": 0.3012,
|
|
"loss/base_kto": 2.409684419631958,
|
|
"metrics/advantage_var": 14463.0234375,
|
|
"rewards/chosen": 2.253114463818438,
|
|
"rewards/margins": 4.25722249214321,
|
|
"rewards/rejected": -2.004108028324772,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 6.378375053405762,
|
|
"kl": 24.94471549987793,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -49503190.90850722,
|
|
"logits/rejected": -48349445.84474886,
|
|
"logps/chosen": -458.17962479935795,
|
|
"logps/rejected": -394.6609589041096,
|
|
"loss": 0.2523,
|
|
"loss/base_kto": 2.0187790393829346,
|
|
"metrics/advantage_var": 16326.5107421875,
|
|
"rewards/chosen": 2.497149863939607,
|
|
"rewards/margins": 4.905280341310707,
|
|
"rewards/rejected": -2.4081304773711,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 6.994624614715576,
|
|
"kl": 13.280878067016602,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -47839512.1509434,
|
|
"logits/rejected": -48610097.29192547,
|
|
"logps/chosen": -460.30483490566036,
|
|
"logps/rejected": -395.1667798913044,
|
|
"loss": 0.2539,
|
|
"loss/base_kto": 2.0312325954437256,
|
|
"metrics/advantage_var": 14454.4248046875,
|
|
"rewards/chosen": 2.519423766705975,
|
|
"rewards/margins": 5.028234248375715,
|
|
"rewards/rejected": -2.50881048166974,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 9.424582481384277,
|
|
"kl": 10.85019302368164,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -49302809.56067588,
|
|
"logits/rejected": -49959888.78855326,
|
|
"logps/chosen": -469.22825460829495,
|
|
"logps/rejected": -394.01599761526234,
|
|
"loss": 0.2785,
|
|
"loss/base_kto": 2.227801561355591,
|
|
"metrics/advantage_var": 16311.119140625,
|
|
"rewards/chosen": 1.9709813928091398,
|
|
"rewards/margins": 4.6200011041565165,
|
|
"rewards/rejected": -2.6490197113473766,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 8.003256797790527,
|
|
"kl": 23.4259033203125,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -49873438.30529595,
|
|
"logits/rejected": -49211558.92163009,
|
|
"logps/chosen": -457.47205996884736,
|
|
"logps/rejected": -405.83365987460814,
|
|
"loss": 0.2568,
|
|
"loss/base_kto": 2.054487943649292,
|
|
"metrics/advantage_var": 13718.9765625,
|
|
"rewards/chosen": 2.581324497115946,
|
|
"rewards/margins": 4.676975196934716,
|
|
"rewards/rejected": -2.0956506998187696,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 7.165423393249512,
|
|
"kl": 24.627613067626953,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -48778415.4517134,
|
|
"logits/rejected": -50606927.448275864,
|
|
"logps/chosen": -446.71582943925233,
|
|
"logps/rejected": -396.57908013322884,
|
|
"loss": 0.2638,
|
|
"loss/base_kto": 2.110078811645508,
|
|
"metrics/advantage_var": 14859.9375,
|
|
"rewards/chosen": 2.3450887804833527,
|
|
"rewards/margins": 4.554141652617953,
|
|
"rewards/rejected": -2.2090528721346003,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 7.8962507247924805,
|
|
"kl": 24.107492446899414,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -48750660.1633888,
|
|
"logits/rejected": -48364727.62520194,
|
|
"logps/chosen": -463.73903177004536,
|
|
"logps/rejected": -390.83587439418415,
|
|
"loss": 0.2409,
|
|
"loss/base_kto": 1.9271656274795532,
|
|
"metrics/advantage_var": 15279.6611328125,
|
|
"rewards/chosen": 2.626690701529406,
|
|
"rewards/margins": 5.226181753124721,
|
|
"rewards/rejected": -2.599491051595315,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 6.696859836578369,
|
|
"kl": 24.49211883544922,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -49706159.587253414,
|
|
"logits/rejected": -49836104.55394525,
|
|
"logps/chosen": -460.4776176024279,
|
|
"logps/rejected": -393.373817431562,
|
|
"loss": 0.2466,
|
|
"loss/base_kto": 1.9727410078048706,
|
|
"metrics/advantage_var": 15380.7998046875,
|
|
"rewards/chosen": 2.8233566631496587,
|
|
"rewards/margins": 5.1379878810693445,
|
|
"rewards/rejected": -2.314631217919686,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 7.559670448303223,
|
|
"kl": 13.5224609375,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -51078631.85691057,
|
|
"logits/rejected": -51562779.33233083,
|
|
"logps/chosen": -487.75752032520325,
|
|
"logps/rejected": -395.12109962406015,
|
|
"loss": 0.2634,
|
|
"loss/base_kto": 2.107135534286499,
|
|
"metrics/advantage_var": 14195.0283203125,
|
|
"rewards/chosen": 2.105270658663618,
|
|
"rewards/margins": 4.8840208789408734,
|
|
"rewards/rejected": -2.7787502202772556,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 7.225378513336182,
|
|
"kl": 26.71965980529785,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -50135112.13476264,
|
|
"logits/rejected": -49574805.84370016,
|
|
"logps/chosen": -444.1115045941807,
|
|
"logps/rejected": -381.9628688197767,
|
|
"loss": 0.2421,
|
|
"loss/base_kto": 1.937116265296936,
|
|
"metrics/advantage_var": 13962.541015625,
|
|
"rewards/chosen": 2.7284518792472245,
|
|
"rewards/margins": 4.976747956799059,
|
|
"rewards/rejected": -2.248296077551834,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 6.193775653839111,
|
|
"kl": 31.8278751373291,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -49843278.52760736,
|
|
"logits/rejected": -49576558.87898089,
|
|
"logps/chosen": -470.5159125766871,
|
|
"logps/rejected": -402.9140376194268,
|
|
"loss": 0.2442,
|
|
"loss/base_kto": 1.9534603357315063,
|
|
"metrics/advantage_var": 14674.25,
|
|
"rewards/chosen": 3.012382647742523,
|
|
"rewards/margins": 5.103156114788204,
|
|
"rewards/rejected": -2.090773467045681,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 6.882968425750732,
|
|
"kl": 24.6142578125,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -49777710.618524335,
|
|
"logits/rejected": -50704320.29860031,
|
|
"logps/chosen": -469.02403846153845,
|
|
"logps/rejected": -415.6784603421462,
|
|
"loss": 0.2486,
|
|
"loss/base_kto": 1.9886268377304077,
|
|
"metrics/advantage_var": 16992.470703125,
|
|
"rewards/chosen": 2.6783355281716053,
|
|
"rewards/margins": 5.319612076091512,
|
|
"rewards/rejected": -2.6412765479199067,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 8.8514986038208,
|
|
"kl": 23.941320419311523,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -52166851.35771065,
|
|
"logits/rejected": -50921553.794162825,
|
|
"logps/chosen": -480.11466613672496,
|
|
"logps/rejected": -391.50662442396316,
|
|
"loss": 0.2476,
|
|
"loss/base_kto": 1.980577826499939,
|
|
"metrics/advantage_var": 14342.166015625,
|
|
"rewards/chosen": 2.738708204988076,
|
|
"rewards/margins": 5.102693000014957,
|
|
"rewards/rejected": -2.3639847950268815,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 8.558167457580566,
|
|
"kl": 17.936880111694336,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -50118090.70031546,
|
|
"logits/rejected": -51773455.851393186,
|
|
"logps/chosen": -455.5176458990536,
|
|
"logps/rejected": -408.96613777089783,
|
|
"loss": 0.2435,
|
|
"loss/base_kto": 1.9483833312988281,
|
|
"metrics/advantage_var": 14109.5107421875,
|
|
"rewards/chosen": 2.2959194243518337,
|
|
"rewards/margins": 5.004700852733219,
|
|
"rewards/rejected": -2.7087814283813856,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 9.361747741699219,
|
|
"kl": 17.557247161865234,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -49804216.25477707,
|
|
"logits/rejected": -50424656.09815951,
|
|
"logps/chosen": -454.2070561305732,
|
|
"logps/rejected": -388.4324674079755,
|
|
"loss": 0.242,
|
|
"loss/base_kto": 1.935681939125061,
|
|
"metrics/advantage_var": 14550.7333984375,
|
|
"rewards/chosen": 2.3316926409484475,
|
|
"rewards/margins": 5.0811738421203225,
|
|
"rewards/rejected": -2.749481201171875,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 9.169776916503906,
|
|
"kl": 20.535934448242188,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -48851373.86506747,
|
|
"logits/rejected": -50709555.78466558,
|
|
"logps/chosen": -436.7430659670165,
|
|
"logps/rejected": -393.1366231647635,
|
|
"loss": 0.2555,
|
|
"loss/base_kto": 2.0440099239349365,
|
|
"metrics/advantage_var": 14528.4658203125,
|
|
"rewards/chosen": 2.7239223405875186,
|
|
"rewards/margins": 4.791540531732503,
|
|
"rewards/rejected": -2.067618191144984,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 8.66065502166748,
|
|
"kl": 25.7312068939209,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -50435637.02009273,
|
|
"logits/rejected": -50470297.276461296,
|
|
"logps/chosen": -479.2921174652241,
|
|
"logps/rejected": -413.2000394944708,
|
|
"loss": 0.2505,
|
|
"loss/base_kto": 2.0043818950653076,
|
|
"metrics/advantage_var": 14897.306640625,
|
|
"rewards/chosen": 2.8491875060374805,
|
|
"rewards/margins": 4.878784832138389,
|
|
"rewards/rejected": -2.0295973261009084,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 8.729217529296875,
|
|
"kl": 12.896934509277344,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -51230771.2,
|
|
"logits/rejected": -48302675.2,
|
|
"logps/chosen": -491.442236328125,
|
|
"logps/rejected": -411.22578125,
|
|
"loss": 0.2505,
|
|
"loss/base_kto": 2.0038585662841797,
|
|
"metrics/advantage_var": 17345.353515625,
|
|
"rewards/chosen": 2.20242862701416,
|
|
"rewards/margins": 5.342057228088379,
|
|
"rewards/rejected": -3.139628601074219,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 7.272515296936035,
|
|
"kl": 20.829553604125977,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -50125110.885758996,
|
|
"logits/rejected": -50515789.079563186,
|
|
"logps/chosen": -442.1692097026604,
|
|
"logps/rejected": -390.7211875975039,
|
|
"loss": 0.2598,
|
|
"loss/base_kto": 2.0783112049102783,
|
|
"metrics/advantage_var": 14066.3505859375,
|
|
"rewards/chosen": 2.8817564676251957,
|
|
"rewards/margins": 4.72465825986876,
|
|
"rewards/rejected": -1.8429017922435647,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 9.126511573791504,
|
|
"kl": 18.096542358398438,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -50236237.91304348,
|
|
"logits/rejected": -50608037.83647799,
|
|
"logps/chosen": -446.0045613354037,
|
|
"logps/rejected": -405.5376130110063,
|
|
"loss": 0.2529,
|
|
"loss/base_kto": 2.023378610610962,
|
|
"metrics/advantage_var": 14046.7861328125,
|
|
"rewards/chosen": 2.22546064483453,
|
|
"rewards/margins": 4.758044205418257,
|
|
"rewards/rejected": -2.5325835605837264,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 7.987075328826904,
|
|
"kl": 33.31155014038086,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -50799396.45696401,
|
|
"logits/rejected": -50986567.08892356,
|
|
"logps/chosen": -466.1683294209703,
|
|
"logps/rejected": -393.43584243369736,
|
|
"loss": 0.2449,
|
|
"loss/base_kto": 1.9588383436203003,
|
|
"metrics/advantage_var": 14299.556640625,
|
|
"rewards/chosen": 2.978382474753032,
|
|
"rewards/margins": 4.966008049111943,
|
|
"rewards/rejected": -1.987625574358912,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 9.176545143127441,
|
|
"kl": 10.651198387145996,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -51713834.40490798,
|
|
"logits/rejected": -51210419.36305732,
|
|
"logps/chosen": -470.2095475460123,
|
|
"logps/rejected": -412.53739550159236,
|
|
"loss": 0.2613,
|
|
"loss/base_kto": 2.0903689861297607,
|
|
"metrics/advantage_var": 14415.9375,
|
|
"rewards/chosen": 2.202017005967216,
|
|
"rewards/margins": 4.79738327133097,
|
|
"rewards/rejected": -2.595366265363754,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 6.654029846191406,
|
|
"kl": 19.941139221191406,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -50187843.47286822,
|
|
"logits/rejected": -51112687.47086614,
|
|
"logps/chosen": -458.1766472868217,
|
|
"logps/rejected": -392.57022637795274,
|
|
"loss": 0.2557,
|
|
"loss/base_kto": 2.0455853939056396,
|
|
"metrics/advantage_var": 14611.9013671875,
|
|
"rewards/chosen": 2.500072674418605,
|
|
"rewards/margins": 4.968738666975199,
|
|
"rewards/rejected": -2.4686659925565944,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 8.606330871582031,
|
|
"kl": 11.365921974182129,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -51410057.46003017,
|
|
"logits/rejected": -51158256.64829822,
|
|
"logps/chosen": -470.796427224736,
|
|
"logps/rejected": -382.7708670988655,
|
|
"loss": 0.2686,
|
|
"loss/base_kto": 2.148906707763672,
|
|
"metrics/advantage_var": 14091.3740234375,
|
|
"rewards/chosen": 2.4390082956259427,
|
|
"rewards/margins": 4.72086251399608,
|
|
"rewards/rejected": -2.2818542183701376,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 7.486184597015381,
|
|
"kl": 27.97551918029785,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -50427826.59527559,
|
|
"logits/rejected": -49912639.900775194,
|
|
"logps/chosen": -449.01761811023624,
|
|
"logps/rejected": -401.06477713178293,
|
|
"loss": 0.2628,
|
|
"loss/base_kto": 2.102313756942749,
|
|
"metrics/advantage_var": 15452.3583984375,
|
|
"rewards/chosen": 2.2096102977362206,
|
|
"rewards/margins": 4.773240347094264,
|
|
"rewards/rejected": -2.563630049358043,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 8.14769458770752,
|
|
"kl": 21.685243606567383,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -51391586.368503936,
|
|
"logits/rejected": -52351553.88527132,
|
|
"logps/chosen": -441.48917322834643,
|
|
"logps/rejected": -404.3329215116279,
|
|
"loss": 0.2538,
|
|
"loss/base_kto": 2.030576467514038,
|
|
"metrics/advantage_var": 14459.845703125,
|
|
"rewards/chosen": 2.7408108929010826,
|
|
"rewards/margins": 4.912655233380734,
|
|
"rewards/rejected": -2.1718443404796512,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 8.10610294342041,
|
|
"kl": 17.0273494720459,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -50129007.825273015,
|
|
"logits/rejected": -52257966.67292645,
|
|
"logps/chosen": -470.134994149766,
|
|
"logps/rejected": -402.0153560250391,
|
|
"loss": 0.263,
|
|
"loss/base_kto": 2.10416316986084,
|
|
"metrics/advantage_var": 14936.8193359375,
|
|
"rewards/chosen": 2.101620392940718,
|
|
"rewards/margins": 4.83664681667409,
|
|
"rewards/rejected": -2.7350264237333723,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 7.606040954589844,
|
|
"kl": 19.883148193359375,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -50607266.01536098,
|
|
"logits/rejected": -51428334.09220985,
|
|
"logps/chosen": -476.41114631336404,
|
|
"logps/rejected": -367.1104680047695,
|
|
"loss": 0.2579,
|
|
"loss/base_kto": 2.0632483959198,
|
|
"metrics/advantage_var": 14255.6953125,
|
|
"rewards/chosen": 2.9189132479478688,
|
|
"rewards/margins": 4.890715199830818,
|
|
"rewards/rejected": -1.971801951882949,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 6.019439697265625,
|
|
"kl": 19.90534782409668,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -51477607.19379845,
|
|
"logits/rejected": -52088490.12913386,
|
|
"logps/chosen": -473.96148255813955,
|
|
"logps/rejected": -417.79862204724407,
|
|
"loss": 0.2387,
|
|
"loss/base_kto": 1.9098351001739502,
|
|
"metrics/advantage_var": 15776.4912109375,
|
|
"rewards/chosen": 2.769668460816376,
|
|
"rewards/margins": 5.314328701650529,
|
|
"rewards/rejected": -2.5446602408341534,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 5.279232978820801,
|
|
"kl": 18.785978317260742,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -49837756.294209704,
|
|
"logits/rejected": -50480509.80343214,
|
|
"logps/chosen": -462.4819053208138,
|
|
"logps/rejected": -391.620027301092,
|
|
"loss": 0.2391,
|
|
"loss/base_kto": 1.912419319152832,
|
|
"metrics/advantage_var": 15797.1923828125,
|
|
"rewards/chosen": 2.4113536470559467,
|
|
"rewards/margins": 5.437673606104308,
|
|
"rewards/rejected": -3.026319959048362,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 6.649168968200684,
|
|
"kl": 25.740583419799805,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -50990672.5129771,
|
|
"logits/rejected": -52533192.2944,
|
|
"logps/chosen": -483.17099236641224,
|
|
"logps/rejected": -405.1268,
|
|
"loss": 0.2562,
|
|
"loss/base_kto": 2.0497548580169678,
|
|
"metrics/advantage_var": 14378.15625,
|
|
"rewards/chosen": 2.62320155951813,
|
|
"rewards/margins": 4.86190702826813,
|
|
"rewards/rejected": -2.23870546875,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 8.85274600982666,
|
|
"kl": 27.409530639648438,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -52023243.90461049,
|
|
"logits/rejected": -52485766.488479264,
|
|
"logps/chosen": -450.50407392686805,
|
|
"logps/rejected": -399.8531105990783,
|
|
"loss": 0.2642,
|
|
"loss/base_kto": 2.113422155380249,
|
|
"metrics/advantage_var": 14363.2626953125,
|
|
"rewards/chosen": 2.7098499292031,
|
|
"rewards/margins": 4.678663683283361,
|
|
"rewards/rejected": -1.968813754080261,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 6.2643303871154785,
|
|
"kl": 16.370676040649414,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -52536850.428794995,
|
|
"logits/rejected": -51779555.2449298,
|
|
"logps/chosen": -460.3179773082942,
|
|
"logps/rejected": -398.3285150156006,
|
|
"loss": 0.2603,
|
|
"loss/base_kto": 2.082447052001953,
|
|
"metrics/advantage_var": 14686.6943359375,
|
|
"rewards/chosen": 2.232792097637911,
|
|
"rewards/margins": 4.775106505057957,
|
|
"rewards/rejected": -2.5423144074200468,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 8.45455551147461,
|
|
"kl": 12.755212783813477,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -51945536.0,
|
|
"logits/rejected": -51340521.6,
|
|
"logps/chosen": -495.90009765625,
|
|
"logps/rejected": -382.887548828125,
|
|
"loss": 0.2633,
|
|
"loss/base_kto": 2.1066510677337646,
|
|
"metrics/advantage_var": 15828.8095703125,
|
|
"rewards/chosen": 2.346480941772461,
|
|
"rewards/margins": 4.982112312316895,
|
|
"rewards/rejected": -2.6356313705444334,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 7.647529125213623,
|
|
"kl": 26.12347984313965,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -50083490.34146342,
|
|
"logits/rejected": -50876591.54285714,
|
|
"logps/chosen": -425.8041666666667,
|
|
"logps/rejected": -411.9215695488722,
|
|
"loss": 0.2553,
|
|
"loss/base_kto": 2.0427498817443848,
|
|
"metrics/advantage_var": 15325.8642578125,
|
|
"rewards/chosen": 2.465984859311484,
|
|
"rewards/margins": 4.981041911120695,
|
|
"rewards/rejected": -2.5150570518092104,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 7.476153373718262,
|
|
"kl": 16.00942039489746,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -52222755.34006734,
|
|
"logits/rejected": -51787650.6122449,
|
|
"logps/chosen": -454.8621632996633,
|
|
"logps/rejected": -391.1714194606414,
|
|
"loss": 0.257,
|
|
"loss/base_kto": 2.055962324142456,
|
|
"metrics/advantage_var": 15469.765625,
|
|
"rewards/chosen": 2.2594590106797137,
|
|
"rewards/margins": 5.041733262456317,
|
|
"rewards/rejected": -2.7822742517766037,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 8.433756828308105,
|
|
"kl": 17.34130859375,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -51585524.34597876,
|
|
"logits/rejected": -50295051.13043478,
|
|
"logps/chosen": -434.56885432473445,
|
|
"logps/rejected": -388.7183977455717,
|
|
"loss": 0.2523,
|
|
"loss/base_kto": 2.018437147140503,
|
|
"metrics/advantage_var": 15157.1318359375,
|
|
"rewards/chosen": 2.2025817037533195,
|
|
"rewards/margins": 5.203248077923005,
|
|
"rewards/rejected": -3.000666374169686,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 7.5897417068481445,
|
|
"kl": 27.72690200805664,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -51326822.87850467,
|
|
"logits/rejected": -51313021.99373041,
|
|
"logps/chosen": -441.95570482866043,
|
|
"logps/rejected": -380.2719190830721,
|
|
"loss": 0.2653,
|
|
"loss/base_kto": 2.1222727298736572,
|
|
"metrics/advantage_var": 14398.7373046875,
|
|
"rewards/chosen": 2.7608041733596185,
|
|
"rewards/margins": 4.658044886831895,
|
|
"rewards/rejected": -1.8972407134722766,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 8.917332649230957,
|
|
"kl": 15.3670654296875,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -52705762.47775947,
|
|
"logits/rejected": -52573042.49628529,
|
|
"logps/chosen": -477.01328253706754,
|
|
"logps/rejected": -394.1614970282318,
|
|
"loss": 0.2483,
|
|
"loss/base_kto": 1.9862979650497437,
|
|
"metrics/advantage_var": 14811.9443359375,
|
|
"rewards/chosen": 2.8682984001750413,
|
|
"rewards/margins": 5.139281253014473,
|
|
"rewards/rejected": -2.2709828528394316,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 6.401049613952637,
|
|
"kl": 20.232099533081055,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -51152623.58974359,
|
|
"logits/rejected": -52514246.06727829,
|
|
"logps/chosen": -471.6942608173077,
|
|
"logps/rejected": -404.40089831804283,
|
|
"loss": 0.2116,
|
|
"loss/base_kto": 1.6930650472640991,
|
|
"metrics/advantage_var": 16739.052734375,
|
|
"rewards/chosen": 2.6238733927408853,
|
|
"rewards/margins": 5.894996783055296,
|
|
"rewards/rejected": -3.2711233903144112,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 6.078398704528809,
|
|
"kl": 18.59099006652832,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -50931055.8755832,
|
|
"logits/rejected": -51348603.78021978,
|
|
"logps/chosen": -450.2767301710731,
|
|
"logps/rejected": -388.2606456043956,
|
|
"loss": 0.2076,
|
|
"loss/base_kto": 1.660752534866333,
|
|
"metrics/advantage_var": 14589.8154296875,
|
|
"rewards/chosen": 2.7858607646651437,
|
|
"rewards/margins": 5.617628710725779,
|
|
"rewards/rejected": -2.8317679460606358,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 11.037747383117676,
|
|
"kl": 21.137937545776367,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -53843410.67540984,
|
|
"logits/rejected": -53166801.38507463,
|
|
"logps/chosen": -465.16198770491803,
|
|
"logps/rejected": -406.10074626865674,
|
|
"loss": 0.2025,
|
|
"loss/base_kto": 1.6196577548980713,
|
|
"metrics/advantage_var": 16674.591796875,
|
|
"rewards/chosen": 2.7636190445696722,
|
|
"rewards/margins": 6.006351670502508,
|
|
"rewards/rejected": -3.242732625932836,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 7.611783027648926,
|
|
"kl": 13.244036674499512,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -52446179.64307693,
|
|
"logits/rejected": -51852775.61904762,
|
|
"logps/chosen": -444.28432692307695,
|
|
"logps/rejected": -401.10828373015875,
|
|
"loss": 0.2162,
|
|
"loss/base_kto": 1.7293108701705933,
|
|
"metrics/advantage_var": 15236.3466796875,
|
|
"rewards/chosen": 2.6056948617788462,
|
|
"rewards/margins": 5.519433003519918,
|
|
"rewards/rejected": -2.9137381417410713,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 8.240866661071777,
|
|
"kl": 18.3961238861084,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -49095924.111455105,
|
|
"logits/rejected": -49097731.23028391,
|
|
"logps/chosen": -460.3483939628483,
|
|
"logps/rejected": -383.85639294164037,
|
|
"loss": 0.2125,
|
|
"loss/base_kto": 1.6999447345733643,
|
|
"metrics/advantage_var": 14634.9404296875,
|
|
"rewards/chosen": 2.5562453137093653,
|
|
"rewards/margins": 5.478640556586928,
|
|
"rewards/rejected": -2.9223952428775632,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 6.910224914550781,
|
|
"kl": 18.553775787353516,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -52505239.25434439,
|
|
"logits/rejected": -52949324.364760436,
|
|
"logps/chosen": -453.1098933649289,
|
|
"logps/rejected": -390.14055255023186,
|
|
"loss": 0.2146,
|
|
"loss/base_kto": 1.7165441513061523,
|
|
"metrics/advantage_var": 17012.947265625,
|
|
"rewards/chosen": 2.5390918123025275,
|
|
"rewards/margins": 5.7191121284250155,
|
|
"rewards/rejected": -3.1800203161224885,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 5.961915969848633,
|
|
"kl": 17.614856719970703,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -50447468.27794562,
|
|
"logits/rejected": -51201342.13592233,
|
|
"logps/chosen": -427.4541163141994,
|
|
"logps/rejected": -407.8164694579288,
|
|
"loss": 0.2066,
|
|
"loss/base_kto": 1.653010606765747,
|
|
"metrics/advantage_var": 14973.6689453125,
|
|
"rewards/chosen": 2.656560892064766,
|
|
"rewards/margins": 5.582759314394863,
|
|
"rewards/rejected": -2.926198422330097,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 3.847012519836426,
|
|
"kl": 23.62422752380371,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -51482255.92626728,
|
|
"logits/rejected": -52466816.61049285,
|
|
"logps/chosen": -459.6874039938556,
|
|
"logps/rejected": -410.0303557233704,
|
|
"loss": 0.2116,
|
|
"loss/base_kto": 1.6924909353256226,
|
|
"metrics/advantage_var": 15904.1904296875,
|
|
"rewards/chosen": 2.9562669510848694,
|
|
"rewards/margins": 5.72156319574405,
|
|
"rewards/rejected": -2.765296244659181,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 9.123897552490234,
|
|
"kl": 23.464460372924805,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -51229029.1712,
|
|
"logits/rejected": -52494498.58931298,
|
|
"logps/chosen": -448.3186,
|
|
"logps/rejected": -407.9988072519084,
|
|
"loss": 0.2057,
|
|
"loss/base_kto": 1.6455026865005493,
|
|
"metrics/advantage_var": 17421.048828125,
|
|
"rewards/chosen": 2.7009166015625,
|
|
"rewards/margins": 5.928064666328721,
|
|
"rewards/rejected": -3.2271480647662214,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 9.547018051147461,
|
|
"kl": 24.59076499938965,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -51683574.138674885,
|
|
"logits/rejected": -52181183.49286846,
|
|
"logps/chosen": -486.013000770416,
|
|
"logps/rejected": -394.2410608161648,
|
|
"loss": 0.2184,
|
|
"loss/base_kto": 1.74701726436615,
|
|
"metrics/advantage_var": 14838.837890625,
|
|
"rewards/chosen": 2.7930456787485554,
|
|
"rewards/margins": 5.485414949375537,
|
|
"rewards/rejected": -2.692369270626981,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 6.961014270782471,
|
|
"kl": 26.59592628479004,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -52061777.361155696,
|
|
"logits/rejected": -51680260.67579909,
|
|
"logps/chosen": -425.9284209470305,
|
|
"logps/rejected": -415.11115867579906,
|
|
"loss": 0.2018,
|
|
"loss/base_kto": 1.614264726638794,
|
|
"metrics/advantage_var": 14626.306640625,
|
|
"rewards/chosen": 2.8435554320701244,
|
|
"rewards/margins": 5.742419156266282,
|
|
"rewards/rejected": -2.898863724196157,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.428321838378906,
|
|
"kl": 20.4986629486084,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -52222980.82260597,
|
|
"logits/rejected": -51456238.88024884,
|
|
"logps/chosen": -455.01800431711143,
|
|
"logps/rejected": -404.5633262052877,
|
|
"loss": 0.2038,
|
|
"loss/base_kto": 1.630610704421997,
|
|
"metrics/advantage_var": 15116.5751953125,
|
|
"rewards/chosen": 2.7454387479150313,
|
|
"rewards/margins": 5.538942860716354,
|
|
"rewards/rejected": -2.793504112801322,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 6.948566436767578,
|
|
"kl": 19.01148796081543,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -52328898.00896861,
|
|
"logits/rejected": -53048723.90180033,
|
|
"logps/chosen": -458.93189461883406,
|
|
"logps/rejected": -408.41865282324056,
|
|
"loss": 0.2116,
|
|
"loss/base_kto": 1.6924980878829956,
|
|
"metrics/advantage_var": 14713.4814453125,
|
|
"rewards/chosen": 2.7260599863368835,
|
|
"rewards/margins": 5.545678024133528,
|
|
"rewards/rejected": -2.8196180377966447,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 6.769558429718018,
|
|
"kl": 19.767709732055664,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -51488283.80830671,
|
|
"logits/rejected": -52025055.90214067,
|
|
"logps/chosen": -455.0748801916933,
|
|
"logps/rejected": -372.0299120795107,
|
|
"loss": 0.1975,
|
|
"loss/base_kto": 1.5798718929290771,
|
|
"metrics/advantage_var": 14985.7646484375,
|
|
"rewards/chosen": 2.8366749918879792,
|
|
"rewards/margins": 5.937662977481443,
|
|
"rewards/rejected": -3.1009879855934632,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 10.205763816833496,
|
|
"kl": 20.990039825439453,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -52205237.57188498,
|
|
"logits/rejected": -52282449.41896024,
|
|
"logps/chosen": -481.22698682108626,
|
|
"logps/rejected": -379.97902331804283,
|
|
"loss": 0.2054,
|
|
"loss/base_kto": 1.643288016319275,
|
|
"metrics/advantage_var": 16388.818359375,
|
|
"rewards/chosen": 2.6462098142971247,
|
|
"rewards/margins": 5.959112235694295,
|
|
"rewards/rejected": -3.312902421397171,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 9.473999977111816,
|
|
"kl": 18.7738094329834,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -50897698.50783699,
|
|
"logits/rejected": -51462290.741433024,
|
|
"logps/chosen": -466.38964537617557,
|
|
"logps/rejected": -401.3882398753894,
|
|
"loss": 0.215,
|
|
"loss/base_kto": 1.71996009349823,
|
|
"metrics/advantage_var": 14777.6533203125,
|
|
"rewards/chosen": 2.3673109096419473,
|
|
"rewards/margins": 5.703103142167259,
|
|
"rewards/rejected": -3.3357922325253115,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 8.424354553222656,
|
|
"kl": 17.50851821899414,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -52425362.86529318,
|
|
"logits/rejected": -53291522.366718024,
|
|
"logps/chosen": -464.77867472266246,
|
|
"logps/rejected": -428.4358628659476,
|
|
"loss": 0.2224,
|
|
"loss/base_kto": 1.7790508270263672,
|
|
"metrics/advantage_var": 15547.15625,
|
|
"rewards/chosen": 2.810249727614897,
|
|
"rewards/margins": 5.644079116578688,
|
|
"rewards/rejected": -2.8338293889637907,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.850090026855469,
|
|
"kl": 23.67220115661621,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -51791914.73515249,
|
|
"logits/rejected": -51904091.17808219,
|
|
"logps/chosen": -465.92596308186194,
|
|
"logps/rejected": -405.92142313546424,
|
|
"loss": 0.1973,
|
|
"loss/base_kto": 1.5781967639923096,
|
|
"metrics/advantage_var": 16292.103515625,
|
|
"rewards/chosen": 2.8144715433136036,
|
|
"rewards/margins": 5.905784967852226,
|
|
"rewards/rejected": -3.0913134245386225,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 5.788835525512695,
|
|
"kl": 12.649649620056152,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -51609774.5631659,
|
|
"logits/rejected": -51973779.10754414,
|
|
"logps/chosen": -453.40577435312025,
|
|
"logps/rejected": -421.79268659711073,
|
|
"loss": 0.1977,
|
|
"loss/base_kto": 1.581206202507019,
|
|
"metrics/advantage_var": 15676.1904296875,
|
|
"rewards/chosen": 2.6842882687642695,
|
|
"rewards/margins": 5.96072932545819,
|
|
"rewards/rejected": -3.2764410566939204,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 5.196254253387451,
|
|
"kl": 18.2519588470459,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -53048049.13548387,
|
|
"logits/rejected": -52975417.406060606,
|
|
"logps/chosen": -464.4367943548387,
|
|
"logps/rejected": -399.4305397727273,
|
|
"loss": 0.2091,
|
|
"loss/base_kto": 1.673047661781311,
|
|
"metrics/advantage_var": 15030.0849609375,
|
|
"rewards/chosen": 2.5911819950226813,
|
|
"rewards/margins": 5.556486652929879,
|
|
"rewards/rejected": -2.965304657907197,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 8.297974586486816,
|
|
"kl": 15.502278327941895,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -52658809.97875569,
|
|
"logits/rejected": -53108066.52495974,
|
|
"logps/chosen": -468.6451062215478,
|
|
"logps/rejected": -410.24074074074076,
|
|
"loss": 0.1953,
|
|
"loss/base_kto": 1.5620167255401611,
|
|
"metrics/advantage_var": 15958.2158203125,
|
|
"rewards/chosen": 2.9076016647500946,
|
|
"rewards/margins": 5.964484071528476,
|
|
"rewards/rejected": -3.0568824067783815,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 6.340052604675293,
|
|
"kl": 22.518726348876953,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -52333680.330683626,
|
|
"logits/rejected": -52139670.21812596,
|
|
"logps/chosen": -461.2084658187599,
|
|
"logps/rejected": -376.0010560675883,
|
|
"loss": 0.2126,
|
|
"loss/base_kto": 1.700911521911621,
|
|
"metrics/advantage_var": 14260.2021484375,
|
|
"rewards/chosen": 2.755645509365064,
|
|
"rewards/margins": 5.5544372570369145,
|
|
"rewards/rejected": -2.798791747671851,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 7.071959018707275,
|
|
"kl": 17.614599227905273,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -51984199.4954955,
|
|
"logits/rejected": -54268844.61237785,
|
|
"logps/chosen": -461.6692942942943,
|
|
"logps/rejected": -391.4291276465798,
|
|
"loss": 0.2117,
|
|
"loss/base_kto": 1.693826675415039,
|
|
"metrics/advantage_var": 15238.3154296875,
|
|
"rewards/chosen": 2.8153856981981984,
|
|
"rewards/margins": 5.722483231980263,
|
|
"rewards/rejected": -2.907097533782064,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 7.8359904289245605,
|
|
"kl": 17.305866241455078,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -52183382.39009288,
|
|
"logits/rejected": -52238788.239747636,
|
|
"logps/chosen": -453.1547503869969,
|
|
"logps/rejected": -426.9443020504732,
|
|
"loss": 0.2045,
|
|
"loss/base_kto": 1.6361528635025024,
|
|
"metrics/advantage_var": 14747.8564453125,
|
|
"rewards/chosen": 2.805559376814048,
|
|
"rewards/margins": 5.680909414397842,
|
|
"rewards/rejected": -2.8753500375837935,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 6.994919776916504,
|
|
"kl": 12.713762283325195,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -52167258.71471471,
|
|
"logits/rejected": -51606157.758957654,
|
|
"logps/chosen": -454.4685623123123,
|
|
"logps/rejected": -402.00356270358304,
|
|
"loss": 0.2012,
|
|
"loss/base_kto": 1.6097615957260132,
|
|
"metrics/advantage_var": 14512.5908203125,
|
|
"rewards/chosen": 2.91284399634009,
|
|
"rewards/margins": 5.728330742764662,
|
|
"rewards/rejected": -2.8154867464245723,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 10.00118637084961,
|
|
"kl": 20.05970001220703,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -51249053.696,
|
|
"logits/rejected": -52149540.3480916,
|
|
"logps/chosen": -480.2795,
|
|
"logps/rejected": -398.30271946564886,
|
|
"loss": 0.2203,
|
|
"loss/base_kto": 1.7626689672470093,
|
|
"metrics/advantage_var": 15542.7998046875,
|
|
"rewards/chosen": 2.693503125,
|
|
"rewards/margins": 5.515547532502385,
|
|
"rewards/rejected": -2.8220444075023856,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 7.515979766845703,
|
|
"kl": 21.391738891601562,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -52715101.09090909,
|
|
"logits/rejected": -52747603.27710843,
|
|
"logps/chosen": -484.2689732142857,
|
|
"logps/rejected": -397.6544145331325,
|
|
"loss": 0.2142,
|
|
"loss/base_kto": 1.7132612466812134,
|
|
"metrics/advantage_var": 15169.9345703125,
|
|
"rewards/chosen": 2.683006782036323,
|
|
"rewards/margins": 5.657759920567008,
|
|
"rewards/rejected": -2.9747531385306853,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.797512531280518,
|
|
"kl": 18.113786697387695,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -52307566.11616954,
|
|
"logits/rejected": -53364216.833592534,
|
|
"logps/chosen": -457.19368131868134,
|
|
"logps/rejected": -406.29393468118195,
|
|
"loss": 0.1993,
|
|
"loss/base_kto": 1.5946701765060425,
|
|
"metrics/advantage_var": 14731.962890625,
|
|
"rewards/chosen": 2.7622787020457222,
|
|
"rewards/margins": 5.704829527339754,
|
|
"rewards/rejected": -2.942550825294032,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 8.445562362670898,
|
|
"kl": 14.823893547058105,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -51118715.24394184,
|
|
"logits/rejected": -50332592.9924357,
|
|
"logps/chosen": -444.25504846526655,
|
|
"logps/rejected": -415.2663577912254,
|
|
"loss": 0.2066,
|
|
"loss/base_kto": 1.6527931690216064,
|
|
"metrics/advantage_var": 14747.3642578125,
|
|
"rewards/chosen": 2.554538806921446,
|
|
"rewards/margins": 5.7892395739883895,
|
|
"rewards/rejected": -3.234700767066944,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 8.256867408752441,
|
|
"kl": 17.39256477355957,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -50655182.7987988,
|
|
"logits/rejected": -51788112.88599349,
|
|
"logps/chosen": -464.0256193693694,
|
|
"logps/rejected": -379.0589881921824,
|
|
"loss": 0.2149,
|
|
"loss/base_kto": 1.7192754745483398,
|
|
"metrics/advantage_var": 15078.853515625,
|
|
"rewards/chosen": 2.6980321385838963,
|
|
"rewards/margins": 5.56999032135059,
|
|
"rewards/rejected": -2.871958182766694,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 12.323614120483398,
|
|
"kl": 13.718680381774902,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -51934669.990697674,
|
|
"logits/rejected": -51647728.27716535,
|
|
"logps/chosen": -473.34796511627906,
|
|
"logps/rejected": -391.9261318897638,
|
|
"loss": 0.2148,
|
|
"loss/base_kto": 1.7184391021728516,
|
|
"metrics/advantage_var": 16194.2373046875,
|
|
"rewards/chosen": 2.701825944767442,
|
|
"rewards/margins": 5.7044957282320095,
|
|
"rewards/rejected": -3.002669783464567,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 7.657737731933594,
|
|
"kl": 16.89313507080078,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -52261126.896988906,
|
|
"logits/rejected": -51801802.74884438,
|
|
"logps/chosen": -453.9925713153724,
|
|
"logps/rejected": -408.42045454545456,
|
|
"loss": 0.2062,
|
|
"loss/base_kto": 1.6494368314743042,
|
|
"metrics/advantage_var": 15846.103515625,
|
|
"rewards/chosen": 2.789609978580626,
|
|
"rewards/margins": 5.766267087775059,
|
|
"rewards/rejected": -2.976657109194434,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 6.649723052978516,
|
|
"kl": 16.165952682495117,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -52194091.591331266,
|
|
"logits/rejected": -52126716.76971609,
|
|
"logps/chosen": -469.999661377709,
|
|
"logps/rejected": -394.8598186119874,
|
|
"loss": 0.2136,
|
|
"loss/base_kto": 1.7085020542144775,
|
|
"metrics/advantage_var": 14259.7578125,
|
|
"rewards/chosen": 2.6862430159152475,
|
|
"rewards/margins": 5.366490999637152,
|
|
"rewards/rejected": -2.6802479837219044,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 7.129169940948486,
|
|
"kl": 17.871610641479492,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -53591976.04907975,
|
|
"logits/rejected": -54577712.91719745,
|
|
"logps/chosen": -475.78930214723925,
|
|
"logps/rejected": -424.4720342356688,
|
|
"loss": 0.2101,
|
|
"loss/base_kto": 1.6811965703964233,
|
|
"metrics/advantage_var": 15833.6982421875,
|
|
"rewards/chosen": 2.7122051964508245,
|
|
"rewards/margins": 5.6900568211522575,
|
|
"rewards/rejected": -2.977851624701433,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 8.79556655883789,
|
|
"kl": 13.786334037780762,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -50803324.00594354,
|
|
"logits/rejected": -51990264.83031301,
|
|
"logps/chosen": -429.8521545319465,
|
|
"logps/rejected": -396.9826503294893,
|
|
"loss": 0.2138,
|
|
"loss/base_kto": 1.710473895072937,
|
|
"metrics/advantage_var": 14456.3095703125,
|
|
"rewards/chosen": 2.7941658734212482,
|
|
"rewards/margins": 5.510056170219024,
|
|
"rewards/rejected": -2.715890296797776,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 6.54217529296875,
|
|
"kl": 13.671980857849121,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -50993193.66510172,
|
|
"logits/rejected": -51762870.914196566,
|
|
"logps/chosen": -463.6511150234742,
|
|
"logps/rejected": -396.309379875195,
|
|
"loss": 0.1945,
|
|
"loss/base_kto": 1.5562013387680054,
|
|
"metrics/advantage_var": 14981.1943359375,
|
|
"rewards/chosen": 2.8755658407912756,
|
|
"rewards/margins": 5.85812178365399,
|
|
"rewards/rejected": -2.9825559428627146,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.645531177520752,
|
|
"kl": 19.706483840942383,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -51603658.47649919,
|
|
"logits/rejected": -50926791.239819005,
|
|
"logps/chosen": -475.88614262560776,
|
|
"logps/rejected": -404.70046191553547,
|
|
"loss": 0.2135,
|
|
"loss/base_kto": 1.708304762840271,
|
|
"metrics/advantage_var": 15029.6640625,
|
|
"rewards/chosen": 2.6179828365579416,
|
|
"rewards/margins": 5.423700477430396,
|
|
"rewards/rejected": -2.8057176408724547,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 6.593998908996582,
|
|
"kl": 23.683530807495117,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -53067135.21472393,
|
|
"logits/rejected": -52689486.26751592,
|
|
"logps/chosen": -448.24467983128835,
|
|
"logps/rejected": -419.56822253184714,
|
|
"loss": 0.211,
|
|
"loss/base_kto": 1.6882925033569336,
|
|
"metrics/advantage_var": 16097.6591796875,
|
|
"rewards/chosen": 2.571448577693635,
|
|
"rewards/margins": 5.724529997814555,
|
|
"rewards/rejected": -3.1530814201209196,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_loss": 0.27245468189473393,
|
|
"train_runtime": 11042.7298,
|
|
"train_samples_per_second": 13.422,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|