1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 10.389772415161133,
|
|
"kl": 47.63533401489258,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -38406302.7123696,
|
|
"logits/rejected": -37807556.30870279,
|
|
"logps/chosen": -457.81594634873323,
|
|
"logps/rejected": -390.4787048440066,
|
|
"loss": 0.4565,
|
|
"loss/base_kto": 3.651801824569702,
|
|
"metrics/advantage_var": 551.3017578125,
|
|
"rewards/chosen": 0.8933472953148286,
|
|
"rewards/margins": 0.2512501854747988,
|
|
"rewards/rejected": 0.6420971098400298,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 13.195425033569336,
|
|
"kl": 12.069128036499023,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37387905.259016395,
|
|
"logits/rejected": -38080683.1761194,
|
|
"logps/chosen": -481.9829918032787,
|
|
"logps/rejected": -372.9957555970149,
|
|
"loss": 0.4578,
|
|
"loss/base_kto": 3.6623947620391846,
|
|
"metrics/advantage_var": 629.2352905273438,
|
|
"rewards/chosen": 0.25825915727459015,
|
|
"rewards/margins": 0.3732206049191797,
|
|
"rewards/rejected": -0.11496144764458956,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 9.603527069091797,
|
|
"kl": 3.2372887134552,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -37479492.266666666,
|
|
"logits/rejected": -38547525.316923074,
|
|
"logps/chosen": -482.2787698412698,
|
|
"logps/rejected": -351.85790865384615,
|
|
"loss": 0.4407,
|
|
"loss/base_kto": 3.5256569385528564,
|
|
"metrics/advantage_var": 952.7435913085938,
|
|
"rewards/chosen": -0.06462832859584264,
|
|
"rewards/margins": 0.5826495034354073,
|
|
"rewards/rejected": -0.64727783203125,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 10.463303565979004,
|
|
"kl": 8.231558799743652,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -37918667.98730159,
|
|
"logits/rejected": -39891054.276923075,
|
|
"logps/chosen": -489.1375992063492,
|
|
"logps/rejected": -385.59139423076925,
|
|
"loss": 0.4274,
|
|
"loss/base_kto": 3.4195592403411865,
|
|
"metrics/advantage_var": 1588.9461669921875,
|
|
"rewards/chosen": 0.16997208368210565,
|
|
"rewards/margins": 0.6811758847638365,
|
|
"rewards/rejected": -0.5112038010817308,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 8.33227825164795,
|
|
"kl": 12.142549514770508,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -38111445.53554502,
|
|
"logits/rejected": -37718256.56877898,
|
|
"logps/chosen": -492.62213665086887,
|
|
"logps/rejected": -366.55549652241115,
|
|
"loss": 0.4183,
|
|
"loss/base_kto": 3.346160888671875,
|
|
"metrics/advantage_var": 2415.298095703125,
|
|
"rewards/chosen": 0.5311772013539199,
|
|
"rewards/margins": 0.9226113652685016,
|
|
"rewards/rejected": -0.39143416391458175,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 9.39903450012207,
|
|
"kl": 24.029155731201172,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -39135715.85141159,
|
|
"logits/rejected": -40351362.741350904,
|
|
"logps/chosen": -472.12286404160477,
|
|
"logps/rejected": -378.63529654036245,
|
|
"loss": 0.3813,
|
|
"loss/base_kto": 3.050102949142456,
|
|
"metrics/advantage_var": 3768.57275390625,
|
|
"rewards/chosen": 1.3469364341973904,
|
|
"rewards/margins": 1.407133280871849,
|
|
"rewards/rejected": -0.060196846674458786,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 9.532487869262695,
|
|
"kl": 19.03123664855957,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -39336960.0,
|
|
"logits/rejected": -40051772.42236025,
|
|
"logps/chosen": -481.5810239779874,
|
|
"logps/rejected": -411.50058229813664,
|
|
"loss": 0.399,
|
|
"loss/base_kto": 3.191612720489502,
|
|
"metrics/advantage_var": 4326.13818359375,
|
|
"rewards/chosen": 0.8986968034468357,
|
|
"rewards/margins": 1.398232784619196,
|
|
"rewards/rejected": -0.49953598117236025,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 8.447214126586914,
|
|
"kl": 20.36687469482422,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -39655028.220858894,
|
|
"logits/rejected": -40515189.401273884,
|
|
"logps/chosen": -449.6458013803681,
|
|
"logps/rejected": -388.88062300955414,
|
|
"loss": 0.3818,
|
|
"loss/base_kto": 3.05460262298584,
|
|
"metrics/advantage_var": 4241.35400390625,
|
|
"rewards/chosen": 1.2029486346098544,
|
|
"rewards/margins": 1.5763467533275097,
|
|
"rewards/rejected": -0.37339811871765527,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 7.352090358734131,
|
|
"kl": 35.560909271240234,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -39940968.540192924,
|
|
"logits/rejected": -40337252.3768997,
|
|
"logps/chosen": -485.1162580385852,
|
|
"logps/rejected": -371.94633358662617,
|
|
"loss": 0.377,
|
|
"loss/base_kto": 3.015608072280884,
|
|
"metrics/advantage_var": 4760.81591796875,
|
|
"rewards/chosen": 1.484773102297277,
|
|
"rewards/margins": 1.578296006064315,
|
|
"rewards/rejected": -0.0935229037670379,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 9.246931076049805,
|
|
"kl": 13.78105354309082,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -42092757.59509202,
|
|
"logits/rejected": -40996766.1656051,
|
|
"logps/chosen": -470.5297162576687,
|
|
"logps/rejected": -393.69737758757964,
|
|
"loss": 0.3788,
|
|
"loss/base_kto": 3.0301249027252197,
|
|
"metrics/advantage_var": 4451.1181640625,
|
|
"rewards/chosen": 0.716448543993242,
|
|
"rewards/margins": 1.7276718207025374,
|
|
"rewards/rejected": -1.0112232767092955,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 8.291376113891602,
|
|
"kl": 29.377546310424805,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -43047583.744,
|
|
"logits/rejected": -41190317.14198473,
|
|
"logps/chosen": -476.7729,
|
|
"logps/rejected": -359.94021946564885,
|
|
"loss": 0.3642,
|
|
"loss/base_kto": 2.9133713245391846,
|
|
"metrics/advantage_var": 5095.07861328125,
|
|
"rewards/chosen": 1.3700134765625,
|
|
"rewards/margins": 1.8142961131321567,
|
|
"rewards/rejected": -0.4442826365696565,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 6.037034511566162,
|
|
"kl": 46.892433166503906,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -41527751.81452859,
|
|
"logits/rejected": -42193924.85308057,
|
|
"logps/chosen": -450.4008887171561,
|
|
"logps/rejected": -365.4776856240126,
|
|
"loss": 0.3526,
|
|
"loss/base_kto": 2.8206701278686523,
|
|
"metrics/advantage_var": 5228.74072265625,
|
|
"rewards/chosen": 1.7855769869348919,
|
|
"rewards/margins": 1.924404125537343,
|
|
"rewards/rejected": -0.13882713860245113,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 8.512300491333008,
|
|
"kl": 14.671371459960938,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -44227913.432646595,
|
|
"logits/rejected": -45005872.91217257,
|
|
"logps/chosen": -486.1079635499208,
|
|
"logps/rejected": -397.2251540832049,
|
|
"loss": 0.3561,
|
|
"loss/base_kto": 2.8489062786102295,
|
|
"metrics/advantage_var": 6176.60205078125,
|
|
"rewards/chosen": 1.0215558567440075,
|
|
"rewards/margins": 2.2437295622344253,
|
|
"rewards/rejected": -1.222173705490418,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 9.194314002990723,
|
|
"kl": 30.27271842956543,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -43314384.7133758,
|
|
"logits/rejected": -43476948.02453988,
|
|
"logps/chosen": -466.47462181528664,
|
|
"logps/rejected": -374.32992235429447,
|
|
"loss": 0.3479,
|
|
"loss/base_kto": 2.782944440841675,
|
|
"metrics/advantage_var": 6720.2744140625,
|
|
"rewards/chosen": 1.835502089968153,
|
|
"rewards/margins": 2.3519556515754667,
|
|
"rewards/rejected": -0.516453561607314,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 8.246805191040039,
|
|
"kl": 23.512361526489258,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -44900505.84037559,
|
|
"logits/rejected": -44444689.5725429,
|
|
"logps/chosen": -470.71112089201876,
|
|
"logps/rejected": -381.8404592433697,
|
|
"loss": 0.3549,
|
|
"loss/base_kto": 2.8391671180725098,
|
|
"metrics/advantage_var": 7269.17431640625,
|
|
"rewards/chosen": 1.6048429247359155,
|
|
"rewards/margins": 2.248600380356406,
|
|
"rewards/rejected": -0.6437574556204905,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 7.926791191101074,
|
|
"kl": 28.59177017211914,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -42740652.972972974,
|
|
"logits/rejected": -43164114.97068404,
|
|
"logps/chosen": -468.4436936936937,
|
|
"logps/rejected": -354.4130191368078,
|
|
"loss": 0.3396,
|
|
"loss/base_kto": 2.716761350631714,
|
|
"metrics/advantage_var": 6583.8837890625,
|
|
"rewards/chosen": 1.8008454010651276,
|
|
"rewards/margins": 2.4791936068303455,
|
|
"rewards/rejected": -0.6783482057652178,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 8.163418769836426,
|
|
"kl": 27.2202091217041,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -43318928.96385542,
|
|
"logits/rejected": -43874051.32467532,
|
|
"logps/chosen": -462.8449736445783,
|
|
"logps/rejected": -392.1993709415584,
|
|
"loss": 0.3595,
|
|
"loss/base_kto": 2.8757095336914062,
|
|
"metrics/advantage_var": 6862.53662109375,
|
|
"rewards/chosen": 1.682490337325866,
|
|
"rewards/margins": 2.2010155476197224,
|
|
"rewards/rejected": -0.5185252102938566,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 8.326271057128906,
|
|
"kl": 32.66901779174805,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -42703674.385964915,
|
|
"logits/rejected": -42824407.620214395,
|
|
"logps/chosen": -483.09006180223287,
|
|
"logps/rejected": -362.03189605666154,
|
|
"loss": 0.3395,
|
|
"loss/base_kto": 2.716346263885498,
|
|
"metrics/advantage_var": 7354.09375,
|
|
"rewards/chosen": 1.8718207205691786,
|
|
"rewards/margins": 2.6472364886886752,
|
|
"rewards/rejected": -0.7754157681194965,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 9.613969802856445,
|
|
"kl": 9.25661849975586,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -44944765.7472,
|
|
"logits/rejected": -45180043.1389313,
|
|
"logps/chosen": -487.5415,
|
|
"logps/rejected": -393.5746183206107,
|
|
"loss": 0.3328,
|
|
"loss/base_kto": 2.6622776985168457,
|
|
"metrics/advantage_var": 8602.853515625,
|
|
"rewards/chosen": 0.8969716796875,
|
|
"rewards/margins": 2.875173835579676,
|
|
"rewards/rejected": -1.9782021558921756,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 8.464766502380371,
|
|
"kl": 28.85038185119629,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -44117981.34153846,
|
|
"logits/rejected": -45089457.168253966,
|
|
"logps/chosen": -472.04365384615386,
|
|
"logps/rejected": -391.177058531746,
|
|
"loss": 0.3282,
|
|
"loss/base_kto": 2.6255855560302734,
|
|
"metrics/advantage_var": 7806.515625,
|
|
"rewards/chosen": 1.9826173753004808,
|
|
"rewards/margins": 2.7731585730621378,
|
|
"rewards/rejected": -0.7905411977616568,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 8.245142936706543,
|
|
"kl": 14.65202522277832,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -44670906.469135806,
|
|
"logits/rejected": -46875116.55696203,
|
|
"logps/chosen": -480.6135223765432,
|
|
"logps/rejected": -370.848397943038,
|
|
"loss": 0.3341,
|
|
"loss/base_kto": 2.672945499420166,
|
|
"metrics/advantage_var": 8595.6455078125,
|
|
"rewards/chosen": 1.4219384840977045,
|
|
"rewards/margins": 2.9975295593671865,
|
|
"rewards/rejected": -1.5755910752694817,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 8.22985553741455,
|
|
"kl": 30.664514541625977,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -43924364.758842446,
|
|
"logits/rejected": -44938856.26747721,
|
|
"logps/chosen": -461.2988344051447,
|
|
"logps/rejected": -384.89855623100306,
|
|
"loss": 0.3369,
|
|
"loss/base_kto": 2.6951730251312256,
|
|
"metrics/advantage_var": 9284.2763671875,
|
|
"rewards/chosen": 1.7856076353999195,
|
|
"rewards/margins": 2.958200957968309,
|
|
"rewards/rejected": -1.172593322568389,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 5.799438953399658,
|
|
"kl": 50.48659896850586,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -44397791.41818182,
|
|
"logits/rejected": -44613483.35483871,
|
|
"logps/chosen": -436.6492424242424,
|
|
"logps/rejected": -382.8592741935484,
|
|
"loss": 0.329,
|
|
"loss/base_kto": 2.632227659225464,
|
|
"metrics/advantage_var": 8154.85498046875,
|
|
"rewards/chosen": 2.547796445904356,
|
|
"rewards/margins": 2.721452651112427,
|
|
"rewards/rejected": -0.17365620520807082,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 8.989084243774414,
|
|
"kl": 29.10886573791504,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -48569896.66126418,
|
|
"logits/rejected": -47298353.03770739,
|
|
"logps/chosen": -482.85595623987035,
|
|
"logps/rejected": -374.7420814479638,
|
|
"loss": 0.3312,
|
|
"loss/base_kto": 2.6498098373413086,
|
|
"metrics/advantage_var": 9052.373046875,
|
|
"rewards/chosen": 1.925774127583063,
|
|
"rewards/margins": 2.9796671579021616,
|
|
"rewards/rejected": -1.0538930303190988,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 7.970582485198975,
|
|
"kl": 34.053043365478516,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -46713117.402756505,
|
|
"logits/rejected": -46109504.91866029,
|
|
"logps/chosen": -475.3674866003063,
|
|
"logps/rejected": -390.2767145135566,
|
|
"loss": 0.3211,
|
|
"loss/base_kto": 2.569037437438965,
|
|
"metrics/advantage_var": 9849.2734375,
|
|
"rewards/chosen": 2.1553962555632658,
|
|
"rewards/margins": 3.101857520920373,
|
|
"rewards/rejected": -0.9464612653571073,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.372785568237305,
|
|
"kl": 26.26869773864746,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -48282539.70731708,
|
|
"logits/rejected": -47864142.76923077,
|
|
"logps/chosen": -460.3074504573171,
|
|
"logps/rejected": -377.1211939102564,
|
|
"loss": 0.3256,
|
|
"loss/base_kto": 2.6048405170440674,
|
|
"metrics/advantage_var": 9842.2724609375,
|
|
"rewards/chosen": 1.9590448519078696,
|
|
"rewards/margins": 3.0523682905033724,
|
|
"rewards/rejected": -1.0933234385955029,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 8.239922523498535,
|
|
"kl": 52.59576416015625,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -48873639.44654088,
|
|
"logits/rejected": -49180035.97515528,
|
|
"logps/chosen": -458.720322327044,
|
|
"logps/rejected": -381.3061432453416,
|
|
"loss": 0.3288,
|
|
"loss/base_kto": 2.630648374557495,
|
|
"metrics/advantage_var": 9683.5693359375,
|
|
"rewards/chosen": 2.6747438442782037,
|
|
"rewards/margins": 2.926806126485053,
|
|
"rewards/rejected": -0.2520622822068493,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 9.227850914001465,
|
|
"kl": 21.218448638916016,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -46331279.295597486,
|
|
"logits/rejected": -47721618.28571428,
|
|
"logps/chosen": -454.13384433962267,
|
|
"logps/rejected": -371.9747185559006,
|
|
"loss": 0.3051,
|
|
"loss/base_kto": 2.4406096935272217,
|
|
"metrics/advantage_var": 10768.3876953125,
|
|
"rewards/chosen": 2.211166285868711,
|
|
"rewards/margins": 3.502381696752834,
|
|
"rewards/rejected": -1.2912154108841227,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 7.125072479248047,
|
|
"kl": 34.98481750488281,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -46227072.38670695,
|
|
"logits/rejected": -46066644.919093855,
|
|
"logps/chosen": -459.86508685800607,
|
|
"logps/rejected": -372.3851132686084,
|
|
"loss": 0.3282,
|
|
"loss/base_kto": 2.6253113746643066,
|
|
"metrics/advantage_var": 11001.2177734375,
|
|
"rewards/chosen": 2.5339547240842144,
|
|
"rewards/margins": 3.2950123092727006,
|
|
"rewards/rejected": -0.7610575851884861,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 7.275957107543945,
|
|
"kl": 33.37077331542969,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -47322094.03508772,
|
|
"logits/rejected": -47008286.57886677,
|
|
"logps/chosen": -459.7950558213716,
|
|
"logps/rejected": -382.4612366003063,
|
|
"loss": 0.3233,
|
|
"loss/base_kto": 2.5865111351013184,
|
|
"metrics/advantage_var": 11704.7568359375,
|
|
"rewards/chosen": 2.4234196664423844,
|
|
"rewards/margins": 3.510266992186546,
|
|
"rewards/rejected": -1.0868473257441615,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 7.228305816650391,
|
|
"kl": 12.117892265319824,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -49392219.51690821,
|
|
"logits/rejected": -47948938.29438543,
|
|
"logps/chosen": -495.15841384863126,
|
|
"logps/rejected": -411.4466995447648,
|
|
"loss": 0.3299,
|
|
"loss/base_kto": 2.6388490200042725,
|
|
"metrics/advantage_var": 12980.6669921875,
|
|
"rewards/chosen": 1.1754036379705113,
|
|
"rewards/margins": 3.4501155594424384,
|
|
"rewards/rejected": -2.274711921471927,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 9.235511779785156,
|
|
"kl": 18.60725212097168,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -47427814.482315116,
|
|
"logits/rejected": -47068841.62917933,
|
|
"logps/chosen": -464.88102893890675,
|
|
"logps/rejected": -397.7166365881459,
|
|
"loss": 0.3285,
|
|
"loss/base_kto": 2.628096342086792,
|
|
"metrics/advantage_var": 11702.447265625,
|
|
"rewards/chosen": 1.6093233850607918,
|
|
"rewards/margins": 3.174098382389344,
|
|
"rewards/rejected": -1.5647749973285525,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 7.23322057723999,
|
|
"kl": 22.347009658813477,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -48391976.585365854,
|
|
"logits/rejected": -47666609.23076923,
|
|
"logps/chosen": -483.9475038109756,
|
|
"logps/rejected": -394.9490184294872,
|
|
"loss": 0.3257,
|
|
"loss/base_kto": 2.6056020259857178,
|
|
"metrics/advantage_var": 13653.556640625,
|
|
"rewards/chosen": 2.2879113918397485,
|
|
"rewards/margins": 3.574918845953831,
|
|
"rewards/rejected": -1.2870074541140826,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 7.010107040405273,
|
|
"kl": 42.226776123046875,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -46929463.439490445,
|
|
"logits/rejected": -45645820.858895704,
|
|
"logps/chosen": -449.6611265923567,
|
|
"logps/rejected": -379.30272239263803,
|
|
"loss": 0.3121,
|
|
"loss/base_kto": 2.4970703125,
|
|
"metrics/advantage_var": 13127.6640625,
|
|
"rewards/chosen": 2.2514016704194866,
|
|
"rewards/margins": 3.6759871181404415,
|
|
"rewards/rejected": -1.4245854477209547,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 6.621373176574707,
|
|
"kl": 38.65568542480469,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -48262849.35347432,
|
|
"logits/rejected": -47633981.307443365,
|
|
"logps/chosen": -451.99679003021146,
|
|
"logps/rejected": -375.6196652508091,
|
|
"loss": 0.325,
|
|
"loss/base_kto": 2.600301742553711,
|
|
"metrics/advantage_var": 11315.232421875,
|
|
"rewards/chosen": 2.659457572637368,
|
|
"rewards/margins": 3.1546796951754192,
|
|
"rewards/rejected": -0.49522212253805115,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 7.346182823181152,
|
|
"kl": 17.64263916015625,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -47635133.896713614,
|
|
"logits/rejected": -46112748.8299532,
|
|
"logps/chosen": -480.8606220657277,
|
|
"logps/rejected": -394.9130752730109,
|
|
"loss": 0.3243,
|
|
"loss/base_kto": 2.5942459106445312,
|
|
"metrics/advantage_var": 12418.57421875,
|
|
"rewards/chosen": 1.482877489546655,
|
|
"rewards/margins": 3.6101688312003213,
|
|
"rewards/rejected": -2.1272913416536663,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 8.256329536437988,
|
|
"kl": 40.221649169921875,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -48079565.4522293,
|
|
"logits/rejected": -48238667.38650307,
|
|
"logps/chosen": -452.71616242038215,
|
|
"logps/rejected": -368.75603911042947,
|
|
"loss": 0.3152,
|
|
"loss/base_kto": 2.521874189376831,
|
|
"metrics/advantage_var": 11227.6708984375,
|
|
"rewards/chosen": 2.4987431422919983,
|
|
"rewards/margins": 3.437864692869548,
|
|
"rewards/rejected": -0.9391215505775499,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 6.824674129486084,
|
|
"kl": 28.154876708984375,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -47101648.47425897,
|
|
"logits/rejected": -47173353.16431925,
|
|
"logps/chosen": -485.76028666146647,
|
|
"logps/rejected": -384.87404636150234,
|
|
"loss": 0.3173,
|
|
"loss/base_kto": 2.5386312007904053,
|
|
"metrics/advantage_var": 13187.3505859375,
|
|
"rewards/chosen": 2.403803832817375,
|
|
"rewards/margins": 3.792621275232281,
|
|
"rewards/rejected": -1.388817442414906,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 6.028614044189453,
|
|
"kl": 24.844501495361328,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -48460178.570982836,
|
|
"logits/rejected": -48257503.0455259,
|
|
"logps/chosen": -469.759360374415,
|
|
"logps/rejected": -379.34561420722133,
|
|
"loss": 0.2968,
|
|
"loss/base_kto": 2.374429702758789,
|
|
"metrics/advantage_var": 13482.1533203125,
|
|
"rewards/chosen": 2.166554943447738,
|
|
"rewards/margins": 4.012237946967655,
|
|
"rewards/rejected": -1.8456830035199177,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 9.08834457397461,
|
|
"kl": 20.602874755859375,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -47292737.37846154,
|
|
"logits/rejected": -46673464.88888889,
|
|
"logps/chosen": -463.24221153846156,
|
|
"logps/rejected": -403.7625248015873,
|
|
"loss": 0.2446,
|
|
"loss/base_kto": 1.9567406177520752,
|
|
"metrics/advantage_var": 14699.2001953125,
|
|
"rewards/chosen": 2.7593828876201925,
|
|
"rewards/margins": 5.1381240520547165,
|
|
"rewards/rejected": -2.378741164434524,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 8.160238265991211,
|
|
"kl": 19.674734115600586,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -48202742.518518515,
|
|
"logits/rejected": -48369903.79746836,
|
|
"logps/chosen": -475.44140625,
|
|
"logps/rejected": -422.9123318829114,
|
|
"loss": 0.2557,
|
|
"loss/base_kto": 2.045527696609497,
|
|
"metrics/advantage_var": 16601.380859375,
|
|
"rewards/chosen": 2.578687126253858,
|
|
"rewards/margins": 5.213243570639241,
|
|
"rewards/rejected": -2.6345564443853835,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 6.527251243591309,
|
|
"kl": 32.78919219970703,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -48127182.11916264,
|
|
"logits/rejected": -48193337.1047041,
|
|
"logps/chosen": -468.6815619967794,
|
|
"logps/rejected": -413.415686646434,
|
|
"loss": 0.2525,
|
|
"loss/base_kto": 2.020085573196411,
|
|
"metrics/advantage_var": 14671.9345703125,
|
|
"rewards/chosen": 2.7362296431536834,
|
|
"rewards/margins": 4.915611133811877,
|
|
"rewards/rejected": -2.1793814906581943,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 7.973196029663086,
|
|
"kl": 8.747920036315918,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -48637005.0048,
|
|
"logits/rejected": -48764981.154198475,
|
|
"logps/chosen": -471.8269,
|
|
"logps/rejected": -418.7538167938931,
|
|
"loss": 0.2555,
|
|
"loss/base_kto": 2.0437705516815186,
|
|
"metrics/advantage_var": 17556.083984375,
|
|
"rewards/chosen": 1.8711177734375,
|
|
"rewards/margins": 5.295105428494752,
|
|
"rewards/rejected": -3.423987655057252,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 7.407348155975342,
|
|
"kl": 25.11842918395996,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -49298159.147335425,
|
|
"logits/rejected": -48831242.367601246,
|
|
"logps/chosen": -449.38700039184954,
|
|
"logps/rejected": -381.8530957943925,
|
|
"loss": 0.2617,
|
|
"loss/base_kto": 2.0939595699310303,
|
|
"metrics/advantage_var": 15131.697265625,
|
|
"rewards/chosen": 2.6031605113636362,
|
|
"rewards/margins": 4.881092176326643,
|
|
"rewards/rejected": -2.277931664963006,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 6.892596244812012,
|
|
"kl": 11.93945598602295,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -49092113.32923077,
|
|
"logits/rejected": -49441834.26031746,
|
|
"logps/chosen": -456.97432692307694,
|
|
"logps/rejected": -407.94742063492066,
|
|
"loss": 0.2669,
|
|
"loss/base_kto": 2.135260820388794,
|
|
"metrics/advantage_var": 16834.97265625,
|
|
"rewards/chosen": 2.1193592247596156,
|
|
"rewards/margins": 4.897466011093941,
|
|
"rewards/rejected": -2.7781067863343254,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 8.031292915344238,
|
|
"kl": 31.970178604125977,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -49624292.079877116,
|
|
"logits/rejected": -49447781.3418124,
|
|
"logps/chosen": -460.8429339477727,
|
|
"logps/rejected": -372.7867647058824,
|
|
"loss": 0.2352,
|
|
"loss/base_kto": 1.8812607526779175,
|
|
"metrics/advantage_var": 14783.8779296875,
|
|
"rewards/chosen": 3.347041960685484,
|
|
"rewards/margins": 5.363860882088504,
|
|
"rewards/rejected": -2.0168189214030208,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 7.753678321838379,
|
|
"kl": 25.85993766784668,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -47928455.12241653,
|
|
"logits/rejected": -47511127.29953917,
|
|
"logps/chosen": -452.45901232114466,
|
|
"logps/rejected": -363.9561491935484,
|
|
"loss": 0.2634,
|
|
"loss/base_kto": 2.107530355453491,
|
|
"metrics/advantage_var": 13522.9609375,
|
|
"rewards/chosen": 2.5884316934121623,
|
|
"rewards/margins": 4.535507556267385,
|
|
"rewards/rejected": -1.9470758628552227,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 8.16309928894043,
|
|
"kl": 20.483793258666992,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -48593606.4,
|
|
"logits/rejected": -49353206.4,
|
|
"logps/chosen": -417.97294921875,
|
|
"logps/rejected": -397.7453857421875,
|
|
"loss": 0.268,
|
|
"loss/base_kto": 2.1443779468536377,
|
|
"metrics/advantage_var": 14067.8466796875,
|
|
"rewards/chosen": 2.1633033752441406,
|
|
"rewards/margins": 4.678990745544434,
|
|
"rewards/rejected": -2.515687370300293,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 6.558415412902832,
|
|
"kl": 28.014001846313477,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -48475016.01284109,
|
|
"logits/rejected": -50391460.82191781,
|
|
"logps/chosen": -488.72893258426967,
|
|
"logps/rejected": -408.69244672754945,
|
|
"loss": 0.2429,
|
|
"loss/base_kto": 1.943566918373108,
|
|
"metrics/advantage_var": 15186.3173828125,
|
|
"rewards/chosen": 2.8327777795194624,
|
|
"rewards/margins": 5.087047267663964,
|
|
"rewards/rejected": -2.2542694881445016,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 7.960604667663574,
|
|
"kl": 13.513842582702637,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -51427026.63291139,
|
|
"logits/rejected": -48594767.01234568,
|
|
"logps/chosen": -449.3992286392405,
|
|
"logps/rejected": -396.8494888117284,
|
|
"loss": 0.2591,
|
|
"loss/base_kto": 2.0725789070129395,
|
|
"metrics/advantage_var": 14900.515625,
|
|
"rewards/chosen": 2.4082622286639634,
|
|
"rewards/margins": 4.899628579756267,
|
|
"rewards/rejected": -2.4913663510923034,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 8.487668991088867,
|
|
"kl": 21.949365615844727,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -49826210.27534039,
|
|
"logits/rejected": -50941670.77221325,
|
|
"logps/chosen": -448.42256051437215,
|
|
"logps/rejected": -397.01726575121165,
|
|
"loss": 0.244,
|
|
"loss/base_kto": 1.9521198272705078,
|
|
"metrics/advantage_var": 14752.8896484375,
|
|
"rewards/chosen": 2.896393884148071,
|
|
"rewards/margins": 5.071983450232482,
|
|
"rewards/rejected": -2.1755895660844105,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 8.264450073242188,
|
|
"kl": 27.853168487548828,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -49997271.798518516,
|
|
"logits/rejected": -50615778.38016529,
|
|
"logps/chosen": -460.35203703703706,
|
|
"logps/rejected": -378.03768078512394,
|
|
"loss": 0.2493,
|
|
"loss/base_kto": 1.994248628616333,
|
|
"metrics/advantage_var": 12929.7939453125,
|
|
"rewards/chosen": 2.9150424985532406,
|
|
"rewards/margins": 4.735268237835266,
|
|
"rewards/rejected": -1.8202257392820247,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 9.001532554626465,
|
|
"kl": 15.930495262145996,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -51689064.8861912,
|
|
"logits/rejected": -51946744.99194847,
|
|
"logps/chosen": -469.39344650986345,
|
|
"logps/rejected": -409.47632347020937,
|
|
"loss": 0.2511,
|
|
"loss/base_kto": 2.008457660675049,
|
|
"metrics/advantage_var": 15899.4736328125,
|
|
"rewards/chosen": 2.12914446308327,
|
|
"rewards/margins": 5.047912272375239,
|
|
"rewards/rejected": -2.9187678092919684,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 8.886826515197754,
|
|
"kl": 18.990217208862305,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -52011945.48694316,
|
|
"logits/rejected": -52379332.17170111,
|
|
"logps/chosen": -479.4702860983103,
|
|
"logps/rejected": -386.3915192766296,
|
|
"loss": 0.2724,
|
|
"loss/base_kto": 2.179483652114868,
|
|
"metrics/advantage_var": 13855.7021484375,
|
|
"rewards/chosen": 2.3804954142065093,
|
|
"rewards/margins": 4.486999530052595,
|
|
"rewards/rejected": -2.106504115846085,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 6.141109943389893,
|
|
"kl": 30.30926513671875,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -50506341.13580247,
|
|
"logits/rejected": -50303163.94936709,
|
|
"logps/chosen": -452.74508101851853,
|
|
"logps/rejected": -372.39700356012656,
|
|
"loss": 0.2546,
|
|
"loss/base_kto": 2.036782741546631,
|
|
"metrics/advantage_var": 14251.7470703125,
|
|
"rewards/chosen": 3.120146009657118,
|
|
"rewards/margins": 4.849291687440939,
|
|
"rewards/rejected": -1.7291456777838212,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 8.584972381591797,
|
|
"kl": 22.3951416015625,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -51018727.88697017,
|
|
"logits/rejected": -50664933.72317263,
|
|
"logps/chosen": -453.2104101255887,
|
|
"logps/rejected": -396.54315707620526,
|
|
"loss": 0.2477,
|
|
"loss/base_kto": 1.9816032648086548,
|
|
"metrics/advantage_var": 16047.587890625,
|
|
"rewards/chosen": 2.28303947029533,
|
|
"rewards/margins": 5.198438858538914,
|
|
"rewards/rejected": -2.9153993882435847,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 7.433993339538574,
|
|
"kl": 17.95124626159668,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -49613046.7856025,
|
|
"logits/rejected": -50742749.653666146,
|
|
"logps/chosen": -476.1836854460094,
|
|
"logps/rejected": -415.5039976599064,
|
|
"loss": 0.2548,
|
|
"loss/base_kto": 2.0387260913848877,
|
|
"metrics/advantage_var": 15328.6748046875,
|
|
"rewards/chosen": 2.675205284441021,
|
|
"rewards/margins": 5.106290594259663,
|
|
"rewards/rejected": -2.4310853098186427,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 6.105855464935303,
|
|
"kl": 17.84329605102539,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -47158237.65853658,
|
|
"logits/rejected": -50338474.666666664,
|
|
"logps/chosen": -455.8099275914634,
|
|
"logps/rejected": -390.8770783253205,
|
|
"loss": 0.2544,
|
|
"loss/base_kto": 2.0349042415618896,
|
|
"metrics/advantage_var": 13977.494140625,
|
|
"rewards/chosen": 2.4411368021150914,
|
|
"rewards/margins": 4.890623363425688,
|
|
"rewards/rejected": -2.449486561310597,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 6.1178879737854,
|
|
"kl": 20.046993255615234,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -50292838.080996886,
|
|
"logits/rejected": -49619740.08777429,
|
|
"logps/chosen": -441.6804906542056,
|
|
"logps/rejected": -386.70307601880876,
|
|
"loss": 0.2609,
|
|
"loss/base_kto": 2.087083101272583,
|
|
"metrics/advantage_var": 13117.759765625,
|
|
"rewards/chosen": 2.7567787051572235,
|
|
"rewards/margins": 4.597649529510671,
|
|
"rewards/rejected": -1.8408708243534482,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 6.218991756439209,
|
|
"kl": 13.484720230102539,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -51252059.69529984,
|
|
"logits/rejected": -50508739.76470588,
|
|
"logps/chosen": -458.5921799027553,
|
|
"logps/rejected": -377.68382352941177,
|
|
"loss": 0.275,
|
|
"loss/base_kto": 2.199876546859741,
|
|
"metrics/advantage_var": 13742.1064453125,
|
|
"rewards/chosen": 1.808982317412885,
|
|
"rewards/margins": 4.709338512324744,
|
|
"rewards/rejected": -2.900356194911859,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 6.822948455810547,
|
|
"kl": 24.661109924316406,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -51494668.190476194,
|
|
"logits/rejected": -51958806.05538461,
|
|
"logps/chosen": -475.4358630952381,
|
|
"logps/rejected": -402.63795673076925,
|
|
"loss": 0.2566,
|
|
"loss/base_kto": 2.0528342723846436,
|
|
"metrics/advantage_var": 14411.15625,
|
|
"rewards/chosen": 2.8241772848462303,
|
|
"rewards/margins": 4.8616267665169035,
|
|
"rewards/rejected": -2.037449481670673,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 5.703562259674072,
|
|
"kl": 35.027462005615234,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -52037708.96732026,
|
|
"logits/rejected": -52339742.65868264,
|
|
"logps/chosen": -462.3002450980392,
|
|
"logps/rejected": -412.0844872754491,
|
|
"loss": 0.2549,
|
|
"loss/base_kto": 2.03886342048645,
|
|
"metrics/advantage_var": 14913.2177734375,
|
|
"rewards/chosen": 2.8195912479575163,
|
|
"rewards/margins": 4.789924872700219,
|
|
"rewards/rejected": -1.9703336247427021,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 9.470081329345703,
|
|
"kl": 24.76580238342285,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -50662217.028213166,
|
|
"logits/rejected": -50396405.632398754,
|
|
"logps/chosen": -473.09370101880876,
|
|
"logps/rejected": -396.8532904984424,
|
|
"loss": 0.232,
|
|
"loss/base_kto": 1.8556301593780518,
|
|
"metrics/advantage_var": 14833.1708984375,
|
|
"rewards/chosen": 2.840141081137343,
|
|
"rewards/margins": 5.319337409749103,
|
|
"rewards/rejected": -2.47919632861176,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 7.3936872482299805,
|
|
"kl": 16.468170166015625,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -51449756.332810044,
|
|
"logits/rejected": -51321140.95178849,
|
|
"logps/chosen": -477.38844191522765,
|
|
"logps/rejected": -398.4009282659409,
|
|
"loss": 0.2407,
|
|
"loss/base_kto": 1.9254833459854126,
|
|
"metrics/advantage_var": 16836.380859375,
|
|
"rewards/chosen": 2.589753299156201,
|
|
"rewards/margins": 5.541966473789269,
|
|
"rewards/rejected": -2.9522131746330675,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 10.445863723754883,
|
|
"kl": 30.038633346557617,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -53743021.82716049,
|
|
"logits/rejected": -51698046.37974683,
|
|
"logps/chosen": -458.8755787037037,
|
|
"logps/rejected": -397.19852650316454,
|
|
"loss": 0.26,
|
|
"loss/base_kto": 2.080324411392212,
|
|
"metrics/advantage_var": 15504.6064453125,
|
|
"rewards/chosen": 2.703242925950039,
|
|
"rewards/margins": 4.670523446618551,
|
|
"rewards/rejected": -1.9672805206685127,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 8.893047332763672,
|
|
"kl": 21.676542282104492,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -51821296.0,
|
|
"logits/rejected": -50792192.0,
|
|
"logps/chosen": -489.333984375,
|
|
"logps/rejected": -384.91728515625,
|
|
"loss": 0.2509,
|
|
"loss/base_kto": 2.0074877738952637,
|
|
"metrics/advantage_var": 14328.572265625,
|
|
"rewards/chosen": 2.416120719909668,
|
|
"rewards/margins": 4.925666618347169,
|
|
"rewards/rejected": -2.5095458984375,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 6.958967208862305,
|
|
"kl": 17.569808959960938,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -49479295.79119086,
|
|
"logits/rejected": -49944173.76911544,
|
|
"logps/chosen": -443.45417006525287,
|
|
"logps/rejected": -394.2795164917541,
|
|
"loss": 0.2545,
|
|
"loss/base_kto": 2.036057472229004,
|
|
"metrics/advantage_var": 15028.1669921875,
|
|
"rewards/chosen": 2.4243399042873164,
|
|
"rewards/margins": 4.9938617839724735,
|
|
"rewards/rejected": -2.5695218796851576,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 7.773034572601318,
|
|
"kl": 23.792673110961914,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -50371183.30434783,
|
|
"logits/rejected": -51787280.10062893,
|
|
"logps/chosen": -442.39780667701865,
|
|
"logps/rejected": -429.15954205974845,
|
|
"loss": 0.2575,
|
|
"loss/base_kto": 2.0603244304656982,
|
|
"metrics/advantage_var": 14647.125,
|
|
"rewards/chosen": 2.392054966517857,
|
|
"rewards/margins": 4.850691364353451,
|
|
"rewards/rejected": -2.4586363978355936,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 6.8994832038879395,
|
|
"kl": 26.021778106689453,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -51890331.00161031,
|
|
"logits/rejected": -52612660.05462822,
|
|
"logps/chosen": -461.7973027375201,
|
|
"logps/rejected": -412.12395675265554,
|
|
"loss": 0.2551,
|
|
"loss/base_kto": 2.040802478790283,
|
|
"metrics/advantage_var": 14155.7734375,
|
|
"rewards/chosen": 2.7738247439865136,
|
|
"rewards/margins": 4.97032312280385,
|
|
"rewards/rejected": -2.1964983788173367,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 8.21739673614502,
|
|
"kl": 22.458288192749023,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -50118678.43505477,
|
|
"logits/rejected": -49249246.4524181,
|
|
"logps/chosen": -443.7821302816901,
|
|
"logps/rejected": -380.8603256630265,
|
|
"loss": 0.2577,
|
|
"loss/base_kto": 2.0616891384124756,
|
|
"metrics/advantage_var": 13250.5341796875,
|
|
"rewards/chosen": 2.6506359118251175,
|
|
"rewards/margins": 4.795738854614704,
|
|
"rewards/rejected": -2.1451029427895865,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 8.37536334991455,
|
|
"kl": 30.660608291625977,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -50617106.16774193,
|
|
"logits/rejected": -50973444.654545456,
|
|
"logps/chosen": -463.07631048387094,
|
|
"logps/rejected": -389.0999053030303,
|
|
"loss": 0.2537,
|
|
"loss/base_kto": 2.029655933380127,
|
|
"metrics/advantage_var": 12852.322265625,
|
|
"rewards/chosen": 2.814442886844758,
|
|
"rewards/margins": 4.745141720888319,
|
|
"rewards/rejected": -1.9306988340435607,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 8.114055633544922,
|
|
"kl": 19.990629196166992,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -52263602.89156626,
|
|
"logits/rejected": -50285930.38961039,
|
|
"logps/chosen": -465.64984939759034,
|
|
"logps/rejected": -403.7388392857143,
|
|
"loss": 0.2475,
|
|
"loss/base_kto": 1.9802414178848267,
|
|
"metrics/advantage_var": 13638.9423828125,
|
|
"rewards/chosen": 2.736863101821348,
|
|
"rewards/margins": 4.915339014336567,
|
|
"rewards/rejected": -2.178475912515219,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 7.683564186096191,
|
|
"kl": 21.631948471069336,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -50590506.86697966,
|
|
"logits/rejected": -50809387.93135726,
|
|
"logps/chosen": -460.38810641627543,
|
|
"logps/rejected": -386.3815815132605,
|
|
"loss": 0.2445,
|
|
"loss/base_kto": 1.9557570219039917,
|
|
"metrics/advantage_var": 13694.1279296875,
|
|
"rewards/chosen": 2.935483260893486,
|
|
"rewards/margins": 5.136769588713786,
|
|
"rewards/rejected": -2.2012863278203003,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 8.713532447814941,
|
|
"kl": 14.259411811828613,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -52655586.43161094,
|
|
"logits/rejected": -51266102.327974275,
|
|
"logps/chosen": -456.429141337386,
|
|
"logps/rejected": -396.64253416398714,
|
|
"loss": 0.2434,
|
|
"loss/base_kto": 1.9473152160644531,
|
|
"metrics/advantage_var": 13935.6845703125,
|
|
"rewards/chosen": 2.8355577463074657,
|
|
"rewards/margins": 5.060517631883229,
|
|
"rewards/rejected": -2.2249598855757635,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 7.959493637084961,
|
|
"kl": 25.214923858642578,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -52473988.3537415,
|
|
"logits/rejected": -52252316.85549133,
|
|
"logps/chosen": -443.0916772959184,
|
|
"logps/rejected": -395.4490606936416,
|
|
"loss": 0.2622,
|
|
"loss/base_kto": 2.0979087352752686,
|
|
"metrics/advantage_var": 14081.1298828125,
|
|
"rewards/chosen": 2.602545705782313,
|
|
"rewards/margins": 4.721474471757295,
|
|
"rewards/rejected": -2.118928765974982,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 6.154829025268555,
|
|
"kl": 13.369463920593262,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -52219029.853658535,
|
|
"logits/rejected": -52134872.615384616,
|
|
"logps/chosen": -477.7408060213415,
|
|
"logps/rejected": -382.8152043269231,
|
|
"loss": 0.2455,
|
|
"loss/base_kto": 1.9636309146881104,
|
|
"metrics/advantage_var": 14574.7529296875,
|
|
"rewards/chosen": 2.274860661204268,
|
|
"rewards/margins": 5.1025322012934105,
|
|
"rewards/rejected": -2.827671540089143,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 7.846475124359131,
|
|
"kl": 16.547452926635742,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -51591907.21893491,
|
|
"logits/rejected": -52209226.59602649,
|
|
"logps/chosen": -461.81850961538464,
|
|
"logps/rejected": -389.6706332781457,
|
|
"loss": 0.2457,
|
|
"loss/base_kto": 1.9656704664230347,
|
|
"metrics/advantage_var": 14242.876953125,
|
|
"rewards/chosen": 2.8158327362241122,
|
|
"rewards/margins": 4.997085727350974,
|
|
"rewards/rejected": -2.1812529911268626,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 5.958191394805908,
|
|
"kl": 29.157068252563477,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -51625569.64510779,
|
|
"logits/rejected": -52310252.65777778,
|
|
"logps/chosen": -456.7199937810945,
|
|
"logps/rejected": -382.9580324074074,
|
|
"loss": 0.2103,
|
|
"loss/base_kto": 1.6824930906295776,
|
|
"metrics/advantage_var": 15329.853515625,
|
|
"rewards/chosen": 3.0148241539697347,
|
|
"rewards/margins": 5.539358659178069,
|
|
"rewards/rejected": -2.5245345052083334,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 6.810640811920166,
|
|
"kl": 18.33027458190918,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -51165085.062801935,
|
|
"logits/rejected": -51187286.23975721,
|
|
"logps/chosen": -443.048309178744,
|
|
"logps/rejected": -402.1879742033384,
|
|
"loss": 0.2023,
|
|
"loss/base_kto": 1.6187645196914673,
|
|
"metrics/advantage_var": 15290.6318359375,
|
|
"rewards/chosen": 2.663107560260668,
|
|
"rewards/margins": 5.793123001506874,
|
|
"rewards/rejected": -3.1300154412462065,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 4.337776184082031,
|
|
"kl": 17.0222225189209,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -49851196.33121019,
|
|
"logits/rejected": -51849448.44171779,
|
|
"logps/chosen": -432.84146098726114,
|
|
"logps/rejected": -406.1946894171779,
|
|
"loss": 0.1933,
|
|
"loss/base_kto": 1.5461317300796509,
|
|
"metrics/advantage_var": 15725.984375,
|
|
"rewards/chosen": 3.019858585041799,
|
|
"rewards/margins": 6.097644708867527,
|
|
"rewards/rejected": -3.0777861238257285,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 5.722031593322754,
|
|
"kl": 18.55806541442871,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -51315540.80495356,
|
|
"logits/rejected": -52570457.64037855,
|
|
"logps/chosen": -461.937886996904,
|
|
"logps/rejected": -408.2663643533123,
|
|
"loss": 0.2,
|
|
"loss/base_kto": 1.600163459777832,
|
|
"metrics/advantage_var": 16523.453125,
|
|
"rewards/chosen": 2.6366288034539473,
|
|
"rewards/margins": 6.083913682446456,
|
|
"rewards/rejected": -3.447284878992508,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 13.217945098876953,
|
|
"kl": 30.77508544921875,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -49535898.27814569,
|
|
"logits/rejected": -50719916.686390534,
|
|
"logps/chosen": -429.2412044701987,
|
|
"logps/rejected": -397.5885724852071,
|
|
"loss": 0.2172,
|
|
"loss/base_kto": 1.7376642227172852,
|
|
"metrics/advantage_var": 14375.2919921875,
|
|
"rewards/chosen": 2.7198288267021935,
|
|
"rewards/margins": 5.497414550398569,
|
|
"rewards/rejected": -2.777585723696376,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 6.928693771362305,
|
|
"kl": 15.719565391540527,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -51029232.047477745,
|
|
"logits/rejected": -51983566.15181518,
|
|
"logps/chosen": -446.40995919881306,
|
|
"logps/rejected": -396.5913778877888,
|
|
"loss": 0.2143,
|
|
"loss/base_kto": 1.7145565748214722,
|
|
"metrics/advantage_var": 15240.5029296875,
|
|
"rewards/chosen": 2.615926227512982,
|
|
"rewards/margins": 5.480235204797429,
|
|
"rewards/rejected": -2.864308977284447,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 7.271140098571777,
|
|
"kl": 19.24259376525879,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -50473962.53892215,
|
|
"logits/rejected": -50950324.705882356,
|
|
"logps/chosen": -447.1455838323353,
|
|
"logps/rejected": -382.8148999183006,
|
|
"loss": 0.1973,
|
|
"loss/base_kto": 1.5785809755325317,
|
|
"metrics/advantage_var": 14588.8984375,
|
|
"rewards/chosen": 2.841123112661396,
|
|
"rewards/margins": 5.781759043140358,
|
|
"rewards/rejected": -2.940635930478962,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.26968765258789,
|
|
"kl": 23.165260314941406,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -51096412.98299845,
|
|
"logits/rejected": -49997028.09478673,
|
|
"logps/chosen": -449.54791344667694,
|
|
"logps/rejected": -388.7860880726698,
|
|
"loss": 0.2058,
|
|
"loss/base_kto": 1.6466184854507446,
|
|
"metrics/advantage_var": 14070.4189453125,
|
|
"rewards/chosen": 3.1233227123985703,
|
|
"rewards/margins": 5.540160238933918,
|
|
"rewards/rejected": -2.4168375265353474,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 7.453213691711426,
|
|
"kl": 24.0378475189209,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -51170121.95555556,
|
|
"logits/rejected": -52540230.10461538,
|
|
"logps/chosen": -449.71170634920634,
|
|
"logps/rejected": -378.6451923076923,
|
|
"loss": 0.2143,
|
|
"loss/base_kto": 1.7145299911499023,
|
|
"metrics/advantage_var": 15124.421875,
|
|
"rewards/chosen": 2.6893165225074407,
|
|
"rewards/margins": 5.637288652916094,
|
|
"rewards/rejected": -2.9479721304086537,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 7.117490291595459,
|
|
"kl": 13.310893058776855,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -53527604.9118541,
|
|
"logits/rejected": -52066139.36977492,
|
|
"logps/chosen": -487.0769376899696,
|
|
"logps/rejected": -395.3713072749196,
|
|
"loss": 0.1942,
|
|
"loss/base_kto": 1.5535629987716675,
|
|
"metrics/advantage_var": 15743.8505859375,
|
|
"rewards/chosen": 2.905693448423252,
|
|
"rewards/margins": 6.0703902430374,
|
|
"rewards/rejected": -3.164696794614148,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 8.523292541503906,
|
|
"kl": 9.435015678405762,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -52503106.983458646,
|
|
"logits/rejected": -52026812.565853655,
|
|
"logps/chosen": -458.8041353383459,
|
|
"logps/rejected": -397.1086636178862,
|
|
"loss": 0.2147,
|
|
"loss/base_kto": 1.7179268598556519,
|
|
"metrics/advantage_var": 15920.5966796875,
|
|
"rewards/chosen": 2.546815158012218,
|
|
"rewards/margins": 5.578676105039657,
|
|
"rewards/rejected": -3.031860947027439,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.27126407623291,
|
|
"kl": 18.656579971313477,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -51284638.84094488,
|
|
"logits/rejected": -52479618.97674418,
|
|
"logps/chosen": -462.7442421259843,
|
|
"logps/rejected": -396.7521560077519,
|
|
"loss": 0.2098,
|
|
"loss/base_kto": 1.6782121658325195,
|
|
"metrics/advantage_var": 16229.3251953125,
|
|
"rewards/chosen": 2.876315860297736,
|
|
"rewards/margins": 5.866616475002194,
|
|
"rewards/rejected": -2.9903006147044575,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 9.956695556640625,
|
|
"kl": 19.218399047851562,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -53085059.28205128,
|
|
"logits/rejected": -53581461.853658535,
|
|
"logps/chosen": -448.7760416666667,
|
|
"logps/rejected": -414.6380049542683,
|
|
"loss": 0.2131,
|
|
"loss/base_kto": 1.7047923803329468,
|
|
"metrics/advantage_var": 15295.837890625,
|
|
"rewards/chosen": 2.752549782777444,
|
|
"rewards/margins": 5.645609303964086,
|
|
"rewards/rejected": -2.8930595211866423,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 6.9843621253967285,
|
|
"kl": 18.419477462768555,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -50652528.64,
|
|
"logits/rejected": -52052174.36335878,
|
|
"logps/chosen": -470.1084,
|
|
"logps/rejected": -411.04742366412216,
|
|
"loss": 0.2113,
|
|
"loss/base_kto": 1.6904957294464111,
|
|
"metrics/advantage_var": 16657.041015625,
|
|
"rewards/chosen": 2.6572134765625,
|
|
"rewards/margins": 5.88140240786021,
|
|
"rewards/rejected": -3.22418893129771,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 7.0096259117126465,
|
|
"kl": 12.720903396606445,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -52172171.779141106,
|
|
"logits/rejected": -53297830.318471335,
|
|
"logps/chosen": -458.0583780674847,
|
|
"logps/rejected": -419.7462679140127,
|
|
"loss": 0.2031,
|
|
"loss/base_kto": 1.6250914335250854,
|
|
"metrics/advantage_var": 16653.916015625,
|
|
"rewards/chosen": 2.894802350939417,
|
|
"rewards/margins": 6.069655438929564,
|
|
"rewards/rejected": -3.1748530879901473,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 8.27895736694336,
|
|
"kl": 17.4371395111084,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -53293252.21556886,
|
|
"logits/rejected": -52639489.67320261,
|
|
"logps/chosen": -440.76974176646706,
|
|
"logps/rejected": -424.7828839869281,
|
|
"loss": 0.1977,
|
|
"loss/base_kto": 1.5816789865493774,
|
|
"metrics/advantage_var": 14772.7255859375,
|
|
"rewards/chosen": 2.7534731562266095,
|
|
"rewards/margins": 5.898639889912271,
|
|
"rewards/rejected": -3.1451667336856617,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 5.753119468688965,
|
|
"kl": 21.079870223999023,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -53754379.906976745,
|
|
"logits/rejected": -52337610.784251966,
|
|
"logps/chosen": -469.2296511627907,
|
|
"logps/rejected": -387.3084153543307,
|
|
"loss": 0.1987,
|
|
"loss/base_kto": 1.589290738105774,
|
|
"metrics/advantage_var": 15189.6533203125,
|
|
"rewards/chosen": 3.0228050054505813,
|
|
"rewards/margins": 5.7660919072099315,
|
|
"rewards/rejected": -2.74328690175935,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.482635498046875,
|
|
"kl": 16.362096786499023,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -52656223.47067669,
|
|
"logits/rejected": -51659437.99674797,
|
|
"logps/chosen": -463.7524906015038,
|
|
"logps/rejected": -418.1763211382114,
|
|
"loss": 0.22,
|
|
"loss/base_kto": 1.7602322101593018,
|
|
"metrics/advantage_var": 15808.220703125,
|
|
"rewards/chosen": 3.0392983802279137,
|
|
"rewards/margins": 5.632142082590718,
|
|
"rewards/rejected": -2.592843702362805,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 8.359546661376953,
|
|
"kl": 20.033288955688477,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -50505083.61370716,
|
|
"logits/rejected": -53133639.42319749,
|
|
"logps/chosen": -479.8252531152648,
|
|
"logps/rejected": -386.9917221786834,
|
|
"loss": 0.2022,
|
|
"loss/base_kto": 1.6174923181533813,
|
|
"metrics/advantage_var": 15608.984375,
|
|
"rewards/chosen": 2.9505763544100465,
|
|
"rewards/margins": 5.919393129549643,
|
|
"rewards/rejected": -2.9688167751395964,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 6.454035758972168,
|
|
"kl": 18.603702545166016,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -51773555.86544342,
|
|
"logits/rejected": -52918828.166134186,
|
|
"logps/chosen": -440.5784594801223,
|
|
"logps/rejected": -401.768794928115,
|
|
"loss": 0.2248,
|
|
"loss/base_kto": 1.7981531620025635,
|
|
"metrics/advantage_var": 14751.2919921875,
|
|
"rewards/chosen": 2.5384420692373855,
|
|
"rewards/margins": 5.27170751639693,
|
|
"rewards/rejected": -2.733265447159545,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 7.05014181137085,
|
|
"kl": 20.547834396362305,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -52540588.00627943,
|
|
"logits/rejected": -52747510.8429238,
|
|
"logps/chosen": -462.1351059654631,
|
|
"logps/rejected": -421.39011469673403,
|
|
"loss": 0.2165,
|
|
"loss/base_kto": 1.732211709022522,
|
|
"metrics/advantage_var": 17090.8125,
|
|
"rewards/chosen": 2.817033273645997,
|
|
"rewards/margins": 5.648834431548408,
|
|
"rewards/rejected": -2.8318011579024107,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 9.174979209899902,
|
|
"kl": 26.36089515686035,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -52650930.716981135,
|
|
"logits/rejected": -53045127.1552795,
|
|
"logps/chosen": -451.7421875,
|
|
"logps/rejected": -429.9204192546584,
|
|
"loss": 0.2084,
|
|
"loss/base_kto": 1.6668694019317627,
|
|
"metrics/advantage_var": 15893.34375,
|
|
"rewards/chosen": 2.9159428818420796,
|
|
"rewards/margins": 5.797485016571311,
|
|
"rewards/rejected": -2.8815421347292314,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 9.659214973449707,
|
|
"kl": 20.512054443359375,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -50370115.347068146,
|
|
"logits/rejected": -52382443.882896766,
|
|
"logps/chosen": -486.13431061806654,
|
|
"logps/rejected": -401.66896186440675,
|
|
"loss": 0.2005,
|
|
"loss/base_kto": 1.6037886142730713,
|
|
"metrics/advantage_var": 16707.20703125,
|
|
"rewards/chosen": 2.9694340580427894,
|
|
"rewards/margins": 6.0927611869189455,
|
|
"rewards/rejected": -3.1233271288761557,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 10.550207138061523,
|
|
"kl": 12.67325210571289,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -52556268.48093842,
|
|
"logits/rejected": -52346379.98662207,
|
|
"logps/chosen": -459.1465359237537,
|
|
"logps/rejected": -410.18653323578593,
|
|
"loss": 0.2122,
|
|
"loss/base_kto": 1.6974483728408813,
|
|
"metrics/advantage_var": 15797.478515625,
|
|
"rewards/chosen": 2.6588947374450145,
|
|
"rewards/margins": 5.798360273607222,
|
|
"rewards/rejected": -3.1394655361622075,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 6.611340045928955,
|
|
"kl": 19.12717628479004,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -53003676.17610063,
|
|
"logits/rejected": -53865535.60248447,
|
|
"logps/chosen": -473.6758549528302,
|
|
"logps/rejected": -414.92342779503105,
|
|
"loss": 0.2046,
|
|
"loss/base_kto": 1.6370079517364502,
|
|
"metrics/advantage_var": 16002.5283203125,
|
|
"rewards/chosen": 2.987971083922956,
|
|
"rewards/margins": 5.849641536538446,
|
|
"rewards/rejected": -2.8616704526154892,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 8.27307415008545,
|
|
"kl": 17.498138427734375,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -52917409.33762058,
|
|
"logits/rejected": -54464972.6443769,
|
|
"logps/chosen": -489.0168810289389,
|
|
"logps/rejected": -361.63445098784194,
|
|
"loss": 0.2184,
|
|
"loss/base_kto": 1.747402548789978,
|
|
"metrics/advantage_var": 15314.0439453125,
|
|
"rewards/chosen": 2.673038593059184,
|
|
"rewards/margins": 5.558903221906069,
|
|
"rewards/rejected": -2.8858646288468845,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 7.73489236831665,
|
|
"kl": 20.97342300415039,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -51807078.4,
|
|
"logits/rejected": -52076121.6,
|
|
"logps/chosen": -439.680224609375,
|
|
"logps/rejected": -410.398681640625,
|
|
"loss": 0.2077,
|
|
"loss/base_kto": 1.6616114377975464,
|
|
"metrics/advantage_var": 16345.5283203125,
|
|
"rewards/chosen": 2.898772430419922,
|
|
"rewards/margins": 5.813119125366211,
|
|
"rewards/rejected": -2.914346694946289,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.893743515014648,
|
|
"kl": 18.39280128479004,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -51742174.08201893,
|
|
"logits/rejected": -53058696.32198142,
|
|
"logps/chosen": -466.2358537066246,
|
|
"logps/rejected": -402.3814096362229,
|
|
"loss": 0.2021,
|
|
"loss/base_kto": 1.6166282892227173,
|
|
"metrics/advantage_var": 16531.447265625,
|
|
"rewards/chosen": 2.782011110323837,
|
|
"rewards/margins": 6.094929475140981,
|
|
"rewards/rejected": -3.312918364817144,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 68.18772888183594,
|
|
"kl": 16.023277282714844,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -51808847.79220779,
|
|
"logits/rejected": -52280751.807228915,
|
|
"logps/chosen": -449.70824878246754,
|
|
"logps/rejected": -390.80294615963857,
|
|
"loss": 0.2201,
|
|
"loss/base_kto": 1.7605113983154297,
|
|
"metrics/advantage_var": 16087.9638671875,
|
|
"rewards/chosen": 2.6439783666040992,
|
|
"rewards/margins": 5.52344900795858,
|
|
"rewards/rejected": -2.8794706413544806,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 6.0411295890808105,
|
|
"kl": 16.658052444458008,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -52695826.73170732,
|
|
"logits/rejected": -53972630.974358976,
|
|
"logps/chosen": -455.8776676829268,
|
|
"logps/rejected": -410.6433293269231,
|
|
"loss": 0.2212,
|
|
"loss/base_kto": 1.7692569494247437,
|
|
"metrics/advantage_var": 14827.8408203125,
|
|
"rewards/chosen": 2.6817597179878048,
|
|
"rewards/margins": 5.596522166029672,
|
|
"rewards/rejected": -2.914762448041867,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 6.565061092376709,
|
|
"kl": 17.83756446838379,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -52485077.19614148,
|
|
"logits/rejected": -51947989.981762916,
|
|
"logps/chosen": -476.3875100482315,
|
|
"logps/rejected": -414.3374810030395,
|
|
"loss": 0.2101,
|
|
"loss/base_kto": 1.6810640096664429,
|
|
"metrics/advantage_var": 19462.033203125,
|
|
"rewards/chosen": 3.0427821119498595,
|
|
"rewards/margins": 6.14089621838033,
|
|
"rewards/rejected": -3.098114106430471,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 6.715926647186279,
|
|
"kl": 19.532516479492188,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -51633677.41140216,
|
|
"logits/rejected": -52861577.12836767,
|
|
"logps/chosen": -422.4470338983051,
|
|
"logps/rejected": -390.1222266244057,
|
|
"loss": 0.2153,
|
|
"loss/base_kto": 1.7227131128311157,
|
|
"metrics/advantage_var": 14244.3701171875,
|
|
"rewards/chosen": 2.758965866958783,
|
|
"rewards/margins": 5.391663067582792,
|
|
"rewards/rejected": -2.6326972006240097,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 7.742791652679443,
|
|
"kl": 20.519500732421875,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -53814366.78603946,
|
|
"logits/rejected": -53390799.355877616,
|
|
"logps/chosen": -480.35399279210924,
|
|
"logps/rejected": -401.73090277777777,
|
|
"loss": 0.1977,
|
|
"loss/base_kto": 1.581398367881775,
|
|
"metrics/advantage_var": 16275.658203125,
|
|
"rewards/chosen": 3.0262747548961495,
|
|
"rewards/margins": 6.12475463550706,
|
|
"rewards/rejected": -3.0984798806109097,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 8.513627052307129,
|
|
"kl": 19.383682250976562,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -51164498.066985644,
|
|
"logits/rejected": -51537223.74272588,
|
|
"logps/chosen": -452.1843102073365,
|
|
"logps/rejected": -391.86485451761104,
|
|
"loss": 0.213,
|
|
"loss/base_kto": 1.7036470174789429,
|
|
"metrics/advantage_var": 14536.775390625,
|
|
"rewards/chosen": 2.9792410380532295,
|
|
"rewards/margins": 5.64744869442287,
|
|
"rewards/rejected": -2.66820765636964,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 7.5082902908325195,
|
|
"kl": 20.06099510192871,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -54347665.12101911,
|
|
"logits/rejected": -53955678.23312883,
|
|
"logps/chosen": -460.7467655254777,
|
|
"logps/rejected": -422.7647622699387,
|
|
"loss": 0.2077,
|
|
"loss/base_kto": 1.6615208387374878,
|
|
"metrics/advantage_var": 17052.80078125,
|
|
"rewards/chosen": 2.729910880896696,
|
|
"rewards/margins": 5.756208795366597,
|
|
"rewards/rejected": -3.0262979144699003,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 6.819154262542725,
|
|
"kl": 19.58155632019043,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -51551179.77125193,
|
|
"logits/rejected": -53412732.96682464,
|
|
"logps/chosen": -456.2817329984544,
|
|
"logps/rejected": -414.9955568720379,
|
|
"loss": 0.2005,
|
|
"loss/base_kto": 1.6039291620254517,
|
|
"metrics/advantage_var": 15772.376953125,
|
|
"rewards/chosen": 2.918591365195131,
|
|
"rewards/margins": 5.92761415473897,
|
|
"rewards/rejected": -3.009022789543839,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.863317489624023,
|
|
"kl": 18.824533462524414,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -51992081.436532505,
|
|
"logits/rejected": -52817858.62460568,
|
|
"logps/chosen": -457.50541795665634,
|
|
"logps/rejected": -387.4557866719243,
|
|
"loss": 0.2272,
|
|
"loss/base_kto": 1.8178986310958862,
|
|
"metrics/advantage_var": 14906.2841796875,
|
|
"rewards/chosen": 2.748334855118034,
|
|
"rewards/margins": 5.385879986476473,
|
|
"rewards/rejected": -2.6375451313584386,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 7.7461771965026855,
|
|
"kl": 20.338605880737305,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -51067443.36569579,
|
|
"logits/rejected": -51839430.7673716,
|
|
"logps/chosen": -480.4406351132686,
|
|
"logps/rejected": -402.48682968277944,
|
|
"loss": 0.2209,
|
|
"loss/base_kto": 1.7669308185577393,
|
|
"metrics/advantage_var": 16044.369140625,
|
|
"rewards/chosen": 2.5220058404126213,
|
|
"rewards/margins": 5.390470837580295,
|
|
"rewards/rejected": -2.8684649971676737,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_loss": 0.2718521337023259,
|
|
"train_runtime": 11043.5973,
|
|
"train_samples_per_second": 13.421,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.960196480656998e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|