796 lines
27 KiB
JSON
796 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9989528795811519,
|
|
"eval_steps": 200,
|
|
"global_step": 477,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0020942408376963353,
|
|
"grad_norm": 334.2563171386719,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.615048885345459,
|
|
"logits/rejected": -0.6184952855110168,
|
|
"logps/chosen": -1.287253737449646,
|
|
"logps/rejected": -1.4317338466644287,
|
|
"loss": 100.0182,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.00019188039004802704,
|
|
"rewards/margins": 0.00016489533300045878,
|
|
"rewards/rejected": 2.698507159948349e-05,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.020942408376963352,
|
|
"grad_norm": 285.50787353515625,
|
|
"learning_rate": 9.375e-08,
|
|
"logits/chosen": -0.6403717398643494,
|
|
"logits/rejected": -0.6338582634925842,
|
|
"logps/chosen": -1.1636602878570557,
|
|
"logps/rejected": -1.3377324342727661,
|
|
"loss": 99.9909,
|
|
"rewards/accuracies": 0.5208333134651184,
|
|
"rewards/chosen": 6.887399649713188e-05,
|
|
"rewards/margins": 5.176978083909489e-05,
|
|
"rewards/rejected": 1.7104217477026395e-05,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.041884816753926704,
|
|
"grad_norm": 435.0426330566406,
|
|
"learning_rate": 1.9791666666666664e-07,
|
|
"logits/chosen": -0.6049723625183105,
|
|
"logits/rejected": -0.6281808614730835,
|
|
"logps/chosen": -1.197880506515503,
|
|
"logps/rejected": -1.3490830659866333,
|
|
"loss": 99.9683,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.00015270955918822438,
|
|
"rewards/margins": 0.00011640439333859831,
|
|
"rewards/rejected": 3.630516948760487e-05,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.06282722513089005,
|
|
"grad_norm": 340.81402587890625,
|
|
"learning_rate": 3.020833333333333e-07,
|
|
"logits/chosen": -0.5800901055335999,
|
|
"logits/rejected": -0.5592249035835266,
|
|
"logps/chosen": -1.1031697988510132,
|
|
"logps/rejected": -1.1418795585632324,
|
|
"loss": 99.8945,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": 0.0005954386433586478,
|
|
"rewards/margins": 0.00042125614709220827,
|
|
"rewards/rejected": 0.0001741824671626091,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.08376963350785341,
|
|
"grad_norm": 417.978271484375,
|
|
"learning_rate": 4.0625e-07,
|
|
"logits/chosen": -0.6128225922584534,
|
|
"logits/rejected": -0.6447689533233643,
|
|
"logps/chosen": -1.1222411394119263,
|
|
"logps/rejected": -1.2407972812652588,
|
|
"loss": 99.5546,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.0018988971132785082,
|
|
"rewards/margins": 0.0013067866675555706,
|
|
"rewards/rejected": 0.0005921103293076158,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.10471204188481675,
|
|
"grad_norm": 316.5003356933594,
|
|
"learning_rate": 4.999932966293553e-07,
|
|
"logits/chosen": -0.6670976877212524,
|
|
"logits/rejected": -0.6841738820075989,
|
|
"logps/chosen": -1.0955612659454346,
|
|
"logps/rejected": -1.2405668497085571,
|
|
"loss": 99.0369,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.00708451634272933,
|
|
"rewards/margins": 0.00436831172555685,
|
|
"rewards/rejected": 0.0027162046171724796,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.1256544502617801,
|
|
"grad_norm": 419.10931396484375,
|
|
"learning_rate": 4.991893270335525e-07,
|
|
"logits/chosen": -0.6393335461616516,
|
|
"logits/rejected": -0.6426213383674622,
|
|
"logps/chosen": -1.1490199565887451,
|
|
"logps/rejected": -1.2731598615646362,
|
|
"loss": 98.2609,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.006263996474444866,
|
|
"rewards/margins": 0.006520474795252085,
|
|
"rewards/rejected": -0.00025647730217315257,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.14659685863874344,
|
|
"grad_norm": 455.3122863769531,
|
|
"learning_rate": 4.970496218214204e-07,
|
|
"logits/chosen": -0.7453044056892395,
|
|
"logits/rejected": -0.7440285682678223,
|
|
"logps/chosen": -1.1631481647491455,
|
|
"logps/rejected": -1.4671075344085693,
|
|
"loss": 97.3229,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": 0.0009300081292167306,
|
|
"rewards/margins": 0.0067420280538499355,
|
|
"rewards/rejected": -0.005812020041048527,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.16753926701570682,
|
|
"grad_norm": 581.3535766601562,
|
|
"learning_rate": 4.935856505068998e-07,
|
|
"logits/chosen": -0.7049443125724792,
|
|
"logits/rejected": -0.7213777303695679,
|
|
"logps/chosen": -1.2213430404663086,
|
|
"logps/rejected": -1.5481337308883667,
|
|
"loss": 95.8171,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.009956983849406242,
|
|
"rewards/margins": 0.012672394514083862,
|
|
"rewards/rejected": -0.022629376500844955,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.18848167539267016,
|
|
"grad_norm": 433.6556396484375,
|
|
"learning_rate": 4.8881598109976e-07,
|
|
"logits/chosen": -0.6699498891830444,
|
|
"logits/rejected": -0.7027503252029419,
|
|
"logps/chosen": -1.3308374881744385,
|
|
"logps/rejected": -1.6979453563690186,
|
|
"loss": 94.2675,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -0.01737907901406288,
|
|
"rewards/margins": 0.019169464707374573,
|
|
"rewards/rejected": -0.036548543721437454,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"grad_norm": 591.3351440429688,
|
|
"learning_rate": 4.827661805750437e-07,
|
|
"logits/chosen": -0.6697665452957153,
|
|
"logits/rejected": -0.6784194111824036,
|
|
"logps/chosen": -1.2227742671966553,
|
|
"logps/rejected": -1.5098785161972046,
|
|
"loss": 94.1616,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.01010363269597292,
|
|
"rewards/margins": 0.01724056527018547,
|
|
"rewards/rejected": -0.027344200760126114,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.23036649214659685,
|
|
"grad_norm": 1046.2178955078125,
|
|
"learning_rate": 4.75468677825789e-07,
|
|
"logits/chosen": -0.6205618977546692,
|
|
"logits/rejected": -0.6708791851997375,
|
|
"logps/chosen": -1.1823982000350952,
|
|
"logps/rejected": -1.7904584407806396,
|
|
"loss": 92.3693,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.014755316078662872,
|
|
"rewards/margins": 0.02819245494902134,
|
|
"rewards/rejected": -0.04294777661561966,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.2513089005235602,
|
|
"grad_norm": 456.5362243652344,
|
|
"learning_rate": 4.669625898336438e-07,
|
|
"logits/chosen": -0.6897668838500977,
|
|
"logits/rejected": -0.6946333646774292,
|
|
"logps/chosen": -1.475966215133667,
|
|
"logps/rejected": -1.7667391300201416,
|
|
"loss": 92.8502,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -0.026928585022687912,
|
|
"rewards/margins": 0.020536581054329872,
|
|
"rewards/rejected": -0.04746516793966293,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.27225130890052357,
|
|
"grad_norm": 493.82305908203125,
|
|
"learning_rate": 4.5729351198915705e-07,
|
|
"logits/chosen": -0.675911009311676,
|
|
"logits/rejected": -0.6610211730003357,
|
|
"logps/chosen": -1.4558229446411133,
|
|
"logps/rejected": -1.6363474130630493,
|
|
"loss": 91.8914,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": -0.024449264630675316,
|
|
"rewards/margins": 0.015649044886231422,
|
|
"rewards/rejected": -0.04009830951690674,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.2931937172774869,
|
|
"grad_norm": 656.6390380859375,
|
|
"learning_rate": 4.4651327368569684e-07,
|
|
"logits/chosen": -0.7115468382835388,
|
|
"logits/rejected": -0.6819766759872437,
|
|
"logps/chosen": -1.3681796789169312,
|
|
"logps/rejected": -1.7001540660858154,
|
|
"loss": 92.2874,
|
|
"rewards/accuracies": 0.5874999761581421,
|
|
"rewards/chosen": -0.013815701007843018,
|
|
"rewards/margins": 0.026623845100402832,
|
|
"rewards/rejected": -0.04043954610824585,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.31413612565445026,
|
|
"grad_norm": 1577.8076171875,
|
|
"learning_rate": 4.346796604970912e-07,
|
|
"logits/chosen": -0.7165893912315369,
|
|
"logits/rejected": -0.7037891745567322,
|
|
"logps/chosen": -1.3247240781784058,
|
|
"logps/rejected": -1.9355542659759521,
|
|
"loss": 90.6388,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.01547580398619175,
|
|
"rewards/margins": 0.04593699425458908,
|
|
"rewards/rejected": -0.06141280010342598,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.33507853403141363,
|
|
"grad_norm": 1120.6865234375,
|
|
"learning_rate": 4.218561044282098e-07,
|
|
"logits/chosen": -0.6731705069541931,
|
|
"logits/rejected": -0.6826528906822205,
|
|
"logps/chosen": -1.5552542209625244,
|
|
"logps/rejected": -2.336169719696045,
|
|
"loss": 89.2392,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.03482538461685181,
|
|
"rewards/margins": 0.05508958175778389,
|
|
"rewards/rejected": -0.0899149626493454,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.35602094240837695,
|
|
"grad_norm": 1590.9273681640625,
|
|
"learning_rate": 4.081113438988443e-07,
|
|
"logits/chosen": -0.6908739805221558,
|
|
"logits/rejected": -0.6841577291488647,
|
|
"logps/chosen": -1.4513384103775024,
|
|
"logps/rejected": -1.9829685688018799,
|
|
"loss": 88.2583,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.0341075137257576,
|
|
"rewards/margins": 0.03856590390205383,
|
|
"rewards/rejected": -0.07267341762781143,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.3769633507853403,
|
|
"grad_norm": 1529.81201171875,
|
|
"learning_rate": 3.935190552834828e-07,
|
|
"logits/chosen": -0.6555457711219788,
|
|
"logits/rejected": -0.6698058843612671,
|
|
"logps/chosen": -1.414902925491333,
|
|
"logps/rejected": -1.8965734243392944,
|
|
"loss": 88.1778,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.021763872355222702,
|
|
"rewards/margins": 0.03738107159733772,
|
|
"rewards/rejected": -0.059144943952560425,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.39790575916230364,
|
|
"grad_norm": 1098.9884033203125,
|
|
"learning_rate": 3.781574579820464e-07,
|
|
"logits/chosen": -0.7295094132423401,
|
|
"logits/rejected": -0.7264297008514404,
|
|
"logps/chosen": -1.3178633451461792,
|
|
"logps/rejected": -2.14937162399292,
|
|
"loss": 87.544,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.02538556233048439,
|
|
"rewards/margins": 0.06075022369623184,
|
|
"rewards/rejected": -0.08613577485084534,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"grad_norm": 920.0095825195312,
|
|
"learning_rate": 3.621088951385353e-07,
|
|
"logits/chosen": -0.7311328649520874,
|
|
"logits/rejected": -0.7470929026603699,
|
|
"logps/chosen": -1.468587040901184,
|
|
"logps/rejected": -2.221226453781128,
|
|
"loss": 87.5502,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.03510420024394989,
|
|
"rewards/margins": 0.05219089239835739,
|
|
"rewards/rejected": -0.08729508519172668,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"eval_logits/chosen": -0.7410492897033691,
|
|
"eval_logits/rejected": -0.7169657945632935,
|
|
"eval_logps/chosen": -1.6581848859786987,
|
|
"eval_logps/rejected": -2.345245361328125,
|
|
"eval_loss": 21.880334854125977,
|
|
"eval_rewards/accuracies": 0.7056451439857483,
|
|
"eval_rewards/chosen": -0.05191723257303238,
|
|
"eval_rewards/margins": 0.04926152899861336,
|
|
"eval_rewards/rejected": -0.10117875784635544,
|
|
"eval_runtime": 44.8829,
|
|
"eval_samples_per_second": 44.56,
|
|
"eval_steps_per_second": 1.404,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.4397905759162304,
|
|
"grad_norm": 895.4285278320312,
|
|
"learning_rate": 3.454593922550693e-07,
|
|
"logits/chosen": -0.6665436029434204,
|
|
"logits/rejected": -0.6757727861404419,
|
|
"logps/chosen": -1.485873818397522,
|
|
"logps/rejected": -2.344456672668457,
|
|
"loss": 86.8513,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.03312455862760544,
|
|
"rewards/margins": 0.06565652042627335,
|
|
"rewards/rejected": -0.09878107905387878,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.4607329842931937,
|
|
"grad_norm": 1199.5086669921875,
|
|
"learning_rate": 3.2829819606729477e-07,
|
|
"logits/chosen": -0.7451668977737427,
|
|
"logits/rejected": -0.7250313758850098,
|
|
"logps/chosen": -1.6201412677764893,
|
|
"logps/rejected": -2.3489794731140137,
|
|
"loss": 85.5906,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -0.046348538249731064,
|
|
"rewards/margins": 0.056174565106630325,
|
|
"rewards/rejected": -0.10252310335636139,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.4816753926701571,
|
|
"grad_norm": 1276.23583984375,
|
|
"learning_rate": 3.1071729615293424e-07,
|
|
"logits/chosen": -0.768721878528595,
|
|
"logits/rejected": -0.7249744534492493,
|
|
"logps/chosen": -1.8909496068954468,
|
|
"logps/rejected": -2.51973557472229,
|
|
"loss": 85.1963,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.06488613784313202,
|
|
"rewards/margins": 0.05823909491300583,
|
|
"rewards/rejected": -0.12312524020671844,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.5026178010471204,
|
|
"grad_norm": 1169.336181640625,
|
|
"learning_rate": 2.9281093183781403e-07,
|
|
"logits/chosen": -0.6923402547836304,
|
|
"logits/rejected": -0.7125403881072998,
|
|
"logps/chosen": -1.8178189992904663,
|
|
"logps/rejected": -2.7802040576934814,
|
|
"loss": 83.9422,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.06791480630636215,
|
|
"rewards/margins": 0.0749489963054657,
|
|
"rewards/rejected": -0.14286379516124725,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5235602094240838,
|
|
"grad_norm": 1323.8421630859375,
|
|
"learning_rate": 2.7467508704251135e-07,
|
|
"logits/chosen": -0.7305997610092163,
|
|
"logits/rejected": -0.7117995023727417,
|
|
"logps/chosen": -1.8664335012435913,
|
|
"logps/rejected": -2.734891653060913,
|
|
"loss": 83.9814,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.0722884088754654,
|
|
"rewards/margins": 0.06869889050722122,
|
|
"rewards/rejected": -0.14098729193210602,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.5445026178010471,
|
|
"grad_norm": 1842.348388671875,
|
|
"learning_rate": 2.5640697577740815e-07,
|
|
"logits/chosen": -0.7306697368621826,
|
|
"logits/rejected": -0.7254212498664856,
|
|
"logps/chosen": -2.0410964488983154,
|
|
"logps/rejected": -2.9278271198272705,
|
|
"loss": 84.5359,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.08307617902755737,
|
|
"rewards/margins": 0.07588212937116623,
|
|
"rewards/rejected": -0.1589583158493042,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.5654450261780105,
|
|
"grad_norm": 1502.2591552734375,
|
|
"learning_rate": 2.381045210440644e-07,
|
|
"logits/chosen": -0.7093442678451538,
|
|
"logits/rejected": -0.6900134682655334,
|
|
"logps/chosen": -2.196712017059326,
|
|
"logps/rejected": -2.765611171722412,
|
|
"loss": 83.7083,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -0.10604780912399292,
|
|
"rewards/margins": 0.04271745681762695,
|
|
"rewards/rejected": -0.14876528084278107,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.5863874345549738,
|
|
"grad_norm": 1244.79345703125,
|
|
"learning_rate": 2.1986582993616925e-07,
|
|
"logits/chosen": -0.7046034336090088,
|
|
"logits/rejected": -0.7077471017837524,
|
|
"logps/chosen": -1.9278895854949951,
|
|
"logps/rejected": -3.006953001022339,
|
|
"loss": 81.093,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.0760059580206871,
|
|
"rewards/margins": 0.0849492996931076,
|
|
"rewards/rejected": -0.1609552651643753,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.6073298429319371,
|
|
"grad_norm": 2431.488037109375,
|
|
"learning_rate": 2.0178866775369774e-07,
|
|
"logits/chosen": -0.7360321879386902,
|
|
"logits/rejected": -0.7089033126831055,
|
|
"logps/chosen": -2.280106544494629,
|
|
"logps/rejected": -3.052873373031616,
|
|
"loss": 84.3058,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -0.10656996071338654,
|
|
"rewards/margins": 0.06549294292926788,
|
|
"rewards/rejected": -0.17206290364265442,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"grad_norm": 2390.089599609375,
|
|
"learning_rate": 1.839699339491937e-07,
|
|
"logits/chosen": -0.7405005097389221,
|
|
"logits/rejected": -0.7184410691261292,
|
|
"logps/chosen": -2.2960636615753174,
|
|
"logps/rejected": -3.015368700027466,
|
|
"loss": 83.439,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.10949590057134628,
|
|
"rewards/margins": 0.06635172665119171,
|
|
"rewards/rejected": -0.17584764957427979,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6492146596858639,
|
|
"grad_norm": 1713.3050537109375,
|
|
"learning_rate": 1.6650514271527465e-07,
|
|
"logits/chosen": -0.6984419822692871,
|
|
"logits/rejected": -0.6775582432746887,
|
|
"logps/chosen": -2.1996333599090576,
|
|
"logps/rejected": -2.9705100059509277,
|
|
"loss": 80.5471,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.09805257618427277,
|
|
"rewards/margins": 0.07093057781457901,
|
|
"rewards/rejected": -0.16898314654827118,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.6701570680628273,
|
|
"grad_norm": 1855.712646484375,
|
|
"learning_rate": 1.4948791099758052e-07,
|
|
"logits/chosen": -0.7052116990089417,
|
|
"logits/rejected": -0.7140225172042847,
|
|
"logps/chosen": -2.282247304916382,
|
|
"logps/rejected": -3.390331745147705,
|
|
"loss": 80.2757,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.107896588742733,
|
|
"rewards/margins": 0.09624334424734116,
|
|
"rewards/rejected": -0.20413991808891296,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.6910994764397905,
|
|
"grad_norm": 1676.298095703125,
|
|
"learning_rate": 1.3300945667758012e-07,
|
|
"logits/chosen": -0.6669484972953796,
|
|
"logits/rejected": -0.6471594572067261,
|
|
"logps/chosen": -2.1016829013824463,
|
|
"logps/rejected": -3.35722279548645,
|
|
"loss": 81.4342,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.08771068602800369,
|
|
"rewards/margins": 0.11208438873291016,
|
|
"rewards/rejected": -0.19979506731033325,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.7120418848167539,
|
|
"grad_norm": 1909.808349609375,
|
|
"learning_rate": 1.1715810961514072e-07,
|
|
"logits/chosen": -0.6822046041488647,
|
|
"logits/rejected": -0.651155948638916,
|
|
"logps/chosen": -2.583623170852661,
|
|
"logps/rejected": -3.5456066131591797,
|
|
"loss": 81.1622,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.13260190188884735,
|
|
"rewards/margins": 0.08422346413135529,
|
|
"rewards/rejected": -0.21682536602020264,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7329842931937173,
|
|
"grad_norm": 1723.5054931640625,
|
|
"learning_rate": 1.0201883817182949e-07,
|
|
"logits/chosen": -0.6957032084465027,
|
|
"logits/rejected": -0.6772996783256531,
|
|
"logps/chosen": -2.392089366912842,
|
|
"logps/rejected": -3.3031153678894043,
|
|
"loss": 81.021,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.12260153144598007,
|
|
"rewards/margins": 0.07098677009344101,
|
|
"rewards/rejected": -0.19358830153942108,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.7539267015706806,
|
|
"grad_norm": 1900.4688720703125,
|
|
"learning_rate": 8.76727937529367e-08,
|
|
"logits/chosen": -0.6489254236221313,
|
|
"logits/rejected": -0.6335070133209229,
|
|
"logps/chosen": -2.3418354988098145,
|
|
"logps/rejected": -3.38752818107605,
|
|
"loss": 82.1297,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -0.11146201193332672,
|
|
"rewards/margins": 0.0862349346280098,
|
|
"rewards/rejected": -0.19769695401191711,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.774869109947644,
|
|
"grad_norm": 1725.67431640625,
|
|
"learning_rate": 7.419687580962222e-08,
|
|
"logits/chosen": -0.6811009645462036,
|
|
"logits/rejected": -0.6841970682144165,
|
|
"logps/chosen": -2.3078436851501465,
|
|
"logps/rejected": -3.5094189643859863,
|
|
"loss": 78.1712,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.11148089170455933,
|
|
"rewards/margins": 0.10181178897619247,
|
|
"rewards/rejected": -0.2132926881313324,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.7958115183246073,
|
|
"grad_norm": 1767.7569580078125,
|
|
"learning_rate": 6.166331963291519e-08,
|
|
"logits/chosen": -0.6415311694145203,
|
|
"logits/rejected": -0.6283861398696899,
|
|
"logps/chosen": -2.2348546981811523,
|
|
"logps/rejected": -3.141282320022583,
|
|
"loss": 81.5469,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.10441361367702484,
|
|
"rewards/margins": 0.08075843751430511,
|
|
"rewards/rejected": -0.18517205119132996,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.8167539267015707,
|
|
"grad_norm": 1984.725341796875,
|
|
"learning_rate": 5.013930914912476e-08,
|
|
"logits/chosen": -0.691740870475769,
|
|
"logits/rejected": -0.6813372373580933,
|
|
"logps/chosen": -2.3861072063446045,
|
|
"logps/rejected": -3.2833335399627686,
|
|
"loss": 81.5221,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.12453154474496841,
|
|
"rewards/margins": 0.07637959718704224,
|
|
"rewards/rejected": -0.20091113448143005,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"grad_norm": 1985.95068359375,
|
|
"learning_rate": 3.968661679220467e-08,
|
|
"logits/chosen": -0.6570695042610168,
|
|
"logits/rejected": -0.6685432195663452,
|
|
"logps/chosen": -2.366621732711792,
|
|
"logps/rejected": -3.2288424968719482,
|
|
"loss": 80.9952,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.11433804035186768,
|
|
"rewards/margins": 0.08440788835287094,
|
|
"rewards/rejected": -0.1987459361553192,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"eval_logits/chosen": -0.677091121673584,
|
|
"eval_logits/rejected": -0.6593382358551025,
|
|
"eval_logps/chosen": -2.3106672763824463,
|
|
"eval_logps/rejected": -3.3355889320373535,
|
|
"eval_loss": 20.19257926940918,
|
|
"eval_rewards/accuracies": 0.7620967626571655,
|
|
"eval_rewards/chosen": -0.11716549098491669,
|
|
"eval_rewards/margins": 0.08304762095212936,
|
|
"eval_rewards/rejected": -0.20021310448646545,
|
|
"eval_runtime": 44.6349,
|
|
"eval_samples_per_second": 44.808,
|
|
"eval_steps_per_second": 1.411,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8586387434554974,
|
|
"grad_norm": 1995.79248046875,
|
|
"learning_rate": 3.036127238347164e-08,
|
|
"logits/chosen": -0.6521174907684326,
|
|
"logits/rejected": -0.6390538811683655,
|
|
"logps/chosen": -2.400747299194336,
|
|
"logps/rejected": -3.1259586811065674,
|
|
"loss": 80.8247,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.12548241019248962,
|
|
"rewards/margins": 0.05977809429168701,
|
|
"rewards/rejected": -0.18526050448417664,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.8795811518324608,
|
|
"grad_norm": 1795.328857421875,
|
|
"learning_rate": 2.2213262793589482e-08,
|
|
"logits/chosen": -0.6523452997207642,
|
|
"logits/rejected": -0.6344730257987976,
|
|
"logps/chosen": -2.3337454795837402,
|
|
"logps/rejected": -3.2163987159729004,
|
|
"loss": 79.558,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -0.12196169048547745,
|
|
"rewards/margins": 0.07503517717123032,
|
|
"rewards/rejected": -0.19699685275554657,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.900523560209424,
|
|
"grad_norm": 2082.081298828125,
|
|
"learning_rate": 1.5286263996730026e-08,
|
|
"logits/chosen": -0.6856145858764648,
|
|
"logits/rejected": -0.6647241115570068,
|
|
"logps/chosen": -2.348968982696533,
|
|
"logps/rejected": -3.445862293243408,
|
|
"loss": 78.859,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.11975270509719849,
|
|
"rewards/margins": 0.08451148122549057,
|
|
"rewards/rejected": -0.20426419377326965,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9214659685863874,
|
|
"grad_norm": 2494.26806640625,
|
|
"learning_rate": 9.617406953185136e-09,
|
|
"logits/chosen": -0.6526113748550415,
|
|
"logits/rejected": -0.6604099869728088,
|
|
"logps/chosen": -2.3740854263305664,
|
|
"logps/rejected": -3.1448159217834473,
|
|
"loss": 81.5522,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -0.12368841469287872,
|
|
"rewards/margins": 0.060592371970415115,
|
|
"rewards/rejected": -0.18428078293800354,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.9424083769633508,
|
|
"grad_norm": 1965.380615234375,
|
|
"learning_rate": 5.2370785753763356e-09,
|
|
"logits/chosen": -0.6635817289352417,
|
|
"logits/rejected": -0.653299868106842,
|
|
"logps/chosen": -2.3846700191497803,
|
|
"logps/rejected": -3.3813278675079346,
|
|
"loss": 79.7652,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.1247202605009079,
|
|
"rewards/margins": 0.08005838841199875,
|
|
"rewards/rejected": -0.20477867126464844,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.9633507853403142,
|
|
"grad_norm": 2279.437744140625,
|
|
"learning_rate": 2.168758844148272e-09,
|
|
"logits/chosen": -0.6574541330337524,
|
|
"logits/rejected": -0.6650590300559998,
|
|
"logps/chosen": -2.2831614017486572,
|
|
"logps/rejected": -3.247992753982544,
|
|
"loss": 82.4212,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -0.11224757134914398,
|
|
"rewards/margins": 0.07813762873411179,
|
|
"rewards/rejected": -0.19038519263267517,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.9842931937172775,
|
|
"grad_norm": 2007.5858154296875,
|
|
"learning_rate": 4.288949484559934e-10,
|
|
"logits/chosen": -0.6454898118972778,
|
|
"logits/rejected": -0.6453719139099121,
|
|
"logps/chosen": -2.3356616497039795,
|
|
"logps/rejected": -3.382112503051758,
|
|
"loss": 79.3744,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -0.11207450926303864,
|
|
"rewards/margins": 0.09411285072565079,
|
|
"rewards/rejected": -0.20618736743927002,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"step": 477,
|
|
"total_flos": 0.0,
|
|
"train_loss": 87.11727690946631,
|
|
"train_runtime": 3396.1479,
|
|
"train_samples_per_second": 18.001,
|
|
"train_steps_per_second": 0.14
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 477,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|