4052 lines
144 KiB
JSON
4052 lines
144 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 2.7512899896800826,
|
||
|
|
"eval_steps": 100,
|
||
|
|
"global_step": 2000,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0,
|
||
|
|
"eval_entropy": 1.6422131361525043,
|
||
|
|
"eval_logits/chosen": -3.145482037066458,
|
||
|
|
"eval_logits/rejected": -3.077003725568922,
|
||
|
|
"eval_logps/chosen": -109.81586472505059,
|
||
|
|
"eval_logps/rejected": -150.2786612167857,
|
||
|
|
"eval_loss": 0.6931471228599548,
|
||
|
|
"eval_mean_token_accuracy": 0.49587641841445873,
|
||
|
|
"eval_num_tokens": 0.0,
|
||
|
|
"eval_rewards/accuracies": 0.0,
|
||
|
|
"eval_rewards/chosen": 0.0,
|
||
|
|
"eval_rewards/margins": 0.0,
|
||
|
|
"eval_rewards/rejected": 0.0,
|
||
|
|
"eval_runtime": 47.3541,
|
||
|
|
"eval_samples_per_second": 12.924,
|
||
|
|
"eval_steps_per_second": 6.462,
|
||
|
|
"step": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.6841309666633606,
|
||
|
|
"epoch": 0.0013759889920880633,
|
||
|
|
"grad_norm": 33.75,
|
||
|
|
"learning_rate": 0.0,
|
||
|
|
"logits/chosen": -3.163112659670131,
|
||
|
|
"logits/rejected": -3.0765846200610514,
|
||
|
|
"logps/chosen": -106.31735277175903,
|
||
|
|
"logps/rejected": -147.35992622375488,
|
||
|
|
"loss": 0.6931471824645996,
|
||
|
|
"mean_token_accuracy": 0.46893012523651123,
|
||
|
|
"num_tokens": 6745.0,
|
||
|
|
"rewards/accuracies": 0.0,
|
||
|
|
"rewards/chosen": 0.0,
|
||
|
|
"rewards/margins": 0.0,
|
||
|
|
"rewards/rejected": 0.0,
|
||
|
|
"step": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.631808305780093,
|
||
|
|
"epoch": 0.013759889920880633,
|
||
|
|
"grad_norm": 11.9375,
|
||
|
|
"learning_rate": 8.256880733944956e-07,
|
||
|
|
"logits/chosen": -3.156389862999097,
|
||
|
|
"logits/rejected": -3.0980865636330024,
|
||
|
|
"logps/chosen": -108.15034267637465,
|
||
|
|
"logps/rejected": -151.11039204067654,
|
||
|
|
"loss": 0.5401247872246636,
|
||
|
|
"mean_token_accuracy": 0.5070789319773515,
|
||
|
|
"num_tokens": 70146.0,
|
||
|
|
"rewards/accuracies": 0.8333333333333334,
|
||
|
|
"rewards/chosen": 0.24961456178344735,
|
||
|
|
"rewards/margins": 0.39007408768197316,
|
||
|
|
"rewards/rejected": -0.1404595249849889,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.4963100239634515,
|
||
|
|
"epoch": 0.027519779841761266,
|
||
|
|
"grad_norm": 0.03759765625,
|
||
|
|
"learning_rate": 1.743119266055046e-06,
|
||
|
|
"logits/chosen": -3.23790955675432,
|
||
|
|
"logits/rejected": -3.0797231596356247,
|
||
|
|
"logps/chosen": -85.77736120223999,
|
||
|
|
"logps/rejected": -178.22046175003052,
|
||
|
|
"loss": 0.047147470712661746,
|
||
|
|
"mean_token_accuracy": 0.5772599957883358,
|
||
|
|
"num_tokens": 145514.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.4378453686833383,
|
||
|
|
"rewards/margins": 4.863506855070591,
|
||
|
|
"rewards/rejected": -2.425661455094814,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3544968962669373,
|
||
|
|
"epoch": 0.0412796697626419,
|
||
|
|
"grad_norm": 0.01318359375,
|
||
|
|
"learning_rate": 2.6605504587155968e-06,
|
||
|
|
"logits/chosen": -3.332316438054062,
|
||
|
|
"logits/rejected": -3.0564629823512917,
|
||
|
|
"logps/chosen": -69.85810537338257,
|
||
|
|
"logps/rejected": -219.8774684906006,
|
||
|
|
"loss": 0.00012127072550356388,
|
||
|
|
"mean_token_accuracy": 0.6386784806847572,
|
||
|
|
"num_tokens": 216437.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 3.659462496638298,
|
||
|
|
"rewards/margins": 10.62372499704361,
|
||
|
|
"rewards/rejected": -6.964262568950653,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3117856681346893,
|
||
|
|
"epoch": 0.05503955968352253,
|
||
|
|
"grad_norm": 0.000865936279296875,
|
||
|
|
"learning_rate": 3.5779816513761473e-06,
|
||
|
|
"logits/chosen": -3.3248741358928164,
|
||
|
|
"logits/rejected": -3.0449267511018827,
|
||
|
|
"logps/chosen": -68.93827650547027,
|
||
|
|
"logps/rejected": -239.04862117767334,
|
||
|
|
"loss": 2.7265999233350157e-05,
|
||
|
|
"mean_token_accuracy": 0.658780874311924,
|
||
|
|
"num_tokens": 291812.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.162062160670757,
|
||
|
|
"rewards/margins": 13.01834270954132,
|
||
|
|
"rewards/rejected": -8.856280529499054,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3213376998901367,
|
||
|
|
"epoch": 0.06879944960440317,
|
||
|
|
"grad_norm": 0.0019683837890625,
|
||
|
|
"learning_rate": 4.4954128440366975e-06,
|
||
|
|
"logits/chosen": -3.3137762086238127,
|
||
|
|
"logits/rejected": -3.042837344703322,
|
||
|
|
"logps/chosen": -65.1577981710434,
|
||
|
|
"logps/rejected": -232.65355396270752,
|
||
|
|
"loss": 2.894022036343813e-05,
|
||
|
|
"mean_token_accuracy": 0.6745641171932221,
|
||
|
|
"num_tokens": 357296.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.219022332131862,
|
||
|
|
"rewards/margins": 12.86244809627533,
|
||
|
|
"rewards/rejected": -8.643425732851028,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.310352511703968,
|
||
|
|
"epoch": 0.0825593395252838,
|
||
|
|
"grad_norm": 0.0012359619140625,
|
||
|
|
"learning_rate": 5.412844036697248e-06,
|
||
|
|
"logits/chosen": -3.316903429840562,
|
||
|
|
"logits/rejected": -3.0264215624626054,
|
||
|
|
"logps/chosen": -66.44462168216705,
|
||
|
|
"logps/rejected": -245.0753698348999,
|
||
|
|
"loss": 1.3988211867399513e-05,
|
||
|
|
"mean_token_accuracy": 0.6762391120195389,
|
||
|
|
"num_tokens": 426934.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.29778570085764,
|
||
|
|
"rewards/margins": 13.754501032829285,
|
||
|
|
"rewards/rejected": -9.456715273857117,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3216734871268272,
|
||
|
|
"epoch": 0.09631922944616443,
|
||
|
|
"grad_norm": 0.0003986358642578125,
|
||
|
|
"learning_rate": 6.330275229357799e-06,
|
||
|
|
"logits/chosen": -3.3179312487999497,
|
||
|
|
"logits/rejected": -3.0427430948262106,
|
||
|
|
"logps/chosen": -66.5002682685852,
|
||
|
|
"logps/rejected": -240.90171585083007,
|
||
|
|
"loss": 3.928586666006595e-05,
|
||
|
|
"mean_token_accuracy": 0.6831246554851532,
|
||
|
|
"num_tokens": 495203.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.366098646819592,
|
||
|
|
"rewards/margins": 13.849577975273132,
|
||
|
|
"rewards/rejected": -9.483479243516921,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3014291919767857,
|
||
|
|
"epoch": 0.11007911936704506,
|
||
|
|
"grad_norm": 0.002532958984375,
|
||
|
|
"learning_rate": 7.247706422018349e-06,
|
||
|
|
"logits/chosen": -3.3127891155448475,
|
||
|
|
"logits/rejected": -3.031499314118935,
|
||
|
|
"logps/chosen": -67.93886163234711,
|
||
|
|
"logps/rejected": -251.54950313568116,
|
||
|
|
"loss": 1.0652156197465956e-05,
|
||
|
|
"mean_token_accuracy": 0.674790358543396,
|
||
|
|
"num_tokens": 570516.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.4089073032140735,
|
||
|
|
"rewards/margins": 13.979358959197999,
|
||
|
|
"rewards/rejected": -9.57045162320137,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3178453385829925,
|
||
|
|
"epoch": 0.1238390092879257,
|
||
|
|
"grad_norm": 0.000690460205078125,
|
||
|
|
"learning_rate": 8.1651376146789e-06,
|
||
|
|
"logits/chosen": -3.3227773672405094,
|
||
|
|
"logits/rejected": -3.040103606137686,
|
||
|
|
"logps/chosen": -65.88669891357422,
|
||
|
|
"logps/rejected": -249.75017547607422,
|
||
|
|
"loss": 1.5238465857692063e-05,
|
||
|
|
"mean_token_accuracy": 0.6740365795791149,
|
||
|
|
"num_tokens": 645105.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.395410162210465,
|
||
|
|
"rewards/margins": 14.197139203548431,
|
||
|
|
"rewards/rejected": -9.80172905921936,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.333089178800583,
|
||
|
|
"epoch": 0.13759889920880633,
|
||
|
|
"grad_norm": 0.007080078125,
|
||
|
|
"learning_rate": 9.08256880733945e-06,
|
||
|
|
"logits/chosen": -3.3304067715679757,
|
||
|
|
"logits/rejected": -3.0432493623910477,
|
||
|
|
"logps/chosen": -64.5229917526245,
|
||
|
|
"logps/rejected": -253.41995105743408,
|
||
|
|
"loss": 8.196531416615471e-06,
|
||
|
|
"mean_token_accuracy": 0.680670789629221,
|
||
|
|
"num_tokens": 716266.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.433503310382366,
|
||
|
|
"rewards/margins": 14.470386946201325,
|
||
|
|
"rewards/rejected": -10.036883610486985,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13759889920880633,
|
||
|
|
"eval_entropy": 1.3168993187106512,
|
||
|
|
"eval_logits/chosen": -3.322066357791712,
|
||
|
|
"eval_logits/rejected": -3.021388443109954,
|
||
|
|
"eval_logps/chosen": -65.62298910602246,
|
||
|
|
"eval_logps/rejected": -252.51850432352302,
|
||
|
|
"eval_loss": 9.113909982261248e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.6737550743459876,
|
||
|
|
"eval_num_tokens": 716266.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 4.419287687033609,
|
||
|
|
"eval_rewards/margins": 14.643272069544574,
|
||
|
|
"eval_rewards/rejected": -10.223984398873023,
|
||
|
|
"eval_runtime": 46.4829,
|
||
|
|
"eval_samples_per_second": 13.166,
|
||
|
|
"eval_steps_per_second": 6.583,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3123450234532357,
|
||
|
|
"epoch": 0.15135878912968695,
|
||
|
|
"grad_norm": 0.0001239776611328125,
|
||
|
|
"learning_rate": 1e-05,
|
||
|
|
"logits/chosen": -3.331090101614931,
|
||
|
|
"logits/rejected": -3.038216613475043,
|
||
|
|
"logps/chosen": -59.395786821842194,
|
||
|
|
"logps/rejected": -254.9130277633667,
|
||
|
|
"loss": 1.392655831295997e-05,
|
||
|
|
"mean_token_accuracy": 0.6849603869020939,
|
||
|
|
"num_tokens": 793653.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.183176264166832,
|
||
|
|
"rewards/margins": 14.316125285625457,
|
||
|
|
"rewards/rejected": -10.132948994636536,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2991145364940166,
|
||
|
|
"epoch": 0.1651186790505676,
|
||
|
|
"grad_norm": 0.000370025634765625,
|
||
|
|
"learning_rate": 1.091743119266055e-05,
|
||
|
|
"logits/chosen": -3.321651732468068,
|
||
|
|
"logits/rejected": -3.013930166082175,
|
||
|
|
"logps/chosen": -64.8379723072052,
|
||
|
|
"logps/rejected": -258.61792945861816,
|
||
|
|
"loss": 1.0242476855637506e-05,
|
||
|
|
"mean_token_accuracy": 0.6761004894971847,
|
||
|
|
"num_tokens": 870785.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.295385938882828,
|
||
|
|
"rewards/margins": 14.680606830120087,
|
||
|
|
"rewards/rejected": -10.385220789909363,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3162065342068672,
|
||
|
|
"epoch": 0.17887856897144822,
|
||
|
|
"grad_norm": 0.0022125244140625,
|
||
|
|
"learning_rate": 1.1834862385321102e-05,
|
||
|
|
"logits/chosen": -3.315425730055486,
|
||
|
|
"logits/rejected": -3.022912476453253,
|
||
|
|
"logps/chosen": -65.0863746881485,
|
||
|
|
"logps/rejected": -252.13726730346679,
|
||
|
|
"loss": 1.1276499571977184e-05,
|
||
|
|
"mean_token_accuracy": 0.6769515044987202,
|
||
|
|
"num_tokens": 944782.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.321224331855774,
|
||
|
|
"rewards/margins": 14.760894215106964,
|
||
|
|
"rewards/rejected": -10.439669924974442,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.323652510344982,
|
||
|
|
"epoch": 0.19263845889232886,
|
||
|
|
"grad_norm": 6.437301635742188e-05,
|
||
|
|
"learning_rate": 1.2752293577981652e-05,
|
||
|
|
"logits/chosen": -3.3024622470454594,
|
||
|
|
"logits/rejected": -3.034751877652379,
|
||
|
|
"logps/chosen": -65.11928598880768,
|
||
|
|
"logps/rejected": -261.44528636932375,
|
||
|
|
"loss": 2.951903297798708e-06,
|
||
|
|
"mean_token_accuracy": 0.6833141274750233,
|
||
|
|
"num_tokens": 1014352.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.368463853001595,
|
||
|
|
"rewards/margins": 15.111195981502533,
|
||
|
|
"rewards/rejected": -10.742732137441635,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2840273901820183,
|
||
|
|
"epoch": 0.20639834881320948,
|
||
|
|
"grad_norm": 0.00396728515625,
|
||
|
|
"learning_rate": 1.3669724770642203e-05,
|
||
|
|
"logits/chosen": -3.3172900030591683,
|
||
|
|
"logits/rejected": -3.0113253204883432,
|
||
|
|
"logps/chosen": -62.9616664648056,
|
||
|
|
"logps/rejected": -264.60894412994384,
|
||
|
|
"loss": 3.129429023829289e-06,
|
||
|
|
"mean_token_accuracy": 0.6832372471690178,
|
||
|
|
"num_tokens": 1088461.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.151188506186008,
|
||
|
|
"rewards/margins": 15.375397801399231,
|
||
|
|
"rewards/rejected": -11.224209105968475,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2966242536902428,
|
||
|
|
"epoch": 0.22015823873409013,
|
||
|
|
"grad_norm": 0.00154876708984375,
|
||
|
|
"learning_rate": 1.4587155963302753e-05,
|
||
|
|
"logits/chosen": -3.3151515082339706,
|
||
|
|
"logits/rejected": -3.018693861306375,
|
||
|
|
"logps/chosen": -62.411375761032104,
|
||
|
|
"logps/rejected": -260.221813583374,
|
||
|
|
"loss": 5.149983917362988e-06,
|
||
|
|
"mean_token_accuracy": 0.6751810491085053,
|
||
|
|
"num_tokens": 1164520.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 4.324609813094139,
|
||
|
|
"rewards/margins": 15.264867746829987,
|
||
|
|
"rewards/rejected": -10.940258049964905,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3339879259467124,
|
||
|
|
"epoch": 0.23391812865497075,
|
||
|
|
"grad_norm": 0.004364013671875,
|
||
|
|
"learning_rate": 1.5504587155963304e-05,
|
||
|
|
"logits/chosen": -3.2697318298829514,
|
||
|
|
"logits/rejected": -3.011607351493888,
|
||
|
|
"logps/chosen": -76.78109130859374,
|
||
|
|
"logps/rejected": -254.79522857666015,
|
||
|
|
"loss": 0.00010178745724260807,
|
||
|
|
"mean_token_accuracy": 0.6379761226475239,
|
||
|
|
"num_tokens": 1236473.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 3.0435921967029573,
|
||
|
|
"rewards/margins": 13.237756145000457,
|
||
|
|
"rewards/rejected": -10.194163882732392,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.34509097635746,
|
||
|
|
"epoch": 0.2476780185758514,
|
||
|
|
"grad_norm": 0.0003337860107421875,
|
||
|
|
"learning_rate": 1.6422018348623852e-05,
|
||
|
|
"logits/chosen": -3.2700272303608466,
|
||
|
|
"logits/rejected": -3.0345531468099116,
|
||
|
|
"logps/chosen": -83.20087404251099,
|
||
|
|
"logps/rejected": -280.89504661560056,
|
||
|
|
"loss": 7.803936023265124e-06,
|
||
|
|
"mean_token_accuracy": 0.5892071291804314,
|
||
|
|
"num_tokens": 1307721.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.824196879565716,
|
||
|
|
"rewards/margins": 15.685904848575593,
|
||
|
|
"rewards/rejected": -12.86170780658722,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2848817110061646,
|
||
|
|
"epoch": 0.26143790849673204,
|
||
|
|
"grad_norm": 0.00023746490478515625,
|
||
|
|
"learning_rate": 1.7339449541284407e-05,
|
||
|
|
"logits/chosen": -3.2647128370329717,
|
||
|
|
"logits/rejected": -3.050985384983876,
|
||
|
|
"logps/chosen": -82.21896734237671,
|
||
|
|
"logps/rejected": -295.75822124481203,
|
||
|
|
"loss": 2.34917170018889e-06,
|
||
|
|
"mean_token_accuracy": 0.5797355584800243,
|
||
|
|
"num_tokens": 1376355.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.341776143759489,
|
||
|
|
"rewards/margins": 17.304658508300783,
|
||
|
|
"rewards/rejected": -14.962882256507873,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2651859059929849,
|
||
|
|
"epoch": 0.27519779841761266,
|
||
|
|
"grad_norm": 3.9577484130859375e-05,
|
||
|
|
"learning_rate": 1.8256880733944955e-05,
|
||
|
|
"logits/chosen": -3.2684643828569073,
|
||
|
|
"logits/rejected": -3.038824479770841,
|
||
|
|
"logps/chosen": -81.89948139190673,
|
||
|
|
"logps/rejected": -302.0743787765503,
|
||
|
|
"loss": 9.769327334652188e-07,
|
||
|
|
"mean_token_accuracy": 0.5773227371275425,
|
||
|
|
"num_tokens": 1448355.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.4392453789711,
|
||
|
|
"rewards/margins": 18.04160568714142,
|
||
|
|
"rewards/rejected": -15.602360212802887,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27519779841761266,
|
||
|
|
"eval_entropy": 1.2808947543692744,
|
||
|
|
"eval_logits/chosen": -3.2662835644748456,
|
||
|
|
"eval_logits/rejected": -3.030983888937842,
|
||
|
|
"eval_logps/chosen": -85.24166726131065,
|
||
|
|
"eval_logps/rejected": -310.94012012357024,
|
||
|
|
"eval_loss": 5.58038095732627e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5686757371706121,
|
||
|
|
"eval_num_tokens": 1448355.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 2.457419808199203,
|
||
|
|
"eval_rewards/margins": 18.523565981123184,
|
||
|
|
"eval_rewards/rejected": -16.066146196103563,
|
||
|
|
"eval_runtime": 46.6414,
|
||
|
|
"eval_samples_per_second": 13.121,
|
||
|
|
"eval_steps_per_second": 6.561,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2722094446420669,
|
||
|
|
"epoch": 0.2889576883384933,
|
||
|
|
"grad_norm": 3.7670135498046875e-05,
|
||
|
|
"learning_rate": 1.9174311926605506e-05,
|
||
|
|
"logits/chosen": -3.274673464599805,
|
||
|
|
"logits/rejected": -3.0529926146478457,
|
||
|
|
"logps/chosen": -80.9079598903656,
|
||
|
|
"logps/rejected": -311.301748085022,
|
||
|
|
"loss": 2.0904919892927864e-07,
|
||
|
|
"mean_token_accuracy": 0.5677489962428808,
|
||
|
|
"num_tokens": 1518187.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.386711198091507,
|
||
|
|
"rewards/margins": 18.499667203426363,
|
||
|
|
"rewards/rejected": -16.112956094741822,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2693706572055816,
|
||
|
|
"epoch": 0.3027175782593739,
|
||
|
|
"grad_norm": 0.00010538101196289062,
|
||
|
|
"learning_rate": 1.9999987193541166e-05,
|
||
|
|
"logits/chosen": -3.2678955318973606,
|
||
|
|
"logits/rejected": -3.0331391966114434,
|
||
|
|
"logps/chosen": -82.6086644411087,
|
||
|
|
"logps/rejected": -318.87992038726804,
|
||
|
|
"loss": 4.3506806832738223e-07,
|
||
|
|
"mean_token_accuracy": 0.5785332221537829,
|
||
|
|
"num_tokens": 1592692.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.4188714638352393,
|
||
|
|
"rewards/margins": 18.80655164718628,
|
||
|
|
"rewards/rejected": -16.387680435180663,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2727026402950288,
|
||
|
|
"epoch": 0.3164774681802546,
|
||
|
|
"grad_norm": 3.814697265625e-05,
|
||
|
|
"learning_rate": 1.9998450458169935e-05,
|
||
|
|
"logits/chosen": -3.263175326214511,
|
||
|
|
"logits/rejected": -3.0319927272644476,
|
||
|
|
"logps/chosen": -89.12346024513245,
|
||
|
|
"logps/rejected": -318.03904724121094,
|
||
|
|
"loss": 2.9918028303654865e-07,
|
||
|
|
"mean_token_accuracy": 0.5649005927145481,
|
||
|
|
"num_tokens": 1665764.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.3706042617559433,
|
||
|
|
"rewards/margins": 18.84476692676544,
|
||
|
|
"rewards/rejected": -16.474162554740907,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2772931650280952,
|
||
|
|
"epoch": 0.3302373581011352,
|
||
|
|
"grad_norm": 0.000888824462890625,
|
||
|
|
"learning_rate": 1.9994352882027634e-05,
|
||
|
|
"logits/chosen": -3.275916983604721,
|
||
|
|
"logits/rejected": -3.045261285523395,
|
||
|
|
"logps/chosen": -85.88168826103211,
|
||
|
|
"logps/rejected": -314.67186374664306,
|
||
|
|
"loss": 7.497599654016085e-07,
|
||
|
|
"mean_token_accuracy": 0.5647643093019724,
|
||
|
|
"num_tokens": 1738039.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.3492226861417294,
|
||
|
|
"rewards/margins": 18.621605122089385,
|
||
|
|
"rewards/rejected": -16.272382295131685,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2810615047812461,
|
||
|
|
"epoch": 0.3439972480220158,
|
||
|
|
"grad_norm": 0.0001277923583984375,
|
||
|
|
"learning_rate": 1.9987695514600882e-05,
|
||
|
|
"logits/chosen": -3.2587914698091525,
|
||
|
|
"logits/rejected": -3.039852993690482,
|
||
|
|
"logps/chosen": -84.9753231048584,
|
||
|
|
"logps/rejected": -316.3932315826416,
|
||
|
|
"loss": 2.315052597623435e-07,
|
||
|
|
"mean_token_accuracy": 0.5717308051884175,
|
||
|
|
"num_tokens": 1809878.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.4452888280153275,
|
||
|
|
"rewards/margins": 18.718373239040375,
|
||
|
|
"rewards/rejected": -16.273084354400634,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2803627982735635,
|
||
|
|
"epoch": 0.35775713794289643,
|
||
|
|
"grad_norm": 1.4841556549072266e-05,
|
||
|
|
"learning_rate": 1.997848006099969e-05,
|
||
|
|
"logits/chosen": -3.243708828438259,
|
||
|
|
"logits/rejected": -3.023015530025309,
|
||
|
|
"logps/chosen": -85.64196882247924,
|
||
|
|
"logps/rejected": -320.8713493347168,
|
||
|
|
"loss": 4.767807695316151e-06,
|
||
|
|
"mean_token_accuracy": 0.5577906873077154,
|
||
|
|
"num_tokens": 1880471.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 2.241638096421957,
|
||
|
|
"rewards/margins": 18.844607853889464,
|
||
|
|
"rewards/rejected": -16.602969908714293,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2728056520223618,
|
||
|
|
"epoch": 0.3715170278637771,
|
||
|
|
"grad_norm": 2.9087066650390625e-05,
|
||
|
|
"learning_rate": 1.9966708881520725e-05,
|
||
|
|
"logits/chosen": -3.1717140539593514,
|
||
|
|
"logits/rejected": -2.9289080783786265,
|
||
|
|
"logps/chosen": -109.57758569717407,
|
||
|
|
"logps/rejected": -343.55820388793944,
|
||
|
|
"loss": 3.192686335751205e-07,
|
||
|
|
"mean_token_accuracy": 0.5119530703872442,
|
||
|
|
"num_tokens": 1954954.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 0.2822640292346478,
|
||
|
|
"rewards/margins": 19.274557650089264,
|
||
|
|
"rewards/rejected": -18.99229363203049,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.258153322339058,
|
||
|
|
"epoch": 0.3852769177846577,
|
||
|
|
"grad_norm": 4.1961669921875e-05,
|
||
|
|
"learning_rate": 1.9952384991042794e-05,
|
||
|
|
"logits/chosen": -3.166911320420769,
|
||
|
|
"logits/rejected": -2.914455899618946,
|
||
|
|
"logps/chosen": -109.23124113082886,
|
||
|
|
"logps/rejected": -342.96191749572756,
|
||
|
|
"loss": 6.130666406534147e-07,
|
||
|
|
"mean_token_accuracy": 0.5128110606223345,
|
||
|
|
"num_tokens": 2027701.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": -0.11331945937126875,
|
||
|
|
"rewards/margins": 19.085047161579134,
|
||
|
|
"rewards/rejected": -19.198366713523864,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2625004753470421,
|
||
|
|
"epoch": 0.39903680770553834,
|
||
|
|
"grad_norm": 0.00022029876708984375,
|
||
|
|
"learning_rate": 1.9935512058254642e-05,
|
||
|
|
"logits/chosen": -3.158640726546935,
|
||
|
|
"logits/rejected": -2.919291940387602,
|
||
|
|
"logps/chosen": -109.37000555992127,
|
||
|
|
"logps/rejected": -348.1477699279785,
|
||
|
|
"loss": 3.579470103431959e-07,
|
||
|
|
"mean_token_accuracy": 0.5032326009124517,
|
||
|
|
"num_tokens": 2097529.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": -0.2131506633013487,
|
||
|
|
"rewards/margins": 19.291283941268922,
|
||
|
|
"rewards/rejected": -19.504434680938722,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2408581115305424,
|
||
|
|
"epoch": 0.41279669762641896,
|
||
|
|
"grad_norm": 0.0006561279296875,
|
||
|
|
"learning_rate": 1.9916094404715338e-05,
|
||
|
|
"logits/chosen": -3.146471768568003,
|
||
|
|
"logits/rejected": -2.9109541862173147,
|
||
|
|
"logps/chosen": -108.3150071144104,
|
||
|
|
"logps/rejected": -346.6201908111572,
|
||
|
|
"loss": 4.624724624591181e-07,
|
||
|
|
"mean_token_accuracy": 0.5017246019095183,
|
||
|
|
"num_tokens": 2164725.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": -0.2834041160531342,
|
||
|
|
"rewards/margins": 19.647798562049864,
|
||
|
|
"rewards/rejected": -19.931202745437623,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41279669762641896,
|
||
|
|
"eval_entropy": 1.2567425892243977,
|
||
|
|
"eval_logits/chosen": -3.1541197386317377,
|
||
|
|
"eval_logits/rejected": -2.9072247599638548,
|
||
|
|
"eval_logps/chosen": -110.04411519119164,
|
||
|
|
"eval_logps/rejected": -345.3444980297213,
|
||
|
|
"eval_loss": 1.5399403991978033e-06,
|
||
|
|
"eval_mean_token_accuracy": 0.503020686164401,
|
||
|
|
"eval_num_tokens": 2164725.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": -0.022825035449190467,
|
||
|
|
"eval_rewards/margins": 19.483759047938328,
|
||
|
|
"eval_rewards/rejected": -19.506584064633238,
|
||
|
|
"eval_runtime": 46.7375,
|
||
|
|
"eval_samples_per_second": 13.094,
|
||
|
|
"eval_steps_per_second": 6.547,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2747977674007416,
|
||
|
|
"epoch": 0.42655658754729964,
|
||
|
|
"grad_norm": 4.363059997558594e-05,
|
||
|
|
"learning_rate": 1.9894137003747404e-05,
|
||
|
|
"logits/chosen": -3.191046102409698,
|
||
|
|
"logits/rejected": -2.934731907158622,
|
||
|
|
"logps/chosen": -105.27848653793335,
|
||
|
|
"logps/rejected": -349.7217990875244,
|
||
|
|
"loss": 3.7774039810756222e-06,
|
||
|
|
"mean_token_accuracy": 0.5230023309588432,
|
||
|
|
"num_tokens": 2240508.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 0.6149379150010645,
|
||
|
|
"rewards/margins": 20.34483733177185,
|
||
|
|
"rewards/rejected": -19.72989946603775,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2644944354891776,
|
||
|
|
"epoch": 0.44031647746818026,
|
||
|
|
"grad_norm": 2.4437904357910156e-05,
|
||
|
|
"learning_rate": 1.9869645479163035e-05,
|
||
|
|
"logits/chosen": -3.200015615413726,
|
||
|
|
"logits/rejected": -2.935862106651629,
|
||
|
|
"logps/chosen": -98.56399579048157,
|
||
|
|
"logps/rejected": -345.60072326660156,
|
||
|
|
"loss": 1.755031689754105e-07,
|
||
|
|
"mean_token_accuracy": 0.5542162295430899,
|
||
|
|
"num_tokens": 2311444.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.1905555572360753,
|
||
|
|
"rewards/margins": 20.78347454071045,
|
||
|
|
"rewards/rejected": -19.592919039726258,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2220299974083901,
|
||
|
|
"epoch": 0.4540763673890609,
|
||
|
|
"grad_norm": 8.158385753631592e-07,
|
||
|
|
"learning_rate": 1.984262610382371e-05,
|
||
|
|
"logits/chosen": -3.2061829556179062,
|
||
|
|
"logits/rejected": -2.9369843729159575,
|
||
|
|
"logps/chosen": -88.93789119720459,
|
||
|
|
"logps/rejected": -350.99997215271,
|
||
|
|
"loss": 1.22557608506213e-08,
|
||
|
|
"mean_token_accuracy": 0.5557950928807258,
|
||
|
|
"num_tokens": 2380803.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3155047172680496,
|
||
|
|
"rewards/margins": 21.637652850151063,
|
||
|
|
"rewards/rejected": -20.322148036956786,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2497497342526913,
|
||
|
|
"epoch": 0.4678362573099415,
|
||
|
|
"grad_norm": 1.4230608940124512e-06,
|
||
|
|
"learning_rate": 1.981308579803356e-05,
|
||
|
|
"logits/chosen": -3.2039305073988444,
|
||
|
|
"logits/rejected": -2.935270721662338,
|
||
|
|
"logps/chosen": -91.58367230892182,
|
||
|
|
"logps/rejected": -352.92971839904783,
|
||
|
|
"loss": 2.8684135600087758e-08,
|
||
|
|
"mean_token_accuracy": 0.5493994299322367,
|
||
|
|
"num_tokens": 2451152.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3400580430403353,
|
||
|
|
"rewards/margins": 21.626568651199342,
|
||
|
|
"rewards/rejected": -20.28651065826416,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2471516221761703,
|
||
|
|
"epoch": 0.48159614723082217,
|
||
|
|
"grad_norm": 8.083879947662354e-07,
|
||
|
|
"learning_rate": 1.9781032127766897e-05,
|
||
|
|
"logits/chosen": -3.192242463166784,
|
||
|
|
"logits/rejected": -2.939047130509837,
|
||
|
|
"logps/chosen": -91.81347761154174,
|
||
|
|
"logps/rejected": -351.9379182815552,
|
||
|
|
"loss": 4.979311825081823e-08,
|
||
|
|
"mean_token_accuracy": 0.5597729679197073,
|
||
|
|
"num_tokens": 2520962.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.1749618555419148,
|
||
|
|
"rewards/margins": 21.238007771968842,
|
||
|
|
"rewards/rejected": -20.063046038150787,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.250276905298233,
|
||
|
|
"epoch": 0.4953560371517028,
|
||
|
|
"grad_norm": 4.7206878662109375e-05,
|
||
|
|
"learning_rate": 1.974647330273043e-05,
|
||
|
|
"logits/chosen": -3.204588056757823,
|
||
|
|
"logits/rejected": -2.9460527693957155,
|
||
|
|
"logps/chosen": -96.47925109863282,
|
||
|
|
"logps/rejected": -357.84326496124265,
|
||
|
|
"loss": 9.702721399662551e-08,
|
||
|
|
"mean_token_accuracy": 0.5596798613667489,
|
||
|
|
"num_tokens": 2596708.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4331018382683396,
|
||
|
|
"rewards/margins": 21.43447951078415,
|
||
|
|
"rewards/rejected": -20.00137746334076,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2356461606919766,
|
||
|
|
"epoch": 0.5091159270725835,
|
||
|
|
"grad_norm": 1.0192394256591797e-05,
|
||
|
|
"learning_rate": 1.9709418174260523e-05,
|
||
|
|
"logits/chosen": -3.202175928278776,
|
||
|
|
"logits/rejected": -2.9334896374443966,
|
||
|
|
"logps/chosen": -93.25146460533142,
|
||
|
|
"logps/rejected": -348.1914405822754,
|
||
|
|
"loss": 1.4943527730792995e-07,
|
||
|
|
"mean_token_accuracy": 0.5434729922562838,
|
||
|
|
"num_tokens": 2663359.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.1101246356964112,
|
||
|
|
"rewards/margins": 21.274962854385375,
|
||
|
|
"rewards/rejected": -20.16483825445175,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2542329356074333,
|
||
|
|
"epoch": 0.5228758169934641,
|
||
|
|
"grad_norm": 2.9087066650390625e-05,
|
||
|
|
"learning_rate": 1.9669876233056173e-05,
|
||
|
|
"logits/chosen": -3.2033071908214827,
|
||
|
|
"logits/rejected": -2.9458752015881497,
|
||
|
|
"logps/chosen": -95.33303580284118,
|
||
|
|
"logps/rejected": -360.0554260253906,
|
||
|
|
"loss": 6.821481406404928e-08,
|
||
|
|
"mean_token_accuracy": 0.5537502180784941,
|
||
|
|
"num_tokens": 2735950.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.1969825938344,
|
||
|
|
"rewards/margins": 21.316751837730408,
|
||
|
|
"rewards/rejected": -20.119769501686097,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2611618615686893,
|
||
|
|
"epoch": 0.5366357069143447,
|
||
|
|
"grad_norm": 9.238719940185547e-06,
|
||
|
|
"learning_rate": 1.962785760674822e-05,
|
||
|
|
"logits/chosen": -3.205445972704493,
|
||
|
|
"logits/rejected": -2.9503712190165086,
|
||
|
|
"logps/chosen": -94.55541400909424,
|
||
|
|
"logps/rejected": -358.6264102935791,
|
||
|
|
"loss": 5.3916863862468745e-08,
|
||
|
|
"mean_token_accuracy": 0.53789919950068,
|
||
|
|
"num_tokens": 2809336.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2039084580726922,
|
||
|
|
"rewards/margins": 21.302520561218262,
|
||
|
|
"rewards/rejected": -20.098612093925475,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2611239463090897,
|
||
|
|
"epoch": 0.5503955968352253,
|
||
|
|
"grad_norm": 3.528594970703125e-05,
|
||
|
|
"learning_rate": 1.9583373057305417e-05,
|
||
|
|
"logits/chosen": -3.213530981959783,
|
||
|
|
"logits/rejected": -2.9513509889366025,
|
||
|
|
"logps/chosen": -97.29888286590577,
|
||
|
|
"logps/rejected": -342.16270599365237,
|
||
|
|
"loss": 7.968355930643156e-08,
|
||
|
|
"mean_token_accuracy": 0.5469969604164362,
|
||
|
|
"num_tokens": 2881281.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.30086164213717,
|
||
|
|
"rewards/margins": 20.653638982772826,
|
||
|
|
"rewards/rejected": -19.352777242660522,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5503955968352253,
|
||
|
|
"eval_entropy": 1.2446555466044182,
|
||
|
|
"eval_logits/chosen": -3.206116940076223,
|
||
|
|
"eval_logits/rejected": -2.932523076753512,
|
||
|
|
"eval_logps/chosen": -96.21067571328356,
|
||
|
|
"eval_logps/rejected": -350.58938982595805,
|
||
|
|
"eval_loss": 1.5858041990668426e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5460730443398157,
|
||
|
|
"eval_num_tokens": 2881281.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.360518928933767,
|
||
|
|
"eval_rewards/margins": 21.391592069388995,
|
||
|
|
"eval_rewards/rejected": -20.031073205611285,
|
||
|
|
"eval_runtime": 47.366,
|
||
|
|
"eval_samples_per_second": 12.921,
|
||
|
|
"eval_steps_per_second": 6.46,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2470418080687522,
|
||
|
|
"epoch": 0.5641554867561059,
|
||
|
|
"grad_norm": 4.857778549194336e-06,
|
||
|
|
"learning_rate": 1.9536433978278034e-05,
|
||
|
|
"logits/chosen": -3.2050009974244076,
|
||
|
|
"logits/rejected": -2.940295918899248,
|
||
|
|
"logps/chosen": -98.1803062915802,
|
||
|
|
"logps/rejected": -353.2428401947021,
|
||
|
|
"loss": 6.607142495340668e-08,
|
||
|
|
"mean_token_accuracy": 0.5483775656670332,
|
||
|
|
"num_tokens": 2951215.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3665587238967418,
|
||
|
|
"rewards/margins": 21.411585366725923,
|
||
|
|
"rewards/rejected": -20.045026814937593,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2767939895391465,
|
||
|
|
"epoch": 0.5779153766769866,
|
||
|
|
"grad_norm": 6.735324859619141e-06,
|
||
|
|
"learning_rate": 1.948705239187969e-05,
|
||
|
|
"logits/chosen": -3.2111276235458264,
|
||
|
|
"logits/rejected": -2.950971977866732,
|
||
|
|
"logps/chosen": -98.60952830314636,
|
||
|
|
"logps/rejected": -356.9093845367432,
|
||
|
|
"loss": 3.040120191144524e-07,
|
||
|
|
"mean_token_accuracy": 0.5495793126523495,
|
||
|
|
"num_tokens": 3025409.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.322571473289281,
|
||
|
|
"rewards/margins": 21.419764494895936,
|
||
|
|
"rewards/rejected": -20.097193121910095,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2340076759457588,
|
||
|
|
"epoch": 0.5916752665978672,
|
||
|
|
"grad_norm": 3.3855438232421875e-05,
|
||
|
|
"learning_rate": 1.9435240945908197e-05,
|
||
|
|
"logits/chosen": -3.201115388808664,
|
||
|
|
"logits/rejected": -2.9359388030639337,
|
||
|
|
"logps/chosen": -93.70817131996155,
|
||
|
|
"logps/rejected": -349.5262786865234,
|
||
|
|
"loss": 3.1302016623158126e-08,
|
||
|
|
"mean_token_accuracy": 0.5430944249033928,
|
||
|
|
"num_tokens": 3095506.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.0892294511198997,
|
||
|
|
"rewards/margins": 21.276236748695375,
|
||
|
|
"rewards/rejected": -20.187007427215576,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2297961503267287,
|
||
|
|
"epoch": 0.6054351565187478,
|
||
|
|
"grad_norm": 8.52346420288086e-06,
|
||
|
|
"learning_rate": 1.9381012910506146e-05,
|
||
|
|
"logits/chosen": -3.209182764942939,
|
||
|
|
"logits/rejected": -2.9383067598099455,
|
||
|
|
"logps/chosen": -94.46383814811706,
|
||
|
|
"logps/rejected": -347.6718967437744,
|
||
|
|
"loss": 1.1145983762617106e-07,
|
||
|
|
"mean_token_accuracy": 0.5509139101952314,
|
||
|
|
"num_tokens": 3167474.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.320781459286809,
|
||
|
|
"rewards/margins": 21.1637371301651,
|
||
|
|
"rewards/rejected": -19.842955470085144,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.238859149813652,
|
||
|
|
"epoch": 0.6191950464396285,
|
||
|
|
"grad_norm": 3.933906555175781e-06,
|
||
|
|
"learning_rate": 1.9324382174762103e-05,
|
||
|
|
"logits/chosen": -3.203365340339883,
|
||
|
|
"logits/rejected": -2.948441983583794,
|
||
|
|
"logps/chosen": -95.94863700866699,
|
||
|
|
"logps/rejected": -354.1850410461426,
|
||
|
|
"loss": 1.6536616385565139e-06,
|
||
|
|
"mean_token_accuracy": 0.5478381294757128,
|
||
|
|
"num_tokens": 3237066.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.1904785027727485,
|
||
|
|
"rewards/margins": 21.46895523071289,
|
||
|
|
"rewards/rejected": -20.278476548194885,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2301946364343166,
|
||
|
|
"epoch": 0.6329549363605091,
|
||
|
|
"grad_norm": 0.00018310546875,
|
||
|
|
"learning_rate": 1.9265363243153302e-05,
|
||
|
|
"logits/chosen": -3.2076860666693108,
|
||
|
|
"logits/rejected": -2.9415288658974417,
|
||
|
|
"logps/chosen": -91.41148219108581,
|
||
|
|
"logps/rejected": -354.1453170776367,
|
||
|
|
"loss": 9.839211543294368e-08,
|
||
|
|
"mean_token_accuracy": 0.5448024846613407,
|
||
|
|
"num_tokens": 3307883.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3097239159047604,
|
||
|
|
"rewards/margins": 21.563816690444945,
|
||
|
|
"rewards/rejected": -20.25409264564514,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2488635182380676,
|
||
|
|
"epoch": 0.6467148262813898,
|
||
|
|
"grad_norm": 4.38690185546875e-05,
|
||
|
|
"learning_rate": 1.9203971231830683e-05,
|
||
|
|
"logits/chosen": -3.2141295170389634,
|
||
|
|
"logits/rejected": -2.938989290807725,
|
||
|
|
"logps/chosen": -95.46482665538788,
|
||
|
|
"logps/rejected": -353.38847160339355,
|
||
|
|
"loss": 8.967840017248819e-08,
|
||
|
|
"mean_token_accuracy": 0.544929426535964,
|
||
|
|
"num_tokens": 3379860.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4560122258961201,
|
||
|
|
"rewards/margins": 21.57176778316498,
|
||
|
|
"rewards/rejected": -20.1157555103302,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2580321684479714,
|
||
|
|
"epoch": 0.6604747162022704,
|
||
|
|
"grad_norm": 1.8700957298278809e-06,
|
||
|
|
"learning_rate": 1.9140221864747304e-05,
|
||
|
|
"logits/chosen": -3.2029551675787014,
|
||
|
|
"logits/rejected": -2.9481494396152192,
|
||
|
|
"logps/chosen": -97.8405794620514,
|
||
|
|
"logps/rejected": -353.5988826751709,
|
||
|
|
"loss": 5.062532181909774e-07,
|
||
|
|
"mean_token_accuracy": 0.5462357386946678,
|
||
|
|
"num_tokens": 3449532.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3817669712938367,
|
||
|
|
"rewards/margins": 21.57014834880829,
|
||
|
|
"rewards/rejected": -20.188381361961365,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2382771328091622,
|
||
|
|
"epoch": 0.674234606123151,
|
||
|
|
"grad_norm": 1.5497207641601562e-06,
|
||
|
|
"learning_rate": 1.9074131469631047e-05,
|
||
|
|
"logits/chosen": -3.200084511483456,
|
||
|
|
"logits/rejected": -2.9279012361838785,
|
||
|
|
"logps/chosen": -96.45669221878052,
|
||
|
|
"logps/rejected": -360.72495155334474,
|
||
|
|
"loss": 2.6530278773861936e-08,
|
||
|
|
"mean_token_accuracy": 0.5516861360520124,
|
||
|
|
"num_tokens": 3520115.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3708888601511717,
|
||
|
|
"rewards/margins": 22.229914927482604,
|
||
|
|
"rewards/rejected": -20.85902613401413,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.231078492105007,
|
||
|
|
"epoch": 0.6879944960440316,
|
||
|
|
"grad_norm": 2.8461217880249023e-06,
|
||
|
|
"learning_rate": 1.9005716973802707e-05,
|
||
|
|
"logits/chosen": -3.1932053921101393,
|
||
|
|
"logits/rejected": -2.9393451806687594,
|
||
|
|
"logps/chosen": -92.85203433036804,
|
||
|
|
"logps/rejected": -356.5406131744385,
|
||
|
|
"loss": 2.374201386601271e-08,
|
||
|
|
"mean_token_accuracy": 0.5493354115635156,
|
||
|
|
"num_tokens": 3586919.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2406577363610267,
|
||
|
|
"rewards/margins": 21.57927179336548,
|
||
|
|
"rewards/rejected": -20.33861403465271,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6879944960440316,
|
||
|
|
"eval_entropy": 1.2378660793008367,
|
||
|
|
"eval_logits/chosen": -3.208702183064752,
|
||
|
|
"eval_logits/rejected": -2.9324433814833553,
|
||
|
|
"eval_logps/chosen": -94.60752372492372,
|
||
|
|
"eval_logps/rejected": -354.49684058295355,
|
||
|
|
"eval_loss": 1.1862820059604928e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5471634168445675,
|
||
|
|
"eval_num_tokens": 3586919.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5208341437574548,
|
||
|
|
"eval_rewards/margins": 21.942652421839096,
|
||
|
|
"eval_rewards/rejected": -20.421818287544003,
|
||
|
|
"eval_runtime": 48.0212,
|
||
|
|
"eval_samples_per_second": 12.744,
|
||
|
|
"eval_steps_per_second": 6.372,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2257853604853153,
|
||
|
|
"epoch": 0.7017543859649122,
|
||
|
|
"grad_norm": 4.6193599700927734e-06,
|
||
|
|
"learning_rate": 1.8934995899840512e-05,
|
||
|
|
"logits/chosen": -3.209992468977989,
|
||
|
|
"logits/rejected": -2.944684840016539,
|
||
|
|
"logps/chosen": -92.81335635185242,
|
||
|
|
"logps/rejected": -354.2427951812744,
|
||
|
|
"loss": 1.865128069766797e-08,
|
||
|
|
"mean_token_accuracy": 0.5567719202488661,
|
||
|
|
"num_tokens": 3658756.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5606805901974439,
|
||
|
|
"rewards/margins": 22.114846849441527,
|
||
|
|
"rewards/rejected": -20.554166197776794,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2566522896289825,
|
||
|
|
"epoch": 0.7155142758857929,
|
||
|
|
"grad_norm": 1.0073184967041016e-05,
|
||
|
|
"learning_rate": 1.886198636109216e-05,
|
||
|
|
"logits/chosen": -3.2030215467134,
|
||
|
|
"logits/rejected": -2.9602792303248324,
|
||
|
|
"logps/chosen": -93.38595032691956,
|
||
|
|
"logps/rejected": -351.78721618652344,
|
||
|
|
"loss": 7.779188422318839e-08,
|
||
|
|
"mean_token_accuracy": 0.5507799163460732,
|
||
|
|
"num_tokens": 3729889.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2870149585418402,
|
||
|
|
"rewards/margins": 21.421838641166687,
|
||
|
|
"rewards/rejected": -20.134823727607728,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.223513600230217,
|
||
|
|
"epoch": 0.7292741658066736,
|
||
|
|
"grad_norm": 1.3113021850585938e-05,
|
||
|
|
"learning_rate": 1.8786707057035556e-05,
|
||
|
|
"logits/chosen": -3.197827496796243,
|
||
|
|
"logits/rejected": -2.939210284225032,
|
||
|
|
"logps/chosen": -90.15343255996704,
|
||
|
|
"logps/rejected": -351.67856483459474,
|
||
|
|
"loss": 3.7565149568763444e-08,
|
||
|
|
"mean_token_accuracy": 0.5483428921550513,
|
||
|
|
"num_tokens": 3797040.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3190150810405612,
|
||
|
|
"rewards/margins": 21.713546490669252,
|
||
|
|
"rewards/rejected": -20.3945313334465,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2207642510533332,
|
||
|
|
"epoch": 0.7430340557275542,
|
||
|
|
"grad_norm": 2.250075340270996e-06,
|
||
|
|
"learning_rate": 1.8709177268489455e-05,
|
||
|
|
"logits/chosen": -3.2128090846377537,
|
||
|
|
"logits/rejected": -2.9295009129558585,
|
||
|
|
"logps/chosen": -91.98796620368958,
|
||
|
|
"logps/rejected": -357.48371505737305,
|
||
|
|
"loss": 4.0783984331937975e-07,
|
||
|
|
"mean_token_accuracy": 0.5455800645053387,
|
||
|
|
"num_tokens": 3873245.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.308496705815196,
|
||
|
|
"rewards/margins": 21.80317234992981,
|
||
|
|
"rewards/rejected": -20.494675588607787,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2357498720288276,
|
||
|
|
"epoch": 0.7567939456484348,
|
||
|
|
"grad_norm": 5.997717380523682e-07,
|
||
|
|
"learning_rate": 1.862941685267516e-05,
|
||
|
|
"logits/chosen": -3.2110368310073283,
|
||
|
|
"logits/rejected": -2.941539484264484,
|
||
|
|
"logps/chosen": -88.533620262146,
|
||
|
|
"logps/rejected": -346.5173583984375,
|
||
|
|
"loss": 2.4679090415702377e-08,
|
||
|
|
"mean_token_accuracy": 0.5551324781030417,
|
||
|
|
"num_tokens": 3947480.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5138933001086117,
|
||
|
|
"rewards/margins": 21.399971747398375,
|
||
|
|
"rewards/rejected": -19.886078691482545,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2267142876982688,
|
||
|
|
"epoch": 0.7705538355693154,
|
||
|
|
"grad_norm": 8.52346420288086e-06,
|
||
|
|
"learning_rate": 1.8547446238130632e-05,
|
||
|
|
"logits/chosen": -3.215176303042424,
|
||
|
|
"logits/rejected": -2.9349409611311232,
|
||
|
|
"logps/chosen": -97.80274991989135,
|
||
|
|
"logps/rejected": -359.84800300598147,
|
||
|
|
"loss": 1.0752984280770761e-08,
|
||
|
|
"mean_token_accuracy": 0.5577903117984533,
|
||
|
|
"num_tokens": 4022803.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5468438116833567,
|
||
|
|
"rewards/margins": 22.347780632972718,
|
||
|
|
"rewards/rejected": -20.800936925411225,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2494088008999824,
|
||
|
|
"epoch": 0.7843137254901961,
|
||
|
|
"grad_norm": 1.9371509552001953e-06,
|
||
|
|
"learning_rate": 1.8463286419478256e-05,
|
||
|
|
"logits/chosen": -3.211196248953847,
|
||
|
|
"logits/rejected": -2.948767881274639,
|
||
|
|
"logps/chosen": -95.37720813751221,
|
||
|
|
"logps/rejected": -348.9063579559326,
|
||
|
|
"loss": 5.1451485205689095e-08,
|
||
|
|
"mean_token_accuracy": 0.5470235139131546,
|
||
|
|
"num_tokens": 4094199.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3498335065320135,
|
||
|
|
"rewards/margins": 21.68865497112274,
|
||
|
|
"rewards/rejected": -20.338821601867675,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2267412588000297,
|
||
|
|
"epoch": 0.7980736154110767,
|
||
|
|
"grad_norm": 0.000453948974609375,
|
||
|
|
"learning_rate": 1.8376958952047606e-05,
|
||
|
|
"logits/chosen": -3.205934045300051,
|
||
|
|
"logits/rejected": -2.926654426249549,
|
||
|
|
"logps/chosen": -93.34790635108948,
|
||
|
|
"logps/rejected": -348.32058773040774,
|
||
|
|
"loss": 1.6058717164924018e-07,
|
||
|
|
"mean_token_accuracy": 0.5513480927795171,
|
||
|
|
"num_tokens": 4165161.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4272053100168705,
|
||
|
|
"rewards/margins": 21.51763561964035,
|
||
|
|
"rewards/rejected": -20.090430426597596,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2365659534931184,
|
||
|
|
"epoch": 0.8118335053319573,
|
||
|
|
"grad_norm": 1.773238182067871e-06,
|
||
|
|
"learning_rate": 1.8288485946354623e-05,
|
||
|
|
"logits/chosen": -3.2136005483338552,
|
||
|
|
"logits/rejected": -2.9395607851534296,
|
||
|
|
"logps/chosen": -92.54814214706421,
|
||
|
|
"logps/rejected": -356.7799785614014,
|
||
|
|
"loss": 9.530706392979483e-08,
|
||
|
|
"mean_token_accuracy": 0.5449393007904291,
|
||
|
|
"num_tokens": 4234086.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4757858499884606,
|
||
|
|
"rewards/margins": 22.288536047935487,
|
||
|
|
"rewards/rejected": -20.81275030374527,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2298890359699726,
|
||
|
|
"epoch": 0.8255933952528379,
|
||
|
|
"grad_norm": 9.179115295410156e-06,
|
||
|
|
"learning_rate": 1.8197890062438593e-05,
|
||
|
|
"logits/chosen": -3.207707571541648,
|
||
|
|
"logits/rejected": -2.946438175072392,
|
||
|
|
"logps/chosen": -90.75098037719727,
|
||
|
|
"logps/rejected": -361.0145431518555,
|
||
|
|
"loss": 1.4683616882393836e-07,
|
||
|
|
"mean_token_accuracy": 0.5618239611387252,
|
||
|
|
"num_tokens": 4310423.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4461902618408202,
|
||
|
|
"rewards/margins": 21.96159906387329,
|
||
|
|
"rewards/rejected": -20.515408909320833,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8255933952528379,
|
||
|
|
"eval_entropy": 1.236954848750744,
|
||
|
|
"eval_logits/chosen": -3.2089251643222068,
|
||
|
|
"eval_logits/rejected": -2.9321425863537924,
|
||
|
|
"eval_logps/chosen": -94.51317416297064,
|
||
|
|
"eval_logps/rejected": -354.90605422873904,
|
||
|
|
"eval_loss": 1.1335689720226583e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5470167230936437,
|
||
|
|
"eval_num_tokens": 4310423.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5302690837069663,
|
||
|
|
"eval_rewards/margins": 21.993008691500993,
|
||
|
|
"eval_rewards/rejected": -20.462739564234916,
|
||
|
|
"eval_runtime": 48.6435,
|
||
|
|
"eval_samples_per_second": 12.581,
|
||
|
|
"eval_steps_per_second": 6.291,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2389264971017837,
|
||
|
|
"epoch": 0.8393532851737187,
|
||
|
|
"grad_norm": 4.8160552978515625e-05,
|
||
|
|
"learning_rate": 1.810519450405835e-05,
|
||
|
|
"logits/chosen": -3.210960691597433,
|
||
|
|
"logits/rejected": -2.935060390790913,
|
||
|
|
"logps/chosen": -93.46412029266358,
|
||
|
|
"logps/rejected": -361.8876728057861,
|
||
|
|
"loss": 3.212771616745158e-07,
|
||
|
|
"mean_token_accuracy": 0.5596938874572516,
|
||
|
|
"num_tokens": 4383786.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4644759917166084,
|
||
|
|
"rewards/margins": 22.466219902038574,
|
||
|
|
"rewards/rejected": -21.001744067668916,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2347271621227265,
|
||
|
|
"epoch": 0.8531131750945993,
|
||
|
|
"grad_norm": 0.000308990478515625,
|
||
|
|
"learning_rate": 1.801042301274927e-05,
|
||
|
|
"logits/chosen": -3.205084209862984,
|
||
|
|
"logits/rejected": -2.928795499158591,
|
||
|
|
"logps/chosen": -93.73975834846496,
|
||
|
|
"logps/rejected": -358.22800674438474,
|
||
|
|
"loss": 8.303017580146843e-08,
|
||
|
|
"mean_token_accuracy": 0.5511996760964394,
|
||
|
|
"num_tokens": 4455322.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4866585440933704,
|
||
|
|
"rewards/margins": 22.33163278102875,
|
||
|
|
"rewards/rejected": -20.844974386692048,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2407884091138839,
|
||
|
|
"epoch": 0.8668730650154799,
|
||
|
|
"grad_norm": 4.76837158203125e-06,
|
||
|
|
"learning_rate": 1.7913599861742475e-05,
|
||
|
|
"logits/chosen": -3.196434341605011,
|
||
|
|
"logits/rejected": -2.9346165308967778,
|
||
|
|
"logps/chosen": -94.46012229919434,
|
||
|
|
"logps/rejected": -360.2606533050537,
|
||
|
|
"loss": 2.6523449037085812e-08,
|
||
|
|
"mean_token_accuracy": 0.553185648098588,
|
||
|
|
"num_tokens": 4524646.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5154777869582177,
|
||
|
|
"rewards/margins": 22.38102035522461,
|
||
|
|
"rewards/rejected": -20.865542578697205,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2456171631813049,
|
||
|
|
"epoch": 0.8806329549363605,
|
||
|
|
"grad_norm": 1.9371509552001953e-06,
|
||
|
|
"learning_rate": 1.7814749849747906e-05,
|
||
|
|
"logits/chosen": -3.2061220261144827,
|
||
|
|
"logits/rejected": -2.933136296525797,
|
||
|
|
"logps/chosen": -93.17085399627686,
|
||
|
|
"logps/rejected": -345.74973487854004,
|
||
|
|
"loss": 4.866203198616858e-07,
|
||
|
|
"mean_token_accuracy": 0.5430413067340851,
|
||
|
|
"num_tokens": 4591613.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4402503899065777,
|
||
|
|
"rewards/margins": 21.325499212741853,
|
||
|
|
"rewards/rejected": -19.885248792171478,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.224543235450983,
|
||
|
|
"epoch": 0.8943928448572411,
|
||
|
|
"grad_norm": 3.203749656677246e-06,
|
||
|
|
"learning_rate": 1.771389829460275e-05,
|
||
|
|
"logits/chosen": -3.21605593581562,
|
||
|
|
"logits/rejected": -2.937328653648714,
|
||
|
|
"logps/chosen": -89.61981456279754,
|
||
|
|
"logps/rejected": -357.4316173553467,
|
||
|
|
"loss": 1.3461053640639876e-07,
|
||
|
|
"mean_token_accuracy": 0.5541570287197828,
|
||
|
|
"num_tokens": 4659715.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5124846341088414,
|
||
|
|
"rewards/margins": 22.234719467163085,
|
||
|
|
"rewards/rejected": -20.72223507165909,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2212258189916612,
|
||
|
|
"epoch": 0.9081527347781218,
|
||
|
|
"grad_norm": 0.00021266937255859375,
|
||
|
|
"learning_rate": 1.7611071026786975e-05,
|
||
|
|
"logits/chosen": -3.193745319470957,
|
||
|
|
"logits/rejected": -2.9283068821586755,
|
||
|
|
"logps/chosen": -94.82682394981384,
|
||
|
|
"logps/rejected": -358.56394958496094,
|
||
|
|
"loss": 1.0348767318646423e-07,
|
||
|
|
"mean_token_accuracy": 0.5502128433436155,
|
||
|
|
"num_tokens": 4726358.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2972642586566507,
|
||
|
|
"rewards/margins": 22.062623143196106,
|
||
|
|
"rewards/rejected": -20.765359020233156,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2308818057179451,
|
||
|
|
"epoch": 0.9219126246990024,
|
||
|
|
"grad_norm": 3.56137752532959e-06,
|
||
|
|
"learning_rate": 1.7506294382807524e-05,
|
||
|
|
"logits/chosen": -3.2158995908819286,
|
||
|
|
"logits/rejected": -2.9366110423420593,
|
||
|
|
"logps/chosen": -86.88749198913574,
|
||
|
|
"logps/rejected": -353.19364166259766,
|
||
|
|
"loss": 2.415685571577342e-08,
|
||
|
|
"mean_token_accuracy": 0.5482585165649653,
|
||
|
|
"num_tokens": 4795784.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.7150141508318484,
|
||
|
|
"rewards/margins": 22.53857171535492,
|
||
|
|
"rewards/rejected": -20.823557567596435,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2435053884983063,
|
||
|
|
"epoch": 0.935672514619883,
|
||
|
|
"grad_norm": 2.7120113372802734e-06,
|
||
|
|
"learning_rate": 1.7399595198452875e-05,
|
||
|
|
"logits/chosen": -3.195924183348283,
|
||
|
|
"logits/rejected": -2.937464334599706,
|
||
|
|
"logps/chosen": -90.25515475273133,
|
||
|
|
"logps/rejected": -365.6073051452637,
|
||
|
|
"loss": 8.592822098307806e-09,
|
||
|
|
"mean_token_accuracy": 0.557866370305419,
|
||
|
|
"num_tokens": 4865396.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5782869830727577,
|
||
|
|
"rewards/margins": 22.804828882217407,
|
||
|
|
"rewards/rejected": -21.22654196023941,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2501905634999275,
|
||
|
|
"epoch": 0.9494324045407637,
|
||
|
|
"grad_norm": 9.655952453613281e-06,
|
||
|
|
"learning_rate": 1.7291000801919807e-05,
|
||
|
|
"logits/chosen": -3.2138292712700163,
|
||
|
|
"logits/rejected": -2.93742158108878,
|
||
|
|
"logps/chosen": -95.1417423248291,
|
||
|
|
"logps/rejected": -363.6313949584961,
|
||
|
|
"loss": 2.6834425170818577e-08,
|
||
|
|
"mean_token_accuracy": 0.5553566936403513,
|
||
|
|
"num_tokens": 4940187.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3729901384562253,
|
||
|
|
"rewards/margins": 22.39214894771576,
|
||
|
|
"rewards/rejected": -21.01915898323059,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2206627286970615,
|
||
|
|
"epoch": 0.9631922944616443,
|
||
|
|
"grad_norm": 1.633167266845703e-05,
|
||
|
|
"learning_rate": 1.7180539006813973e-05,
|
||
|
|
"logits/chosen": -3.2035463098229506,
|
||
|
|
"logits/rejected": -2.933229209285195,
|
||
|
|
"logps/chosen": -89.3526448249817,
|
||
|
|
"logps/rejected": -358.9783546447754,
|
||
|
|
"loss": 9.578674564636459e-09,
|
||
|
|
"mean_token_accuracy": 0.5566618263721466,
|
||
|
|
"num_tokens": 5007712.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5610000245273112,
|
||
|
|
"rewards/margins": 22.329492390155792,
|
||
|
|
"rewards/rejected": -20.768492436408998,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9631922944616443,
|
||
|
|
"eval_entropy": 1.2349938719101201,
|
||
|
|
"eval_logits/chosen": -3.2095329852412915,
|
||
|
|
"eval_logits/rejected": -2.930840597158518,
|
||
|
|
"eval_logps/chosen": -93.99583950541378,
|
||
|
|
"eval_logps/rejected": -357.03034364787584,
|
||
|
|
"eval_loss": 1.0149173590434657e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5478390133848378,
|
||
|
|
"eval_num_tokens": 5007712.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5820025781449538,
|
||
|
|
"eval_rewards/margins": 22.257171013775995,
|
||
|
|
"eval_rewards/rejected": -20.67516856100045,
|
||
|
|
"eval_runtime": 49.0639,
|
||
|
|
"eval_samples_per_second": 12.474,
|
||
|
|
"eval_steps_per_second": 6.237,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2194250263273716,
|
||
|
|
"epoch": 0.976952184382525,
|
||
|
|
"grad_norm": 9.08970832824707e-07,
|
||
|
|
"learning_rate": 1.7068238105026167e-05,
|
||
|
|
"logits/chosen": -3.20297050334334,
|
||
|
|
"logits/rejected": -2.9201588284997833,
|
||
|
|
"logps/chosen": -92.99847078323364,
|
||
|
|
"logps/rejected": -361.15418434143066,
|
||
|
|
"loss": 1.1712397451901779e-08,
|
||
|
|
"mean_token_accuracy": 0.5613618675619364,
|
||
|
|
"num_tokens": 5080481.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.535900639742613,
|
||
|
|
"rewards/margins": 22.68755226135254,
|
||
|
|
"rewards/rejected": -21.15165193080902,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.256634522974491,
|
||
|
|
"epoch": 0.9907120743034056,
|
||
|
|
"grad_norm": 1.1086463928222656e-05,
|
||
|
|
"learning_rate": 1.6954126859486123e-05,
|
||
|
|
"logits/chosen": -3.2114024947419955,
|
||
|
|
"logits/rejected": -2.9574638812658067,
|
||
|
|
"logps/chosen": -93.72816143035888,
|
||
|
|
"logps/rejected": -361.8413818359375,
|
||
|
|
"loss": 5.007818799640518e-08,
|
||
|
|
"mean_token_accuracy": 0.5503214780241251,
|
||
|
|
"num_tokens": 5148455.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.52496765749529,
|
||
|
|
"rewards/margins": 22.30840722322464,
|
||
|
|
"rewards/rejected": -20.783439564704896,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2511392663686702,
|
||
|
|
"epoch": 1.0041279669762642,
|
||
|
|
"grad_norm": 1.817941665649414e-06,
|
||
|
|
"learning_rate": 1.6838234496795632e-05,
|
||
|
|
"logits/chosen": -3.208696360325024,
|
||
|
|
"logits/rejected": -2.938165258133346,
|
||
|
|
"logps/chosen": -94.96354098197742,
|
||
|
|
"logps/rejected": -359.0692009559044,
|
||
|
|
"loss": 8.297415092783922e-08,
|
||
|
|
"mean_token_accuracy": 0.5419222819499481,
|
||
|
|
"num_tokens": 5217649.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5244833397177548,
|
||
|
|
"rewards/margins": 22.175074552878357,
|
||
|
|
"rewards/rejected": -20.650591312310635,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2317955523729325,
|
||
|
|
"epoch": 1.0178878568971448,
|
||
|
|
"grad_norm": 0.0002613067626953125,
|
||
|
|
"learning_rate": 1.67205906997429e-05,
|
||
|
|
"logits/chosen": -3.193782121863356,
|
||
|
|
"logits/rejected": -2.939726684571892,
|
||
|
|
"logps/chosen": -98.34705619812011,
|
||
|
|
"logps/rejected": -357.925959777832,
|
||
|
|
"loss": 8.716393722352222e-08,
|
||
|
|
"mean_token_accuracy": 0.5531163465231657,
|
||
|
|
"num_tokens": 5287632.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4051031542010606,
|
||
|
|
"rewards/margins": 22.358167839050292,
|
||
|
|
"rewards/rejected": -20.953064560890198,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2180696174502372,
|
||
|
|
"epoch": 1.0316477468180254,
|
||
|
|
"grad_norm": 9.47713851928711e-06,
|
||
|
|
"learning_rate": 1.660122559970011e-05,
|
||
|
|
"logits/chosen": -3.1999433298614353,
|
||
|
|
"logits/rejected": -2.9382047283630732,
|
||
|
|
"logps/chosen": -91.44652361869812,
|
||
|
|
"logps/rejected": -355.18139419555666,
|
||
|
|
"loss": 3.3117328257503684e-08,
|
||
|
|
"mean_token_accuracy": 0.5552334699779748,
|
||
|
|
"num_tokens": 5359755.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4298429565504194,
|
||
|
|
"rewards/margins": 22.017473244667052,
|
||
|
|
"rewards/rejected": -20.587630295753478,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2397649094462395,
|
||
|
|
"epoch": 1.045407636738906,
|
||
|
|
"grad_norm": 7.450580596923828e-06,
|
||
|
|
"learning_rate": 1.6480169768906044e-05,
|
||
|
|
"logits/chosen": -3.201849871038413,
|
||
|
|
"logits/rejected": -2.942824107119408,
|
||
|
|
"logps/chosen": -94.76110773086548,
|
||
|
|
"logps/rejected": -354.3981412887573,
|
||
|
|
"loss": 6.155430583021371e-08,
|
||
|
|
"mean_token_accuracy": 0.5530436370521784,
|
||
|
|
"num_tokens": 5430991.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3717019651085138,
|
||
|
|
"rewards/margins": 21.781417500972747,
|
||
|
|
"rewards/rejected": -20.40971568822861,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2326208755373955,
|
||
|
|
"epoch": 1.0591675266597866,
|
||
|
|
"grad_norm": 2.1457672119140625e-06,
|
||
|
|
"learning_rate": 1.6357454212635798e-05,
|
||
|
|
"logits/chosen": -3.1993549874995666,
|
||
|
|
"logits/rejected": -2.9411404620494945,
|
||
|
|
"logps/chosen": -92.11231818199158,
|
||
|
|
"logps/rejected": -353.9612159729004,
|
||
|
|
"loss": 3.313938350402168e-08,
|
||
|
|
"mean_token_accuracy": 0.5549323987215757,
|
||
|
|
"num_tokens": 5495381.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6085279668681323,
|
||
|
|
"rewards/margins": 22.037530279159547,
|
||
|
|
"rewards/rejected": -20.429002428054808,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2556634098291397,
|
||
|
|
"epoch": 1.0729274165806673,
|
||
|
|
"grad_norm": 1.996755599975586e-06,
|
||
|
|
"learning_rate": 1.6233110361259616e-05,
|
||
|
|
"logits/chosen": -3.2060114017402634,
|
||
|
|
"logits/rejected": -2.9339854814430995,
|
||
|
|
"logps/chosen": -95.85781788825989,
|
||
|
|
"logps/rejected": -363.0917518615723,
|
||
|
|
"loss": 1.4122252878223662e-08,
|
||
|
|
"mean_token_accuracy": 0.5444140575826169,
|
||
|
|
"num_tokens": 5565468.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5266890531405806,
|
||
|
|
"rewards/margins": 22.333020043373107,
|
||
|
|
"rewards/rejected": -20.806330823898314,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.234340064227581,
|
||
|
|
"epoch": 1.086687306501548,
|
||
|
|
"grad_norm": 0.0002460479736328125,
|
||
|
|
"learning_rate": 1.610717006219282e-05,
|
||
|
|
"logits/chosen": -3.20874590791633,
|
||
|
|
"logits/rejected": -2.9412810311356044,
|
||
|
|
"logps/chosen": -92.43198065757751,
|
||
|
|
"logps/rejected": -358.45581817626953,
|
||
|
|
"loss": 1.5261637145158602e-07,
|
||
|
|
"mean_token_accuracy": 0.5527918070554734,
|
||
|
|
"num_tokens": 5639764.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5685145473107696,
|
||
|
|
"rewards/margins": 22.137369656562804,
|
||
|
|
"rewards/rejected": -20.568855047225952,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2221609637141229,
|
||
|
|
"epoch": 1.1004471964224287,
|
||
|
|
"grad_norm": 2.0116567611694336e-06,
|
||
|
|
"learning_rate": 1.5979665571738908e-05,
|
||
|
|
"logits/chosen": -3.2068134907404735,
|
||
|
|
"logits/rejected": -2.93636597301207,
|
||
|
|
"logps/chosen": -98.61505641937256,
|
||
|
|
"logps/rejected": -354.62015495300295,
|
||
|
|
"loss": 1.0114164297192474e-07,
|
||
|
|
"mean_token_accuracy": 0.552211070433259,
|
||
|
|
"num_tokens": 5713950.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5757936414331197,
|
||
|
|
"rewards/margins": 22.034456276893614,
|
||
|
|
"rewards/rejected": -20.4586629986763,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1004471964224287,
|
||
|
|
"eval_entropy": 1.2343543432896433,
|
||
|
|
"eval_logits/chosen": -3.209811952371513,
|
||
|
|
"eval_logits/rejected": -2.930982134660661,
|
||
|
|
"eval_logps/chosen": -93.95115639805014,
|
||
|
|
"eval_logps/rejected": -357.3008830749911,
|
||
|
|
"eval_loss": 1.0059742550083683e-07,
|
||
|
|
"eval_mean_token_accuracy": 0.5480056442076864,
|
||
|
|
"eval_num_tokens": 5713950.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5864708358604533,
|
||
|
|
"eval_rewards/margins": 22.288693184946098,
|
||
|
|
"eval_rewards/rejected": -20.702222353492687,
|
||
|
|
"eval_runtime": 49.896,
|
||
|
|
"eval_samples_per_second": 12.266,
|
||
|
|
"eval_steps_per_second": 6.133,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2298845812678336,
|
||
|
|
"epoch": 1.1142070863433093,
|
||
|
|
"grad_norm": 2.60770320892334e-06,
|
||
|
|
"learning_rate": 1.5850629546827977e-05,
|
||
|
|
"logits/chosen": -3.2075300177973256,
|
||
|
|
"logits/rejected": -2.9281061202537826,
|
||
|
|
"logps/chosen": -90.7022216796875,
|
||
|
|
"logps/rejected": -358.142036819458,
|
||
|
|
"loss": 2.4776889517852398e-08,
|
||
|
|
"mean_token_accuracy": 0.5414935078471899,
|
||
|
|
"num_tokens": 5783814.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.41559614893049,
|
||
|
|
"rewards/margins": 22.017850613594057,
|
||
|
|
"rewards/rejected": -20.602254581451415,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2270297072827816,
|
||
|
|
"epoch": 1.12796697626419,
|
||
|
|
"grad_norm": 1.4543533325195312e-05,
|
||
|
|
"learning_rate": 1.5720095036652486e-05,
|
||
|
|
"logits/chosen": -3.20386469479259,
|
||
|
|
"logits/rejected": -2.933272782764143,
|
||
|
|
"logps/chosen": -89.29530630111694,
|
||
|
|
"logps/rejected": -357.4134010314941,
|
||
|
|
"loss": 1.849425927957782e-08,
|
||
|
|
"mean_token_accuracy": 0.5607247911393642,
|
||
|
|
"num_tokens": 5852617.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.58894761223346,
|
||
|
|
"rewards/margins": 22.511793541908265,
|
||
|
|
"rewards/rejected": -20.922845947742463,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2241738960146904,
|
||
|
|
"epoch": 1.1417268661850706,
|
||
|
|
"grad_norm": 7.599592208862305e-07,
|
||
|
|
"learning_rate": 1.5588095474202597e-05,
|
||
|
|
"logits/chosen": -3.2111792369422907,
|
||
|
|
"logits/rejected": -2.939014422286019,
|
||
|
|
"logps/chosen": -96.20790944099426,
|
||
|
|
"logps/rejected": -361.97329750061033,
|
||
|
|
"loss": 8.978912546808715e-08,
|
||
|
|
"mean_token_accuracy": 0.5441163565963507,
|
||
|
|
"num_tokens": 5922241.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.36717337500304,
|
||
|
|
"rewards/margins": 22.38765777349472,
|
||
|
|
"rewards/rejected": -21.020484483242036,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2215713784098625,
|
||
|
|
"epoch": 1.1554867561059512,
|
||
|
|
"grad_norm": 2.207234501838684e-07,
|
||
|
|
"learning_rate": 1.5454664667703148e-05,
|
||
|
|
"logits/chosen": -3.207690730104448,
|
||
|
|
"logits/rejected": -2.9357031268257865,
|
||
|
|
"logps/chosen": -91.12561550140381,
|
||
|
|
"logps/rejected": -355.57129364013673,
|
||
|
|
"loss": 2.337985023359579e-08,
|
||
|
|
"mean_token_accuracy": 0.5488512497395277,
|
||
|
|
"num_tokens": 5985778.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4972319712862372,
|
||
|
|
"rewards/margins": 22.422241616249085,
|
||
|
|
"rewards/rejected": -20.92500967979431,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.258409896492958,
|
||
|
|
"epoch": 1.1692466460268318,
|
||
|
|
"grad_norm": 5.550682544708252e-07,
|
||
|
|
"learning_rate": 1.531983679195462e-05,
|
||
|
|
"logits/chosen": -3.226955490438931,
|
||
|
|
"logits/rejected": -2.945719325954898,
|
||
|
|
"logps/chosen": -95.44598877429962,
|
||
|
|
"logps/rejected": -366.71283493041994,
|
||
|
|
"loss": 4.6230468342400854e-09,
|
||
|
|
"mean_token_accuracy": 0.550101426243782,
|
||
|
|
"num_tokens": 6064389.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.780148933455348,
|
||
|
|
"rewards/margins": 22.78461835384369,
|
||
|
|
"rewards/rejected": -21.004469299316405,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2130897045135498,
|
||
|
|
"epoch": 1.1830065359477124,
|
||
|
|
"grad_norm": 1.233816146850586e-05,
|
||
|
|
"learning_rate": 1.5183646379580108e-05,
|
||
|
|
"logits/chosen": -3.2055889715400085,
|
||
|
|
"logits/rejected": -2.930225354490301,
|
||
|
|
"logps/chosen": -86.66465277671814,
|
||
|
|
"logps/rejected": -355.93296508789064,
|
||
|
|
"loss": 3.668304771053954e-08,
|
||
|
|
"mean_token_accuracy": 0.5639011237770319,
|
||
|
|
"num_tokens": 6136897.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5607147116214037,
|
||
|
|
"rewards/margins": 22.342380928993226,
|
||
|
|
"rewards/rejected": -20.781666159629822,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2411843314766884,
|
||
|
|
"epoch": 1.196766425868593,
|
||
|
|
"grad_norm": 0.00017261505126953125,
|
||
|
|
"learning_rate": 1.5046128312180735e-05,
|
||
|
|
"logits/chosen": -3.2021021642991614,
|
||
|
|
"logits/rejected": -2.9457325465225077,
|
||
|
|
"logps/chosen": -95.11519818305969,
|
||
|
|
"logps/rejected": -357.90135345458987,
|
||
|
|
"loss": 1.1841185596495051e-07,
|
||
|
|
"mean_token_accuracy": 0.5472324322909117,
|
||
|
|
"num_tokens": 6206206.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3694644190371037,
|
||
|
|
"rewards/margins": 21.965344977378845,
|
||
|
|
"rewards/rejected": -20.595880722999574,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.234082168340683,
|
||
|
|
"epoch": 1.2105263157894737,
|
||
|
|
"grad_norm": 1.0207295417785645e-06,
|
||
|
|
"learning_rate": 1.4907317811401609e-05,
|
||
|
|
"logits/chosen": -3.2189416776847666,
|
||
|
|
"logits/rejected": -2.9338163368790253,
|
||
|
|
"logps/chosen": -91.24591059684754,
|
||
|
|
"logps/rejected": -364.68142623901366,
|
||
|
|
"loss": 1.026212856913844e-08,
|
||
|
|
"mean_token_accuracy": 0.5534303054213524,
|
||
|
|
"num_tokens": 6279994.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.7311106204986573,
|
||
|
|
"rewards/margins": 22.925514221191406,
|
||
|
|
"rewards/rejected": -21.194403612613677,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2325060382485389,
|
||
|
|
"epoch": 1.2242862057103543,
|
||
|
|
"grad_norm": 8.67992639541626e-07,
|
||
|
|
"learning_rate": 1.4767250429910754e-05,
|
||
|
|
"logits/chosen": -3.210597847276174,
|
||
|
|
"logits/rejected": -2.9429847990942415,
|
||
|
|
"logps/chosen": -88.37438547611237,
|
||
|
|
"logps/rejected": -357.3102340698242,
|
||
|
|
"loss": 1.2495095802478317e-08,
|
||
|
|
"mean_token_accuracy": 0.5496666703373194,
|
||
|
|
"num_tokens": 6353089.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4847000421956182,
|
||
|
|
"rewards/margins": 22.177469778060914,
|
||
|
|
"rewards/rejected": -20.692769980430604,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2352219231426715,
|
||
|
|
"epoch": 1.238046095631235,
|
||
|
|
"grad_norm": 4.696846008300781e-05,
|
||
|
|
"learning_rate": 1.4625962042293212e-05,
|
||
|
|
"logits/chosen": -3.200865099898492,
|
||
|
|
"logits/rejected": -2.9249128824488015,
|
||
|
|
"logps/chosen": -97.9710922241211,
|
||
|
|
"logps/rejected": -356.9550182342529,
|
||
|
|
"loss": 4.1527565031174164e-08,
|
||
|
|
"mean_token_accuracy": 0.54386255890131,
|
||
|
|
"num_tokens": 6423626.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4338832331821323,
|
||
|
|
"rewards/margins": 22.15778750181198,
|
||
|
|
"rewards/rejected": -20.72390434741974,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.238046095631235,
|
||
|
|
"eval_entropy": 1.2343237298765992,
|
||
|
|
"eval_logits/chosen": -3.2097772658901613,
|
||
|
|
"eval_logits/rejected": -2.930761755546573,
|
||
|
|
"eval_logps/chosen": -93.91436023338169,
|
||
|
|
"eval_logps/rejected": -357.41727881338085,
|
||
|
|
"eval_loss": 9.47132434703235e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5485258311812394,
|
||
|
|
"eval_num_tokens": 6423626.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.590150504939112,
|
||
|
|
"eval_rewards/margins": 22.30401264764125,
|
||
|
|
"eval_rewards/rejected": -20.713862163568635,
|
||
|
|
"eval_runtime": 51.361,
|
||
|
|
"eval_samples_per_second": 11.916,
|
||
|
|
"eval_steps_per_second": 5.958,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2400694966316224,
|
||
|
|
"epoch": 1.2518059855521155,
|
||
|
|
"grad_norm": 2.7120113372802734e-06,
|
||
|
|
"learning_rate": 1.44834888358627e-05,
|
||
|
|
"logits/chosen": -3.218890531500263,
|
||
|
|
"logits/rejected": -2.9362049726909754,
|
||
|
|
"logps/chosen": -89.21487245559692,
|
||
|
|
"logps/rejected": -356.256791305542,
|
||
|
|
"loss": 2.0597786942744277e-08,
|
||
|
|
"mean_token_accuracy": 0.5380889017134904,
|
||
|
|
"num_tokens": 6490312.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4560846026055514,
|
||
|
|
"rewards/margins": 22.216789221763612,
|
||
|
|
"rewards/rejected": -20.76070455312729,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2324326142668725,
|
||
|
|
"epoch": 1.2655658754729961,
|
||
|
|
"grad_norm": 5.900859832763672e-06,
|
||
|
|
"learning_rate": 1.4339867301393204e-05,
|
||
|
|
"logits/chosen": -3.2058008679050474,
|
||
|
|
"logits/rejected": -2.9334829204994004,
|
||
|
|
"logps/chosen": -90.68490617275238,
|
||
|
|
"logps/rejected": -352.11963920593263,
|
||
|
|
"loss": 3.130011236862629e-08,
|
||
|
|
"mean_token_accuracy": 0.5447090461850166,
|
||
|
|
"num_tokens": 6557228.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4577140986919404,
|
||
|
|
"rewards/margins": 22.14238922595978,
|
||
|
|
"rewards/rejected": -20.68467507362366,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.236050571501255,
|
||
|
|
"epoch": 1.2793257653938768,
|
||
|
|
"grad_norm": 2.428889274597168e-06,
|
||
|
|
"learning_rate": 1.41951342237728e-05,
|
||
|
|
"logits/chosen": -3.2088376395139306,
|
||
|
|
"logits/rejected": -2.9461211800539613,
|
||
|
|
"logps/chosen": -94.4995069026947,
|
||
|
|
"logps/rejected": -360.5341571807861,
|
||
|
|
"loss": 1.521080861266455e-08,
|
||
|
|
"mean_token_accuracy": 0.5474192447960377,
|
||
|
|
"num_tokens": 6628952.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4530406618490814,
|
||
|
|
"rewards/margins": 22.35727779865265,
|
||
|
|
"rewards/rejected": -20.9042368888855,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2395083367824555,
|
||
|
|
"epoch": 1.2930856553147576,
|
||
|
|
"grad_norm": 0.0003032684326171875,
|
||
|
|
"learning_rate": 1.4049326672582186e-05,
|
||
|
|
"logits/chosen": -3.2151515899768377,
|
||
|
|
"logits/rejected": -2.9367740687639,
|
||
|
|
"logps/chosen": -90.78844072818757,
|
||
|
|
"logps/rejected": -352.5623176574707,
|
||
|
|
"loss": 9.801140095078154e-08,
|
||
|
|
"mean_token_accuracy": 0.5518607906997204,
|
||
|
|
"num_tokens": 6694115.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5609057364985346,
|
||
|
|
"rewards/margins": 22.212734603881835,
|
||
|
|
"rewards/rejected": -20.65182896852493,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2343350157141686,
|
||
|
|
"epoch": 1.306845545235638,
|
||
|
|
"grad_norm": 4.4345855712890625e-05,
|
||
|
|
"learning_rate": 1.390248199260027e-05,
|
||
|
|
"logits/chosen": -3.2039629217549477,
|
||
|
|
"logits/rejected": -2.9385683242624454,
|
||
|
|
"logps/chosen": -93.59937400817871,
|
||
|
|
"logps/rejected": -358.15326766967775,
|
||
|
|
"loss": 3.500546768009371e-08,
|
||
|
|
"mean_token_accuracy": 0.5573053862899542,
|
||
|
|
"num_tokens": 6766186.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5620607184246182,
|
||
|
|
"rewards/margins": 22.00651559829712,
|
||
|
|
"rewards/rejected": -20.44445481300354,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2210247091948987,
|
||
|
|
"epoch": 1.3206054351565188,
|
||
|
|
"grad_norm": 3.501772880554199e-07,
|
||
|
|
"learning_rate": 1.3754637794239303e-05,
|
||
|
|
"logits/chosen": -3.1871140209822464,
|
||
|
|
"logits/rejected": -2.9242806765491824,
|
||
|
|
"logps/chosen": -92.62435631752014,
|
||
|
|
"logps/rejected": -361.9002456665039,
|
||
|
|
"loss": 2.329596213712648e-08,
|
||
|
|
"mean_token_accuracy": 0.5678409364074468,
|
||
|
|
"num_tokens": 6833190.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4230625696480275,
|
||
|
|
"rewards/margins": 22.756867027282716,
|
||
|
|
"rewards/rejected": -21.333804500102996,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2278488397598266,
|
||
|
|
"epoch": 1.3343653250773992,
|
||
|
|
"grad_norm": 1.1444091796875e-05,
|
||
|
|
"learning_rate": 1.3605831943911923e-05,
|
||
|
|
"logits/chosen": -3.20925780292652,
|
||
|
|
"logits/rejected": -2.948473146985262,
|
||
|
|
"logps/chosen": -95.62888393402099,
|
||
|
|
"logps/rejected": -354.1160026550293,
|
||
|
|
"loss": 1.2392845860631496e-08,
|
||
|
|
"mean_token_accuracy": 0.5485764220356941,
|
||
|
|
"num_tokens": 6900474.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.473580021597445,
|
||
|
|
"rewards/margins": 22.098388063907624,
|
||
|
|
"rewards/rejected": -20.624807834625244,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2570519968867302,
|
||
|
|
"epoch": 1.34812521499828,
|
||
|
|
"grad_norm": 1.7136335372924805e-06,
|
||
|
|
"learning_rate": 1.3456102554332717e-05,
|
||
|
|
"logits/chosen": -3.2114092807728825,
|
||
|
|
"logits/rejected": -2.9454987107403863,
|
||
|
|
"logps/chosen": -99.15215730667114,
|
||
|
|
"logps/rejected": -366.17869873046874,
|
||
|
|
"loss": 7.894200848568289e-08,
|
||
|
|
"mean_token_accuracy": 0.5519696664065122,
|
||
|
|
"num_tokens": 6978618.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4953111509792507,
|
||
|
|
"rewards/margins": 22.495365333557128,
|
||
|
|
"rewards/rejected": -21.000054240226746,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.239316475391388,
|
||
|
|
"epoch": 1.3618851049191607,
|
||
|
|
"grad_norm": 2.491474151611328e-05,
|
||
|
|
"learning_rate": 1.3305487974756601e-05,
|
||
|
|
"logits/chosen": -3.2142083873833514,
|
||
|
|
"logits/rejected": -2.9419815492384034,
|
||
|
|
"logps/chosen": -96.75212671756745,
|
||
|
|
"logps/rejected": -360.60957794189454,
|
||
|
|
"loss": 6.680644446532824e-08,
|
||
|
|
"mean_token_accuracy": 0.551985003426671,
|
||
|
|
"num_tokens": 7054612.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4342084027826787,
|
||
|
|
"rewards/margins": 22.23996684551239,
|
||
|
|
"rewards/rejected": -20.805758452415468,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.239124795794487,
|
||
|
|
"epoch": 1.3756449948400413,
|
||
|
|
"grad_norm": 1.9222497940063477e-06,
|
||
|
|
"learning_rate": 1.3154026781156697e-05,
|
||
|
|
"logits/chosen": -3.20129278083637,
|
||
|
|
"logits/rejected": -2.9428580494057384,
|
||
|
|
"logps/chosen": -93.50243964195252,
|
||
|
|
"logps/rejected": -360.9033599853516,
|
||
|
|
"loss": 7.131031338758476e-08,
|
||
|
|
"mean_token_accuracy": 0.5573917739093304,
|
||
|
|
"num_tokens": 7126656.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5027323237620294,
|
||
|
|
"rewards/margins": 22.239959049224854,
|
||
|
|
"rewards/rejected": -20.73722686767578,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3756449948400413,
|
||
|
|
"eval_entropy": 1.2343096980472017,
|
||
|
|
"eval_logits/chosen": -3.2097842934074605,
|
||
|
|
"eval_logits/rejected": -2.930549885680492,
|
||
|
|
"eval_logps/chosen": -93.8916666492138,
|
||
|
|
"eval_logps/rejected": -357.49578269011056,
|
||
|
|
"eval_loss": 9.931152789022235e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5481461427764955,
|
||
|
|
"eval_num_tokens": 7126656.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5924198131497,
|
||
|
|
"eval_rewards/margins": 22.314132154377457,
|
||
|
|
"eval_rewards/rejected": -20.721712392919205,
|
||
|
|
"eval_runtime": 52.097,
|
||
|
|
"eval_samples_per_second": 11.747,
|
||
|
|
"eval_steps_per_second": 5.874,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2653382867574692,
|
||
|
|
"epoch": 1.389404884760922,
|
||
|
|
"grad_norm": 2.1010637283325195e-06,
|
||
|
|
"learning_rate": 1.300175776634409e-05,
|
||
|
|
"logits/chosen": -3.2129485356574863,
|
||
|
|
"logits/rejected": -2.954222626255789,
|
||
|
|
"logps/chosen": -92.84323039054871,
|
||
|
|
"logps/rejected": -361.7887050628662,
|
||
|
|
"loss": 2.7653899792312586e-08,
|
||
|
|
"mean_token_accuracy": 0.5468194488435983,
|
||
|
|
"num_tokens": 7201540.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3965744260232895,
|
||
|
|
"rewards/margins": 21.90877208709717,
|
||
|
|
"rewards/rejected": -20.512197828292848,
|
||
|
|
"step": 1010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2396557286381722,
|
||
|
|
"epoch": 1.4031647746818026,
|
||
|
|
"grad_norm": 3.9637088775634766e-06,
|
||
|
|
"learning_rate": 1.2848719930032083e-05,
|
||
|
|
"logits/chosen": -3.194560362905095,
|
||
|
|
"logits/rejected": -2.9334940937288825,
|
||
|
|
"logps/chosen": -89.86825790405274,
|
||
|
|
"logps/rejected": -355.7735252380371,
|
||
|
|
"loss": 2.492495525530103e-08,
|
||
|
|
"mean_token_accuracy": 0.5589596729725599,
|
||
|
|
"num_tokens": 7274153.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.48170037958771,
|
||
|
|
"rewards/margins": 22.06132137775421,
|
||
|
|
"rewards/rejected": -20.57962087392807,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2463233694434166,
|
||
|
|
"epoch": 1.4169246646026832,
|
||
|
|
"grad_norm": 9.238719940185547e-06,
|
||
|
|
"learning_rate": 1.269495246884743e-05,
|
||
|
|
"logits/chosen": -3.2081193037821327,
|
||
|
|
"logits/rejected": -2.9385929283645895,
|
||
|
|
"logps/chosen": -90.74200019836425,
|
||
|
|
"logps/rejected": -359.9956394195557,
|
||
|
|
"loss": 3.979102700668591e-08,
|
||
|
|
"mean_token_accuracy": 0.5487319093197585,
|
||
|
|
"num_tokens": 7344517.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6196894589811563,
|
||
|
|
"rewards/margins": 22.411945462226868,
|
||
|
|
"rewards/rejected": -20.792256021499632,
|
||
|
|
"step": 1030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2206041820347309,
|
||
|
|
"epoch": 1.4306845545235638,
|
||
|
|
"grad_norm": 1.8835067749023438e-05,
|
||
|
|
"learning_rate": 1.2540494766291151e-05,
|
||
|
|
"logits/chosen": -3.2122664430868433,
|
||
|
|
"logits/rejected": -2.940406184250665,
|
||
|
|
"logps/chosen": -95.85816128253937,
|
||
|
|
"logps/rejected": -356.52550678253175,
|
||
|
|
"loss": 2.728211256908253e-08,
|
||
|
|
"mean_token_accuracy": 0.5568196009844542,
|
||
|
|
"num_tokens": 7416599.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.451601443067193,
|
||
|
|
"rewards/margins": 22.1426992893219,
|
||
|
|
"rewards/rejected": -20.69109789133072,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2360372744500636,
|
||
|
|
"epoch": 1.4444444444444444,
|
||
|
|
"grad_norm": 1.436471939086914e-05,
|
||
|
|
"learning_rate": 1.2385386382651503e-05,
|
||
|
|
"logits/chosen": -3.2093854919794973,
|
||
|
|
"logits/rejected": -2.9219334413945957,
|
||
|
|
"logps/chosen": -96.11689052581787,
|
||
|
|
"logps/rejected": -367.30119705200195,
|
||
|
|
"loss": 7.263046342131929e-09,
|
||
|
|
"mean_token_accuracy": 0.5502486415207386,
|
||
|
|
"num_tokens": 7487006.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4450448545627297,
|
||
|
|
"rewards/margins": 22.97078468799591,
|
||
|
|
"rewards/rejected": -21.52573983669281,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2560921475291251,
|
||
|
|
"epoch": 1.458204334365325,
|
||
|
|
"grad_norm": 5.21540641784668e-06,
|
||
|
|
"learning_rate": 1.2229667044871603e-05,
|
||
|
|
"logits/chosen": -3.2238342267568925,
|
||
|
|
"logits/rejected": -2.954690942822835,
|
||
|
|
"logps/chosen": -95.17089319229126,
|
||
|
|
"logps/rejected": -358.49853858947756,
|
||
|
|
"loss": 1.2689355344264186e-07,
|
||
|
|
"mean_token_accuracy": 0.5482176937162876,
|
||
|
|
"num_tokens": 7567622.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6461190782487392,
|
||
|
|
"rewards/margins": 22.06815986633301,
|
||
|
|
"rewards/rejected": -20.422040891647338,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2410757690668106,
|
||
|
|
"epoch": 1.4719642242862057,
|
||
|
|
"grad_norm": 2.3603439331054688e-05,
|
||
|
|
"learning_rate": 1.2073376636374467e-05,
|
||
|
|
"logits/chosen": -3.2181867929946835,
|
||
|
|
"logits/rejected": -2.9455415466787125,
|
||
|
|
"logps/chosen": -95.13691272735596,
|
||
|
|
"logps/rejected": -360.3422981262207,
|
||
|
|
"loss": 6.41182055005629e-08,
|
||
|
|
"mean_token_accuracy": 0.5514874663203955,
|
||
|
|
"num_tokens": 7640270.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5654873698949814,
|
||
|
|
"rewards/margins": 22.5183806180954,
|
||
|
|
"rewards/rejected": -20.95289328098297,
|
||
|
|
"step": 1070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.223815706372261,
|
||
|
|
"epoch": 1.4857241142070863,
|
||
|
|
"grad_norm": 7.562339305877686e-07,
|
||
|
|
"learning_rate": 1.191655518684788e-05,
|
||
|
|
"logits/chosen": -3.213393593248744,
|
||
|
|
"logits/rejected": -2.9516763163029864,
|
||
|
|
"logps/chosen": -85.92939200401307,
|
||
|
|
"logps/rejected": -354.56640129089357,
|
||
|
|
"loss": 5.5562270517839354e-08,
|
||
|
|
"mean_token_accuracy": 0.5542758256196976,
|
||
|
|
"num_tokens": 7714968.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.601221213489771,
|
||
|
|
"rewards/margins": 22.047427451610567,
|
||
|
|
"rewards/rejected": -20.446206474304198,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2295074120163918,
|
||
|
|
"epoch": 1.499484004127967,
|
||
|
|
"grad_norm": 2.652406692504883e-06,
|
||
|
|
"learning_rate": 1.1759242861991855e-05,
|
||
|
|
"logits/chosen": -3.208135379165089,
|
||
|
|
"logits/rejected": -2.9249312915782903,
|
||
|
|
"logps/chosen": -91.00948677062988,
|
||
|
|
"logps/rejected": -348.90424194335935,
|
||
|
|
"loss": 7.343320476138616e-08,
|
||
|
|
"mean_token_accuracy": 0.5440276902168989,
|
||
|
|
"num_tokens": 7783981.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6290188884362578,
|
||
|
|
"rewards/margins": 21.78258399963379,
|
||
|
|
"rewards/rejected": -20.1535649061203,
|
||
|
|
"step": 1090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2454005569219588,
|
||
|
|
"epoch": 1.5132438940488475,
|
||
|
|
"grad_norm": 4.351139068603516e-06,
|
||
|
|
"learning_rate": 1.1601479953231251e-05,
|
||
|
|
"logits/chosen": -3.2080805895734223,
|
||
|
|
"logits/rejected": -2.940618116061205,
|
||
|
|
"logps/chosen": -95.01053881645203,
|
||
|
|
"logps/rejected": -350.12212295532225,
|
||
|
|
"loss": 2.5034836426129914e-08,
|
||
|
|
"mean_token_accuracy": 0.5546459078788757,
|
||
|
|
"num_tokens": 7847253.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6118897259235383,
|
||
|
|
"rewards/margins": 22.07096242904663,
|
||
|
|
"rewards/rejected": -20.459072756767274,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5132438940488475,
|
||
|
|
"eval_entropy": 1.2339863264872357,
|
||
|
|
"eval_logits/chosen": -3.2096496999814828,
|
||
|
|
"eval_logits/rejected": -2.930534343593307,
|
||
|
|
"eval_logps/chosen": -93.90245786679337,
|
||
|
|
"eval_logps/rejected": -357.6898516486673,
|
||
|
|
"eval_loss": 9.994760574727479e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5476265988903108,
|
||
|
|
"eval_num_tokens": 7847253.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5913407134707847,
|
||
|
|
"eval_rewards/margins": 22.332460110483606,
|
||
|
|
"eval_rewards/rejected": -20.741119384765625,
|
||
|
|
"eval_runtime": 53.2502,
|
||
|
|
"eval_samples_per_second": 11.493,
|
||
|
|
"eval_steps_per_second": 5.746,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2328468561172485,
|
||
|
|
"epoch": 1.5270037839697284,
|
||
|
|
"grad_norm": 3.874301910400391e-06,
|
||
|
|
"learning_rate": 1.1443306867396164e-05,
|
||
|
|
"logits/chosen": -3.2139558810032733,
|
||
|
|
"logits/rejected": -2.936539280420114,
|
||
|
|
"logps/chosen": -87.71727166175842,
|
||
|
|
"logps/rejected": -350.89677238464355,
|
||
|
|
"loss": 3.940974124816421e-08,
|
||
|
|
"mean_token_accuracy": 0.5458600226789713,
|
||
|
|
"num_tokens": 7914853.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.639497240446508,
|
||
|
|
"rewards/margins": 21.940470695495605,
|
||
|
|
"rewards/rejected": -20.300973522663117,
|
||
|
|
"step": 1110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2204807847738266,
|
||
|
|
"epoch": 1.5407636738906088,
|
||
|
|
"grad_norm": 1.049041748046875e-05,
|
||
|
|
"learning_rate": 1.1284764116372788e-05,
|
||
|
|
"logits/chosen": -3.1927449778795194,
|
||
|
|
"logits/rejected": -2.930753806827826,
|
||
|
|
"logps/chosen": -89.22110188007355,
|
||
|
|
"logps/rejected": -357.37607154846194,
|
||
|
|
"loss": 4.953578809363535e-08,
|
||
|
|
"mean_token_accuracy": 0.5495975099503994,
|
||
|
|
"num_tokens": 7984117.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3059963096864522,
|
||
|
|
"rewards/margins": 22.082072138786316,
|
||
|
|
"rewards/rejected": -20.776075863838194,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2255059480667114,
|
||
|
|
"epoch": 1.5545235638114896,
|
||
|
|
"grad_norm": 1.5273690223693848e-07,
|
||
|
|
"learning_rate": 1.1125892306727362e-05,
|
||
|
|
"logits/chosen": -3.213309220424828,
|
||
|
|
"logits/rejected": -2.9230614100860066,
|
||
|
|
"logps/chosen": -99.92382683753968,
|
||
|
|
"logps/rejected": -364.37585296630857,
|
||
|
|
"loss": 2.4999820880111656e-08,
|
||
|
|
"mean_token_accuracy": 0.5512117072939873,
|
||
|
|
"num_tokens": 8055365.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5203925285488367,
|
||
|
|
"rewards/margins": 22.711718726158143,
|
||
|
|
"rewards/rejected": -21.191326379776,
|
||
|
|
"step": 1130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2493881568312646,
|
||
|
|
"epoch": 1.56828345373237,
|
||
|
|
"grad_norm": 2.6345252990722656e-05,
|
||
|
|
"learning_rate": 1.096673212930585e-05,
|
||
|
|
"logits/chosen": -3.216281187013641,
|
||
|
|
"logits/rejected": -2.943805298211413,
|
||
|
|
"logps/chosen": -89.30674161911011,
|
||
|
|
"logps/rejected": -357.40013275146487,
|
||
|
|
"loss": 5.5321879699477e-08,
|
||
|
|
"mean_token_accuracy": 0.5533328153192997,
|
||
|
|
"num_tokens": 8128635.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4588562190067023,
|
||
|
|
"rewards/margins": 21.81429543495178,
|
||
|
|
"rewards/rejected": -20.35543934106827,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2265586256980896,
|
||
|
|
"epoch": 1.5820433436532508,
|
||
|
|
"grad_norm": 6.668269634246826e-07,
|
||
|
|
"learning_rate": 1.0807324348812065e-05,
|
||
|
|
"logits/chosen": -3.1953608613402964,
|
||
|
|
"logits/rejected": -2.931393446827495,
|
||
|
|
"logps/chosen": -91.45233364105225,
|
||
|
|
"logps/rejected": -355.4373069763184,
|
||
|
|
"loss": 1.3633338369345438e-08,
|
||
|
|
"mean_token_accuracy": 0.5539218068122864,
|
||
|
|
"num_tokens": 8193593.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4279992308467626,
|
||
|
|
"rewards/margins": 22.249031281471254,
|
||
|
|
"rewards/rejected": -20.821032071113585,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2320809975266456,
|
||
|
|
"epoch": 1.5958032335741315,
|
||
|
|
"grad_norm": 2.944469451904297e-05,
|
||
|
|
"learning_rate": 1.0647709793366843e-05,
|
||
|
|
"logits/chosen": -3.1978028499428817,
|
||
|
|
"logits/rejected": -2.924464054780402,
|
||
|
|
"logps/chosen": -91.58377666473389,
|
||
|
|
"logps/rejected": -357.0462505340576,
|
||
|
|
"loss": 1.6318038831286684e-08,
|
||
|
|
"mean_token_accuracy": 0.5578478880226612,
|
||
|
|
"num_tokens": 8262660.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5524121232330799,
|
||
|
|
"rewards/margins": 22.37659764289856,
|
||
|
|
"rewards/rejected": -20.824185419082642,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2370183646678925,
|
||
|
|
"epoch": 1.609563123495012,
|
||
|
|
"grad_norm": 2.467632293701172e-05,
|
||
|
|
"learning_rate": 1.0487929344051005e-05,
|
||
|
|
"logits/chosen": -3.2050213860609924,
|
||
|
|
"logits/rejected": -2.9398967008651766,
|
||
|
|
"logps/chosen": -90.65321626663209,
|
||
|
|
"logps/rejected": -364.4670093536377,
|
||
|
|
"loss": 1.9538390461093513e-08,
|
||
|
|
"mean_token_accuracy": 0.5525426868349314,
|
||
|
|
"num_tokens": 8337583.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4652544744312763,
|
||
|
|
"rewards/margins": 22.466693067550658,
|
||
|
|
"rewards/rejected": -21.001438570022582,
|
||
|
|
"step": 1170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2341183677315712,
|
||
|
|
"epoch": 1.6233230134158927,
|
||
|
|
"grad_norm": 6.67572021484375e-06,
|
||
|
|
"learning_rate": 1.0328023924434731e-05,
|
||
|
|
"logits/chosen": -3.222561275162723,
|
||
|
|
"logits/rejected": -2.9361925600402694,
|
||
|
|
"logps/chosen": -94.25461287498474,
|
||
|
|
"logps/rejected": -365.5099418640137,
|
||
|
|
"loss": 8.563840765418718e-08,
|
||
|
|
"mean_token_accuracy": 0.5535896979272366,
|
||
|
|
"num_tokens": 8408914.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.589152716472745,
|
||
|
|
"rewards/margins": 22.772674751281738,
|
||
|
|
"rewards/rejected": -21.18352210521698,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2446356549859048,
|
||
|
|
"epoch": 1.6370829033367733,
|
||
|
|
"grad_norm": 4.2282044887542725e-07,
|
||
|
|
"learning_rate": 1.0168034490096068e-05,
|
||
|
|
"logits/chosen": -3.2167415967969895,
|
||
|
|
"logits/rejected": -2.9377228327041256,
|
||
|
|
"logps/chosen": -91.5360399723053,
|
||
|
|
"logps/rejected": -365.2527450561523,
|
||
|
|
"loss": 3.7552769072135564e-08,
|
||
|
|
"mean_token_accuracy": 0.5535377282649279,
|
||
|
|
"num_tokens": 8477253.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6271376270800828,
|
||
|
|
"rewards/margins": 22.49522852897644,
|
||
|
|
"rewards/rejected": -20.868090736865998,
|
||
|
|
"step": 1190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.238689461350441,
|
||
|
|
"epoch": 1.650842793257654,
|
||
|
|
"grad_norm": 4.1425228118896484e-06,
|
||
|
|
"learning_rate": 1.0008002018131222e-05,
|
||
|
|
"logits/chosen": -3.2078820613389305,
|
||
|
|
"logits/rejected": -2.953614994023544,
|
||
|
|
"logps/chosen": -92.05451779365539,
|
||
|
|
"logps/rejected": -362.38125648498533,
|
||
|
|
"loss": 6.920008672750555e-08,
|
||
|
|
"mean_token_accuracy": 0.55484758913517,
|
||
|
|
"num_tokens": 8553796.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4701514573767782,
|
||
|
|
"rewards/margins": 22.34501588344574,
|
||
|
|
"rewards/rejected": -20.874864387512208,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.650842793257654,
|
||
|
|
"eval_entropy": 1.2338554866952833,
|
||
|
|
"eval_logits/chosen": -3.2098196420074143,
|
||
|
|
"eval_logits/rejected": -2.930539210739882,
|
||
|
|
"eval_logps/chosen": -93.88720486834158,
|
||
|
|
"eval_logps/rejected": -357.73581810246884,
|
||
|
|
"eval_loss": 9.633028241751163e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5478904262866849,
|
||
|
|
"eval_num_tokens": 8553796.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5928660004819726,
|
||
|
|
"eval_rewards/margins": 22.33858197654774,
|
||
|
|
"eval_rewards/rejected": -20.745715982773724,
|
||
|
|
"eval_runtime": 53.4054,
|
||
|
|
"eval_samples_per_second": 11.46,
|
||
|
|
"eval_steps_per_second": 5.73,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2338899701833725,
|
||
|
|
"epoch": 1.6646026831785345,
|
||
|
|
"grad_norm": 0.00341796875,
|
||
|
|
"learning_rate": 9.84796749665937e-06,
|
||
|
|
"logits/chosen": -3.2209590147273546,
|
||
|
|
"logits/rejected": -2.926932008123792,
|
||
|
|
"logps/chosen": -92.47018933296204,
|
||
|
|
"logps/rejected": -364.83294715881345,
|
||
|
|
"loss": 4.6182253754523117e-07,
|
||
|
|
"mean_token_accuracy": 0.550224631279707,
|
||
|
|
"num_tokens": 8627941.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.383915454056114,
|
||
|
|
"rewards/margins": 22.66803867816925,
|
||
|
|
"rewards/rejected": -21.28412308692932,
|
||
|
|
"step": 1210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.219579754024744,
|
||
|
|
"epoch": 1.6783625730994152,
|
||
|
|
"grad_norm": 3.5762786865234375e-05,
|
||
|
|
"learning_rate": 9.687971914324607e-06,
|
||
|
|
"logits/chosen": -3.208296211567128,
|
||
|
|
"logits/rejected": -2.918006915466585,
|
||
|
|
"logps/chosen": -92.74604415893555,
|
||
|
|
"logps/rejected": -358.02383651733396,
|
||
|
|
"loss": 1.0100798135681543e-07,
|
||
|
|
"mean_token_accuracy": 0.5535013288259506,
|
||
|
|
"num_tokens": 8699432.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5753983343020082,
|
||
|
|
"rewards/margins": 22.859029364585876,
|
||
|
|
"rewards/rejected": -21.283631086349487,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2542573563754558,
|
||
|
|
"epoch": 1.6921224630202958,
|
||
|
|
"grad_norm": 6.288290023803711e-06,
|
||
|
|
"learning_rate": 9.528056249797801e-06,
|
||
|
|
"logits/chosen": -3.204274880340815,
|
||
|
|
"logits/rejected": -2.9406342860725028,
|
||
|
|
"logps/chosen": -97.6590735912323,
|
||
|
|
"logps/rejected": -361.1593727111816,
|
||
|
|
"loss": 1.0381215531651833e-08,
|
||
|
|
"mean_token_accuracy": 0.5510105274617672,
|
||
|
|
"num_tokens": 8770988.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5584924587979914,
|
||
|
|
"rewards/margins": 22.114416193962096,
|
||
|
|
"rewards/rejected": -20.555923748016358,
|
||
|
|
"step": 1230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2104025840759278,
|
||
|
|
"epoch": 1.7058823529411766,
|
||
|
|
"grad_norm": 6.079673767089844e-06,
|
||
|
|
"learning_rate": 9.36826146128095e-06,
|
||
|
|
"logits/chosen": -3.194136407385158,
|
||
|
|
"logits/rejected": -2.92752970132409,
|
||
|
|
"logps/chosen": -89.4238208770752,
|
||
|
|
"logps/rejected": -367.14545822143555,
|
||
|
|
"loss": 8.342782820136563e-09,
|
||
|
|
"mean_token_accuracy": 0.5553437374532223,
|
||
|
|
"num_tokens": 8839680.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4051247568801046,
|
||
|
|
"rewards/margins": 23.100000309944154,
|
||
|
|
"rewards/rejected": -21.694875502586363,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2199999384582043,
|
||
|
|
"epoch": 1.719642242862057,
|
||
|
|
"grad_norm": 1.4156103134155273e-06,
|
||
|
|
"learning_rate": 9.208628476016851e-06,
|
||
|
|
"logits/chosen": -3.2222275132636247,
|
||
|
|
"logits/rejected": -2.9386455534159923,
|
||
|
|
"logps/chosen": -88.96111073493958,
|
||
|
|
"logps/rejected": -359.295943069458,
|
||
|
|
"loss": 4.11268274547183e-08,
|
||
|
|
"mean_token_accuracy": 0.559587674215436,
|
||
|
|
"num_tokens": 8914608.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5049045903608202,
|
||
|
|
"rewards/margins": 22.171202778816223,
|
||
|
|
"rewards/rejected": -20.666298282146453,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2306214161217213,
|
||
|
|
"epoch": 1.7334021327829379,
|
||
|
|
"grad_norm": 2.905726432800293e-06,
|
||
|
|
"learning_rate": 9.049198179806639e-06,
|
||
|
|
"logits/chosen": -3.202567711907225,
|
||
|
|
"logits/rejected": -2.9338417581047036,
|
||
|
|
"logps/chosen": -92.89101815223694,
|
||
|
|
"logps/rejected": -363.7858856201172,
|
||
|
|
"loss": 1.9706928355844867e-08,
|
||
|
|
"mean_token_accuracy": 0.5425420433282853,
|
||
|
|
"num_tokens": 8988203.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5503723615780474,
|
||
|
|
"rewards/margins": 22.790516090393066,
|
||
|
|
"rewards/rejected": -21.240143823623658,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2146451964974403,
|
||
|
|
"epoch": 1.7471620227038183,
|
||
|
|
"grad_norm": 3.0040740966796875e-05,
|
||
|
|
"learning_rate": 8.890011406537962e-06,
|
||
|
|
"logits/chosen": -3.1990021274553615,
|
||
|
|
"logits/rejected": -2.9203509087743083,
|
||
|
|
"logps/chosen": -98.18809704780578,
|
||
|
|
"logps/rejected": -361.4477626800537,
|
||
|
|
"loss": 4.6641707740491255e-08,
|
||
|
|
"mean_token_accuracy": 0.5551731705665588,
|
||
|
|
"num_tokens": 9061240.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2511278545483946,
|
||
|
|
"rewards/margins": 22.257884681224823,
|
||
|
|
"rewards/rejected": -21.00675710439682,
|
||
|
|
"step": 1270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.24142100289464,
|
||
|
|
"epoch": 1.760921912624699,
|
||
|
|
"grad_norm": 5.692243576049805e-06,
|
||
|
|
"learning_rate": 8.731108927726456e-06,
|
||
|
|
"logits/chosen": -3.207833236413998,
|
||
|
|
"logits/rejected": -2.946940461163376,
|
||
|
|
"logps/chosen": -94.46060605049134,
|
||
|
|
"logps/rejected": -364.594189453125,
|
||
|
|
"loss": 2.9929975653431026e-08,
|
||
|
|
"mean_token_accuracy": 0.547786720469594,
|
||
|
|
"num_tokens": 9137771.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4769228994846344,
|
||
|
|
"rewards/margins": 22.50814254283905,
|
||
|
|
"rewards/rejected": -21.031219601631165,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2370137855410577,
|
||
|
|
"epoch": 1.7746818025455795,
|
||
|
|
"grad_norm": 9.015202522277832e-07,
|
||
|
|
"learning_rate": 8.57253144207319e-06,
|
||
|
|
"logits/chosen": -3.2113461060224298,
|
||
|
|
"logits/rejected": -2.947444498776847,
|
||
|
|
"logps/chosen": -92.91609952449798,
|
||
|
|
"logps/rejected": -361.47128639221194,
|
||
|
|
"loss": 2.0866957584075864e-08,
|
||
|
|
"mean_token_accuracy": 0.5487309109419585,
|
||
|
|
"num_tokens": 9205311.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4436738032847642,
|
||
|
|
"rewards/margins": 22.531071925163268,
|
||
|
|
"rewards/rejected": -21.08739824295044,
|
||
|
|
"step": 1290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2511209398508072,
|
||
|
|
"epoch": 1.7884416924664603,
|
||
|
|
"grad_norm": 1.5944242477416992e-06,
|
||
|
|
"learning_rate": 8.414319565040767e-06,
|
||
|
|
"logits/chosen": -3.2049623372893365,
|
||
|
|
"logits/rejected": -2.939238119128755,
|
||
|
|
"logps/chosen": -90.87593531608582,
|
||
|
|
"logps/rejected": -362.4408033370972,
|
||
|
|
"loss": 1.4193443576004939e-08,
|
||
|
|
"mean_token_accuracy": 0.550374123826623,
|
||
|
|
"num_tokens": 9276318.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4517353810369968,
|
||
|
|
"rewards/margins": 22.48921357393265,
|
||
|
|
"rewards/rejected": -21.037478232383727,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7884416924664603,
|
||
|
|
"eval_entropy": 1.231158983863257,
|
||
|
|
"eval_logits/chosen": -3.2081173073045157,
|
||
|
|
"eval_logits/rejected": -2.9273825382166865,
|
||
|
|
"eval_logps/chosen": -94.19876768074784,
|
||
|
|
"eval_logps/rejected": -359.8468223023259,
|
||
|
|
"eval_loss": 9.118837596133744e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5473919115424936,
|
||
|
|
"eval_num_tokens": 9276318.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5617097491045404,
|
||
|
|
"eval_rewards/margins": 22.518526158301658,
|
||
|
|
"eval_rewards/rejected": -20.956816346037623,
|
||
|
|
"eval_runtime": 54.4555,
|
||
|
|
"eval_samples_per_second": 11.239,
|
||
|
|
"eval_steps_per_second": 5.619,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.215628532320261,
|
||
|
|
"epoch": 1.8022015823873407,
|
||
|
|
"grad_norm": 3.170967102050781e-05,
|
||
|
|
"learning_rate": 8.256513818450732e-06,
|
||
|
|
"logits/chosen": -3.2075363655001468,
|
||
|
|
"logits/rejected": -2.942603792959435,
|
||
|
|
"logps/chosen": -88.86163854598999,
|
||
|
|
"logps/rejected": -366.01112480163573,
|
||
|
|
"loss": 2.054591732303379e-08,
|
||
|
|
"mean_token_accuracy": 0.5597702454775572,
|
||
|
|
"num_tokens": 9352151.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.412201857380569,
|
||
|
|
"rewards/margins": 22.351779055595397,
|
||
|
|
"rewards/rejected": -20.939577162265778,
|
||
|
|
"step": 1310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2229851864278316,
|
||
|
|
"epoch": 1.8159614723082216,
|
||
|
|
"grad_norm": 2.2530555725097656e-05,
|
||
|
|
"learning_rate": 8.099154620104965e-06,
|
||
|
|
"logits/chosen": -3.2086390495519246,
|
||
|
|
"logits/rejected": -2.9236085096964888,
|
||
|
|
"logps/chosen": -92.50195686817169,
|
||
|
|
"logps/rejected": -357.3075210571289,
|
||
|
|
"loss": 8.650430345369387e-08,
|
||
|
|
"mean_token_accuracy": 0.5526527386158705,
|
||
|
|
"num_tokens": 9423542.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6557919790968298,
|
||
|
|
"rewards/margins": 22.534833192825317,
|
||
|
|
"rewards/rejected": -20.87904134988785,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.230076377093792,
|
||
|
|
"epoch": 1.8297213622291022,
|
||
|
|
"grad_norm": 7.497146725654602e-08,
|
||
|
|
"learning_rate": 7.942282273433732e-06,
|
||
|
|
"logits/chosen": -3.194487966932489,
|
||
|
|
"logits/rejected": -2.9350706353122984,
|
||
|
|
"logps/chosen": -88.21589787006378,
|
||
|
|
"logps/rejected": -364.98791046142577,
|
||
|
|
"loss": 5.49991739262623e-08,
|
||
|
|
"mean_token_accuracy": 0.559753555431962,
|
||
|
|
"num_tokens": 9490789.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3836110661271959,
|
||
|
|
"rewards/margins": 22.71821274757385,
|
||
|
|
"rewards/rejected": -21.33460156917572,
|
||
|
|
"step": 1330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2405915766954423,
|
||
|
|
"epoch": 1.8434812521499828,
|
||
|
|
"grad_norm": 2.076849341392517e-07,
|
||
|
|
"learning_rate": 7.785936957173026e-06,
|
||
|
|
"logits/chosen": -3.2164539935100174,
|
||
|
|
"logits/rejected": -2.9248479991999337,
|
||
|
|
"logps/chosen": -89.55370953083039,
|
||
|
|
"logps/rejected": -362.57230072021486,
|
||
|
|
"loss": 1.4500157874408615e-08,
|
||
|
|
"mean_token_accuracy": 0.5470205157995224,
|
||
|
|
"num_tokens": 9556753.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5294401086866856,
|
||
|
|
"rewards/margins": 22.728866374492647,
|
||
|
|
"rewards/rejected": -21.19942626953125,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2255703970789908,
|
||
|
|
"epoch": 1.8572411420708634,
|
||
|
|
"grad_norm": 2.0265579223632812e-06,
|
||
|
|
"learning_rate": 7.630158715073813e-06,
|
||
|
|
"logits/chosen": -3.208081273055396,
|
||
|
|
"logits/rejected": -2.9360034267996244,
|
||
|
|
"logps/chosen": -94.43741364479065,
|
||
|
|
"logps/rejected": -355.30568141937255,
|
||
|
|
"loss": 3.467322358119418e-08,
|
||
|
|
"mean_token_accuracy": 0.5444860707968473,
|
||
|
|
"num_tokens": 9627619.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4317927658557892,
|
||
|
|
"rewards/margins": 22.225133633613588,
|
||
|
|
"rewards/rejected": -20.79334100484848,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2434842370450496,
|
||
|
|
"epoch": 1.871001031991744,
|
||
|
|
"grad_norm": 8.121132850646973e-07,
|
||
|
|
"learning_rate": 7.4749874456458945e-06,
|
||
|
|
"logits/chosen": -3.212613006457194,
|
||
|
|
"logits/rejected": -2.9251595540262754,
|
||
|
|
"logps/chosen": -91.65022368431092,
|
||
|
|
"logps/rejected": -357.33582496643066,
|
||
|
|
"loss": 1.5621509419361246e-08,
|
||
|
|
"mean_token_accuracy": 0.5496355626732111,
|
||
|
|
"num_tokens": 9701352.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5653375163674355,
|
||
|
|
"rewards/margins": 22.173942589759825,
|
||
|
|
"rewards/rejected": -20.608605122566225,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2268347784876823,
|
||
|
|
"epoch": 1.8847609219126247,
|
||
|
|
"grad_norm": 2.1338462829589844e-05,
|
||
|
|
"learning_rate": 7.3204628919389485e-06,
|
||
|
|
"logits/chosen": -3.2073015388607997,
|
||
|
|
"logits/rejected": -2.933093921397797,
|
||
|
|
"logps/chosen": -96.34979357719422,
|
||
|
|
"logps/rejected": -358.03315315246584,
|
||
|
|
"loss": 2.1239095815417386e-08,
|
||
|
|
"mean_token_accuracy": 0.5452436875551939,
|
||
|
|
"num_tokens": 9770885.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5329102117568254,
|
||
|
|
"rewards/margins": 22.605379486083983,
|
||
|
|
"rewards/rejected": -21.07246913909912,
|
||
|
|
"step": 1370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2450409799814224,
|
||
|
|
"epoch": 1.8985208118335053,
|
||
|
|
"grad_norm": 2.518296241760254e-06,
|
||
|
|
"learning_rate": 7.166624631363386e-06,
|
||
|
|
"logits/chosen": -3.2212513769590827,
|
||
|
|
"logits/rejected": -2.9404137559127967,
|
||
|
|
"logps/chosen": -92.94262833595276,
|
||
|
|
"logps/rejected": -358.9796733856201,
|
||
|
|
"loss": 6.693504133181704e-08,
|
||
|
|
"mean_token_accuracy": 0.5537132691591978,
|
||
|
|
"num_tokens": 9844261.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5716754218563438,
|
||
|
|
"rewards/margins": 22.33404486179352,
|
||
|
|
"rewards/rejected": -20.76236960887909,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.243330615758896,
|
||
|
|
"epoch": 1.912280701754386,
|
||
|
|
"grad_norm": 3.4868717193603516e-06,
|
||
|
|
"learning_rate": 7.01351206555364e-06,
|
||
|
|
"logits/chosen": -3.202905518009641,
|
||
|
|
"logits/rejected": -2.95122810678823,
|
||
|
|
"logps/chosen": -92.49315547943115,
|
||
|
|
"logps/rejected": -368.03791732788085,
|
||
|
|
"loss": 5.302155727804347e-08,
|
||
|
|
"mean_token_accuracy": 0.5490838758647442,
|
||
|
|
"num_tokens": 9916718.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.408531033806503,
|
||
|
|
"rewards/margins": 22.847691893577576,
|
||
|
|
"rewards/rejected": -21.439160919189455,
|
||
|
|
"step": 1390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2381026059389115,
|
||
|
|
"epoch": 1.9260405916752665,
|
||
|
|
"grad_norm": 0.00011587142944335938,
|
||
|
|
"learning_rate": 6.861164410276469e-06,
|
||
|
|
"logits/chosen": -3.209368117524053,
|
||
|
|
"logits/rejected": -2.9519032131182743,
|
||
|
|
"logps/chosen": -90.15022115707397,
|
||
|
|
"logps/rejected": -360.4969120025635,
|
||
|
|
"loss": 3.583730858736089e-08,
|
||
|
|
"mean_token_accuracy": 0.554393558204174,
|
||
|
|
"num_tokens": 9992414.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.602032061293721,
|
||
|
|
"rewards/margins": 22.397946190834045,
|
||
|
|
"rewards/rejected": -20.795914244651794,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9260405916752665,
|
||
|
|
"eval_entropy": 1.2311562377642962,
|
||
|
|
"eval_logits/chosen": -3.208069794657062,
|
||
|
|
"eval_logits/rejected": -2.9272682126130256,
|
||
|
|
"eval_logps/chosen": -94.2417087305605,
|
||
|
|
"eval_logps/rejected": -359.93737563588263,
|
||
|
|
"eval_loss": 9.088540053880934e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5476740352273767,
|
||
|
|
"eval_num_tokens": 9992414.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5574156118715121,
|
||
|
|
"eval_rewards/margins": 22.523287224613764,
|
||
|
|
"eval_rewards/rejected": -20.9658716644337,
|
||
|
|
"eval_runtime": 55.0297,
|
||
|
|
"eval_samples_per_second": 11.121,
|
||
|
|
"eval_steps_per_second": 5.561,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2308048702776433,
|
||
|
|
"epoch": 1.9398004815961474,
|
||
|
|
"grad_norm": 2.0563602447509766e-06,
|
||
|
|
"learning_rate": 6.709620685386857e-06,
|
||
|
|
"logits/chosen": -3.21866885172357,
|
||
|
|
"logits/rejected": -2.924553737218475,
|
||
|
|
"logps/chosen": -95.29532494544983,
|
||
|
|
"logps/rejected": -351.67724494934083,
|
||
|
|
"loss": 6.201777864589531e-08,
|
||
|
|
"mean_token_accuracy": 0.5582938764244318,
|
||
|
|
"num_tokens": 10073883.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5281909180805087,
|
||
|
|
"rewards/margins": 21.83339207172394,
|
||
|
|
"rewards/rejected": -20.30520099401474,
|
||
|
|
"step": 1410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2180306166410446,
|
||
|
|
"epoch": 1.9535603715170278,
|
||
|
|
"grad_norm": 3.241002559661865e-07,
|
||
|
|
"learning_rate": 6.558919704834121e-06,
|
||
|
|
"logits/chosen": -3.2067047975104517,
|
||
|
|
"logits/rejected": -2.931083820952747,
|
||
|
|
"logps/chosen": -89.81032228469849,
|
||
|
|
"logps/rejected": -363.26050567626953,
|
||
|
|
"loss": 1.7645916727815347e-08,
|
||
|
|
"mean_token_accuracy": 0.5559442441910505,
|
||
|
|
"num_tokens": 10145775.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3647572711110114,
|
||
|
|
"rewards/margins": 22.6029949426651,
|
||
|
|
"rewards/rejected": -21.238237845897675,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2391163140535355,
|
||
|
|
"epoch": 1.9673202614379086,
|
||
|
|
"grad_norm": 4.880130290985107e-07,
|
||
|
|
"learning_rate": 6.409100066720724e-06,
|
||
|
|
"logits/chosen": -3.208245858330298,
|
||
|
|
"logits/rejected": -2.927146088570569,
|
||
|
|
"logps/chosen": -96.24866271018982,
|
||
|
|
"logps/rejected": -374.9543224334717,
|
||
|
|
"loss": 3.9638800330976666e-08,
|
||
|
|
"mean_token_accuracy": 0.5497354336082936,
|
||
|
|
"num_tokens": 10217226.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.399499694549013,
|
||
|
|
"rewards/margins": 23.0203608751297,
|
||
|
|
"rewards/rejected": -21.620861136913298,
|
||
|
|
"step": 1430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.227453202009201,
|
||
|
|
"epoch": 1.981080151358789,
|
||
|
|
"grad_norm": 5.4836273193359375e-06,
|
||
|
|
"learning_rate": 6.260200143416385e-06,
|
||
|
|
"logits/chosen": -3.213358957807857,
|
||
|
|
"logits/rejected": -2.9454548998081966,
|
||
|
|
"logps/chosen": -90.53418622016906,
|
||
|
|
"logps/rejected": -359.1464733123779,
|
||
|
|
"loss": 2.0817584811538837e-08,
|
||
|
|
"mean_token_accuracy": 0.5452344041317702,
|
||
|
|
"num_tokens": 10289925.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5044855073094368,
|
||
|
|
"rewards/margins": 22.441794085502625,
|
||
|
|
"rewards/rejected": -20.937308418750764,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2387429229915141,
|
||
|
|
"epoch": 1.9948400412796699,
|
||
|
|
"grad_norm": 1.2218952178955078e-06,
|
||
|
|
"learning_rate": 6.112258071729999e-06,
|
||
|
|
"logits/chosen": -3.2053667412365074,
|
||
|
|
"logits/rejected": -2.9334120491094025,
|
||
|
|
"logps/chosen": -96.13532671928405,
|
||
|
|
"logps/rejected": -364.0920166015625,
|
||
|
|
"loss": 1.0310853326700454e-08,
|
||
|
|
"mean_token_accuracy": 0.5550614651292562,
|
||
|
|
"num_tokens": 10365037.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5084194824099542,
|
||
|
|
"rewards/margins": 22.576842379570007,
|
||
|
|
"rewards/rejected": -21.068422937393187,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2330439381110363,
|
||
|
|
"epoch": 2.0082559339525283,
|
||
|
|
"grad_norm": 8.754432201385498e-07,
|
||
|
|
"learning_rate": 5.965311743141904e-06,
|
||
|
|
"logits/chosen": -3.208575671654014,
|
||
|
|
"logits/rejected": -2.9326438852599463,
|
||
|
|
"logps/chosen": -91.21437239035582,
|
||
|
|
"logps/rejected": -363.5683351174379,
|
||
|
|
"loss": 2.8951188824066777e-08,
|
||
|
|
"mean_token_accuracy": 0.5454204422541153,
|
||
|
|
"num_tokens": 10438552.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3954702159151053,
|
||
|
|
"rewards/margins": 22.230517093951885,
|
||
|
|
"rewards/rejected": -20.83504707385332,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2330299749970437,
|
||
|
|
"epoch": 2.022015823873409,
|
||
|
|
"grad_norm": 3.166496753692627e-07,
|
||
|
|
"learning_rate": 5.819398794098965e-06,
|
||
|
|
"logits/chosen": -3.1973997733416484,
|
||
|
|
"logits/rejected": -2.9382980669061474,
|
||
|
|
"logps/chosen": -93.53767242431641,
|
||
|
|
"logps/rejected": -366.40000381469724,
|
||
|
|
"loss": 5.971180172537061e-09,
|
||
|
|
"mean_token_accuracy": 0.5506532412022352,
|
||
|
|
"num_tokens": 10508891.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3291676957160234,
|
||
|
|
"rewards/margins": 22.63092885017395,
|
||
|
|
"rewards/rejected": -21.30176111459732,
|
||
|
|
"step": 1470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2254330806434155,
|
||
|
|
"epoch": 2.0357757137942896,
|
||
|
|
"grad_norm": 1.8477439880371094e-06,
|
||
|
|
"learning_rate": 5.674556596374993e-06,
|
||
|
|
"logits/chosen": -3.2044453379794327,
|
||
|
|
"logits/rejected": -2.9349608519596306,
|
||
|
|
"logps/chosen": -96.1356906414032,
|
||
|
|
"logps/rejected": -366.6023094177246,
|
||
|
|
"loss": 1.3861522063507437e-08,
|
||
|
|
"mean_token_accuracy": 0.5465929575264454,
|
||
|
|
"num_tokens": 10581981.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3926833651959896,
|
||
|
|
"rewards/margins": 22.933032035827637,
|
||
|
|
"rewards/rejected": -21.540348649024963,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2419807285070419,
|
||
|
|
"epoch": 2.0495356037151704,
|
||
|
|
"grad_norm": 3.635883331298828e-06,
|
||
|
|
"learning_rate": 5.530822247498926e-06,
|
||
|
|
"logits/chosen": -3.203796424428083,
|
||
|
|
"logits/rejected": -2.9470883391626623,
|
||
|
|
"logps/chosen": -92.74581410884858,
|
||
|
|
"logps/rejected": -362.62787170410155,
|
||
|
|
"loss": 6.57019540994952e-08,
|
||
|
|
"mean_token_accuracy": 0.5511626075953245,
|
||
|
|
"num_tokens": 10654203.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5442042749375104,
|
||
|
|
"rewards/margins": 22.415554237365722,
|
||
|
|
"rewards/rejected": -20.87134999036789,
|
||
|
|
"step": 1490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2162199713289739,
|
||
|
|
"epoch": 2.063295493636051,
|
||
|
|
"grad_norm": 2.682209014892578e-06,
|
||
|
|
"learning_rate": 5.388232561253301e-06,
|
||
|
|
"logits/chosen": -3.1954805100926533,
|
||
|
|
"logits/rejected": -2.9325630461122247,
|
||
|
|
"logps/chosen": -98.36348547935486,
|
||
|
|
"logps/rejected": -367.20038681030275,
|
||
|
|
"loss": 4.9491802656120853e-08,
|
||
|
|
"mean_token_accuracy": 0.5584958054125309,
|
||
|
|
"num_tokens": 10724322.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2897070567123592,
|
||
|
|
"rewards/margins": 22.65980746746063,
|
||
|
|
"rewards/rejected": -21.370100212097167,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.063295493636051,
|
||
|
|
"eval_entropy": 1.2309571816251168,
|
||
|
|
"eval_logits/chosen": -3.208181417563463,
|
||
|
|
"eval_logits/rejected": -2.927348986347919,
|
||
|
|
"eval_logps/chosen": -94.22639578775643,
|
||
|
|
"eval_logps/rejected": -359.94768000584025,
|
||
|
|
"eval_loss": 9.004938306134136e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5473804468033361,
|
||
|
|
"eval_num_tokens": 10724322.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5589469410235586,
|
||
|
|
"eval_rewards/margins": 22.525849155351228,
|
||
|
|
"eval_rewards/rejected": -20.966902284061206,
|
||
|
|
"eval_runtime": 56.0305,
|
||
|
|
"eval_samples_per_second": 10.923,
|
||
|
|
"eval_steps_per_second": 5.461,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2275754123926164,
|
||
|
|
"epoch": 2.0770553835569316,
|
||
|
|
"grad_norm": 1.4826655387878418e-06,
|
||
|
|
"learning_rate": 5.24682405824535e-06,
|
||
|
|
"logits/chosen": -3.2175959418317355,
|
||
|
|
"logits/rejected": -2.9383502084395783,
|
||
|
|
"logps/chosen": -92.50881714820862,
|
||
|
|
"logps/rejected": -371.0841016769409,
|
||
|
|
"loss": 2.6978818823408802e-08,
|
||
|
|
"mean_token_accuracy": 0.55126539953053,
|
||
|
|
"num_tokens": 10800237.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4649929033592344,
|
||
|
|
"rewards/margins": 22.734590768814087,
|
||
|
|
"rewards/rejected": -21.269598031044005,
|
||
|
|
"step": 1510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2253120809793472,
|
||
|
|
"epoch": 2.090815273477812,
|
||
|
|
"grad_norm": 1.5124678611755371e-06,
|
||
|
|
"learning_rate": 5.106632956553195e-06,
|
||
|
|
"logits/chosen": -3.1973274281094786,
|
||
|
|
"logits/rejected": -2.9305039646676687,
|
||
|
|
"logps/chosen": -90.93314514160156,
|
||
|
|
"logps/rejected": -365.3414768218994,
|
||
|
|
"loss": 1.938408900059585e-08,
|
||
|
|
"mean_token_accuracy": 0.5518848109990359,
|
||
|
|
"num_tokens": 10871898.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.426638696854934,
|
||
|
|
"rewards/margins": 22.47057647705078,
|
||
|
|
"rewards/rejected": -21.043937778472902,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.233763538300991,
|
||
|
|
"epoch": 2.104575163398693,
|
||
|
|
"grad_norm": 2.2165477275848389e-07,
|
||
|
|
"learning_rate": 4.967695162449564e-06,
|
||
|
|
"logits/chosen": -3.205103492080074,
|
||
|
|
"logits/rejected": -2.9324556564221185,
|
||
|
|
"logps/chosen": -91.14850578308105,
|
||
|
|
"logps/rejected": -364.838455581665,
|
||
|
|
"loss": 3.638180317011574e-08,
|
||
|
|
"mean_token_accuracy": 0.5595021311193704,
|
||
|
|
"num_tokens": 10941931.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4626295033842325,
|
||
|
|
"rewards/margins": 22.897762179374695,
|
||
|
|
"rewards/rejected": -21.43513250350952,
|
||
|
|
"step": 1530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2216242380440234,
|
||
|
|
"epoch": 2.1183350533195733,
|
||
|
|
"grad_norm": 5.3942203521728516e-06,
|
||
|
|
"learning_rate": 4.830046261205322e-06,
|
||
|
|
"logits/chosen": -3.2007257440208505,
|
||
|
|
"logits/rejected": -2.921705737174231,
|
||
|
|
"logps/chosen": -86.60471982955933,
|
||
|
|
"logps/rejected": -351.9167873382568,
|
||
|
|
"loss": 1.817294901229616e-08,
|
||
|
|
"mean_token_accuracy": 0.5593262724578381,
|
||
|
|
"num_tokens": 11007312.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4826515832915903,
|
||
|
|
"rewards/margins": 21.87851518392563,
|
||
|
|
"rewards/rejected": -20.395863604545593,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2366574123501777,
|
||
|
|
"epoch": 2.132094943240454,
|
||
|
|
"grad_norm": 3.427267074584961e-06,
|
||
|
|
"learning_rate": 4.693721507975237e-06,
|
||
|
|
"logits/chosen": -3.2084120035887294,
|
||
|
|
"logits/rejected": -2.9482167846435265,
|
||
|
|
"logps/chosen": -88.57906460762024,
|
||
|
|
"logps/rejected": -358.5270462036133,
|
||
|
|
"loss": 3.240177193220006e-08,
|
||
|
|
"mean_token_accuracy": 0.5516232702881098,
|
||
|
|
"num_tokens": 11084161.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.534480356797576,
|
||
|
|
"rewards/margins": 21.942111873626708,
|
||
|
|
"rewards/rejected": -20.40763156414032,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2354340434074402,
|
||
|
|
"epoch": 2.1458548331613345,
|
||
|
|
"grad_norm": 4.559755325317383e-06,
|
||
|
|
"learning_rate": 4.558755818768313e-06,
|
||
|
|
"logits/chosen": -3.2086426875744265,
|
||
|
|
"logits/rejected": -2.930807029613111,
|
||
|
|
"logps/chosen": -97.39483575820923,
|
||
|
|
"logps/rejected": -363.5681007385254,
|
||
|
|
"loss": 7.447766847690218e-08,
|
||
|
|
"mean_token_accuracy": 0.541871952265501,
|
||
|
|
"num_tokens": 11162793.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2772513107396661,
|
||
|
|
"rewards/margins": 22.164503431320192,
|
||
|
|
"rewards/rejected": -20.887252259254456,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.249245548248291,
|
||
|
|
"epoch": 2.1596147230822154,
|
||
|
|
"grad_norm": 1.2069940567016602e-06,
|
||
|
|
"learning_rate": 4.425183761504959e-06,
|
||
|
|
"logits/chosen": -3.215003107093657,
|
||
|
|
"logits/rejected": -2.93700778830525,
|
||
|
|
"logps/chosen": -96.33298497200012,
|
||
|
|
"logps/rejected": -364.01307106018066,
|
||
|
|
"loss": 2.6582091550153562e-08,
|
||
|
|
"mean_token_accuracy": 0.5509772460907698,
|
||
|
|
"num_tokens": 11233429.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.645207718014717,
|
||
|
|
"rewards/margins": 22.903429484367372,
|
||
|
|
"rewards/rejected": -21.258221745491028,
|
||
|
|
"step": 1570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2453189969062806,
|
||
|
|
"epoch": 2.173374613003096,
|
||
|
|
"grad_norm": 4.00543212890625e-05,
|
||
|
|
"learning_rate": 4.293039547163317e-06,
|
||
|
|
"logits/chosen": -3.203883989884722,
|
||
|
|
"logits/rejected": -2.9435132493578235,
|
||
|
|
"logps/chosen": -96.69977416992188,
|
||
|
|
"logps/rejected": -356.1233158111572,
|
||
|
|
"loss": 1.787334326763812e-07,
|
||
|
|
"mean_token_accuracy": 0.5541251346468925,
|
||
|
|
"num_tokens": 11306856.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.564822706952691,
|
||
|
|
"rewards/margins": 22.051422691345216,
|
||
|
|
"rewards/rejected": -20.486599802970886,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2330203399062156,
|
||
|
|
"epoch": 2.1871345029239766,
|
||
|
|
"grad_norm": 9.059906005859375e-06,
|
||
|
|
"learning_rate": 4.162357021017024e-06,
|
||
|
|
"logits/chosen": -3.2046981821484644,
|
||
|
|
"logits/rejected": -2.9331812450409367,
|
||
|
|
"logps/chosen": -96.95174942016601,
|
||
|
|
"logps/rejected": -360.5036224365234,
|
||
|
|
"loss": 1.6826841431338834e-08,
|
||
|
|
"mean_token_accuracy": 0.5593572478741408,
|
||
|
|
"num_tokens": 11382019.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5032939210534095,
|
||
|
|
"rewards/margins": 22.360925555229187,
|
||
|
|
"rewards/rejected": -20.857631814479827,
|
||
|
|
"step": 1590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2257887080311776,
|
||
|
|
"epoch": 2.2008943928448574,
|
||
|
|
"grad_norm": 1.259148120880127e-06,
|
||
|
|
"learning_rate": 4.033169653966643e-06,
|
||
|
|
"logits/chosen": -3.194304146133926,
|
||
|
|
"logits/rejected": -2.929467087673367,
|
||
|
|
"logps/chosen": -90.85989444255829,
|
||
|
|
"logps/rejected": -357.8446460723877,
|
||
|
|
"loss": 1.9598689959821058e-08,
|
||
|
|
"mean_token_accuracy": 0.5546761188656092,
|
||
|
|
"num_tokens": 11450315.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3403418456204235,
|
||
|
|
"rewards/margins": 22.388022828102113,
|
||
|
|
"rewards/rejected": -21.0476811170578,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2008943928448574,
|
||
|
|
"eval_entropy": 1.2307752834426031,
|
||
|
|
"eval_logits/chosen": -3.208144838054297,
|
||
|
|
"eval_logits/rejected": -2.927330668770716,
|
||
|
|
"eval_logps/chosen": -94.22898377935871,
|
||
|
|
"eval_logps/rejected": -360.023319817836,
|
||
|
|
"eval_loss": 8.89587852270779e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5471583587281844,
|
||
|
|
"eval_num_tokens": 11450315.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5586880991856258,
|
||
|
|
"eval_rewards/margins": 22.533154122969684,
|
||
|
|
"eval_rewards/rejected": -20.974466049593257,
|
||
|
|
"eval_runtime": 57.2229,
|
||
|
|
"eval_samples_per_second": 10.695,
|
||
|
|
"eval_steps_per_second": 5.348,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2212409198284149,
|
||
|
|
"epoch": 2.214654282765738,
|
||
|
|
"grad_norm": 1.0073184967041016e-05,
|
||
|
|
"learning_rate": 3.905510533966959e-06,
|
||
|
|
"logits/chosen": -3.214314473031172,
|
||
|
|
"logits/rejected": -2.9391068323773606,
|
||
|
|
"logps/chosen": -91.97887825965881,
|
||
|
|
"logps/rejected": -363.52599411010743,
|
||
|
|
"loss": 1.9494005698561522e-08,
|
||
|
|
"mean_token_accuracy": 0.5564212694764137,
|
||
|
|
"num_tokens": 11526978.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.525117878522724,
|
||
|
|
"rewards/margins": 22.85406105518341,
|
||
|
|
"rewards/rejected": -21.328943157196044,
|
||
|
|
"step": 1610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2366786934435368,
|
||
|
|
"epoch": 2.2284141726866187,
|
||
|
|
"grad_norm": 1.1324882507324219e-05,
|
||
|
|
"learning_rate": 3.7794123575523646e-06,
|
||
|
|
"logits/chosen": -3.2020117108366244,
|
||
|
|
"logits/rejected": -2.941936707092504,
|
||
|
|
"logps/chosen": -95.76478714942932,
|
||
|
|
"logps/rejected": -358.69820404052734,
|
||
|
|
"loss": 6.403648740160861e-08,
|
||
|
|
"mean_token_accuracy": 0.5473370250314474,
|
||
|
|
"num_tokens": 11597783.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4697597675025462,
|
||
|
|
"rewards/margins": 22.156175112724306,
|
||
|
|
"rewards/rejected": -20.686415362358094,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2357837766408921,
|
||
|
|
"epoch": 2.242174062607499,
|
||
|
|
"grad_norm": 3.266334533691406e-05,
|
||
|
|
"learning_rate": 3.6549074214625004e-06,
|
||
|
|
"logits/chosen": -3.207695843054857,
|
||
|
|
"logits/rejected": -2.931019461868485,
|
||
|
|
"logps/chosen": -95.36823711395263,
|
||
|
|
"logps/rejected": -364.51545028686525,
|
||
|
|
"loss": 1.6160251448127384e-08,
|
||
|
|
"mean_token_accuracy": 0.5486490651965141,
|
||
|
|
"num_tokens": 11667690.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.452584043238312,
|
||
|
|
"rewards/margins": 22.741941976547242,
|
||
|
|
"rewards/rejected": -21.28935799598694,
|
||
|
|
"step": 1630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2502774253487587,
|
||
|
|
"epoch": 2.25593395252838,
|
||
|
|
"grad_norm": 1.2040138244628906e-05,
|
||
|
|
"learning_rate": 3.532027614370309e-06,
|
||
|
|
"logits/chosen": -3.2043903527517044,
|
||
|
|
"logits/rejected": -2.9334508568932933,
|
||
|
|
"logps/chosen": -92.87388520240783,
|
||
|
|
"logps/rejected": -353.56325378417966,
|
||
|
|
"loss": 1.9827470509881095e-08,
|
||
|
|
"mean_token_accuracy": 0.5545094706118107,
|
||
|
|
"num_tokens": 11736002.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.63247759337537,
|
||
|
|
"rewards/margins": 22.165972471237183,
|
||
|
|
"rewards/rejected": -20.53349506855011,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.223061728477478,
|
||
|
|
"epoch": 2.2696938424492603,
|
||
|
|
"grad_norm": 3.9674341678619385e-07,
|
||
|
|
"learning_rate": 3.4108044087145764e-06,
|
||
|
|
"logits/chosen": -3.212080244158158,
|
||
|
|
"logits/rejected": -2.9380040624704558,
|
||
|
|
"logps/chosen": -91.98516411781311,
|
||
|
|
"logps/rejected": -363.08466148376465,
|
||
|
|
"loss": 2.2882196049067714e-08,
|
||
|
|
"mean_token_accuracy": 0.5499565359205008,
|
||
|
|
"num_tokens": 11806524.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5206007588654757,
|
||
|
|
"rewards/margins": 22.941864371299744,
|
||
|
|
"rewards/rejected": -21.42126350402832,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2240382507443428,
|
||
|
|
"epoch": 2.283453732370141,
|
||
|
|
"grad_norm": 4.9173831939697266e-06,
|
||
|
|
"learning_rate": 3.2912688526390747e-06,
|
||
|
|
"logits/chosen": -3.201106158061068,
|
||
|
|
"logits/rejected": -2.94364090708882,
|
||
|
|
"logps/chosen": -90.90681099891663,
|
||
|
|
"logps/rejected": -357.92090034484863,
|
||
|
|
"loss": 3.113183311143075e-08,
|
||
|
|
"mean_token_accuracy": 0.5508798561990261,
|
||
|
|
"num_tokens": 11877466.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3562086425721644,
|
||
|
|
"rewards/margins": 22.361026167869568,
|
||
|
|
"rewards/rejected": -21.004817533493043,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.229287575185299,
|
||
|
|
"epoch": 2.2972136222910216,
|
||
|
|
"grad_norm": 5.662441253662109e-07,
|
||
|
|
"learning_rate": 3.173451562040428e-06,
|
||
|
|
"logits/chosen": -3.2092518895630406,
|
||
|
|
"logits/rejected": -2.9406627581018197,
|
||
|
|
"logps/chosen": -93.75379481315613,
|
||
|
|
"logps/rejected": -355.3395336151123,
|
||
|
|
"loss": 1.1599519211813458e-07,
|
||
|
|
"mean_token_accuracy": 0.5449749272316694,
|
||
|
|
"num_tokens": 11946957.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5091715334914624,
|
||
|
|
"rewards/margins": 22.31476628780365,
|
||
|
|
"rewards/rejected": -20.80559469461441,
|
||
|
|
"step": 1670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2276184171438218,
|
||
|
|
"epoch": 2.3109735122119024,
|
||
|
|
"grad_norm": 1.6093254089355469e-06,
|
||
|
|
"learning_rate": 3.0573827127266388e-06,
|
||
|
|
"logits/chosen": -3.211323672958944,
|
||
|
|
"logits/rejected": -2.9291713412088,
|
||
|
|
"logps/chosen": -92.78370037078858,
|
||
|
|
"logps/rejected": -362.05942516326905,
|
||
|
|
"loss": 4.019505581709382e-08,
|
||
|
|
"mean_token_accuracy": 0.5594781070947648,
|
||
|
|
"num_tokens": 12019258.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4811159919947385,
|
||
|
|
"rewards/margins": 22.234603309631346,
|
||
|
|
"rewards/rejected": -20.753487145900728,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.243286745250225,
|
||
|
|
"epoch": 2.324733402132783,
|
||
|
|
"grad_norm": 4.116445779800415e-07,
|
||
|
|
"learning_rate": 2.9430920326883406e-06,
|
||
|
|
"logits/chosen": -3.198338749012816,
|
||
|
|
"logits/rejected": -2.936215425086737,
|
||
|
|
"logps/chosen": -97.7242082118988,
|
||
|
|
"logps/rejected": -358.0124053955078,
|
||
|
|
"loss": 3.1689586421634884e-08,
|
||
|
|
"mean_token_accuracy": 0.536598714068532,
|
||
|
|
"num_tokens": 12086564.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2556954478845,
|
||
|
|
"rewards/margins": 22.314459955692293,
|
||
|
|
"rewards/rejected": -21.058764576911926,
|
||
|
|
"step": 1690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2481802843511105,
|
||
|
|
"epoch": 2.3384932920536636,
|
||
|
|
"grad_norm": 1.080334186553955e-06,
|
||
|
|
"learning_rate": 2.830608794484756e-06,
|
||
|
|
"logits/chosen": -3.1943609337265597,
|
||
|
|
"logits/rejected": -2.9488356300852865,
|
||
|
|
"logps/chosen": -89.5389600276947,
|
||
|
|
"logps/rejected": -359.9536491394043,
|
||
|
|
"loss": 2.3795587367203554e-08,
|
||
|
|
"mean_token_accuracy": 0.5535460270941257,
|
||
|
|
"num_tokens": 12158539.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.362912526819855,
|
||
|
|
"rewards/margins": 22.151762986183165,
|
||
|
|
"rewards/rejected": -20.78885051012039,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3384932920536636,
|
||
|
|
"eval_entropy": 1.2307304270516812,
|
||
|
|
"eval_logits/chosen": -3.2081578712434293,
|
||
|
|
"eval_logits/rejected": -2.9272117784230485,
|
||
|
|
"eval_logps/chosen": -94.22525123521393,
|
||
|
|
"eval_logps/rejected": -360.04312303331164,
|
||
|
|
"eval_loss": 8.878492252506476e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5476229925171222,
|
||
|
|
"eval_num_tokens": 12158539.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5590613652142435,
|
||
|
|
"eval_rewards/margins": 22.535507769366497,
|
||
|
|
"eval_rewards/rejected": -20.976446469624836,
|
||
|
|
"eval_runtime": 56.6188,
|
||
|
|
"eval_samples_per_second": 10.809,
|
||
|
|
"eval_steps_per_second": 5.405,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2194796189665795,
|
||
|
|
"epoch": 2.352253181974544,
|
||
|
|
"grad_norm": 2.905726432800293e-06,
|
||
|
|
"learning_rate": 2.7199618077463007e-06,
|
||
|
|
"logits/chosen": -3.2052349518897323,
|
||
|
|
"logits/rejected": -2.938724784978354,
|
||
|
|
"logps/chosen": -97.47988610267639,
|
||
|
|
"logps/rejected": -361.4190784454346,
|
||
|
|
"loss": 1.1008413025592744e-08,
|
||
|
|
"mean_token_accuracy": 0.5539489138871432,
|
||
|
|
"num_tokens": 12228074.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5363455599173903,
|
||
|
|
"rewards/margins": 22.715231454372407,
|
||
|
|
"rewards/rejected": -21.178886151313783,
|
||
|
|
"step": 1710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2418527334928513,
|
||
|
|
"epoch": 2.366013071895425,
|
||
|
|
"grad_norm": 9.012222290039062e-05,
|
||
|
|
"learning_rate": 2.6111794117957612e-06,
|
||
|
|
"logits/chosen": -3.211556303584559,
|
||
|
|
"logits/rejected": -2.93789795511538,
|
||
|
|
"logps/chosen": -93.4774594783783,
|
||
|
|
"logps/rejected": -356.88815116882324,
|
||
|
|
"loss": 4.893282152806933e-08,
|
||
|
|
"mean_token_accuracy": 0.547554525360465,
|
||
|
|
"num_tokens": 12292110.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5211389977484941,
|
||
|
|
"rewards/margins": 22.537738013267518,
|
||
|
|
"rewards/rejected": -21.016598844528197,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.240048848092556,
|
||
|
|
"epoch": 2.3797729618163057,
|
||
|
|
"grad_norm": 4.506111145019531e-05,
|
||
|
|
"learning_rate": 2.504289468389899e-06,
|
||
|
|
"logits/chosen": -3.206205666054303,
|
||
|
|
"logits/rejected": -2.943638671152749,
|
||
|
|
"logps/chosen": -90.50992622375489,
|
||
|
|
"logps/rejected": -362.01258277893066,
|
||
|
|
"loss": 8.840201530802005e-08,
|
||
|
|
"mean_token_accuracy": 0.5487963829189539,
|
||
|
|
"num_tokens": 12358835.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.542650154232979,
|
||
|
|
"rewards/margins": 22.581651854515076,
|
||
|
|
"rewards/rejected": -21.039001631736756,
|
||
|
|
"step": 1730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2210087485611438,
|
||
|
|
"epoch": 2.393532851737186,
|
||
|
|
"grad_norm": 5.650520324707031e-05,
|
||
|
|
"learning_rate": 2.3993193545834182e-06,
|
||
|
|
"logits/chosen": -3.2105181749109306,
|
||
|
|
"logits/rejected": -2.931494112684553,
|
||
|
|
"logps/chosen": -92.28900375366212,
|
||
|
|
"logps/rejected": -364.4213104248047,
|
||
|
|
"loss": 7.528050787186657e-08,
|
||
|
|
"mean_token_accuracy": 0.5661897122859955,
|
||
|
|
"num_tokens": 12433046.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4046879611909389,
|
||
|
|
"rewards/margins": 22.549579215049743,
|
||
|
|
"rewards/rejected": -21.14489116668701,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.230967465043068,
|
||
|
|
"epoch": 2.4072927416580665,
|
||
|
|
"grad_norm": 4.4405460357666016e-06,
|
||
|
|
"learning_rate": 2.2962959557170284e-06,
|
||
|
|
"logits/chosen": -3.214696198045111,
|
||
|
|
"logits/rejected": -2.92904874748245,
|
||
|
|
"logps/chosen": -96.72074069976807,
|
||
|
|
"logps/rejected": -371.5300548553467,
|
||
|
|
"loss": 7.795250667186338e-09,
|
||
|
|
"mean_token_accuracy": 0.5476579256355762,
|
||
|
|
"num_tokens": 12510093.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5278369845356792,
|
||
|
|
"rewards/margins": 22.89517023563385,
|
||
|
|
"rewards/rejected": -21.367333316802977,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2341578394174575,
|
||
|
|
"epoch": 2.4210526315789473,
|
||
|
|
"grad_norm": 6.765127182006836e-06,
|
||
|
|
"learning_rate": 2.1952456585314894e-06,
|
||
|
|
"logits/chosen": -3.212110355605135,
|
||
|
|
"logits/rejected": -2.9278311870022096,
|
||
|
|
"logps/chosen": -94.76394276618957,
|
||
|
|
"logps/rejected": -366.80492820739744,
|
||
|
|
"loss": 1.8708394122768367e-08,
|
||
|
|
"mean_token_accuracy": 0.5509970057755709,
|
||
|
|
"num_tokens": 12582912.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5956923257559539,
|
||
|
|
"rewards/margins": 22.885032773017883,
|
||
|
|
"rewards/rejected": -21.289340591430665,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2361514404416085,
|
||
|
|
"epoch": 2.434812521499828,
|
||
|
|
"grad_norm": 7.37607479095459e-07,
|
||
|
|
"learning_rate": 2.0961943444093204e-06,
|
||
|
|
"logits/chosen": -3.215165102721952,
|
||
|
|
"logits/rejected": -2.928150886837859,
|
||
|
|
"logps/chosen": -96.75569877624511,
|
||
|
|
"logps/rejected": -358.2433895111084,
|
||
|
|
"loss": 1.789561281384522e-08,
|
||
|
|
"mean_token_accuracy": 0.5518970482051373,
|
||
|
|
"num_tokens": 12653031.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4577152743004262,
|
||
|
|
"rewards/margins": 22.373184776306154,
|
||
|
|
"rewards/rejected": -20.915469527244568,
|
||
|
|
"step": 1770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2188403598964215,
|
||
|
|
"epoch": 2.4485724114207086,
|
||
|
|
"grad_norm": 7.271766662597656e-06,
|
||
|
|
"learning_rate": 1.9991673827459835e-06,
|
||
|
|
"logits/chosen": -3.1989314300688627,
|
||
|
|
"logits/rejected": -2.9344233134383053,
|
||
|
|
"logps/chosen": -92.87870864868164,
|
||
|
|
"logps/rejected": -356.7177398681641,
|
||
|
|
"loss": 4.4659608988695255e-08,
|
||
|
|
"mean_token_accuracy": 0.5518682572990656,
|
||
|
|
"num_tokens": 12725637.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4662983087822794,
|
||
|
|
"rewards/margins": 22.44385893344879,
|
||
|
|
"rewards/rejected": -20.977560567855836,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.246582841873169,
|
||
|
|
"epoch": 2.4623323013415894,
|
||
|
|
"grad_norm": 6.628036499023438e-05,
|
||
|
|
"learning_rate": 1.9041896244521584e-06,
|
||
|
|
"logits/chosen": -3.203583763033077,
|
||
|
|
"logits/rejected": -2.9414372493750873,
|
||
|
|
"logps/chosen": -89.14191975593567,
|
||
|
|
"logps/rejected": -357.88146629333494,
|
||
|
|
"loss": 2.0817780921333907e-08,
|
||
|
|
"mean_token_accuracy": 0.5476832162588835,
|
||
|
|
"num_tokens": 12793669.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.506299409084022,
|
||
|
|
"rewards/margins": 22.251666259765624,
|
||
|
|
"rewards/rejected": -20.745366990566254,
|
||
|
|
"step": 1790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2349150389432908,
|
||
|
|
"epoch": 2.47609219126247,
|
||
|
|
"grad_norm": 3.5762786865234375e-05,
|
||
|
|
"learning_rate": 1.8112853955888543e-06,
|
||
|
|
"logits/chosen": -3.2071731207232768,
|
||
|
|
"logits/rejected": -2.936215464591402,
|
||
|
|
"logps/chosen": -90.68877263069153,
|
||
|
|
"logps/rejected": -355.58887882232665,
|
||
|
|
"loss": 3.7140389963497e-08,
|
||
|
|
"mean_token_accuracy": 0.5518446356058121,
|
||
|
|
"num_tokens": 12863146.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4937029276043177,
|
||
|
|
"rewards/margins": 22.40243979692459,
|
||
|
|
"rewards/rejected": -20.90873694419861,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.47609219126247,
|
||
|
|
"eval_entropy": 1.2310942107555913,
|
||
|
|
"eval_logits/chosen": -3.208092843314108,
|
||
|
|
"eval_logits/rejected": -2.9271828156565594,
|
||
|
|
"eval_logps/chosen": -94.19586368635589,
|
||
|
|
"eval_logps/rejected": -359.9574696659263,
|
||
|
|
"eval_loss": 9.066506834187749e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5472219296140608,
|
||
|
|
"eval_num_tokens": 12863146.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5620001425443133,
|
||
|
|
"eval_rewards/margins": 22.529881178164015,
|
||
|
|
"eval_rewards/rejected": -20.967881090500775,
|
||
|
|
"eval_runtime": 58.2977,
|
||
|
|
"eval_samples_per_second": 10.498,
|
||
|
|
"eval_steps_per_second": 5.249,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2429441884160042,
|
||
|
|
"epoch": 2.4898520811833507,
|
||
|
|
"grad_norm": 3.293156623840332e-06,
|
||
|
|
"learning_rate": 1.7204784911369043e-06,
|
||
|
|
"logits/chosen": -3.2072891811345925,
|
||
|
|
"logits/rejected": -2.9434615661283723,
|
||
|
|
"logps/chosen": -91.1972602367401,
|
||
|
|
"logps/rejected": -373.37213764190676,
|
||
|
|
"loss": 7.088502798069385e-08,
|
||
|
|
"mean_token_accuracy": 0.5530083872377872,
|
||
|
|
"num_tokens": 12939453.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6164010954089463,
|
||
|
|
"rewards/margins": 23.070138931274414,
|
||
|
|
"rewards/rejected": -21.45373786687851,
|
||
|
|
"step": 1810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2349141895771027,
|
||
|
|
"epoch": 2.503611971104231,
|
||
|
|
"grad_norm": 2.0563602447509766e-06,
|
||
|
|
"learning_rate": 1.6317921689025307e-06,
|
||
|
|
"logits/chosen": -3.2110918760815195,
|
||
|
|
"logits/rejected": -2.9284428531035287,
|
||
|
|
"logps/chosen": -93.45727353096008,
|
||
|
|
"logps/rejected": -359.75511894226076,
|
||
|
|
"loss": 1.1874520851051784e-07,
|
||
|
|
"mean_token_accuracy": 0.5524209752678871,
|
||
|
|
"num_tokens": 13005628.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4790690172463656,
|
||
|
|
"rewards/margins": 22.616058588027954,
|
||
|
|
"rewards/rejected": -21.136989569664003,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2337662652134895,
|
||
|
|
"epoch": 2.517371861025112,
|
||
|
|
"grad_norm": 3.0547380447387695e-06,
|
||
|
|
"learning_rate": 1.5452491435604467e-06,
|
||
|
|
"logits/chosen": -3.2113447433852804,
|
||
|
|
"logits/rejected": -2.940791876036429,
|
||
|
|
"logps/chosen": -88.05490498542785,
|
||
|
|
"logps/rejected": -356.84777145385743,
|
||
|
|
"loss": 3.411617228721298e-08,
|
||
|
|
"mean_token_accuracy": 0.5570485234260559,
|
||
|
|
"num_tokens": 13079021.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5968380993232132,
|
||
|
|
"rewards/margins": 22.486863994598387,
|
||
|
|
"rewards/rejected": -20.890025854110718,
|
||
|
|
"step": 1830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2261295109987258,
|
||
|
|
"epoch": 2.5311317509459923,
|
||
|
|
"grad_norm": 1.1026859283447266e-05,
|
||
|
|
"learning_rate": 1.460871580836095e-06,
|
||
|
|
"logits/chosen": -3.213007126401073,
|
||
|
|
"logits/rejected": -2.9374573601576284,
|
||
|
|
"logps/chosen": -90.18659546375275,
|
||
|
|
"logps/rejected": -362.702806854248,
|
||
|
|
"loss": 7.099046683833876e-09,
|
||
|
|
"mean_token_accuracy": 0.5493871968239545,
|
||
|
|
"num_tokens": 13147233.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5181418016552926,
|
||
|
|
"rewards/margins": 22.71970386505127,
|
||
|
|
"rewards/rejected": -21.201562070846556,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2428302437067031,
|
||
|
|
"epoch": 2.544891640866873,
|
||
|
|
"grad_norm": 1.633167266845703e-05,
|
||
|
|
"learning_rate": 1.3786810918284877e-06,
|
||
|
|
"logits/chosen": -3.2022471548135534,
|
||
|
|
"logits/rejected": -2.9372537430710937,
|
||
|
|
"logps/chosen": -95.1559627532959,
|
||
|
|
"logps/rejected": -360.5957592010498,
|
||
|
|
"loss": 1.7248763128918654e-08,
|
||
|
|
"mean_token_accuracy": 0.5445155739784241,
|
||
|
|
"num_tokens": 13217763.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5004485316574574,
|
||
|
|
"rewards/margins": 22.449256420135498,
|
||
|
|
"rewards/rejected": -20.948807954788208,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.235001514852047,
|
||
|
|
"epoch": 2.5586515307877535,
|
||
|
|
"grad_norm": 3.4868717193603516e-06,
|
||
|
|
"learning_rate": 1.2986987274750829e-06,
|
||
|
|
"logits/chosen": -3.196346249542863,
|
||
|
|
"logits/rejected": -2.929766749069544,
|
||
|
|
"logps/chosen": -91.7070778131485,
|
||
|
|
"logps/rejected": -363.7645969390869,
|
||
|
|
"loss": 1.4013249938216177e-08,
|
||
|
|
"mean_token_accuracy": 0.5452596392482519,
|
||
|
|
"num_tokens": 13291621.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5715331200510263,
|
||
|
|
"rewards/margins": 22.702108526229857,
|
||
|
|
"rewards/rejected": -21.130575299263,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2162961602210998,
|
||
|
|
"epoch": 2.5724114207086344,
|
||
|
|
"grad_norm": 1.1995434761047363e-06,
|
||
|
|
"learning_rate": 1.220944973160133e-06,
|
||
|
|
"logits/chosen": -3.213586161821438,
|
||
|
|
"logits/rejected": -2.9203404315066175,
|
||
|
|
"logps/chosen": -89.05219874382018,
|
||
|
|
"logps/rejected": -359.9140110015869,
|
||
|
|
"loss": 4.524198402577895e-08,
|
||
|
|
"mean_token_accuracy": 0.5608073528856039,
|
||
|
|
"num_tokens": 13360260.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5491325071081519,
|
||
|
|
"rewards/margins": 22.66431896686554,
|
||
|
|
"rewards/rejected": -21.1151864528656,
|
||
|
|
"step": 1870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2236336886882782,
|
||
|
|
"epoch": 2.586171310629515,
|
||
|
|
"grad_norm": 3.7997961044311523e-07,
|
||
|
|
"learning_rate": 1.1454397434679022e-06,
|
||
|
|
"logits/chosen": -3.2053671717959227,
|
||
|
|
"logits/rejected": -2.925974566084328,
|
||
|
|
"logps/chosen": -92.41997270584106,
|
||
|
|
"logps/rejected": -361.17205810546875,
|
||
|
|
"loss": 5.012340125176706e-08,
|
||
|
|
"mean_token_accuracy": 0.5515494048595428,
|
||
|
|
"num_tokens": 13431334.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5517112540081144,
|
||
|
|
"rewards/margins": 22.80179090499878,
|
||
|
|
"rewards/rejected": -21.250079607963563,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2449517458677293,
|
||
|
|
"epoch": 2.5999312005503956,
|
||
|
|
"grad_norm": 1.2218952178955078e-06,
|
||
|
|
"learning_rate": 1.0722023770820622e-06,
|
||
|
|
"logits/chosen": -3.208692543622597,
|
||
|
|
"logits/rejected": -2.951054135339878,
|
||
|
|
"logps/chosen": -94.28899235725403,
|
||
|
|
"logps/rejected": -367.7868989944458,
|
||
|
|
"loss": 1.7088214576688187e-08,
|
||
|
|
"mean_token_accuracy": 0.541762464120984,
|
||
|
|
"num_tokens": 13501056.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.5825941693037748,
|
||
|
|
"rewards/margins": 22.99413607120514,
|
||
|
|
"rewards/rejected": -21.41154201030731,
|
||
|
|
"step": 1890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2336580827832222,
|
||
|
|
"epoch": 2.613691090471276,
|
||
|
|
"grad_norm": 2.086162567138672e-06,
|
||
|
|
"learning_rate": 1.0012516318326014e-06,
|
||
|
|
"logits/chosen": -3.2122659432292493,
|
||
|
|
"logits/rejected": -2.9348672913748013,
|
||
|
|
"logps/chosen": -94.13200654983521,
|
||
|
|
"logps/rejected": -360.9386867523193,
|
||
|
|
"loss": 1.2763308632202097e-08,
|
||
|
|
"mean_token_accuracy": 0.5477464541792869,
|
||
|
|
"num_tokens": 13570872.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4218829076737165,
|
||
|
|
"rewards/margins": 22.134320437908173,
|
||
|
|
"rewards/rejected": -20.71243748664856,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.613691090471276,
|
||
|
|
"eval_entropy": 1.231101697761249,
|
||
|
|
"eval_logits/chosen": -3.207994031495441,
|
||
|
|
"eval_logits/rejected": -2.9272202394592175,
|
||
|
|
"eval_logps/chosen": -94.21152074975905,
|
||
|
|
"eval_logps/rejected": -359.98469304103475,
|
||
|
|
"eval_loss": 8.744856216935659e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5469447381161396,
|
||
|
|
"eval_num_tokens": 13570872.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5604344270879926,
|
||
|
|
"eval_rewards/margins": 22.53103791966158,
|
||
|
|
"eval_rewards/rejected": -20.970603593813827,
|
||
|
|
"eval_runtime": 59.2861,
|
||
|
|
"eval_samples_per_second": 10.323,
|
||
|
|
"eval_steps_per_second": 5.161,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.227197167277336,
|
||
|
|
"epoch": 2.627450980392157,
|
||
|
|
"grad_norm": 2.428889274597168e-06,
|
||
|
|
"learning_rate": 9.326056798914895e-07,
|
||
|
|
"logits/chosen": -3.2164457588964277,
|
||
|
|
"logits/rejected": -2.942774935824112,
|
||
|
|
"logps/chosen": -84.70766513347625,
|
||
|
|
"logps/rejected": -362.8008171081543,
|
||
|
|
"loss": 1.0640263781169779e-08,
|
||
|
|
"mean_token_accuracy": 0.5610819097608328,
|
||
|
|
"num_tokens": 13647574.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4893750285729765,
|
||
|
|
"rewards/margins": 22.473388147354125,
|
||
|
|
"rewards/rejected": -20.984013080596924,
|
||
|
|
"step": 1910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2400737136602402,
|
||
|
|
"epoch": 2.6412108703130377,
|
||
|
|
"grad_norm": 5.811452865600586e-06,
|
||
|
|
"learning_rate": 8.662821031183566e-07,
|
||
|
|
"logits/chosen": -3.2042261467779616,
|
||
|
|
"logits/rejected": -2.9372568023284353,
|
||
|
|
"logps/chosen": -92.49354066848755,
|
||
|
|
"logps/rejected": -353.8560489654541,
|
||
|
|
"loss": 6.457948416027648e-08,
|
||
|
|
"mean_token_accuracy": 0.5623761888593435,
|
||
|
|
"num_tokens": 13715499.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.6017520420253277,
|
||
|
|
"rewards/margins": 22.434368634223937,
|
||
|
|
"rewards/rejected": -20.832616329193115,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2138798840343952,
|
||
|
|
"epoch": 2.654970760233918,
|
||
|
|
"grad_norm": 7.3015689849853516e-06,
|
||
|
|
"learning_rate": 8.022978885573662e-07,
|
||
|
|
"logits/chosen": -3.204353447210667,
|
||
|
|
"logits/rejected": -2.9360457918068823,
|
||
|
|
"logps/chosen": -94.42889518737793,
|
||
|
|
"logps/rejected": -361.6750244140625,
|
||
|
|
"loss": 1.7699032639484357e-08,
|
||
|
|
"mean_token_accuracy": 0.5568779963999987,
|
||
|
|
"num_tokens": 13788452.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.4762537024915219,
|
||
|
|
"rewards/margins": 22.71884949207306,
|
||
|
|
"rewards/rejected": -21.242595887184144,
|
||
|
|
"step": 1930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2365102216601371,
|
||
|
|
"epoch": 2.6687306501547985,
|
||
|
|
"grad_norm": 3.203749656677246e-06,
|
||
|
|
"learning_rate": 7.406694240864276e-07,
|
||
|
|
"logits/chosen": -3.191136028588488,
|
||
|
|
"logits/rejected": -2.9315919526530343,
|
||
|
|
"logps/chosen": -95.14295864105225,
|
||
|
|
"logps/rejected": -363.475435256958,
|
||
|
|
"loss": 5.760826979894773e-08,
|
||
|
|
"mean_token_accuracy": 0.5441556949168443,
|
||
|
|
"num_tokens": 13859857.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.2871343778446316,
|
||
|
|
"rewards/margins": 22.71376383304596,
|
||
|
|
"rewards/rejected": -21.42662957906723,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2252389818429947,
|
||
|
|
"epoch": 2.6824905400756793,
|
||
|
|
"grad_norm": 7.987022399902344e-06,
|
||
|
|
"learning_rate": 6.814124942198596e-07,
|
||
|
|
"logits/chosen": -3.2188158142258727,
|
||
|
|
"logits/rejected": -2.925274084382701,
|
||
|
|
"logps/chosen": -91.77870345115662,
|
||
|
|
"logps/rejected": -363.1586544036865,
|
||
|
|
"loss": 1.2229202184244059e-08,
|
||
|
|
"mean_token_accuracy": 0.5488399602472782,
|
||
|
|
"num_tokens": 13932968.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.640056242980063,
|
||
|
|
"rewards/margins": 22.991242933273316,
|
||
|
|
"rewards/rejected": -21.351186680793763,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2329418018460274,
|
||
|
|
"epoch": 2.69625042999656,
|
||
|
|
"grad_norm": 0.000152587890625,
|
||
|
|
"learning_rate": 6.245422760656173e-07,
|
||
|
|
"logits/chosen": -3.2162328344583018,
|
||
|
|
"logits/rejected": -2.931803809545019,
|
||
|
|
"logps/chosen": -93.28221130371094,
|
||
|
|
"logps/rejected": -357.8889636993408,
|
||
|
|
"loss": 1.2377095117699355e-07,
|
||
|
|
"mean_token_accuracy": 0.5366842743009329,
|
||
|
|
"num_tokens": 14001238.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.387164283171296,
|
||
|
|
"rewards/margins": 22.26145691871643,
|
||
|
|
"rewards/rejected": -20.87429268360138,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2324133336544036,
|
||
|
|
"epoch": 2.7100103199174406,
|
||
|
|
"grad_norm": 1.0132789611816406e-06,
|
||
|
|
"learning_rate": 5.700733354380672e-07,
|
||
|
|
"logits/chosen": -3.2115706194311855,
|
||
|
|
"logits/rejected": -2.9334242400065627,
|
||
|
|
"logps/chosen": -92.92520937919616,
|
||
|
|
"logps/rejected": -358.1720157623291,
|
||
|
|
"loss": 1.562668217047758e-08,
|
||
|
|
"mean_token_accuracy": 0.552143507450819,
|
||
|
|
"num_tokens": 14071079.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.521490297280252,
|
||
|
|
"rewards/margins": 22.51833870410919,
|
||
|
|
"rewards/rejected": -20.996848475933074,
|
||
|
|
"step": 1970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2193569228053094,
|
||
|
|
"epoch": 2.7237702098383214,
|
||
|
|
"grad_norm": 0.00011682510375976562,
|
||
|
|
"learning_rate": 5.180196231273316e-07,
|
||
|
|
"logits/chosen": -3.206004179527697,
|
||
|
|
"logits/rejected": -2.9225641136769545,
|
||
|
|
"logps/chosen": -92.5201644897461,
|
||
|
|
"logps/rejected": -355.29096069335935,
|
||
|
|
"loss": 1.1564081887627254e-07,
|
||
|
|
"mean_token_accuracy": 0.550432949885726,
|
||
|
|
"num_tokens": 14138316.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.48047917149961,
|
||
|
|
"rewards/margins": 22.245127046108244,
|
||
|
|
"rewards/rejected": -20.76464786529541,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1910540789365769,
|
||
|
|
"epoch": 2.737530099759202,
|
||
|
|
"grad_norm": 2.396106719970703e-05,
|
||
|
|
"learning_rate": 4.68394471326169e-07,
|
||
|
|
"logits/chosen": -3.198829245112273,
|
||
|
|
"logits/rejected": -2.9225497128968745,
|
||
|
|
"logps/chosen": -88.18752355575562,
|
||
|
|
"logps/rejected": -352.7966428756714,
|
||
|
|
"loss": 8.365720418623824e-08,
|
||
|
|
"mean_token_accuracy": 0.5554662171751261,
|
||
|
|
"num_tokens": 14206436.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.3859461545944214,
|
||
|
|
"rewards/margins": 22.12413411140442,
|
||
|
|
"rewards/rejected": -20.738187992572783,
|
||
|
|
"step": 1990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2391421049833298,
|
||
|
|
"epoch": 2.7512899896800826,
|
||
|
|
"grad_norm": 1.0073184967041016e-05,
|
||
|
|
"learning_rate": 4.21210590215273e-07,
|
||
|
|
"logits/chosen": -3.2132494305346264,
|
||
|
|
"logits/rejected": -2.9325754681344756,
|
||
|
|
"logps/chosen": -92.24013915061951,
|
||
|
|
"logps/rejected": -357.6173038482666,
|
||
|
|
"loss": 3.3882781735883325e-08,
|
||
|
|
"mean_token_accuracy": 0.5463012181222439,
|
||
|
|
"num_tokens": 14275387.0,
|
||
|
|
"rewards/accuracies": 1.0,
|
||
|
|
"rewards/chosen": 1.347787582874298,
|
||
|
|
"rewards/margins": 22.133311522006988,
|
||
|
|
"rewards/rejected": -20.7855238199234,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7512899896800826,
|
||
|
|
"eval_entropy": 1.2307997451109045,
|
||
|
|
"eval_logits/chosen": -3.2081586983940737,
|
||
|
|
"eval_logits/rejected": -2.9273201189702758,
|
||
|
|
"eval_logps/chosen": -94.20940250197268,
|
||
|
|
"eval_logps/rejected": -360.0332946777344,
|
||
|
|
"eval_loss": 9.027248637494267e-08,
|
||
|
|
"eval_mean_token_accuracy": 0.5471953758422066,
|
||
|
|
"eval_num_tokens": 14275387.0,
|
||
|
|
"eval_rewards/accuracies": 1.0,
|
||
|
|
"eval_rewards/chosen": 1.5606462517692372,
|
||
|
|
"eval_rewards/margins": 22.536109949249067,
|
||
|
|
"eval_rewards/rejected": -20.975463770573434,
|
||
|
|
"eval_runtime": 59.956,
|
||
|
|
"eval_samples_per_second": 10.207,
|
||
|
|
"eval_steps_per_second": 5.104,
|
||
|
|
"step": 2000
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 2181,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 200,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 6.758176153518735e+17,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|