7f970a698a
Model: W-61/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-s_star-0.4-20260425-111846 Source: Original Platform
9141 lines
335 KiB
JSON
9141 lines
335 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9989528795811519,
|
|
"eval_steps": 200,
|
|
"global_step": 477,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0020942408376963353,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.02793481945991516,
|
|
"fcm_dpo/q_t": 0.500069797039032,
|
|
"grad_norm": 28.588966369628906,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.5898098945617676,
|
|
"logits/rejected": -0.604260265827179,
|
|
"logps/chosen": -275.28570556640625,
|
|
"logps/ref_chosen": -275.2312927246094,
|
|
"logps/ref_rejected": -222.9380340576172,
|
|
"logps/rejected": -222.96453857421875,
|
|
"loss": 5.5463,
|
|
"margin_dpo/margin_mean": -0.02793477475643158,
|
|
"margin_dpo/margin_std": 0.5724214911460876,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.004188481675392671,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.014312177896499634,
|
|
"fcm_dpo/q_t": 0.4999642074108124,
|
|
"grad_norm": 27.87852668762207,
|
|
"learning_rate": 1.0416666666666666e-08,
|
|
"logits/chosen": -0.6574729681015015,
|
|
"logits/rejected": -0.6464410424232483,
|
|
"logps/chosen": -264.7165222167969,
|
|
"logps/ref_chosen": -264.7611083984375,
|
|
"logps/ref_rejected": -242.5597686767578,
|
|
"logps/rejected": -242.52951049804688,
|
|
"loss": 5.5446,
|
|
"margin_dpo/margin_mean": 0.014312252402305603,
|
|
"margin_dpo/margin_std": 0.6423971652984619,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.0062827225130890054,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.016503050923347473,
|
|
"fcm_dpo/q_t": 0.5000412464141846,
|
|
"grad_norm": 25.815200805664062,
|
|
"learning_rate": 2.083333333333333e-08,
|
|
"logits/chosen": -0.6840308308601379,
|
|
"logits/rejected": -0.7351540327072144,
|
|
"logps/chosen": -274.10198974609375,
|
|
"logps/ref_chosen": -274.1018981933594,
|
|
"logps/ref_rejected": -286.5882568359375,
|
|
"logps/rejected": -286.5718688964844,
|
|
"loss": 5.5459,
|
|
"margin_dpo/margin_mean": -0.016506418585777283,
|
|
"margin_dpo/margin_std": 0.6721947193145752,
|
|
"step": 3
|
|
},
|
|
{
|
|
"epoch": 0.008376963350785341,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.011121675372123718,
|
|
"fcm_dpo/q_t": 0.5000278353691101,
|
|
"grad_norm": 32.197547912597656,
|
|
"learning_rate": 3.125e-08,
|
|
"logits/chosen": -0.6205219626426697,
|
|
"logits/rejected": -0.6148541569709778,
|
|
"logps/chosen": -329.8299255371094,
|
|
"logps/ref_chosen": -329.8382568359375,
|
|
"logps/ref_rejected": -303.2850646972656,
|
|
"logps/rejected": -303.26556396484375,
|
|
"loss": 5.5457,
|
|
"margin_dpo/margin_mean": -0.011121734976768494,
|
|
"margin_dpo/margin_std": 0.7242615222930908,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.010471204188481676,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.055549949407577515,
|
|
"fcm_dpo/q_t": 0.5001388788223267,
|
|
"grad_norm": 29.520116806030273,
|
|
"learning_rate": 4.166666666666666e-08,
|
|
"logits/chosen": -0.5685195922851562,
|
|
"logits/rejected": -0.5843265652656555,
|
|
"logps/chosen": -301.7572021484375,
|
|
"logps/ref_chosen": -301.7389221191406,
|
|
"logps/ref_rejected": -274.7654724121094,
|
|
"logps/rejected": -274.7281799316406,
|
|
"loss": 5.5475,
|
|
"margin_dpo/margin_mean": -0.05554959177970886,
|
|
"margin_dpo/margin_std": 0.8365975022315979,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.012565445026178011,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.024648115038871765,
|
|
"fcm_dpo/q_t": 0.49993836879730225,
|
|
"grad_norm": 28.228336334228516,
|
|
"learning_rate": 5.208333333333333e-08,
|
|
"logits/chosen": -0.6921972036361694,
|
|
"logits/rejected": -0.6560333967208862,
|
|
"logps/chosen": -285.6768493652344,
|
|
"logps/ref_chosen": -285.6946716308594,
|
|
"logps/ref_rejected": -245.8200225830078,
|
|
"logps/rejected": -245.8268585205078,
|
|
"loss": 5.5442,
|
|
"margin_dpo/margin_mean": 0.024649202823638916,
|
|
"margin_dpo/margin_std": 0.704728901386261,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.014659685863874346,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.06559588015079498,
|
|
"fcm_dpo/q_t": 0.5001639127731323,
|
|
"grad_norm": 28.551637649536133,
|
|
"learning_rate": 6.25e-08,
|
|
"logits/chosen": -0.5759603381156921,
|
|
"logits/rejected": -0.6086133718490601,
|
|
"logps/chosen": -264.6998596191406,
|
|
"logps/ref_chosen": -264.65545654296875,
|
|
"logps/ref_rejected": -253.10305786132812,
|
|
"logps/rejected": -253.0819091796875,
|
|
"loss": 5.5478,
|
|
"margin_dpo/margin_mean": -0.06559541821479797,
|
|
"margin_dpo/margin_std": 0.6130937933921814,
|
|
"step": 7
|
|
},
|
|
{
|
|
"epoch": 0.016753926701570682,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.028539016842842102,
|
|
"fcm_dpo/q_t": 0.49992865324020386,
|
|
"grad_norm": 30.765186309814453,
|
|
"learning_rate": 7.291666666666667e-08,
|
|
"logits/chosen": -0.6858725547790527,
|
|
"logits/rejected": -0.6940003037452698,
|
|
"logps/chosen": -354.1387023925781,
|
|
"logps/ref_chosen": -354.1887512207031,
|
|
"logps/ref_rejected": -282.9112243652344,
|
|
"logps/rejected": -282.8896484375,
|
|
"loss": 5.5441,
|
|
"margin_dpo/margin_mean": 0.02854001522064209,
|
|
"margin_dpo/margin_std": 0.676353394985199,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.018848167539267015,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.09649568796157837,
|
|
"fcm_dpo/q_t": 0.500241219997406,
|
|
"grad_norm": 27.911169052124023,
|
|
"learning_rate": 8.333333333333333e-08,
|
|
"logits/chosen": -0.6349136829376221,
|
|
"logits/rejected": -0.6529811024665833,
|
|
"logps/chosen": -285.5809326171875,
|
|
"logps/ref_chosen": -285.5502014160156,
|
|
"logps/ref_rejected": -267.99664306640625,
|
|
"logps/rejected": -267.9308776855469,
|
|
"loss": 5.5491,
|
|
"margin_dpo/margin_mean": -0.09649543464183807,
|
|
"margin_dpo/margin_std": 0.650297224521637,
|
|
"step": 9
|
|
},
|
|
{
|
|
"epoch": 0.020942408376963352,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.019019827246665955,
|
|
"fcm_dpo/q_t": 0.4999524652957916,
|
|
"grad_norm": 26.725051879882812,
|
|
"learning_rate": 9.375e-08,
|
|
"logits/chosen": -0.6896336674690247,
|
|
"logits/rejected": -0.6844220757484436,
|
|
"logps/chosen": -251.85069274902344,
|
|
"logps/ref_chosen": -251.91238403320312,
|
|
"logps/ref_rejected": -226.45260620117188,
|
|
"logps/rejected": -226.40988159179688,
|
|
"loss": 5.5445,
|
|
"margin_dpo/margin_mean": 0.019019678235054016,
|
|
"margin_dpo/margin_std": 0.6863389015197754,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.023036649214659685,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.03224755823612213,
|
|
"fcm_dpo/q_t": 0.5000805854797363,
|
|
"grad_norm": 28.949674606323242,
|
|
"learning_rate": 1.0416666666666667e-07,
|
|
"logits/chosen": -0.5962199568748474,
|
|
"logits/rejected": -0.6516202092170715,
|
|
"logps/chosen": -301.0753479003906,
|
|
"logps/ref_chosen": -301.08343505859375,
|
|
"logps/ref_rejected": -259.546630859375,
|
|
"logps/rejected": -259.5062561035156,
|
|
"loss": 5.5465,
|
|
"margin_dpo/margin_mean": -0.032247647643089294,
|
|
"margin_dpo/margin_std": 0.6402597427368164,
|
|
"step": 11
|
|
},
|
|
{
|
|
"epoch": 0.025130890052356022,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.1078348308801651,
|
|
"fcm_dpo/q_t": 0.4997304081916809,
|
|
"grad_norm": 30.103239059448242,
|
|
"learning_rate": 1.1458333333333332e-07,
|
|
"logits/chosen": -0.577997624874115,
|
|
"logits/rejected": -0.5377943515777588,
|
|
"logps/chosen": -287.52789306640625,
|
|
"logps/ref_chosen": -287.548095703125,
|
|
"logps/ref_rejected": -277.37945556640625,
|
|
"logps/rejected": -277.46710205078125,
|
|
"loss": 5.5409,
|
|
"margin_dpo/margin_mean": 0.10783536732196808,
|
|
"margin_dpo/margin_std": 0.7473784685134888,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.027225130890052355,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.10099650919437408,
|
|
"fcm_dpo/q_t": 0.49974748492240906,
|
|
"grad_norm": 27.242509841918945,
|
|
"learning_rate": 1.25e-07,
|
|
"logits/chosen": -0.6565054059028625,
|
|
"logits/rejected": -0.6640400290489197,
|
|
"logps/chosen": -270.64508056640625,
|
|
"logps/ref_chosen": -270.6664123535156,
|
|
"logps/ref_rejected": -274.6546936035156,
|
|
"logps/rejected": -274.7343444824219,
|
|
"loss": 5.5412,
|
|
"margin_dpo/margin_mean": 0.1009967178106308,
|
|
"margin_dpo/margin_std": 0.676527738571167,
|
|
"step": 13
|
|
},
|
|
{
|
|
"epoch": 0.02931937172774869,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": -0.02157723903656006,
|
|
"fcm_dpo/q_t": 0.5000539422035217,
|
|
"grad_norm": 28.256738662719727,
|
|
"learning_rate": 1.3541666666666666e-07,
|
|
"logits/chosen": -0.62161785364151,
|
|
"logits/rejected": -0.6502420902252197,
|
|
"logps/chosen": -281.6171569824219,
|
|
"logps/ref_chosen": -281.59320068359375,
|
|
"logps/ref_rejected": -263.52215576171875,
|
|
"logps/rejected": -263.5245056152344,
|
|
"loss": 5.5461,
|
|
"margin_dpo/margin_mean": -0.021577835083007812,
|
|
"margin_dpo/margin_std": 0.698745846748352,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.031413612565445025,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.05705377459526062,
|
|
"fcm_dpo/q_t": 0.4998573660850525,
|
|
"grad_norm": 30.230676651000977,
|
|
"learning_rate": 1.4583333333333335e-07,
|
|
"logits/chosen": -0.6337839365005493,
|
|
"logits/rejected": -0.6448744535446167,
|
|
"logps/chosen": -298.41180419921875,
|
|
"logps/ref_chosen": -298.45343017578125,
|
|
"logps/ref_rejected": -227.17832946777344,
|
|
"logps/rejected": -227.19375610351562,
|
|
"loss": 5.5429,
|
|
"margin_dpo/margin_mean": 0.05705252289772034,
|
|
"margin_dpo/margin_std": 0.6579793691635132,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.033507853403141365,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.08925226330757141,
|
|
"fcm_dpo/q_t": 0.4997768998146057,
|
|
"grad_norm": 30.15053939819336,
|
|
"learning_rate": 1.5624999999999999e-07,
|
|
"logits/chosen": -0.6029733419418335,
|
|
"logits/rejected": -0.6001195907592773,
|
|
"logps/chosen": -293.8938293457031,
|
|
"logps/ref_chosen": -293.96661376953125,
|
|
"logps/ref_rejected": -250.78443908691406,
|
|
"logps/rejected": -250.80093383789062,
|
|
"loss": 5.5417,
|
|
"margin_dpo/margin_mean": 0.0892522782087326,
|
|
"margin_dpo/margin_std": 0.725162148475647,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.0356020942408377,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.04146842658519745,
|
|
"fcm_dpo/q_t": 0.4998963177204132,
|
|
"grad_norm": 27.696533203125,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": -0.5717862248420715,
|
|
"logits/rejected": -0.5980989336967468,
|
|
"logps/chosen": -262.3448791503906,
|
|
"logps/ref_chosen": -262.39398193359375,
|
|
"logps/ref_rejected": -248.500244140625,
|
|
"logps/rejected": -248.49258422851562,
|
|
"loss": 5.5436,
|
|
"margin_dpo/margin_mean": 0.041468992829322815,
|
|
"margin_dpo/margin_std": 0.6475922465324402,
|
|
"step": 17
|
|
},
|
|
{
|
|
"epoch": 0.03769633507853403,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.0832410454750061,
|
|
"fcm_dpo/q_t": 0.4997918903827667,
|
|
"grad_norm": 29.68451690673828,
|
|
"learning_rate": 1.7708333333333334e-07,
|
|
"logits/chosen": -0.6132781505584717,
|
|
"logits/rejected": -0.6193888187408447,
|
|
"logps/chosen": -293.6752014160156,
|
|
"logps/ref_chosen": -293.709228515625,
|
|
"logps/ref_rejected": -274.5875244140625,
|
|
"logps/rejected": -274.63677978515625,
|
|
"loss": 5.5419,
|
|
"margin_dpo/margin_mean": 0.0832410603761673,
|
|
"margin_dpo/margin_std": 0.6698707938194275,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.039790575916230364,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.008256226778030396,
|
|
"fcm_dpo/q_t": 0.4999793469905853,
|
|
"grad_norm": 28.128124237060547,
|
|
"learning_rate": 1.875e-07,
|
|
"logits/chosen": -0.6208982467651367,
|
|
"logits/rejected": -0.6168572902679443,
|
|
"logps/chosen": -280.22943115234375,
|
|
"logps/ref_chosen": -280.26568603515625,
|
|
"logps/ref_rejected": -259.9742736816406,
|
|
"logps/rejected": -259.9462890625,
|
|
"loss": 5.5449,
|
|
"margin_dpo/margin_mean": 0.008254945278167725,
|
|
"margin_dpo/margin_std": 0.6279901266098022,
|
|
"step": 19
|
|
},
|
|
{
|
|
"epoch": 0.041884816753926704,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.07622025907039642,
|
|
"fcm_dpo/q_t": 0.4998094439506531,
|
|
"grad_norm": 29.674253463745117,
|
|
"learning_rate": 1.9791666666666664e-07,
|
|
"logits/chosen": -0.6202477812767029,
|
|
"logits/rejected": -0.6521307826042175,
|
|
"logps/chosen": -303.79779052734375,
|
|
"logps/ref_chosen": -303.8954162597656,
|
|
"logps/ref_rejected": -260.214599609375,
|
|
"logps/rejected": -260.1932067871094,
|
|
"loss": 5.5422,
|
|
"margin_dpo/margin_mean": 0.07622019946575165,
|
|
"margin_dpo/margin_std": 0.7053053379058838,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.04397905759162304,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.10494795441627502,
|
|
"fcm_dpo/q_t": 0.4997376799583435,
|
|
"grad_norm": 35.037410736083984,
|
|
"learning_rate": 2.0833333333333333e-07,
|
|
"logits/chosen": -0.6494017243385315,
|
|
"logits/rejected": -0.6782788038253784,
|
|
"logps/chosen": -301.4276123046875,
|
|
"logps/ref_chosen": -301.5334777832031,
|
|
"logps/ref_rejected": -280.28900146484375,
|
|
"logps/rejected": -280.2880554199219,
|
|
"loss": 5.541,
|
|
"margin_dpo/margin_mean": 0.10494862496852875,
|
|
"margin_dpo/margin_std": 0.7810779213905334,
|
|
"step": 21
|
|
},
|
|
{
|
|
"epoch": 0.04607329842931937,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.06129841506481171,
|
|
"fcm_dpo/q_t": 0.49984675645828247,
|
|
"grad_norm": 25.279335021972656,
|
|
"learning_rate": 2.1875e-07,
|
|
"logits/chosen": -0.6577863693237305,
|
|
"logits/rejected": -0.6595051288604736,
|
|
"logps/chosen": -259.9060974121094,
|
|
"logps/ref_chosen": -259.9951477050781,
|
|
"logps/ref_rejected": -243.0721435546875,
|
|
"logps/rejected": -243.04441833496094,
|
|
"loss": 5.5428,
|
|
"margin_dpo/margin_mean": 0.061297982931137085,
|
|
"margin_dpo/margin_std": 0.7572940587997437,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.048167539267015703,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.15443375706672668,
|
|
"fcm_dpo/q_t": 0.4996139407157898,
|
|
"grad_norm": 28.133821487426758,
|
|
"learning_rate": 2.2916666666666663e-07,
|
|
"logits/chosen": -0.6106438636779785,
|
|
"logits/rejected": -0.6437903046607971,
|
|
"logps/chosen": -282.0675048828125,
|
|
"logps/ref_chosen": -282.1807556152344,
|
|
"logps/ref_rejected": -265.0758056640625,
|
|
"logps/rejected": -265.11700439453125,
|
|
"loss": 5.5391,
|
|
"margin_dpo/margin_mean": 0.1544336974620819,
|
|
"margin_dpo/margin_std": 0.8530293107032776,
|
|
"step": 23
|
|
},
|
|
{
|
|
"epoch": 0.050261780104712044,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.24844954907894135,
|
|
"fcm_dpo/q_t": 0.49937885999679565,
|
|
"grad_norm": 29.64801597595215,
|
|
"learning_rate": 2.3958333333333335e-07,
|
|
"logits/chosen": -0.6590055823326111,
|
|
"logits/rejected": -0.5712109208106995,
|
|
"logps/chosen": -300.9676513671875,
|
|
"logps/ref_chosen": -301.17962646484375,
|
|
"logps/ref_rejected": -302.12786865234375,
|
|
"logps/rejected": -302.1643371582031,
|
|
"loss": 5.5353,
|
|
"margin_dpo/margin_mean": 0.2484499216079712,
|
|
"margin_dpo/margin_std": 0.7506996393203735,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.05235602094240838,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.15574315190315247,
|
|
"fcm_dpo/q_t": 0.49961066246032715,
|
|
"grad_norm": 26.359050750732422,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -0.5938785076141357,
|
|
"logits/rejected": -0.6017901301383972,
|
|
"logps/chosen": -246.56497192382812,
|
|
"logps/ref_chosen": -246.74649047851562,
|
|
"logps/ref_rejected": -235.55638122558594,
|
|
"logps/rejected": -235.53057861328125,
|
|
"loss": 5.539,
|
|
"margin_dpo/margin_mean": 0.15574237704277039,
|
|
"margin_dpo/margin_std": 0.7371436953544617,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.05445026178010471,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.15086300671100616,
|
|
"fcm_dpo/q_t": 0.4996228516101837,
|
|
"grad_norm": 28.607776641845703,
|
|
"learning_rate": 2.604166666666667e-07,
|
|
"logits/chosen": -0.6509619951248169,
|
|
"logits/rejected": -0.6650726795196533,
|
|
"logps/chosen": -281.9699401855469,
|
|
"logps/ref_chosen": -282.1955871582031,
|
|
"logps/ref_rejected": -235.3135528564453,
|
|
"logps/rejected": -235.23878479003906,
|
|
"loss": 5.5392,
|
|
"margin_dpo/margin_mean": 0.1508627086877823,
|
|
"margin_dpo/margin_std": 0.83431077003479,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.05654450261780105,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.20935215055942535,
|
|
"fcm_dpo/q_t": 0.4994766116142273,
|
|
"grad_norm": 27.72915267944336,
|
|
"learning_rate": 2.708333333333333e-07,
|
|
"logits/chosen": -0.6564357280731201,
|
|
"logits/rejected": -0.6764754056930542,
|
|
"logps/chosen": -323.56011962890625,
|
|
"logps/ref_chosen": -323.8563537597656,
|
|
"logps/ref_rejected": -245.968017578125,
|
|
"logps/rejected": -245.8811492919922,
|
|
"loss": 5.5369,
|
|
"margin_dpo/margin_mean": 0.20935235917568207,
|
|
"margin_dpo/margin_std": 0.7932536005973816,
|
|
"step": 27
|
|
},
|
|
{
|
|
"epoch": 0.05863874345549738,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.18029074370861053,
|
|
"fcm_dpo/q_t": 0.4995492696762085,
|
|
"grad_norm": 26.313339233398438,
|
|
"learning_rate": 2.8125e-07,
|
|
"logits/chosen": -0.6219736337661743,
|
|
"logits/rejected": -0.6311923861503601,
|
|
"logps/chosen": -247.9824981689453,
|
|
"logps/ref_chosen": -248.24673461914062,
|
|
"logps/ref_rejected": -240.0382080078125,
|
|
"logps/rejected": -239.95425415039062,
|
|
"loss": 5.5381,
|
|
"margin_dpo/margin_mean": 0.18029029667377472,
|
|
"margin_dpo/margin_std": 0.8702433705329895,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.060732984293193716,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.23714801669120789,
|
|
"fcm_dpo/q_t": 0.4994071424007416,
|
|
"grad_norm": 29.599367141723633,
|
|
"learning_rate": 2.916666666666667e-07,
|
|
"logits/chosen": -0.5967488884925842,
|
|
"logits/rejected": -0.6196838617324829,
|
|
"logps/chosen": -317.91094970703125,
|
|
"logps/ref_chosen": -318.2564392089844,
|
|
"logps/ref_rejected": -286.75848388671875,
|
|
"logps/rejected": -286.650146484375,
|
|
"loss": 5.5358,
|
|
"margin_dpo/margin_mean": 0.23714756965637207,
|
|
"margin_dpo/margin_std": 0.8616628646850586,
|
|
"step": 29
|
|
},
|
|
{
|
|
"epoch": 0.06282722513089005,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.2596416473388672,
|
|
"fcm_dpo/q_t": 0.4993509352207184,
|
|
"grad_norm": 28.952259063720703,
|
|
"learning_rate": 3.020833333333333e-07,
|
|
"logits/chosen": -0.5958232283592224,
|
|
"logits/rejected": -0.6129447221755981,
|
|
"logps/chosen": -252.67791748046875,
|
|
"logps/ref_chosen": -253.0491485595703,
|
|
"logps/ref_rejected": -261.30029296875,
|
|
"logps/rejected": -261.188720703125,
|
|
"loss": 5.5349,
|
|
"margin_dpo/margin_mean": 0.2596414089202881,
|
|
"margin_dpo/margin_std": 0.9200783967971802,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.06492146596858639,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.31609046459198,
|
|
"fcm_dpo/q_t": 0.49920982122421265,
|
|
"grad_norm": 25.55852699279785,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"logits/chosen": -0.6523265838623047,
|
|
"logits/rejected": -0.6868293881416321,
|
|
"logps/chosen": -247.73052978515625,
|
|
"logps/ref_chosen": -248.15301513671875,
|
|
"logps/ref_rejected": -203.17703247070312,
|
|
"logps/rejected": -203.07061767578125,
|
|
"loss": 5.5327,
|
|
"margin_dpo/margin_mean": 0.316089928150177,
|
|
"margin_dpo/margin_std": 1.0166584253311157,
|
|
"step": 31
|
|
},
|
|
{
|
|
"epoch": 0.06701570680628273,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.20415076613426208,
|
|
"fcm_dpo/q_t": 0.49948960542678833,
|
|
"grad_norm": 29.79863166809082,
|
|
"learning_rate": 3.2291666666666666e-07,
|
|
"logits/chosen": -0.6170097589492798,
|
|
"logits/rejected": -0.6221901774406433,
|
|
"logps/chosen": -305.0362548828125,
|
|
"logps/ref_chosen": -305.5399475097656,
|
|
"logps/ref_rejected": -267.6527099609375,
|
|
"logps/rejected": -267.3531799316406,
|
|
"loss": 5.5371,
|
|
"margin_dpo/margin_mean": 0.20415127277374268,
|
|
"margin_dpo/margin_std": 0.90806645154953,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.06910994764397906,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.4111286997795105,
|
|
"fcm_dpo/q_t": 0.49897223711013794,
|
|
"grad_norm": 28.34008026123047,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -0.6372838020324707,
|
|
"logits/rejected": -0.6494584083557129,
|
|
"logps/chosen": -285.63262939453125,
|
|
"logps/ref_chosen": -286.2335205078125,
|
|
"logps/ref_rejected": -255.38748168945312,
|
|
"logps/rejected": -255.19772338867188,
|
|
"loss": 5.5289,
|
|
"margin_dpo/margin_mean": 0.41112855076789856,
|
|
"margin_dpo/margin_std": 1.0520519018173218,
|
|
"step": 33
|
|
},
|
|
{
|
|
"epoch": 0.0712041884816754,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.7322409152984619,
|
|
"fcm_dpo/q_t": 0.49816954135894775,
|
|
"grad_norm": 31.091548919677734,
|
|
"learning_rate": 3.4375e-07,
|
|
"logits/chosen": -0.6315101385116577,
|
|
"logits/rejected": -0.6417458653450012,
|
|
"logps/chosen": -340.8562927246094,
|
|
"logps/ref_chosen": -341.5920104980469,
|
|
"logps/ref_rejected": -278.8866882324219,
|
|
"logps/rejected": -278.8831787109375,
|
|
"loss": 5.5161,
|
|
"margin_dpo/margin_mean": 0.7322409749031067,
|
|
"margin_dpo/margin_std": 1.1406543254852295,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.07329842931937172,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.43320757150650024,
|
|
"fcm_dpo/q_t": 0.49891701340675354,
|
|
"grad_norm": 26.510805130004883,
|
|
"learning_rate": 3.541666666666667e-07,
|
|
"logits/chosen": -0.6248103380203247,
|
|
"logits/rejected": -0.6445828080177307,
|
|
"logps/chosen": -264.3880310058594,
|
|
"logps/ref_chosen": -265.0795593261719,
|
|
"logps/ref_rejected": -264.4876708984375,
|
|
"logps/rejected": -264.2293701171875,
|
|
"loss": 5.528,
|
|
"margin_dpo/margin_mean": 0.4332066774368286,
|
|
"margin_dpo/margin_std": 1.2199397087097168,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.07539267015706806,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.6611357927322388,
|
|
"fcm_dpo/q_t": 0.49834728240966797,
|
|
"grad_norm": 31.89276885986328,
|
|
"learning_rate": 3.645833333333333e-07,
|
|
"logits/chosen": -0.6019506454467773,
|
|
"logits/rejected": -0.6194975972175598,
|
|
"logps/chosen": -296.47509765625,
|
|
"logps/ref_chosen": -297.3261413574219,
|
|
"logps/ref_rejected": -282.09515380859375,
|
|
"logps/rejected": -281.9052734375,
|
|
"loss": 5.519,
|
|
"margin_dpo/margin_mean": 0.6611351370811462,
|
|
"margin_dpo/margin_std": 1.3979815244674683,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.0774869109947644,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.5799505710601807,
|
|
"fcm_dpo/q_t": 0.4985502362251282,
|
|
"grad_norm": 30.829147338867188,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -0.5941240191459656,
|
|
"logits/rejected": -0.6079503297805786,
|
|
"logps/chosen": -313.24591064453125,
|
|
"logps/ref_chosen": -314.0340270996094,
|
|
"logps/ref_rejected": -299.3437805175781,
|
|
"logps/rejected": -299.1355895996094,
|
|
"loss": 5.5223,
|
|
"margin_dpo/margin_mean": 0.579948902130127,
|
|
"margin_dpo/margin_std": 1.535064458847046,
|
|
"step": 37
|
|
},
|
|
{
|
|
"epoch": 0.07958115183246073,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.6605011820793152,
|
|
"fcm_dpo/q_t": 0.49834883213043213,
|
|
"grad_norm": 28.47408676147461,
|
|
"learning_rate": 3.8541666666666665e-07,
|
|
"logits/chosen": -0.6379660964012146,
|
|
"logits/rejected": -0.6501216292381287,
|
|
"logps/chosen": -281.49658203125,
|
|
"logps/ref_chosen": -282.54119873046875,
|
|
"logps/ref_rejected": -269.7773132324219,
|
|
"logps/rejected": -269.3931884765625,
|
|
"loss": 5.5191,
|
|
"margin_dpo/margin_mean": 0.6605010032653809,
|
|
"margin_dpo/margin_std": 1.4664862155914307,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.08167539267015707,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.1984353065490723,
|
|
"fcm_dpo/q_t": 0.49700427055358887,
|
|
"grad_norm": 29.65317726135254,
|
|
"learning_rate": 3.958333333333333e-07,
|
|
"logits/chosen": -0.6123558878898621,
|
|
"logits/rejected": -0.6260115504264832,
|
|
"logps/chosen": -275.47564697265625,
|
|
"logps/ref_chosen": -276.7729187011719,
|
|
"logps/ref_rejected": -249.95889282226562,
|
|
"logps/rejected": -249.8600311279297,
|
|
"loss": 5.4977,
|
|
"margin_dpo/margin_mean": 1.198434829711914,
|
|
"margin_dpo/margin_std": 1.7556183338165283,
|
|
"step": 39
|
|
},
|
|
{
|
|
"epoch": 0.08376963350785341,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.7856162786483765,
|
|
"fcm_dpo/q_t": 0.49803632497787476,
|
|
"grad_norm": 27.415355682373047,
|
|
"learning_rate": 4.0625e-07,
|
|
"logits/chosen": -0.623888373374939,
|
|
"logits/rejected": -0.6597940921783447,
|
|
"logps/chosen": -283.21929931640625,
|
|
"logps/ref_chosen": -284.30706787109375,
|
|
"logps/ref_rejected": -244.4459991455078,
|
|
"logps/rejected": -244.14381408691406,
|
|
"loss": 5.5142,
|
|
"margin_dpo/margin_mean": 0.7856165170669556,
|
|
"margin_dpo/margin_std": 1.7853032350540161,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.08586387434554973,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.7766204476356506,
|
|
"fcm_dpo/q_t": 0.49805861711502075,
|
|
"grad_norm": 30.24774742126465,
|
|
"learning_rate": 4.1666666666666667e-07,
|
|
"logits/chosen": -0.6275569200515747,
|
|
"logits/rejected": -0.6529408097267151,
|
|
"logps/chosen": -292.7530822753906,
|
|
"logps/ref_chosen": -293.8151550292969,
|
|
"logps/ref_rejected": -252.16815185546875,
|
|
"logps/rejected": -251.88272094726562,
|
|
"loss": 5.5145,
|
|
"margin_dpo/margin_mean": 0.7766217589378357,
|
|
"margin_dpo/margin_std": 1.7000435590744019,
|
|
"step": 41
|
|
},
|
|
{
|
|
"epoch": 0.08795811518324607,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.9282649755477905,
|
|
"fcm_dpo/q_t": 0.49767956137657166,
|
|
"grad_norm": 27.3597412109375,
|
|
"learning_rate": 4.270833333333333e-07,
|
|
"logits/chosen": -0.6338014602661133,
|
|
"logits/rejected": -0.6516577005386353,
|
|
"logps/chosen": -251.51942443847656,
|
|
"logps/ref_chosen": -252.76023864746094,
|
|
"logps/ref_rejected": -261.0414733886719,
|
|
"logps/rejected": -260.7288818359375,
|
|
"loss": 5.5087,
|
|
"margin_dpo/margin_mean": 0.928265392780304,
|
|
"margin_dpo/margin_std": 2.2327194213867188,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.09005235602094241,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.1566113233566284,
|
|
"fcm_dpo/q_t": 0.4971088171005249,
|
|
"grad_norm": 29.90976905822754,
|
|
"learning_rate": 4.375e-07,
|
|
"logits/chosen": -0.5936682820320129,
|
|
"logits/rejected": -0.609065055847168,
|
|
"logps/chosen": -315.6018981933594,
|
|
"logps/ref_chosen": -316.8347473144531,
|
|
"logps/ref_rejected": -273.7649230957031,
|
|
"logps/rejected": -273.68865966796875,
|
|
"loss": 5.4996,
|
|
"margin_dpo/margin_mean": 1.1566113233566284,
|
|
"margin_dpo/margin_std": 2.265800952911377,
|
|
"step": 43
|
|
},
|
|
{
|
|
"epoch": 0.09214659685863874,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.5960203409194946,
|
|
"fcm_dpo/q_t": 0.49601075053215027,
|
|
"grad_norm": 30.911487579345703,
|
|
"learning_rate": 4.479166666666667e-07,
|
|
"logits/chosen": -0.6010478734970093,
|
|
"logits/rejected": -0.6001090407371521,
|
|
"logps/chosen": -285.37725830078125,
|
|
"logps/ref_chosen": -286.8757019042969,
|
|
"logps/ref_rejected": -282.4681396484375,
|
|
"logps/rejected": -282.56573486328125,
|
|
"loss": 5.4826,
|
|
"margin_dpo/margin_mean": 1.5960203409194946,
|
|
"margin_dpo/margin_std": 3.0894060134887695,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.09424083769633508,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.3747289180755615,
|
|
"fcm_dpo/q_t": 0.49656426906585693,
|
|
"grad_norm": 28.888906478881836,
|
|
"learning_rate": 4.5833333333333327e-07,
|
|
"logits/chosen": -0.6945655345916748,
|
|
"logits/rejected": -0.7191615104675293,
|
|
"logps/chosen": -322.55853271484375,
|
|
"logps/ref_chosen": -324.2633972167969,
|
|
"logps/ref_rejected": -293.09466552734375,
|
|
"logps/rejected": -292.7645263671875,
|
|
"loss": 5.4912,
|
|
"margin_dpo/margin_mean": 1.3747289180755615,
|
|
"margin_dpo/margin_std": 2.859666109085083,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.09633507853403141,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.5242335796356201,
|
|
"fcm_dpo/q_t": 0.49619045853614807,
|
|
"grad_norm": 30.363393783569336,
|
|
"learning_rate": 4.6874999999999996e-07,
|
|
"logits/chosen": -0.6261583566665649,
|
|
"logits/rejected": -0.6395462155342102,
|
|
"logps/chosen": -296.660888671875,
|
|
"logps/ref_chosen": -298.3357238769531,
|
|
"logps/ref_rejected": -267.66204833984375,
|
|
"logps/rejected": -267.51141357421875,
|
|
"loss": 5.4852,
|
|
"margin_dpo/margin_mean": 1.5242342948913574,
|
|
"margin_dpo/margin_std": 2.66381573677063,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.09842931937172775,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 0.9342592358589172,
|
|
"fcm_dpo/q_t": 0.49766474962234497,
|
|
"grad_norm": 26.27431869506836,
|
|
"learning_rate": 4.791666666666667e-07,
|
|
"logits/chosen": -0.6025406718254089,
|
|
"logits/rejected": -0.6231379508972168,
|
|
"logps/chosen": -261.08758544921875,
|
|
"logps/ref_chosen": -262.5669250488281,
|
|
"logps/ref_rejected": -258.70989990234375,
|
|
"logps/rejected": -258.1648254394531,
|
|
"loss": 5.5091,
|
|
"margin_dpo/margin_mean": 0.9342588186264038,
|
|
"margin_dpo/margin_std": 3.3158531188964844,
|
|
"step": 47
|
|
},
|
|
{
|
|
"epoch": 0.10052356020942409,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.5198674201965332,
|
|
"fcm_dpo/q_t": 0.49620240926742554,
|
|
"grad_norm": 27.544031143188477,
|
|
"learning_rate": 4.895833333333333e-07,
|
|
"logits/chosen": -0.6120340824127197,
|
|
"logits/rejected": -0.6370552182197571,
|
|
"logps/chosen": -267.6554870605469,
|
|
"logps/ref_chosen": -269.4932556152344,
|
|
"logps/ref_rejected": -241.888916015625,
|
|
"logps/rejected": -241.5709991455078,
|
|
"loss": 5.4857,
|
|
"margin_dpo/margin_mean": 1.5198673009872437,
|
|
"margin_dpo/margin_std": 2.993640422821045,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.10261780104712041,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.6754217147827148,
|
|
"fcm_dpo/q_t": 0.49581262469291687,
|
|
"grad_norm": 27.63544273376465,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -0.6648722887039185,
|
|
"logits/rejected": -0.649385929107666,
|
|
"logps/chosen": -255.69033813476562,
|
|
"logps/ref_chosen": -257.8844909667969,
|
|
"logps/ref_rejected": -256.8912048339844,
|
|
"logps/rejected": -256.3724670410156,
|
|
"loss": 5.4797,
|
|
"margin_dpo/margin_mean": 1.6754218339920044,
|
|
"margin_dpo/margin_std": 3.419741630554199,
|
|
"step": 49
|
|
},
|
|
{
|
|
"epoch": 0.10471204188481675,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.1309083700180054,
|
|
"fcm_dpo/q_t": 0.4971740245819092,
|
|
"grad_norm": 28.015737533569336,
|
|
"learning_rate": 4.999932966293553e-07,
|
|
"logits/chosen": -0.6291318535804749,
|
|
"logits/rejected": -0.6510509252548218,
|
|
"logps/chosen": -299.6326599121094,
|
|
"logps/ref_chosen": -301.62884521484375,
|
|
"logps/ref_rejected": -298.2716064453125,
|
|
"logps/rejected": -297.40631103515625,
|
|
"loss": 5.5017,
|
|
"margin_dpo/margin_mean": 1.1309083700180054,
|
|
"margin_dpo/margin_std": 3.866288423538208,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.1068062827225131,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.9230157136917114,
|
|
"fcm_dpo/q_t": 0.49519532918930054,
|
|
"grad_norm": 29.120933532714844,
|
|
"learning_rate": 4.999731868769026e-07,
|
|
"logits/chosen": -0.6170083284378052,
|
|
"logits/rejected": -0.6109319925308228,
|
|
"logps/chosen": -267.2721862792969,
|
|
"logps/ref_chosen": -269.37237548828125,
|
|
"logps/ref_rejected": -297.0167541503906,
|
|
"logps/rejected": -296.8395690917969,
|
|
"loss": 5.4711,
|
|
"margin_dpo/margin_mean": 1.9230164289474487,
|
|
"margin_dpo/margin_std": 4.57507848739624,
|
|
"step": 51
|
|
},
|
|
{
|
|
"epoch": 0.10890052356020942,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.091094493865967,
|
|
"fcm_dpo/q_t": 0.4922794699668884,
|
|
"grad_norm": 30.30776023864746,
|
|
"learning_rate": 4.99939671821067e-07,
|
|
"logits/chosen": -0.6616694331169128,
|
|
"logits/rejected": -0.6685453653335571,
|
|
"logps/chosen": -304.083251953125,
|
|
"logps/ref_chosen": -306.9028015136719,
|
|
"logps/ref_rejected": -281.24737548828125,
|
|
"logps/rejected": -281.5189208984375,
|
|
"loss": 5.4251,
|
|
"margin_dpo/margin_mean": 3.0910940170288086,
|
|
"margin_dpo/margin_std": 4.87042760848999,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.11099476439790576,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 2.1641690731048584,
|
|
"fcm_dpo/q_t": 0.4945937991142273,
|
|
"grad_norm": 29.97233772277832,
|
|
"learning_rate": 4.998927532591591e-07,
|
|
"logits/chosen": -0.6528322100639343,
|
|
"logits/rejected": -0.6935089230537415,
|
|
"logps/chosen": -283.1597595214844,
|
|
"logps/ref_chosen": -285.9759521484375,
|
|
"logps/ref_rejected": -273.9073486328125,
|
|
"logps/rejected": -273.2553405761719,
|
|
"loss": 5.4621,
|
|
"margin_dpo/margin_mean": 2.164168119430542,
|
|
"margin_dpo/margin_std": 5.062405586242676,
|
|
"step": 53
|
|
},
|
|
{
|
|
"epoch": 0.1130890052356021,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.8245185613632202,
|
|
"fcm_dpo/q_t": 0.4954439401626587,
|
|
"grad_norm": 26.48580551147461,
|
|
"learning_rate": 4.998324337072792e-07,
|
|
"logits/chosen": -0.698631763458252,
|
|
"logits/rejected": -0.7079989314079285,
|
|
"logps/chosen": -303.81121826171875,
|
|
"logps/ref_chosen": -306.504638671875,
|
|
"logps/ref_rejected": -272.67431640625,
|
|
"logps/rejected": -271.8053894042969,
|
|
"loss": 5.4757,
|
|
"margin_dpo/margin_mean": 1.824518084526062,
|
|
"margin_dpo/margin_std": 5.36137580871582,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.11518324607329843,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 2.2451283931732178,
|
|
"fcm_dpo/q_t": 0.494392067193985,
|
|
"grad_norm": 24.85249137878418,
|
|
"learning_rate": 4.997587164001815e-07,
|
|
"logits/chosen": -0.6399841904640198,
|
|
"logits/rejected": -0.6441481113433838,
|
|
"logps/chosen": -220.5376434326172,
|
|
"logps/ref_chosen": -222.33013916015625,
|
|
"logps/ref_rejected": -206.59571838378906,
|
|
"logps/rejected": -207.04835510253906,
|
|
"loss": 5.4583,
|
|
"margin_dpo/margin_mean": 2.245128870010376,
|
|
"margin_dpo/margin_std": 4.873081207275391,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.11727748691099477,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.144686698913574,
|
|
"fcm_dpo/q_t": 0.4921509027481079,
|
|
"grad_norm": 27.462751388549805,
|
|
"learning_rate": 4.996716052911017e-07,
|
|
"logits/chosen": -0.6191872358322144,
|
|
"logits/rejected": -0.6348315477371216,
|
|
"logps/chosen": -247.60455322265625,
|
|
"logps/ref_chosen": -250.47816467285156,
|
|
"logps/ref_rejected": -228.25848388671875,
|
|
"logps/rejected": -228.52955627441406,
|
|
"loss": 5.4246,
|
|
"margin_dpo/margin_mean": 3.144686698913574,
|
|
"margin_dpo/margin_std": 5.893349647521973,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.1193717277486911,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.213151454925537,
|
|
"fcm_dpo/q_t": 0.48947957158088684,
|
|
"grad_norm": 30.760799407958984,
|
|
"learning_rate": 4.99571105051544e-07,
|
|
"logits/chosen": -0.7002225518226624,
|
|
"logits/rejected": -0.6711569428443909,
|
|
"logps/chosen": -311.23614501953125,
|
|
"logps/ref_chosen": -315.1195373535156,
|
|
"logps/ref_rejected": -272.755615234375,
|
|
"logps/rejected": -273.08538818359375,
|
|
"loss": 5.3819,
|
|
"margin_dpo/margin_mean": 4.213151454925537,
|
|
"margin_dpo/margin_std": 5.628840446472168,
|
|
"step": 57
|
|
},
|
|
{
|
|
"epoch": 0.12146596858638743,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 2.343231439590454,
|
|
"fcm_dpo/q_t": 0.4941456913948059,
|
|
"grad_norm": 27.43522834777832,
|
|
"learning_rate": 4.994572210710314e-07,
|
|
"logits/chosen": -0.6193488836288452,
|
|
"logits/rejected": -0.6438428163528442,
|
|
"logps/chosen": -262.74346923828125,
|
|
"logps/ref_chosen": -265.1816711425781,
|
|
"logps/ref_rejected": -268.2203369140625,
|
|
"logps/rejected": -268.1253662109375,
|
|
"loss": 5.4556,
|
|
"margin_dpo/margin_mean": 2.343231201171875,
|
|
"margin_dpo/margin_std": 5.691303253173828,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.12356020942408377,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 1.5638056993484497,
|
|
"fcm_dpo/q_t": 0.4960971474647522,
|
|
"grad_norm": 29.727506637573242,
|
|
"learning_rate": 4.993299594568162e-07,
|
|
"logits/chosen": -0.6027001142501831,
|
|
"logits/rejected": -0.595492959022522,
|
|
"logps/chosen": -284.2311096191406,
|
|
"logps/ref_chosen": -286.35394287109375,
|
|
"logps/ref_rejected": -260.6757507324219,
|
|
"logps/rejected": -260.11669921875,
|
|
"loss": 5.4886,
|
|
"margin_dpo/margin_mean": 1.5638054609298706,
|
|
"margin_dpo/margin_std": 6.958956241607666,
|
|
"step": 59
|
|
},
|
|
{
|
|
"epoch": 0.1256544502617801,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.373918056488037,
|
|
"fcm_dpo/q_t": 0.4915754795074463,
|
|
"grad_norm": 27.950477600097656,
|
|
"learning_rate": 4.991893270335525e-07,
|
|
"logits/chosen": -0.6873760223388672,
|
|
"logits/rejected": -0.7125962376594543,
|
|
"logps/chosen": -255.85675048828125,
|
|
"logps/ref_chosen": -258.74859619140625,
|
|
"logps/ref_rejected": -255.04893493652344,
|
|
"logps/rejected": -255.53097534179688,
|
|
"loss": 5.4167,
|
|
"margin_dpo/margin_mean": 3.3739190101623535,
|
|
"margin_dpo/margin_std": 7.061320781707764,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.12774869109947645,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.1508805751800537,
|
|
"fcm_dpo/q_t": 0.4921364188194275,
|
|
"grad_norm": 29.970989227294922,
|
|
"learning_rate": 4.990353313429303e-07,
|
|
"logits/chosen": -0.6524519920349121,
|
|
"logits/rejected": -0.6696836948394775,
|
|
"logps/chosen": -275.4600524902344,
|
|
"logps/ref_chosen": -278.4678955078125,
|
|
"logps/ref_rejected": -252.02720642089844,
|
|
"logps/rejected": -252.170166015625,
|
|
"loss": 5.4269,
|
|
"margin_dpo/margin_mean": 3.1508798599243164,
|
|
"margin_dpo/margin_std": 7.672506809234619,
|
|
"step": 61
|
|
},
|
|
{
|
|
"epoch": 0.12984293193717278,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.335423469543457,
|
|
"fcm_dpo/q_t": 0.4916812479496002,
|
|
"grad_norm": 26.69674301147461,
|
|
"learning_rate": 4.988679806432711e-07,
|
|
"logits/chosen": -0.6011494994163513,
|
|
"logits/rejected": -0.6419218182563782,
|
|
"logps/chosen": -269.00103759765625,
|
|
"logps/ref_chosen": -272.92431640625,
|
|
"logps/ref_rejected": -260.7935485839844,
|
|
"logps/rejected": -260.2056884765625,
|
|
"loss": 5.4187,
|
|
"margin_dpo/margin_mean": 3.3354249000549316,
|
|
"margin_dpo/margin_std": 7.46706485748291,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.1319371727748691,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.5092599391937256,
|
|
"fcm_dpo/q_t": 0.4912412762641907,
|
|
"grad_norm": 28.325027465820312,
|
|
"learning_rate": 4.986872839090852e-07,
|
|
"logits/chosen": -0.6640771627426147,
|
|
"logits/rejected": -0.6706352233886719,
|
|
"logps/chosen": -273.694580078125,
|
|
"logps/ref_chosen": -277.0889892578125,
|
|
"logps/ref_rejected": -273.3413391113281,
|
|
"logps/rejected": -273.4562072753906,
|
|
"loss": 5.4118,
|
|
"margin_dpo/margin_mean": 3.509258985519409,
|
|
"margin_dpo/margin_std": 7.294081687927246,
|
|
"step": 63
|
|
},
|
|
{
|
|
"epoch": 0.13403141361256546,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.153190612792969,
|
|
"fcm_dpo/q_t": 0.489638090133667,
|
|
"grad_norm": 28.40380859375,
|
|
"learning_rate": 4.9849325083059e-07,
|
|
"logits/chosen": -0.6381584405899048,
|
|
"logits/rejected": -0.6338601112365723,
|
|
"logps/chosen": -279.7315368652344,
|
|
"logps/ref_chosen": -283.8244934082031,
|
|
"logps/ref_rejected": -263.29351806640625,
|
|
"logps/rejected": -263.3537902832031,
|
|
"loss": 5.3882,
|
|
"margin_dpo/margin_mean": 4.1531901359558105,
|
|
"margin_dpo/margin_std": 8.279900550842285,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.13612565445026178,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.30912184715271,
|
|
"fcm_dpo/q_t": 0.4917401969432831,
|
|
"grad_norm": 27.938804626464844,
|
|
"learning_rate": 4.982858918131906e-07,
|
|
"logits/chosen": -0.7063099145889282,
|
|
"logits/rejected": -0.674233078956604,
|
|
"logps/chosen": -261.4156188964844,
|
|
"logps/ref_chosen": -264.8699645996094,
|
|
"logps/ref_rejected": -268.5076904296875,
|
|
"logps/rejected": -268.3625183105469,
|
|
"loss": 5.4195,
|
|
"margin_dpo/margin_mean": 3.3091213703155518,
|
|
"margin_dpo/margin_std": 7.275178909301758,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.1382198952879581,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.092304229736328,
|
|
"fcm_dpo/q_t": 0.48980259895324707,
|
|
"grad_norm": 28.009225845336914,
|
|
"learning_rate": 4.980652179769217e-07,
|
|
"logits/chosen": -0.6750394105911255,
|
|
"logits/rejected": -0.6948624849319458,
|
|
"logps/chosen": -269.80352783203125,
|
|
"logps/ref_chosen": -272.9283142089844,
|
|
"logps/ref_rejected": -280.94696044921875,
|
|
"logps/rejected": -281.9145202636719,
|
|
"loss": 5.3937,
|
|
"margin_dpo/margin_mean": 4.09230375289917,
|
|
"margin_dpo/margin_std": 10.11854362487793,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.14031413612565444,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 3.718635320663452,
|
|
"fcm_dpo/q_t": 0.4907230734825134,
|
|
"grad_norm": 25.479549407958984,
|
|
"learning_rate": 4.978312411558517e-07,
|
|
"logits/chosen": -0.6987938284873962,
|
|
"logits/rejected": -0.7324379086494446,
|
|
"logps/chosen": -262.1153564453125,
|
|
"logps/ref_chosen": -266.18695068359375,
|
|
"logps/ref_rejected": -250.17405700683594,
|
|
"logps/rejected": -249.82113647460938,
|
|
"loss": 5.4061,
|
|
"margin_dpo/margin_mean": 3.718634843826294,
|
|
"margin_dpo/margin_std": 8.839105606079102,
|
|
"step": 67
|
|
},
|
|
{
|
|
"epoch": 0.1424083769633508,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.682402610778809,
|
|
"fcm_dpo/q_t": 0.48835432529449463,
|
|
"grad_norm": 27.78085708618164,
|
|
"learning_rate": 4.975839738974473e-07,
|
|
"logits/chosen": -0.6808131337165833,
|
|
"logits/rejected": -0.6937141418457031,
|
|
"logps/chosen": -294.8902587890625,
|
|
"logps/ref_chosen": -297.9385986328125,
|
|
"logps/ref_rejected": -261.5141296386719,
|
|
"logps/rejected": -263.148193359375,
|
|
"loss": 5.3723,
|
|
"margin_dpo/margin_mean": 4.682403564453125,
|
|
"margin_dpo/margin_std": 10.64492130279541,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.14450261780104712,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.0140380859375,
|
|
"fcm_dpo/q_t": 0.48501238226890564,
|
|
"grad_norm": 28.59217643737793,
|
|
"learning_rate": 4.97323429461901e-07,
|
|
"logits/chosen": -0.6912984251976013,
|
|
"logits/rejected": -0.7229157090187073,
|
|
"logps/chosen": -261.6896667480469,
|
|
"logps/ref_chosen": -265.6175231933594,
|
|
"logps/ref_rejected": -236.8287353515625,
|
|
"logps/rejected": -238.91488647460938,
|
|
"loss": 5.3182,
|
|
"margin_dpo/margin_mean": 6.014038562774658,
|
|
"margin_dpo/margin_std": 9.668503761291504,
|
|
"step": 69
|
|
},
|
|
{
|
|
"epoch": 0.14659685863874344,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 5.854832649230957,
|
|
"fcm_dpo/q_t": 0.48542001843452454,
|
|
"grad_norm": 28.611467361450195,
|
|
"learning_rate": 4.970496218214204e-07,
|
|
"logits/chosen": -0.6638342142105103,
|
|
"logits/rejected": -0.6955633759498596,
|
|
"logps/chosen": -291.9162902832031,
|
|
"logps/ref_chosen": -296.2259216308594,
|
|
"logps/ref_rejected": -254.68496704101562,
|
|
"logps/rejected": -256.2301330566406,
|
|
"loss": 5.3275,
|
|
"margin_dpo/margin_mean": 5.854830741882324,
|
|
"margin_dpo/margin_std": 11.032973289489746,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.1486910994764398,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 5.318414211273193,
|
|
"fcm_dpo/q_t": 0.48674529790878296,
|
|
"grad_norm": 28.393184661865234,
|
|
"learning_rate": 4.967625656594781e-07,
|
|
"logits/chosen": -0.6533457040786743,
|
|
"logits/rejected": -0.6424985527992249,
|
|
"logps/chosen": -283.6885681152344,
|
|
"logps/ref_chosen": -288.92724609375,
|
|
"logps/ref_rejected": -278.6405334472656,
|
|
"logps/rejected": -278.72027587890625,
|
|
"loss": 5.3521,
|
|
"margin_dpo/margin_mean": 5.318414688110352,
|
|
"margin_dpo/margin_std": 12.410322189331055,
|
|
"step": 71
|
|
},
|
|
{
|
|
"epoch": 0.15078534031413612,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 5.140360355377197,
|
|
"fcm_dpo/q_t": 0.48718106746673584,
|
|
"grad_norm": 28.1318359375,
|
|
"learning_rate": 4.964622763700252e-07,
|
|
"logits/chosen": -0.6930279731750488,
|
|
"logits/rejected": -0.7042161226272583,
|
|
"logps/chosen": -233.69351196289062,
|
|
"logps/ref_chosen": -237.0452880859375,
|
|
"logps/ref_rejected": -252.7946319580078,
|
|
"logps/rejected": -254.58322143554688,
|
|
"loss": 5.3546,
|
|
"margin_dpo/margin_mean": 5.140359878540039,
|
|
"margin_dpo/margin_std": 10.367585182189941,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.15287958115183245,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.793428897857666,
|
|
"fcm_dpo/q_t": 0.48804718255996704,
|
|
"grad_norm": 27.781818389892578,
|
|
"learning_rate": 4.961487700566646e-07,
|
|
"logits/chosen": -0.6512953042984009,
|
|
"logits/rejected": -0.6683382987976074,
|
|
"logps/chosen": -268.6753845214844,
|
|
"logps/ref_chosen": -273.0531005859375,
|
|
"logps/ref_rejected": -246.8330841064453,
|
|
"logps/rejected": -247.24880981445312,
|
|
"loss": 5.372,
|
|
"margin_dpo/margin_mean": 4.793429374694824,
|
|
"margin_dpo/margin_std": 12.23983383178711,
|
|
"step": 73
|
|
},
|
|
{
|
|
"epoch": 0.1549738219895288,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 4.438821792602539,
|
|
"fcm_dpo/q_t": 0.4889482259750366,
|
|
"grad_norm": 30.2846736907959,
|
|
"learning_rate": 4.958220635317885e-07,
|
|
"logits/chosen": -0.7208205461502075,
|
|
"logits/rejected": -0.6990806460380554,
|
|
"logps/chosen": -338.8956298828125,
|
|
"logps/ref_chosen": -342.2818908691406,
|
|
"logps/ref_rejected": -330.0293884277344,
|
|
"logps/rejected": -331.0819091796875,
|
|
"loss": 5.3847,
|
|
"margin_dpo/margin_mean": 4.438822269439697,
|
|
"margin_dpo/margin_std": 11.789268493652344,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.15706806282722513,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.654679298400879,
|
|
"fcm_dpo/q_t": 0.4834195673465729,
|
|
"grad_norm": 29.386520385742188,
|
|
"learning_rate": 4.954821743156767e-07,
|
|
"logits/chosen": -0.6316950917243958,
|
|
"logits/rejected": -0.632973849773407,
|
|
"logps/chosen": -262.26519775390625,
|
|
"logps/ref_chosen": -266.8641662597656,
|
|
"logps/ref_rejected": -276.8699951171875,
|
|
"logps/rejected": -278.92572021484375,
|
|
"loss": 5.2967,
|
|
"margin_dpo/margin_mean": 6.654679775238037,
|
|
"margin_dpo/margin_std": 10.98399829864502,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.15916230366492146,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 7.279214859008789,
|
|
"fcm_dpo/q_t": 0.48191386461257935,
|
|
"grad_norm": 29.20782470703125,
|
|
"learning_rate": 4.951291206355559e-07,
|
|
"logits/chosen": -0.7203505039215088,
|
|
"logits/rejected": -0.7281088829040527,
|
|
"logps/chosen": -277.01470947265625,
|
|
"logps/ref_chosen": -281.174560546875,
|
|
"logps/ref_rejected": -263.6067199707031,
|
|
"logps/rejected": -266.72607421875,
|
|
"loss": 5.2772,
|
|
"margin_dpo/margin_mean": 7.279214859008789,
|
|
"margin_dpo/margin_std": 12.870685577392578,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.1612565445026178,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 5.79923152923584,
|
|
"fcm_dpo/q_t": 0.4856007397174835,
|
|
"grad_norm": 32.94340133666992,
|
|
"learning_rate": 4.947629214246236e-07,
|
|
"logits/chosen": -0.5545709729194641,
|
|
"logits/rejected": -0.5626035928726196,
|
|
"logps/chosen": -302.3695068359375,
|
|
"logps/ref_chosen": -306.09527587890625,
|
|
"logps/ref_rejected": -253.49569702148438,
|
|
"logps/rejected": -255.5691375732422,
|
|
"loss": 5.3375,
|
|
"margin_dpo/margin_mean": 5.79923152923584,
|
|
"margin_dpo/margin_std": 13.905439376831055,
|
|
"step": 77
|
|
},
|
|
{
|
|
"epoch": 0.16335078534031414,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.666434288024902,
|
|
"fcm_dpo/q_t": 0.478497177362442,
|
|
"grad_norm": 29.634822845458984,
|
|
"learning_rate": 4.943835963210323e-07,
|
|
"logits/chosen": -0.677687406539917,
|
|
"logits/rejected": -0.6719750165939331,
|
|
"logps/chosen": -253.00672912597656,
|
|
"logps/ref_chosen": -256.90234375,
|
|
"logps/ref_rejected": -211.57154846191406,
|
|
"logps/rejected": -216.3424072265625,
|
|
"loss": 5.2278,
|
|
"margin_dpo/margin_mean": 8.666435241699219,
|
|
"margin_dpo/margin_std": 14.475006103515625,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.16544502617801046,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.294734954833984,
|
|
"fcm_dpo/q_t": 0.47941556572914124,
|
|
"grad_norm": 29.799663543701172,
|
|
"learning_rate": 4.939911656668361e-07,
|
|
"logits/chosen": -0.6619881987571716,
|
|
"logits/rejected": -0.6830765604972839,
|
|
"logps/chosen": -263.20843505859375,
|
|
"logps/ref_chosen": -266.2735595703125,
|
|
"logps/ref_rejected": -251.57257080078125,
|
|
"logps/rejected": -256.80218505859375,
|
|
"loss": 5.2444,
|
|
"margin_dpo/margin_mean": 8.294734001159668,
|
|
"margin_dpo/margin_std": 15.015451431274414,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.16753926701570682,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 6.695713043212891,
|
|
"fcm_dpo/q_t": 0.48338207602500916,
|
|
"grad_norm": 28.94121742248535,
|
|
"learning_rate": 4.935856505068998e-07,
|
|
"logits/chosen": -0.666534423828125,
|
|
"logits/rejected": -0.6961286067962646,
|
|
"logps/chosen": -285.9317932128906,
|
|
"logps/ref_chosen": -287.8509826660156,
|
|
"logps/ref_rejected": -256.0766296386719,
|
|
"logps/rejected": -260.8531494140625,
|
|
"loss": 5.3016,
|
|
"margin_dpo/margin_mean": 6.695713043212891,
|
|
"margin_dpo/margin_std": 13.224631309509277,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.16963350785340314,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 7.167357444763184,
|
|
"fcm_dpo/q_t": 0.482227087020874,
|
|
"grad_norm": 28.103904724121094,
|
|
"learning_rate": 4.93167072587771e-07,
|
|
"logits/chosen": -0.6446385383605957,
|
|
"logits/rejected": -0.6438944935798645,
|
|
"logps/chosen": -266.1433410644531,
|
|
"logps/ref_chosen": -268.5232238769531,
|
|
"logps/ref_rejected": -237.81137084960938,
|
|
"logps/rejected": -242.59884643554688,
|
|
"loss": 5.2934,
|
|
"margin_dpo/margin_mean": 7.167357921600342,
|
|
"margin_dpo/margin_std": 16.743844985961914,
|
|
"step": 81
|
|
},
|
|
{
|
|
"epoch": 0.17172774869109947,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 7.904582977294922,
|
|
"fcm_dpo/q_t": 0.480448454618454,
|
|
"grad_norm": 27.851560592651367,
|
|
"learning_rate": 4.92735454356513e-07,
|
|
"logits/chosen": -0.7309600710868835,
|
|
"logits/rejected": -0.7388311624526978,
|
|
"logps/chosen": -276.9690246582031,
|
|
"logps/ref_chosen": -279.36395263671875,
|
|
"logps/ref_rejected": -236.51365661621094,
|
|
"logps/rejected": -242.02334594726562,
|
|
"loss": 5.2636,
|
|
"margin_dpo/margin_mean": 7.904582977294922,
|
|
"margin_dpo/margin_std": 15.73293685913086,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.17382198952879582,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.677565574645996,
|
|
"fcm_dpo/q_t": 0.4784577488899231,
|
|
"grad_norm": 30.75320816040039,
|
|
"learning_rate": 4.922908189595017e-07,
|
|
"logits/chosen": -0.6892061233520508,
|
|
"logits/rejected": -0.6728148460388184,
|
|
"logps/chosen": -273.9486083984375,
|
|
"logps/ref_chosen": -274.21923828125,
|
|
"logps/ref_rejected": -276.2212219238281,
|
|
"logps/rejected": -284.628173828125,
|
|
"loss": 5.2424,
|
|
"margin_dpo/margin_mean": 8.67756462097168,
|
|
"margin_dpo/margin_std": 18.219993591308594,
|
|
"step": 83
|
|
},
|
|
{
|
|
"epoch": 0.17591623036649215,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 7.32714319229126,
|
|
"fcm_dpo/q_t": 0.48182541131973267,
|
|
"grad_norm": 29.90275764465332,
|
|
"learning_rate": 4.918331902411841e-07,
|
|
"logits/chosen": -0.7316781282424927,
|
|
"logits/rejected": -0.7458916902542114,
|
|
"logps/chosen": -293.8858337402344,
|
|
"logps/ref_chosen": -294.3975524902344,
|
|
"logps/ref_rejected": -279.81884765625,
|
|
"logps/rejected": -286.63421630859375,
|
|
"loss": 5.2887,
|
|
"margin_dpo/margin_mean": 7.327144145965576,
|
|
"margin_dpo/margin_std": 16.910655975341797,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.17801047120418848,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 5.808302879333496,
|
|
"fcm_dpo/q_t": 0.48558884859085083,
|
|
"grad_norm": 29.378389358520508,
|
|
"learning_rate": 4.913625927427995e-07,
|
|
"logits/chosen": -0.6668398380279541,
|
|
"logits/rejected": -0.6751588582992554,
|
|
"logps/chosen": -245.18560791015625,
|
|
"logps/ref_chosen": -243.66220092773438,
|
|
"logps/ref_rejected": -263.9421691894531,
|
|
"logps/rejected": -271.2738952636719,
|
|
"loss": 5.3432,
|
|
"margin_dpo/margin_mean": 5.808302402496338,
|
|
"margin_dpo/margin_std": 15.501203536987305,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.18010471204188483,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.855512619018555,
|
|
"fcm_dpo/q_t": 0.47801530361175537,
|
|
"grad_norm": 34.95083236694336,
|
|
"learning_rate": 4.908790517010636e-07,
|
|
"logits/chosen": -0.6866583228111267,
|
|
"logits/rejected": -0.6797590255737305,
|
|
"logps/chosen": -308.28826904296875,
|
|
"logps/ref_chosen": -309.4306945800781,
|
|
"logps/ref_rejected": -290.91278076171875,
|
|
"logps/rejected": -298.6258544921875,
|
|
"loss": 5.2308,
|
|
"margin_dpo/margin_mean": 8.855510711669922,
|
|
"margin_dpo/margin_std": 17.44927406311035,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.18219895287958116,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 10.284191131591797,
|
|
"fcm_dpo/q_t": 0.4746079742908478,
|
|
"grad_norm": 29.6323184967041,
|
|
"learning_rate": 4.903825930468148e-07,
|
|
"logits/chosen": -0.7473527193069458,
|
|
"logits/rejected": -0.7407711744308472,
|
|
"logps/chosen": -278.1808166503906,
|
|
"logps/ref_chosen": -278.0277099609375,
|
|
"logps/ref_rejected": -245.70123291015625,
|
|
"logps/rejected": -256.13848876953125,
|
|
"loss": 5.1829,
|
|
"margin_dpo/margin_mean": 10.284191131591797,
|
|
"margin_dpo/margin_std": 18.875999450683594,
|
|
"step": 87
|
|
},
|
|
{
|
|
"epoch": 0.18429319371727748,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.930685997009277,
|
|
"fcm_dpo/q_t": 0.47792893648147583,
|
|
"grad_norm": 28.799327850341797,
|
|
"learning_rate": 4.898732434036243e-07,
|
|
"logits/chosen": -0.7647247314453125,
|
|
"logits/rejected": -0.7806797027587891,
|
|
"logps/chosen": -268.60662841796875,
|
|
"logps/ref_chosen": -266.5148010253906,
|
|
"logps/ref_rejected": -265.90081787109375,
|
|
"logps/rejected": -276.9233093261719,
|
|
"loss": 5.2343,
|
|
"margin_dpo/margin_mean": 8.930685997009277,
|
|
"margin_dpo/margin_std": 19.163759231567383,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.18638743455497384,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.23874282836914,
|
|
"fcm_dpo/q_t": 0.4771117866039276,
|
|
"grad_norm": 30.643220901489258,
|
|
"learning_rate": 4.893510300863676e-07,
|
|
"logits/chosen": -0.7400345802307129,
|
|
"logits/rejected": -0.7301192283630371,
|
|
"logps/chosen": -265.6995544433594,
|
|
"logps/ref_chosen": -265.6893005371094,
|
|
"logps/ref_rejected": -251.49314880371094,
|
|
"logps/rejected": -260.7421569824219,
|
|
"loss": 5.2214,
|
|
"margin_dpo/margin_mean": 9.238741874694824,
|
|
"margin_dpo/margin_std": 18.232431411743164,
|
|
"step": 89
|
|
},
|
|
{
|
|
"epoch": 0.18848167539267016,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 8.30599594116211,
|
|
"fcm_dpo/q_t": 0.4794096052646637,
|
|
"grad_norm": 29.981016159057617,
|
|
"learning_rate": 4.8881598109976e-07,
|
|
"logits/chosen": -0.7483118176460266,
|
|
"logits/rejected": -0.7575773000717163,
|
|
"logps/chosen": -308.5583801269531,
|
|
"logps/ref_chosen": -307.4250183105469,
|
|
"logps/ref_rejected": -265.7172546386719,
|
|
"logps/rejected": -275.1565856933594,
|
|
"loss": 5.2543,
|
|
"margin_dpo/margin_mean": 8.305996894836426,
|
|
"margin_dpo/margin_std": 17.779386520385742,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.1905759162303665,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.575455665588379,
|
|
"fcm_dpo/q_t": 0.47638604044914246,
|
|
"grad_norm": 33.001312255859375,
|
|
"learning_rate": 4.882681251368548e-07,
|
|
"logits/chosen": -0.6704283356666565,
|
|
"logits/rejected": -0.6869698166847229,
|
|
"logps/chosen": -237.86941528320312,
|
|
"logps/ref_chosen": -235.74098205566406,
|
|
"logps/ref_rejected": -226.6428985595703,
|
|
"logps/rejected": -238.34674072265625,
|
|
"loss": 5.2137,
|
|
"margin_dpo/margin_mean": 9.575454711914062,
|
|
"margin_dpo/margin_std": 19.24980926513672,
|
|
"step": 91
|
|
},
|
|
{
|
|
"epoch": 0.19267015706806281,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 10.090764999389648,
|
|
"fcm_dpo/q_t": 0.475067675113678,
|
|
"grad_norm": 33.0310173034668,
|
|
"learning_rate": 4.877074915775048e-07,
|
|
"logits/chosen": -0.7317771911621094,
|
|
"logits/rejected": -0.7149327993392944,
|
|
"logps/chosen": -286.60858154296875,
|
|
"logps/ref_chosen": -283.4475402832031,
|
|
"logps/ref_rejected": -273.134033203125,
|
|
"logps/rejected": -286.38580322265625,
|
|
"loss": 5.1997,
|
|
"margin_dpo/margin_mean": 10.090764999389648,
|
|
"margin_dpo/margin_std": 21.30156707763672,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.19476439790575917,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.687959671020508,
|
|
"fcm_dpo/q_t": 0.4760984778404236,
|
|
"grad_norm": 28.729263305664062,
|
|
"learning_rate": 4.871341104867864e-07,
|
|
"logits/chosen": -0.7301860451698303,
|
|
"logits/rejected": -0.7536556124687195,
|
|
"logps/chosen": -235.7203826904297,
|
|
"logps/ref_chosen": -233.33714294433594,
|
|
"logps/ref_rejected": -230.54273986816406,
|
|
"logps/rejected": -242.61395263671875,
|
|
"loss": 5.2079,
|
|
"margin_dpo/margin_mean": 9.687959671020508,
|
|
"margin_dpo/margin_std": 19.470970153808594,
|
|
"step": 93
|
|
},
|
|
{
|
|
"epoch": 0.1968586387434555,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 9.32542610168457,
|
|
"fcm_dpo/q_t": 0.47693106532096863,
|
|
"grad_norm": 31.511959075927734,
|
|
"learning_rate": 4.865480126133871e-07,
|
|
"logits/chosen": -0.6908999681472778,
|
|
"logits/rejected": -0.7126898169517517,
|
|
"logps/chosen": -297.0324401855469,
|
|
"logps/ref_chosen": -294.6528015136719,
|
|
"logps/ref_rejected": -283.657958984375,
|
|
"logps/rejected": -295.3630065917969,
|
|
"loss": 5.2322,
|
|
"margin_dpo/margin_mean": 9.325427055358887,
|
|
"margin_dpo/margin_std": 21.890119552612305,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.19895287958115182,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 10.206320762634277,
|
|
"fcm_dpo/q_t": 0.4748663306236267,
|
|
"grad_norm": 33.60295867919922,
|
|
"learning_rate": 4.859492293879573e-07,
|
|
"logits/chosen": -0.7269071936607361,
|
|
"logits/rejected": -0.7481443881988525,
|
|
"logps/chosen": -314.99395751953125,
|
|
"logps/ref_chosen": -311.6697082519531,
|
|
"logps/ref_rejected": -262.7471923828125,
|
|
"logps/rejected": -276.27777099609375,
|
|
"loss": 5.2042,
|
|
"margin_dpo/margin_mean": 10.206320762634277,
|
|
"margin_dpo/margin_std": 22.42072296142578,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.20104712041884817,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 10.304360389709473,
|
|
"fcm_dpo/q_t": 0.4746476709842682,
|
|
"grad_norm": 34.0047721862793,
|
|
"learning_rate": 4.853377929214243e-07,
|
|
"logits/chosen": -0.6979155540466309,
|
|
"logits/rejected": -0.7102517485618591,
|
|
"logps/chosen": -287.2977294921875,
|
|
"logps/ref_chosen": -282.55596923828125,
|
|
"logps/ref_rejected": -242.71588134765625,
|
|
"logps/rejected": -257.7619934082031,
|
|
"loss": 5.2021,
|
|
"margin_dpo/margin_mean": 10.304360389709473,
|
|
"margin_dpo/margin_std": 23.36362648010254,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.2031413612565445,
|
|
"fcm_dpo/beta": 0.009999998845160007,
|
|
"fcm_dpo/delta": 0.0,
|
|
"fcm_dpo/margin": 12.058700561523438,
|
|
"fcm_dpo/q_t": 0.47029611468315125,
|
|
"grad_norm": 32.09419250488281,
|
|
"learning_rate": 4.847137360032699e-07,
|
|
"logits/chosen": -0.7502722144126892,
|
|
"logits/rejected": -0.7376055717468262,
|
|
"logps/chosen": -307.92425537109375,
|
|
"logps/ref_chosen": -303.57781982421875,
|
|
"logps/ref_rejected": -264.22491455078125,
|
|
"logps/rejected": -280.6300964355469,
|
|
"loss": 5.1301,
|
|
"margin_dpo/margin_mean": 12.05870246887207,
|
|
"margin_dpo/margin_std": 22.335546493530273,
|
|
"step": 97
|
|
},
|
|
{
|
|
"epoch": 0.20523560209424083,
|
|
"fcm_dpo/beta": 0.010166899301111698,
|
|
"fcm_dpo/delta": 0.03736456483602524,
|
|
"fcm_dpo/margin": 12.270679473876953,
|
|
"fcm_dpo/q_t": 0.4697565734386444,
|
|
"grad_norm": 35.15353012084961,
|
|
"learning_rate": 4.84077092099773e-07,
|
|
"logits/chosen": -0.7689064145088196,
|
|
"logits/rejected": -0.7808451056480408,
|
|
"logps/chosen": -291.73529052734375,
|
|
"logps/ref_chosen": -286.8303527832031,
|
|
"logps/ref_rejected": -278.08331298828125,
|
|
"logps/rejected": -295.2589416503906,
|
|
"loss": 5.1298,
|
|
"margin_dpo/margin_mean": 12.270679473876953,
|
|
"margin_dpo/margin_std": 22.752973556518555,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.20732984293193718,
|
|
"fcm_dpo/beta": 0.010414928197860718,
|
|
"fcm_dpo/delta": 0.021457988768815994,
|
|
"fcm_dpo/margin": 12.633167266845703,
|
|
"fcm_dpo/q_t": 0.46785154938697815,
|
|
"grad_norm": 33.9055290222168,
|
|
"learning_rate": 4.834278953522137e-07,
|
|
"logits/chosen": -0.7366681098937988,
|
|
"logits/rejected": -0.7495771646499634,
|
|
"logps/chosen": -285.1845703125,
|
|
"logps/ref_chosen": -279.92120361328125,
|
|
"logps/ref_rejected": -250.3365478515625,
|
|
"logps/rejected": -268.2330627441406,
|
|
"loss": 5.121,
|
|
"margin_dpo/margin_mean": 12.633166313171387,
|
|
"margin_dpo/margin_std": 27.084667205810547,
|
|
"step": 99
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"fcm_dpo/beta": 0.010724691674113274,
|
|
"fcm_dpo/delta": 0.03860073536634445,
|
|
"fcm_dpo/margin": 12.622820854187012,
|
|
"fcm_dpo/q_t": 0.4671909511089325,
|
|
"grad_norm": 37.84049987792969,
|
|
"learning_rate": 4.827661805750437e-07,
|
|
"logits/chosen": -0.785658597946167,
|
|
"logits/rejected": -0.7990057468414307,
|
|
"logps/chosen": -304.5786437988281,
|
|
"logps/ref_chosen": -296.8276672363281,
|
|
"logps/ref_rejected": -275.56146240234375,
|
|
"logps/rejected": -295.9351806640625,
|
|
"loss": 5.0963,
|
|
"margin_dpo/margin_mean": 12.622820854187012,
|
|
"margin_dpo/margin_std": 24.152620315551758,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.21151832460732983,
|
|
"fcm_dpo/beta": 0.010941779240965843,
|
|
"fcm_dpo/delta": 0.03345339000225067,
|
|
"fcm_dpo/margin": 15.069441795349121,
|
|
"fcm_dpo/q_t": 0.4600520133972168,
|
|
"grad_norm": 35.19692611694336,
|
|
"learning_rate": 4.820919832540181e-07,
|
|
"logits/chosen": -0.7683415412902832,
|
|
"logits/rejected": -0.7820852398872375,
|
|
"logps/chosen": -257.931396484375,
|
|
"logps/ref_chosen": -252.74203491210938,
|
|
"logps/ref_rejected": -276.4185485839844,
|
|
"logps/rejected": -296.6773986816406,
|
|
"loss": 5.0018,
|
|
"margin_dpo/margin_mean": 15.069441795349121,
|
|
"margin_dpo/margin_std": 26.246978759765625,
|
|
"step": 101
|
|
},
|
|
{
|
|
"epoch": 0.2136125654450262,
|
|
"fcm_dpo/beta": 0.011548048816621304,
|
|
"fcm_dpo/delta": 0.08071689307689667,
|
|
"fcm_dpo/margin": 14.844532012939453,
|
|
"fcm_dpo/q_t": 0.45827770233154297,
|
|
"grad_norm": 37.058868408203125,
|
|
"learning_rate": 4.814053395442932e-07,
|
|
"logits/chosen": -0.7501267194747925,
|
|
"logits/rejected": -0.7462649941444397,
|
|
"logps/chosen": -224.66371154785156,
|
|
"logps/ref_chosen": -219.5537109375,
|
|
"logps/ref_rejected": -231.90853881835938,
|
|
"logps/rejected": -251.86306762695312,
|
|
"loss": 4.9782,
|
|
"margin_dpo/margin_mean": 14.844533920288086,
|
|
"margin_dpo/margin_std": 25.003633499145508,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.2157068062827225,
|
|
"fcm_dpo/beta": 0.01229560561478138,
|
|
"fcm_dpo/delta": 0.06470398604869843,
|
|
"fcm_dpo/margin": 13.61020278930664,
|
|
"fcm_dpo/q_t": 0.4601737856864929,
|
|
"grad_norm": 39.8343391418457,
|
|
"learning_rate": 4.807062862684873e-07,
|
|
"logits/chosen": -0.7709028124809265,
|
|
"logits/rejected": -0.7678197026252747,
|
|
"logps/chosen": -264.3942565917969,
|
|
"logps/ref_chosen": -259.6750793457031,
|
|
"logps/ref_rejected": -278.7400817871094,
|
|
"logps/rejected": -297.0694885253906,
|
|
"loss": 5.0107,
|
|
"margin_dpo/margin_mean": 13.61020278930664,
|
|
"margin_dpo/margin_std": 25.62775993347168,
|
|
"step": 103
|
|
},
|
|
{
|
|
"epoch": 0.21780104712041884,
|
|
"fcm_dpo/beta": 0.01250280998647213,
|
|
"fcm_dpo/delta": 0.023114658892154694,
|
|
"fcm_dpo/margin": 10.093114852905273,
|
|
"fcm_dpo/q_t": 0.4694807529449463,
|
|
"grad_norm": 41.128822326660156,
|
|
"learning_rate": 4.799948609147061e-07,
|
|
"logits/chosen": -0.765766978263855,
|
|
"logits/rejected": -0.771574079990387,
|
|
"logps/chosen": -276.9814147949219,
|
|
"logps/ref_chosen": -267.9741516113281,
|
|
"logps/ref_rejected": -230.5306396484375,
|
|
"logps/rejected": -249.6309814453125,
|
|
"loss": 5.1671,
|
|
"margin_dpo/margin_mean": 10.09311580657959,
|
|
"margin_dpo/margin_std": 26.334136962890625,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.2198952879581152,
|
|
"fcm_dpo/beta": 0.012972178868949413,
|
|
"fcm_dpo/delta": 0.06148073077201843,
|
|
"fcm_dpo/margin": 20.653295516967773,
|
|
"fcm_dpo/q_t": 0.4356263279914856,
|
|
"grad_norm": 43.76247024536133,
|
|
"learning_rate": 4.792711016345321e-07,
|
|
"logits/chosen": -0.7641775608062744,
|
|
"logits/rejected": -0.7758720517158508,
|
|
"logps/chosen": -327.5227966308594,
|
|
"logps/ref_chosen": -322.25482177734375,
|
|
"logps/ref_rejected": -279.02978515625,
|
|
"logps/rejected": -304.9510498046875,
|
|
"loss": 4.6816,
|
|
"margin_dpo/margin_mean": 20.653295516967773,
|
|
"margin_dpo/margin_std": 27.019716262817383,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.22198952879581152,
|
|
"fcm_dpo/beta": 0.01369519904255867,
|
|
"fcm_dpo/delta": 0.049693141132593155,
|
|
"fcm_dpo/margin": 12.526891708374023,
|
|
"fcm_dpo/q_t": 0.4589945077896118,
|
|
"grad_norm": 50.769107818603516,
|
|
"learning_rate": 4.785350472409791e-07,
|
|
"logits/chosen": -0.7395763397216797,
|
|
"logits/rejected": -0.7750528454780579,
|
|
"logps/chosen": -308.6379089355469,
|
|
"logps/ref_chosen": -296.15777587890625,
|
|
"logps/ref_rejected": -266.2691650390625,
|
|
"logps/rejected": -291.2762145996094,
|
|
"loss": 5.0575,
|
|
"margin_dpo/margin_mean": 12.52688980102539,
|
|
"margin_dpo/margin_std": 29.427845001220703,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.22408376963350785,
|
|
"fcm_dpo/beta": 0.014684994705021381,
|
|
"fcm_dpo/delta": 0.11428901553153992,
|
|
"fcm_dpo/margin": 19.656145095825195,
|
|
"fcm_dpo/q_t": 0.431806743144989,
|
|
"grad_norm": 52.60951232910156,
|
|
"learning_rate": 4.777867372064105e-07,
|
|
"logits/chosen": -0.780152440071106,
|
|
"logits/rejected": -0.7732595205307007,
|
|
"logps/chosen": -311.265869140625,
|
|
"logps/ref_chosen": -306.996337890625,
|
|
"logps/ref_rejected": -296.79412841796875,
|
|
"logps/rejected": -320.7198181152344,
|
|
"loss": 4.6426,
|
|
"margin_dpo/margin_mean": 19.656145095825195,
|
|
"margin_dpo/margin_std": 27.61237335205078,
|
|
"step": 107
|
|
},
|
|
{
|
|
"epoch": 0.2261780104712042,
|
|
"fcm_dpo/beta": 0.015535693615674973,
|
|
"fcm_dpo/delta": 0.054226309061050415,
|
|
"fcm_dpo/margin": 18.248409271240234,
|
|
"fcm_dpo/q_t": 0.4343521296977997,
|
|
"grad_norm": 162.06214904785156,
|
|
"learning_rate": 4.770262116604223e-07,
|
|
"logits/chosen": -0.7690107822418213,
|
|
"logits/rejected": -0.781175434589386,
|
|
"logps/chosen": -300.4916687011719,
|
|
"logps/ref_chosen": -295.1526794433594,
|
|
"logps/ref_rejected": -235.974853515625,
|
|
"logps/rejected": -259.562255859375,
|
|
"loss": 4.7324,
|
|
"margin_dpo/margin_mean": 18.248409271240234,
|
|
"margin_dpo/margin_std": 30.292400360107422,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.22827225130890053,
|
|
"fcm_dpo/beta": 0.016129182651638985,
|
|
"fcm_dpo/delta": 0.03493024408817291,
|
|
"fcm_dpo/margin": 20.186677932739258,
|
|
"fcm_dpo/q_t": 0.4253409504890442,
|
|
"grad_norm": 57.66378402709961,
|
|
"learning_rate": 4.7625351138769166e-07,
|
|
"logits/chosen": -0.7831578850746155,
|
|
"logits/rejected": -0.7787821888923645,
|
|
"logps/chosen": -333.96337890625,
|
|
"logps/ref_chosen": -325.9248046875,
|
|
"logps/ref_rejected": -279.15423583984375,
|
|
"logps/rejected": -307.3795166015625,
|
|
"loss": 4.6025,
|
|
"margin_dpo/margin_mean": 20.186674118041992,
|
|
"margin_dpo/margin_std": 31.418649673461914,
|
|
"step": 109
|
|
},
|
|
{
|
|
"epoch": 0.23036649214659685,
|
|
"fcm_dpo/beta": 0.016638344153761864,
|
|
"fcm_dpo/delta": 0.03626322001218796,
|
|
"fcm_dpo/margin": 18.989017486572266,
|
|
"fcm_dpo/q_t": 0.42738083004951477,
|
|
"grad_norm": 53.80938720703125,
|
|
"learning_rate": 4.75468677825789e-07,
|
|
"logits/chosen": -0.8024147748947144,
|
|
"logits/rejected": -0.7910180687904358,
|
|
"logps/chosen": -282.9509582519531,
|
|
"logps/ref_chosen": -274.439208984375,
|
|
"logps/ref_rejected": -260.0552062988281,
|
|
"logps/rejected": -287.5560302734375,
|
|
"loss": 4.6842,
|
|
"margin_dpo/margin_mean": 18.989017486572266,
|
|
"margin_dpo/margin_std": 33.461700439453125,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.2324607329842932,
|
|
"fcm_dpo/beta": 0.016925374045968056,
|
|
"fcm_dpo/delta": 0.004534606821835041,
|
|
"fcm_dpo/margin": 20.09602928161621,
|
|
"fcm_dpo/q_t": 0.4227144718170166,
|
|
"grad_norm": 59.05382537841797,
|
|
"learning_rate": 4.7467175306295647e-07,
|
|
"logits/chosen": -0.8369516134262085,
|
|
"logits/rejected": -0.8165923953056335,
|
|
"logps/chosen": -338.4478454589844,
|
|
"logps/ref_chosen": -329.2361755371094,
|
|
"logps/ref_rejected": -287.82830810546875,
|
|
"logps/rejected": -317.135986328125,
|
|
"loss": 4.6353,
|
|
"margin_dpo/margin_mean": 20.09602928161621,
|
|
"margin_dpo/margin_std": 33.968441009521484,
|
|
"step": 111
|
|
},
|
|
{
|
|
"epoch": 0.23455497382198953,
|
|
"fcm_dpo/beta": 0.016810838133096695,
|
|
"fcm_dpo/delta": 0.002743429969996214,
|
|
"fcm_dpo/margin": 12.950404167175293,
|
|
"fcm_dpo/q_t": 0.45069649815559387,
|
|
"grad_norm": 67.04423522949219,
|
|
"learning_rate": 4.7386277983585053e-07,
|
|
"logits/chosen": -0.7323642373085022,
|
|
"logits/rejected": -0.762016773223877,
|
|
"logps/chosen": -271.5980529785156,
|
|
"logps/ref_chosen": -257.0593566894531,
|
|
"logps/ref_rejected": -272.9595031738281,
|
|
"logps/rejected": -300.4485778808594,
|
|
"loss": 5.1077,
|
|
"margin_dpo/margin_mean": 12.950403213500977,
|
|
"margin_dpo/margin_std": 35.27339172363281,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.23664921465968586,
|
|
"fcm_dpo/beta": 0.016340255737304688,
|
|
"fcm_dpo/delta": -0.055206988006830215,
|
|
"fcm_dpo/margin": 23.974803924560547,
|
|
"fcm_dpo/q_t": 0.41226544976234436,
|
|
"grad_norm": 59.851139068603516,
|
|
"learning_rate": 4.7304180152725024e-07,
|
|
"logits/chosen": -0.7996099591255188,
|
|
"logits/rejected": -0.8028220534324646,
|
|
"logps/chosen": -297.7246398925781,
|
|
"logps/ref_chosen": -286.0416564941406,
|
|
"logps/ref_rejected": -270.374267578125,
|
|
"logps/rejected": -306.0320739746094,
|
|
"loss": 4.5109,
|
|
"margin_dpo/margin_mean": 23.974803924560547,
|
|
"margin_dpo/margin_std": 38.330360412597656,
|
|
"step": 113
|
|
},
|
|
{
|
|
"epoch": 0.2387434554973822,
|
|
"fcm_dpo/beta": 0.01674072816967964,
|
|
"fcm_dpo/delta": 0.04583574831485748,
|
|
"fcm_dpo/margin": 13.43380355834961,
|
|
"fcm_dpo/q_t": 0.44763025641441345,
|
|
"grad_norm": 63.377777099609375,
|
|
"learning_rate": 4.7220886216373085e-07,
|
|
"logits/chosen": -0.844326913356781,
|
|
"logits/rejected": -0.8458607196807861,
|
|
"logps/chosen": -275.0863037109375,
|
|
"logps/ref_chosen": -260.0084533691406,
|
|
"logps/ref_rejected": -246.67190551757812,
|
|
"logps/rejected": -275.18353271484375,
|
|
"loss": 5.0337,
|
|
"margin_dpo/margin_mean": 13.433801651000977,
|
|
"margin_dpo/margin_std": 34.03593444824219,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.24083769633507854,
|
|
"fcm_dpo/beta": 0.01667495258152485,
|
|
"fcm_dpo/delta": 0.022258806973695755,
|
|
"fcm_dpo/margin": 13.424970626831055,
|
|
"fcm_dpo/q_t": 0.45159363746643066,
|
|
"grad_norm": 71.67733001708984,
|
|
"learning_rate": 4.7136400641330245e-07,
|
|
"logits/chosen": -0.8421843647956848,
|
|
"logits/rejected": -0.8031139969825745,
|
|
"logps/chosen": -315.6531677246094,
|
|
"logps/ref_chosen": -299.4229736328125,
|
|
"logps/ref_rejected": -272.1186828613281,
|
|
"logps/rejected": -301.77386474609375,
|
|
"loss": 5.0871,
|
|
"margin_dpo/margin_mean": 13.424969673156738,
|
|
"margin_dpo/margin_std": 37.24310302734375,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.24293193717277486,
|
|
"fcm_dpo/beta": 0.01778252422809601,
|
|
"fcm_dpo/delta": 0.06407217681407928,
|
|
"fcm_dpo/margin": 17.605409622192383,
|
|
"fcm_dpo/q_t": 0.4284539520740509,
|
|
"grad_norm": 64.84068298339844,
|
|
"learning_rate": 4.70507279583015e-07,
|
|
"logits/chosen": -0.8544034957885742,
|
|
"logits/rejected": -0.8170161843299866,
|
|
"logps/chosen": -291.0081481933594,
|
|
"logps/ref_chosen": -279.263916015625,
|
|
"logps/ref_rejected": -253.6192169189453,
|
|
"logps/rejected": -282.9688720703125,
|
|
"loss": 4.744,
|
|
"margin_dpo/margin_mean": 17.60540771484375,
|
|
"margin_dpo/margin_std": 33.50556182861328,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.2450261780104712,
|
|
"fcm_dpo/beta": 0.017902228981256485,
|
|
"fcm_dpo/delta": -0.05058136209845543,
|
|
"fcm_dpo/margin": 19.721925735473633,
|
|
"fcm_dpo/q_t": 0.41956445574760437,
|
|
"grad_norm": 74.53150177001953,
|
|
"learning_rate": 4.6963872761652834e-07,
|
|
"logits/chosen": -0.8143470287322998,
|
|
"logits/rejected": -0.8171401619911194,
|
|
"logps/chosen": -273.25616455078125,
|
|
"logps/ref_chosen": -259.2248840332031,
|
|
"logps/ref_rejected": -229.3042755126953,
|
|
"logps/rejected": -263.0574951171875,
|
|
"loss": 4.6214,
|
|
"margin_dpo/margin_mean": 19.721923828125,
|
|
"margin_dpo/margin_std": 30.71808624267578,
|
|
"step": 117
|
|
},
|
|
{
|
|
"epoch": 0.24712041884816754,
|
|
"fcm_dpo/beta": 0.01726110279560089,
|
|
"fcm_dpo/delta": -0.024961750954389572,
|
|
"fcm_dpo/margin": 22.500560760498047,
|
|
"fcm_dpo/q_t": 0.4144339859485626,
|
|
"grad_norm": 71.345458984375,
|
|
"learning_rate": 4.687583970916486e-07,
|
|
"logits/chosen": -0.7954049110412598,
|
|
"logits/rejected": -0.7841992378234863,
|
|
"logps/chosen": -285.67327880859375,
|
|
"logps/ref_chosen": -267.0707092285156,
|
|
"logps/ref_rejected": -272.7322082519531,
|
|
"logps/rejected": -313.8353271484375,
|
|
"loss": 4.6027,
|
|
"margin_dpo/margin_mean": 22.50056266784668,
|
|
"margin_dpo/margin_std": 39.91600036621094,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.24921465968586387,
|
|
"fcm_dpo/beta": 0.017213156446814537,
|
|
"fcm_dpo/delta": 0.016311757266521454,
|
|
"fcm_dpo/margin": 16.51951026916504,
|
|
"fcm_dpo/q_t": 0.43743908405303955,
|
|
"grad_norm": 75.18436431884766,
|
|
"learning_rate": 4.6786633521783005e-07,
|
|
"logits/chosen": -0.8822775483131409,
|
|
"logits/rejected": -0.8836984634399414,
|
|
"logps/chosen": -347.19122314453125,
|
|
"logps/ref_chosen": -324.6766357421875,
|
|
"logps/ref_rejected": -306.0322265625,
|
|
"logps/rejected": -345.0663146972656,
|
|
"loss": 4.9579,
|
|
"margin_dpo/margin_mean": 16.519508361816406,
|
|
"margin_dpo/margin_std": 38.816429138183594,
|
|
"step": 119
|
|
},
|
|
{
|
|
"epoch": 0.2513089005235602,
|
|
"fcm_dpo/beta": 0.017053598538041115,
|
|
"fcm_dpo/delta": -0.027553297579288483,
|
|
"fcm_dpo/margin": 17.9389591217041,
|
|
"fcm_dpo/q_t": 0.43243440985679626,
|
|
"grad_norm": 66.44527435302734,
|
|
"learning_rate": 4.669625898336438e-07,
|
|
"logits/chosen": -0.8248355388641357,
|
|
"logits/rejected": -0.839232325553894,
|
|
"logps/chosen": -334.0506286621094,
|
|
"logps/ref_chosen": -315.2617492675781,
|
|
"logps/ref_rejected": -265.32501220703125,
|
|
"logps/rejected": -302.0528564453125,
|
|
"loss": 4.869,
|
|
"margin_dpo/margin_mean": 17.9389591217041,
|
|
"margin_dpo/margin_std": 38.128028869628906,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.2534031413612565,
|
|
"fcm_dpo/beta": 0.01718924008309841,
|
|
"fcm_dpo/delta": 0.06284664571285248,
|
|
"fcm_dpo/margin": 14.12536334991455,
|
|
"fcm_dpo/q_t": 0.4474913775920868,
|
|
"grad_norm": 73.3908462524414,
|
|
"learning_rate": 4.6604720940421207e-07,
|
|
"logits/chosen": -0.8356261253356934,
|
|
"logits/rejected": -0.8507847785949707,
|
|
"logps/chosen": -246.18655395507812,
|
|
"logps/ref_chosen": -222.99609375,
|
|
"logps/ref_rejected": -226.92860412597656,
|
|
"logps/rejected": -264.244384765625,
|
|
"loss": 5.0392,
|
|
"margin_dpo/margin_mean": 14.125362396240234,
|
|
"margin_dpo/margin_std": 37.465171813964844,
|
|
"step": 121
|
|
},
|
|
{
|
|
"epoch": 0.2554973821989529,
|
|
"fcm_dpo/beta": 0.017536453902721405,
|
|
"fcm_dpo/delta": 0.005643364042043686,
|
|
"fcm_dpo/margin": 18.017147064208984,
|
|
"fcm_dpo/q_t": 0.4305107891559601,
|
|
"grad_norm": 70.16292572021484,
|
|
"learning_rate": 4.651202430186092e-07,
|
|
"logits/chosen": -0.89496248960495,
|
|
"logits/rejected": -0.8574784994125366,
|
|
"logps/chosen": -299.5643310546875,
|
|
"logps/ref_chosen": -276.02630615234375,
|
|
"logps/ref_rejected": -277.97418212890625,
|
|
"logps/rejected": -319.52935791015625,
|
|
"loss": 4.8959,
|
|
"margin_dpo/margin_mean": 18.017147064208984,
|
|
"margin_dpo/margin_std": 41.55096435546875,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.25759162303664923,
|
|
"fcm_dpo/beta": 0.01756231114268303,
|
|
"fcm_dpo/delta": -0.03471625596284866,
|
|
"fcm_dpo/margin": 23.458398818969727,
|
|
"fcm_dpo/q_t": 0.40726321935653687,
|
|
"grad_norm": 73.8258056640625,
|
|
"learning_rate": 4.6418174038722924e-07,
|
|
"logits/chosen": -0.8168793320655823,
|
|
"logits/rejected": -0.8113296031951904,
|
|
"logps/chosen": -346.0722351074219,
|
|
"logps/ref_chosen": -328.1546325683594,
|
|
"logps/ref_rejected": -280.6911315917969,
|
|
"logps/rejected": -322.06719970703125,
|
|
"loss": 4.5056,
|
|
"margin_dpo/margin_mean": 23.458398818969727,
|
|
"margin_dpo/margin_std": 37.95882034301758,
|
|
"step": 123
|
|
},
|
|
{
|
|
"epoch": 0.25968586387434556,
|
|
"fcm_dpo/beta": 0.017159288749098778,
|
|
"fcm_dpo/delta": 0.005674887448549271,
|
|
"fcm_dpo/margin": 19.5401668548584,
|
|
"fcm_dpo/q_t": 0.42332780361175537,
|
|
"grad_norm": 76.41766357421875,
|
|
"learning_rate": 4.6323175183912023e-07,
|
|
"logits/chosen": -0.8458907604217529,
|
|
"logits/rejected": -0.8133480548858643,
|
|
"logps/chosen": -295.9541015625,
|
|
"logps/ref_chosen": -275.6961975097656,
|
|
"logps/ref_rejected": -225.361572265625,
|
|
"logps/rejected": -265.1595764160156,
|
|
"loss": 4.7102,
|
|
"margin_dpo/margin_mean": 19.540164947509766,
|
|
"margin_dpo/margin_std": 35.17292785644531,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.2617801047120419,
|
|
"fcm_dpo/beta": 0.017188817262649536,
|
|
"fcm_dpo/delta": -0.03201991319656372,
|
|
"fcm_dpo/margin": 18.89151382446289,
|
|
"fcm_dpo/q_t": 0.43093550205230713,
|
|
"grad_norm": 80.25518035888672,
|
|
"learning_rate": 4.6227032831928483e-07,
|
|
"logits/chosen": -0.8065889477729797,
|
|
"logits/rejected": -0.7634490728378296,
|
|
"logps/chosen": -299.75115966796875,
|
|
"logps/ref_chosen": -278.06976318359375,
|
|
"logps/ref_rejected": -265.63873291015625,
|
|
"logps/rejected": -306.21160888671875,
|
|
"loss": 4.9278,
|
|
"margin_dpo/margin_mean": 18.89151382446289,
|
|
"margin_dpo/margin_std": 44.717071533203125,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.2638743455497382,
|
|
"fcm_dpo/beta": 0.01680522970855236,
|
|
"fcm_dpo/delta": 0.005955208092927933,
|
|
"fcm_dpo/margin": 20.775150299072266,
|
|
"fcm_dpo/q_t": 0.4213416278362274,
|
|
"grad_norm": 78.756103515625,
|
|
"learning_rate": 4.612975213859487e-07,
|
|
"logits/chosen": -0.8176358938217163,
|
|
"logits/rejected": -0.8314664959907532,
|
|
"logps/chosen": -339.98236083984375,
|
|
"logps/ref_chosen": -321.3960876464844,
|
|
"logps/ref_rejected": -285.37664794921875,
|
|
"logps/rejected": -324.7380676269531,
|
|
"loss": 4.6984,
|
|
"margin_dpo/margin_mean": 20.775150299072266,
|
|
"margin_dpo/margin_std": 39.32340621948242,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.26596858638743454,
|
|
"fcm_dpo/beta": 0.01681065373122692,
|
|
"fcm_dpo/delta": -0.07578819990158081,
|
|
"fcm_dpo/margin": 24.14862823486328,
|
|
"fcm_dpo/q_t": 0.4106398820877075,
|
|
"grad_norm": 79.24022674560547,
|
|
"learning_rate": 4.603133832077953e-07,
|
|
"logits/chosen": -0.8952986598014832,
|
|
"logits/rejected": -0.8467426300048828,
|
|
"logps/chosen": -322.97332763671875,
|
|
"logps/ref_chosen": -306.55877685546875,
|
|
"logps/ref_rejected": -274.8651428222656,
|
|
"logps/rejected": -315.4283447265625,
|
|
"loss": 4.5653,
|
|
"margin_dpo/margin_mean": 24.14862632751465,
|
|
"margin_dpo/margin_std": 39.488861083984375,
|
|
"step": 127
|
|
},
|
|
{
|
|
"epoch": 0.2680628272251309,
|
|
"fcm_dpo/beta": 0.015498126856982708,
|
|
"fcm_dpo/delta": -0.057604167610406876,
|
|
"fcm_dpo/margin": 27.36457061767578,
|
|
"fcm_dpo/q_t": 0.40523380041122437,
|
|
"grad_norm": 59.929752349853516,
|
|
"learning_rate": 4.5931796656116837e-07,
|
|
"logits/chosen": -0.7822437286376953,
|
|
"logits/rejected": -0.783670961856842,
|
|
"logps/chosen": -276.9512634277344,
|
|
"logps/ref_chosen": -265.3973693847656,
|
|
"logps/ref_rejected": -250.9737548828125,
|
|
"logps/rejected": -289.8922119140625,
|
|
"loss": 4.4608,
|
|
"margin_dpo/margin_mean": 27.364572525024414,
|
|
"margin_dpo/margin_std": 43.219398498535156,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.27015706806282724,
|
|
"fcm_dpo/beta": 0.015127632766962051,
|
|
"fcm_dpo/delta": -0.0237518809735775,
|
|
"fcm_dpo/margin": 24.08989906311035,
|
|
"fcm_dpo/q_t": 0.4185941815376282,
|
|
"grad_norm": 64.50642395019531,
|
|
"learning_rate": 4.5831132482724193e-07,
|
|
"logits/chosen": -0.8073078989982605,
|
|
"logits/rejected": -0.809032678604126,
|
|
"logps/chosen": -315.49456787109375,
|
|
"logps/ref_chosen": -303.158447265625,
|
|
"logps/ref_rejected": -275.9891052246094,
|
|
"logps/rejected": -312.4151306152344,
|
|
"loss": 4.622,
|
|
"margin_dpo/margin_mean": 24.089900970458984,
|
|
"margin_dpo/margin_std": 42.023170471191406,
|
|
"step": 129
|
|
},
|
|
{
|
|
"epoch": 0.27225130890052357,
|
|
"fcm_dpo/beta": 0.015055293217301369,
|
|
"fcm_dpo/delta": 0.02385116182267666,
|
|
"fcm_dpo/margin": 23.168506622314453,
|
|
"fcm_dpo/q_t": 0.42154157161712646,
|
|
"grad_norm": 69.50651550292969,
|
|
"learning_rate": 4.5729351198915705e-07,
|
|
"logits/chosen": -0.7816063761711121,
|
|
"logits/rejected": -0.8228086829185486,
|
|
"logps/chosen": -300.2805480957031,
|
|
"logps/ref_chosen": -286.4073486328125,
|
|
"logps/ref_rejected": -294.38665771484375,
|
|
"logps/rejected": -331.42840576171875,
|
|
"loss": 4.6517,
|
|
"margin_dpo/margin_mean": 23.168506622314453,
|
|
"margin_dpo/margin_std": 41.75603103637695,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.2743455497382199,
|
|
"fcm_dpo/beta": 0.01558979507535696,
|
|
"fcm_dpo/delta": 0.03358662500977516,
|
|
"fcm_dpo/margin": 19.491979598999023,
|
|
"fcm_dpo/q_t": 0.4319177567958832,
|
|
"grad_norm": 82.41316986083984,
|
|
"learning_rate": 4.5626458262912735e-07,
|
|
"logits/chosen": -0.855750560760498,
|
|
"logits/rejected": -0.8051818013191223,
|
|
"logps/chosen": -327.0265808105469,
|
|
"logps/ref_chosen": -311.5650634765625,
|
|
"logps/ref_rejected": -291.62432861328125,
|
|
"logps/rejected": -326.577880859375,
|
|
"loss": 4.863,
|
|
"margin_dpo/margin_mean": 19.491979598999023,
|
|
"margin_dpo/margin_std": 42.24031066894531,
|
|
"step": 131
|
|
},
|
|
{
|
|
"epoch": 0.2764397905759162,
|
|
"fcm_dpo/beta": 0.015864521265029907,
|
|
"fcm_dpo/delta": -0.031978119164705276,
|
|
"fcm_dpo/margin": 27.09807014465332,
|
|
"fcm_dpo/q_t": 0.4063335061073303,
|
|
"grad_norm": 95.45539093017578,
|
|
"learning_rate": 4.5522459192551166e-07,
|
|
"logits/chosen": -0.8062165379524231,
|
|
"logits/rejected": -0.7902382612228394,
|
|
"logps/chosen": -281.4801025390625,
|
|
"logps/ref_chosen": -270.0818176269531,
|
|
"logps/ref_rejected": -284.3084411621094,
|
|
"logps/rejected": -322.8048095703125,
|
|
"loss": 4.5304,
|
|
"margin_dpo/margin_mean": 27.098068237304688,
|
|
"margin_dpo/margin_std": 45.48096466064453,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.27853403141361255,
|
|
"fcm_dpo/beta": 0.015226936899125576,
|
|
"fcm_dpo/delta": -0.009090131148695946,
|
|
"fcm_dpo/margin": 24.486173629760742,
|
|
"fcm_dpo/q_t": 0.41608279943466187,
|
|
"grad_norm": 59.91682434082031,
|
|
"learning_rate": 4.541735956498554e-07,
|
|
"logits/chosen": -0.8516502976417542,
|
|
"logits/rejected": -0.8550176024436951,
|
|
"logps/chosen": -298.2793884277344,
|
|
"logps/ref_chosen": -285.6213684082031,
|
|
"logps/ref_rejected": -251.19386291503906,
|
|
"logps/rejected": -288.3380432128906,
|
|
"loss": 4.607,
|
|
"margin_dpo/margin_mean": 24.486173629760742,
|
|
"margin_dpo/margin_std": 42.33135223388672,
|
|
"step": 133
|
|
},
|
|
{
|
|
"epoch": 0.2806282722513089,
|
|
"fcm_dpo/beta": 0.015293042175471783,
|
|
"fcm_dpo/delta": 0.0003851773217320442,
|
|
"fcm_dpo/margin": 19.430198669433594,
|
|
"fcm_dpo/q_t": 0.43152743577957153,
|
|
"grad_norm": 61.967952728271484,
|
|
"learning_rate": 4.5311165016389914e-07,
|
|
"logits/chosen": -0.8527645468711853,
|
|
"logits/rejected": -0.8533920049667358,
|
|
"logps/chosen": -341.70172119140625,
|
|
"logps/ref_chosen": -318.92083740234375,
|
|
"logps/ref_rejected": -293.1894836425781,
|
|
"logps/rejected": -335.4005432128906,
|
|
"loss": 4.8138,
|
|
"margin_dpo/margin_mean": 19.430200576782227,
|
|
"margin_dpo/margin_std": 38.5192985534668,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.28272251308900526,
|
|
"fcm_dpo/beta": 0.015089647844433784,
|
|
"fcm_dpo/delta": -0.019326163455843925,
|
|
"fcm_dpo/margin": 22.623905181884766,
|
|
"fcm_dpo/q_t": 0.4212813675403595,
|
|
"grad_norm": 82.32707214355469,
|
|
"learning_rate": 4.520388124165564e-07,
|
|
"logits/chosen": -0.7468621134757996,
|
|
"logits/rejected": -0.7879329323768616,
|
|
"logps/chosen": -311.9961853027344,
|
|
"logps/ref_chosen": -292.8217468261719,
|
|
"logps/ref_rejected": -269.2896728515625,
|
|
"logps/rejected": -311.0880432128906,
|
|
"loss": 4.6979,
|
|
"margin_dpo/margin_mean": 22.623905181884766,
|
|
"margin_dpo/margin_std": 40.231651306152344,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.2848167539267016,
|
|
"fcm_dpo/beta": 0.014683660119771957,
|
|
"fcm_dpo/delta": -0.03437824174761772,
|
|
"fcm_dpo/margin": 25.665916442871094,
|
|
"fcm_dpo/q_t": 0.4163571298122406,
|
|
"grad_norm": 78.79556274414062,
|
|
"learning_rate": 4.5095513994085974e-07,
|
|
"logits/chosen": -0.808585524559021,
|
|
"logits/rejected": -0.8038730621337891,
|
|
"logps/chosen": -292.9419860839844,
|
|
"logps/ref_chosen": -272.8525390625,
|
|
"logps/ref_rejected": -252.68202209472656,
|
|
"logps/rejected": -298.4374084472656,
|
|
"loss": 4.6275,
|
|
"margin_dpo/margin_mean": 25.665916442871094,
|
|
"margin_dpo/margin_std": 45.39299392700195,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.2869109947643979,
|
|
"fcm_dpo/beta": 0.014740757644176483,
|
|
"fcm_dpo/delta": 0.006549544632434845,
|
|
"fcm_dpo/margin": 22.0756893157959,
|
|
"fcm_dpo/q_t": 0.4280848503112793,
|
|
"grad_norm": 67.095458984375,
|
|
"learning_rate": 4.498606908508753e-07,
|
|
"logits/chosen": -0.8490076661109924,
|
|
"logits/rejected": -0.8347117304801941,
|
|
"logps/chosen": -323.59130859375,
|
|
"logps/ref_chosen": -300.7522277832031,
|
|
"logps/ref_rejected": -286.1935119628906,
|
|
"logps/rejected": -331.1082458496094,
|
|
"loss": 4.7574,
|
|
"margin_dpo/margin_mean": 22.07569122314453,
|
|
"margin_dpo/margin_std": 43.516868591308594,
|
|
"step": 137
|
|
},
|
|
{
|
|
"epoch": 0.28900523560209423,
|
|
"fcm_dpo/beta": 0.01454901322722435,
|
|
"fcm_dpo/delta": -0.0002657044678926468,
|
|
"fcm_dpo/margin": 24.885833740234375,
|
|
"fcm_dpo/q_t": 0.42203420400619507,
|
|
"grad_norm": 63.38542556762695,
|
|
"learning_rate": 4.487555238385862e-07,
|
|
"logits/chosen": -0.7799475789070129,
|
|
"logits/rejected": -0.7601336240768433,
|
|
"logps/chosen": -309.79620361328125,
|
|
"logps/ref_chosen": -288.9369812011719,
|
|
"logps/ref_rejected": -263.7076416015625,
|
|
"logps/rejected": -309.45269775390625,
|
|
"loss": 4.7118,
|
|
"margin_dpo/margin_mean": 24.885831832885742,
|
|
"margin_dpo/margin_std": 49.14541244506836,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.29109947643979056,
|
|
"fcm_dpo/beta": 0.014541015960276127,
|
|
"fcm_dpo/delta": 0.017182698473334312,
|
|
"fcm_dpo/margin": 16.95909309387207,
|
|
"fcm_dpo/q_t": 0.4451315104961395,
|
|
"grad_norm": 65.62071228027344,
|
|
"learning_rate": 4.476396981707453e-07,
|
|
"logits/chosen": -0.8039760589599609,
|
|
"logits/rejected": -0.8343940377235413,
|
|
"logps/chosen": -290.50616455078125,
|
|
"logps/ref_chosen": -270.0443115234375,
|
|
"logps/ref_rejected": -267.3226013183594,
|
|
"logps/rejected": -304.7435302734375,
|
|
"loss": 5.0197,
|
|
"margin_dpo/margin_mean": 16.95909309387207,
|
|
"margin_dpo/margin_std": 42.556758880615234,
|
|
"step": 139
|
|
},
|
|
{
|
|
"epoch": 0.2931937172774869,
|
|
"fcm_dpo/beta": 0.014604137279093266,
|
|
"fcm_dpo/delta": -0.03599924594163895,
|
|
"fcm_dpo/margin": 26.53323745727539,
|
|
"fcm_dpo/q_t": 0.4121074080467224,
|
|
"grad_norm": 72.81695556640625,
|
|
"learning_rate": 4.4651327368569684e-07,
|
|
"logits/chosen": -0.8876050710678101,
|
|
"logits/rejected": -0.856640636920929,
|
|
"logps/chosen": -302.76434326171875,
|
|
"logps/ref_chosen": -282.9555969238281,
|
|
"logps/ref_rejected": -251.17181396484375,
|
|
"logps/rejected": -297.5137939453125,
|
|
"loss": 4.495,
|
|
"margin_dpo/margin_mean": 26.53323745727539,
|
|
"margin_dpo/margin_std": 40.214508056640625,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.29528795811518327,
|
|
"fcm_dpo/beta": 0.01404612883925438,
|
|
"fcm_dpo/delta": -0.04051290452480316,
|
|
"fcm_dpo/margin": 29.221651077270508,
|
|
"fcm_dpo/q_t": 0.4060809910297394,
|
|
"grad_norm": 59.52610778808594,
|
|
"learning_rate": 4.453763107901675e-07,
|
|
"logits/chosen": -0.8155990242958069,
|
|
"logits/rejected": -0.8117971420288086,
|
|
"logps/chosen": -314.1397399902344,
|
|
"logps/ref_chosen": -296.3001708984375,
|
|
"logps/ref_rejected": -279.8486633300781,
|
|
"logps/rejected": -326.9098815917969,
|
|
"loss": 4.4761,
|
|
"margin_dpo/margin_mean": 29.221649169921875,
|
|
"margin_dpo/margin_std": 45.766151428222656,
|
|
"step": 141
|
|
},
|
|
{
|
|
"epoch": 0.2973821989528796,
|
|
"fcm_dpo/beta": 0.01387177873402834,
|
|
"fcm_dpo/delta": -0.0016752388328313828,
|
|
"fcm_dpo/margin": 23.28960418701172,
|
|
"fcm_dpo/q_t": 0.42782607674598694,
|
|
"grad_norm": 62.42996597290039,
|
|
"learning_rate": 4.4422887045602674e-07,
|
|
"logits/chosen": -0.8308397531509399,
|
|
"logits/rejected": -0.8334007263183594,
|
|
"logps/chosen": -321.20721435546875,
|
|
"logps/ref_chosen": -300.56585693359375,
|
|
"logps/ref_rejected": -231.43316650390625,
|
|
"logps/rejected": -275.3641357421875,
|
|
"loss": 4.8108,
|
|
"margin_dpo/margin_mean": 23.28960418701172,
|
|
"margin_dpo/margin_std": 48.74913024902344,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.2994764397905759,
|
|
"fcm_dpo/beta": 0.01358854304999113,
|
|
"fcm_dpo/delta": -0.025667672976851463,
|
|
"fcm_dpo/margin": 28.89480209350586,
|
|
"fcm_dpo/q_t": 0.4115155041217804,
|
|
"grad_norm": 58.460384368896484,
|
|
"learning_rate": 4.4307101421701755e-07,
|
|
"logits/chosen": -0.8297260403633118,
|
|
"logits/rejected": -0.8115658760070801,
|
|
"logps/chosen": -318.246337890625,
|
|
"logps/ref_chosen": -296.73236083984375,
|
|
"logps/ref_rejected": -266.45257568359375,
|
|
"logps/rejected": -316.8614196777344,
|
|
"loss": 4.4811,
|
|
"margin_dpo/margin_mean": 28.894800186157227,
|
|
"margin_dpo/margin_std": 43.21503448486328,
|
|
"step": 143
|
|
},
|
|
{
|
|
"epoch": 0.30157068062827225,
|
|
"fcm_dpo/beta": 0.01343129575252533,
|
|
"fcm_dpo/delta": -0.006690308451652527,
|
|
"fcm_dpo/margin": 23.162403106689453,
|
|
"fcm_dpo/q_t": 0.43112730979919434,
|
|
"grad_norm": 63.20121383666992,
|
|
"learning_rate": 4.419028041654559e-07,
|
|
"logits/chosen": -0.878182590007782,
|
|
"logits/rejected": -0.864521324634552,
|
|
"logps/chosen": -323.18084716796875,
|
|
"logps/ref_chosen": -298.843994140625,
|
|
"logps/ref_rejected": -266.120849609375,
|
|
"logps/rejected": -313.6200866699219,
|
|
"loss": 4.8203,
|
|
"margin_dpo/margin_mean": 23.162403106689453,
|
|
"margin_dpo/margin_std": 48.521446228027344,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.3036649214659686,
|
|
"fcm_dpo/beta": 0.012913926504552364,
|
|
"fcm_dpo/delta": -0.08783036470413208,
|
|
"fcm_dpo/margin": 31.798524856567383,
|
|
"fcm_dpo/q_t": 0.4062502682209015,
|
|
"grad_norm": 59.772727966308594,
|
|
"learning_rate": 4.4072430294890166e-07,
|
|
"logits/chosen": -0.8632069826126099,
|
|
"logits/rejected": -0.8655521273612976,
|
|
"logps/chosen": -296.8944091796875,
|
|
"logps/ref_chosen": -275.7528381347656,
|
|
"logps/ref_rejected": -214.74807739257812,
|
|
"logps/rejected": -267.68817138671875,
|
|
"loss": 4.4332,
|
|
"margin_dpo/margin_mean": 31.798524856567383,
|
|
"margin_dpo/margin_std": 45.571964263916016,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.3057591623036649,
|
|
"fcm_dpo/beta": 0.01257642637938261,
|
|
"fcm_dpo/delta": -0.011518063023686409,
|
|
"fcm_dpo/margin": 30.71509552001953,
|
|
"fcm_dpo/q_t": 0.412579208612442,
|
|
"grad_norm": 73.49806213378906,
|
|
"learning_rate": 4.395355737667985e-07,
|
|
"logits/chosen": -0.8509379625320435,
|
|
"logits/rejected": -0.8461902737617493,
|
|
"logps/chosen": -308.1560363769531,
|
|
"logps/ref_chosen": -277.09820556640625,
|
|
"logps/ref_rejected": -265.41046142578125,
|
|
"logps/rejected": -327.1833190917969,
|
|
"loss": 4.493,
|
|
"margin_dpo/margin_mean": 30.71509552001953,
|
|
"margin_dpo/margin_std": 45.785736083984375,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.3078534031413613,
|
|
"fcm_dpo/beta": 0.01252201572060585,
|
|
"fcm_dpo/delta": -0.038640253245830536,
|
|
"fcm_dpo/margin": 23.62162971496582,
|
|
"fcm_dpo/q_t": 0.4322171211242676,
|
|
"grad_norm": 72.45833587646484,
|
|
"learning_rate": 4.3833668036708483e-07,
|
|
"logits/chosen": -0.8362576365470886,
|
|
"logits/rejected": -0.8324323892593384,
|
|
"logps/chosen": -325.21533203125,
|
|
"logps/ref_chosen": -291.4185791015625,
|
|
"logps/ref_rejected": -253.43051147460938,
|
|
"logps/rejected": -310.84893798828125,
|
|
"loss": 4.8467,
|
|
"margin_dpo/margin_mean": 23.621631622314453,
|
|
"margin_dpo/margin_std": 47.34939193725586,
|
|
"step": 147
|
|
},
|
|
{
|
|
"epoch": 0.3099476439790576,
|
|
"fcm_dpo/beta": 0.012431098148226738,
|
|
"fcm_dpo/delta": 0.056200552731752396,
|
|
"fcm_dpo/margin": 23.877735137939453,
|
|
"fcm_dpo/q_t": 0.43532854318618774,
|
|
"grad_norm": 64.35401916503906,
|
|
"learning_rate": 4.3712768704277524e-07,
|
|
"logits/chosen": -0.8974530100822449,
|
|
"logits/rejected": -0.8999665379524231,
|
|
"logps/chosen": -275.08160400390625,
|
|
"logps/ref_chosen": -236.74850463867188,
|
|
"logps/ref_rejected": -231.4674072265625,
|
|
"logps/rejected": -293.67816162109375,
|
|
"loss": 4.865,
|
|
"margin_dpo/margin_mean": 23.877735137939453,
|
|
"margin_dpo/margin_std": 53.183143615722656,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.31204188481675393,
|
|
"fcm_dpo/beta": 0.012634760700166225,
|
|
"fcm_dpo/delta": -0.0065022800117731094,
|
|
"fcm_dpo/margin": 32.123836517333984,
|
|
"fcm_dpo/q_t": 0.4069136083126068,
|
|
"grad_norm": 76.46810150146484,
|
|
"learning_rate": 4.3590865862851263e-07,
|
|
"logits/chosen": -0.8669228553771973,
|
|
"logits/rejected": -0.8517608046531677,
|
|
"logps/chosen": -360.485595703125,
|
|
"logps/ref_chosen": -319.9284973144531,
|
|
"logps/ref_rejected": -308.20233154296875,
|
|
"logps/rejected": -380.88330078125,
|
|
"loss": 4.3739,
|
|
"margin_dpo/margin_mean": 32.123836517333984,
|
|
"margin_dpo/margin_std": 43.10367202758789,
|
|
"step": 149
|
|
},
|
|
{
|
|
"epoch": 0.31413612565445026,
|
|
"fcm_dpo/beta": 0.012791416607797146,
|
|
"fcm_dpo/delta": 0.025544868782162666,
|
|
"fcm_dpo/margin": 24.68185806274414,
|
|
"fcm_dpo/q_t": 0.4278425872325897,
|
|
"grad_norm": 66.41766357421875,
|
|
"learning_rate": 4.346796604970912e-07,
|
|
"logits/chosen": -0.8340862989425659,
|
|
"logits/rejected": -0.8192029595375061,
|
|
"logps/chosen": -324.4963073730469,
|
|
"logps/ref_chosen": -276.3182373046875,
|
|
"logps/ref_rejected": -273.02215576171875,
|
|
"logps/rejected": -345.8821105957031,
|
|
"loss": 4.734,
|
|
"margin_dpo/margin_mean": 24.68185806274414,
|
|
"margin_dpo/margin_std": 46.04266357421875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.3162303664921466,
|
|
"fcm_dpo/beta": 0.012728921137750149,
|
|
"fcm_dpo/delta": -0.04246210679411888,
|
|
"fcm_dpo/margin": 34.62612533569336,
|
|
"fcm_dpo/q_t": 0.40107759833335876,
|
|
"grad_norm": 93.15609741210938,
|
|
"learning_rate": 4.3344075855595097e-07,
|
|
"logits/chosen": -0.8574318289756775,
|
|
"logits/rejected": -0.8527010679244995,
|
|
"logps/chosen": -344.9148254394531,
|
|
"logps/ref_chosen": -297.31280517578125,
|
|
"logps/ref_rejected": -266.1003723144531,
|
|
"logps/rejected": -348.3284912109375,
|
|
"loss": 4.3664,
|
|
"margin_dpo/margin_mean": 34.62612533569336,
|
|
"margin_dpo/margin_std": 49.68989562988281,
|
|
"step": 151
|
|
},
|
|
{
|
|
"epoch": 0.3183246073298429,
|
|
"fcm_dpo/beta": 0.012182039208710194,
|
|
"fcm_dpo/delta": -0.0288827046751976,
|
|
"fcm_dpo/margin": 30.36262321472168,
|
|
"fcm_dpo/q_t": 0.4171737730503082,
|
|
"grad_norm": 97.927978515625,
|
|
"learning_rate": 4.3219201924364323e-07,
|
|
"logits/chosen": -0.8693718314170837,
|
|
"logits/rejected": -0.8675329685211182,
|
|
"logps/chosen": -310.8759765625,
|
|
"logps/ref_chosen": -270.2470397949219,
|
|
"logps/ref_rejected": -269.7749328613281,
|
|
"logps/rejected": -340.7665100097656,
|
|
"loss": 4.5782,
|
|
"margin_dpo/margin_mean": 30.36262321472168,
|
|
"margin_dpo/margin_std": 49.9327278137207,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.3204188481675393,
|
|
"fcm_dpo/beta": 0.01159929670393467,
|
|
"fcm_dpo/delta": -0.1059812605381012,
|
|
"fcm_dpo/margin": 43.1450080871582,
|
|
"fcm_dpo/q_t": 0.3860260844230652,
|
|
"grad_norm": 74.16168212890625,
|
|
"learning_rate": 4.309335095262675e-07,
|
|
"logits/chosen": -0.8602730631828308,
|
|
"logits/rejected": -0.8485139608383179,
|
|
"logps/chosen": -318.30548095703125,
|
|
"logps/ref_chosen": -273.779052734375,
|
|
"logps/ref_rejected": -280.9530944824219,
|
|
"logps/rejected": -368.62457275390625,
|
|
"loss": 4.1014,
|
|
"margin_dpo/margin_mean": 43.1450080871582,
|
|
"margin_dpo/margin_std": 48.90004348754883,
|
|
"step": 153
|
|
},
|
|
{
|
|
"epoch": 0.3225130890052356,
|
|
"fcm_dpo/beta": 0.010896118357777596,
|
|
"fcm_dpo/delta": -0.05441652983427048,
|
|
"fcm_dpo/margin": 35.19595718383789,
|
|
"fcm_dpo/q_t": 0.41286537051200867,
|
|
"grad_norm": 69.13275909423828,
|
|
"learning_rate": 4.2966529689388064e-07,
|
|
"logits/chosen": -0.8889592289924622,
|
|
"logits/rejected": -0.8696526288986206,
|
|
"logps/chosen": -331.47686767578125,
|
|
"logps/ref_chosen": -289.9031982421875,
|
|
"logps/ref_rejected": -261.5166320800781,
|
|
"logps/rejected": -338.2862854003906,
|
|
"loss": 4.5747,
|
|
"margin_dpo/margin_mean": 35.19595718383789,
|
|
"margin_dpo/margin_std": 58.20466232299805,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.32460732984293195,
|
|
"fcm_dpo/beta": 0.010797424241900444,
|
|
"fcm_dpo/delta": 0.02106129191815853,
|
|
"fcm_dpo/margin": 30.947917938232422,
|
|
"fcm_dpo/q_t": 0.42601996660232544,
|
|
"grad_norm": 76.80484008789062,
|
|
"learning_rate": 4.2838744935687716e-07,
|
|
"logits/chosen": -0.8353626132011414,
|
|
"logits/rejected": -0.8337759375572205,
|
|
"logps/chosen": -328.4182434082031,
|
|
"logps/ref_chosen": -285.8612060546875,
|
|
"logps/ref_rejected": -300.1272888183594,
|
|
"logps/rejected": -373.63226318359375,
|
|
"loss": 4.6686,
|
|
"margin_dpo/margin_mean": 30.947919845581055,
|
|
"margin_dpo/margin_std": 54.85850143432617,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.3267015706806283,
|
|
"fcm_dpo/beta": 0.010698702186346054,
|
|
"fcm_dpo/delta": -0.09174332022666931,
|
|
"fcm_dpo/margin": 43.385841369628906,
|
|
"fcm_dpo/q_t": 0.3962228298187256,
|
|
"grad_norm": 61.48815155029297,
|
|
"learning_rate": 4.271000354423425e-07,
|
|
"logits/chosen": -0.8628277778625488,
|
|
"logits/rejected": -0.8613920211791992,
|
|
"logps/chosen": -312.9440002441406,
|
|
"logps/ref_chosen": -279.0354919433594,
|
|
"logps/ref_rejected": -244.2198486328125,
|
|
"logps/rejected": -321.5141906738281,
|
|
"loss": 4.3529,
|
|
"margin_dpo/margin_mean": 43.38584518432617,
|
|
"margin_dpo/margin_std": 63.006675720214844,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.3287958115183246,
|
|
"fcm_dpo/beta": 0.010013958439230919,
|
|
"fcm_dpo/delta": -0.013928013853728771,
|
|
"fcm_dpo/margin": 33.83953094482422,
|
|
"fcm_dpo/q_t": 0.422372430562973,
|
|
"grad_norm": 63.8918571472168,
|
|
"learning_rate": 4.258031241903777e-07,
|
|
"logits/chosen": -0.9240726232528687,
|
|
"logits/rejected": -0.9229752421379089,
|
|
"logps/chosen": -305.9951171875,
|
|
"logps/ref_chosen": -270.830322265625,
|
|
"logps/ref_rejected": -259.08319091796875,
|
|
"logps/rejected": -328.08758544921875,
|
|
"loss": 4.618,
|
|
"margin_dpo/margin_mean": 33.83953094482422,
|
|
"margin_dpo/margin_std": 55.008338928222656,
|
|
"step": 157
|
|
},
|
|
{
|
|
"epoch": 0.3308900523560209,
|
|
"fcm_dpo/beta": 0.010049000382423401,
|
|
"fcm_dpo/delta": -0.018396489322185516,
|
|
"fcm_dpo/margin": 36.628936767578125,
|
|
"fcm_dpo/q_t": 0.4149770736694336,
|
|
"grad_norm": 68.94039916992188,
|
|
"learning_rate": 4.2449678515039743e-07,
|
|
"logits/chosen": -0.8562602996826172,
|
|
"logits/rejected": -0.8422697186470032,
|
|
"logps/chosen": -323.416748046875,
|
|
"logps/ref_chosen": -289.9663391113281,
|
|
"logps/ref_rejected": -271.335693359375,
|
|
"logps/rejected": -341.4150390625,
|
|
"loss": 4.5504,
|
|
"margin_dpo/margin_mean": 36.628936767578125,
|
|
"margin_dpo/margin_std": 56.774017333984375,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.33298429319371725,
|
|
"fcm_dpo/beta": 0.01005611103028059,
|
|
"fcm_dpo/delta": 0.04300171136856079,
|
|
"fcm_dpo/margin": 23.479984283447266,
|
|
"fcm_dpo/q_t": 0.4478485584259033,
|
|
"grad_norm": 74.50262451171875,
|
|
"learning_rate": 4.2318108837739986e-07,
|
|
"logits/chosen": -0.9407361745834351,
|
|
"logits/rejected": -0.8954955339431763,
|
|
"logps/chosen": -359.2971496582031,
|
|
"logps/ref_chosen": -321.37835693359375,
|
|
"logps/ref_rejected": -250.45652770996094,
|
|
"logps/rejected": -311.85528564453125,
|
|
"loss": 5.0421,
|
|
"margin_dpo/margin_mean": 23.4799861907959,
|
|
"margin_dpo/margin_std": 59.843711853027344,
|
|
"step": 159
|
|
},
|
|
{
|
|
"epoch": 0.33507853403141363,
|
|
"fcm_dpo/beta": 0.010049490258097649,
|
|
"fcm_dpo/delta": -0.03839590400457382,
|
|
"fcm_dpo/margin": 43.411376953125,
|
|
"fcm_dpo/q_t": 0.39909377694129944,
|
|
"grad_norm": 58.37568664550781,
|
|
"learning_rate": 4.218561044282098e-07,
|
|
"logits/chosen": -0.8778911828994751,
|
|
"logits/rejected": -0.883208155632019,
|
|
"logps/chosen": -304.81475830078125,
|
|
"logps/ref_chosen": -276.28350830078125,
|
|
"logps/ref_rejected": -262.7477722167969,
|
|
"logps/rejected": -334.6904296875,
|
|
"loss": 4.2303,
|
|
"margin_dpo/margin_mean": 43.411376953125,
|
|
"margin_dpo/margin_std": 48.92018508911133,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.33717277486910996,
|
|
"fcm_dpo/beta": 0.009891795925796032,
|
|
"fcm_dpo/delta": -0.013076556846499443,
|
|
"fcm_dpo/margin": 36.996002197265625,
|
|
"fcm_dpo/q_t": 0.4181225299835205,
|
|
"grad_norm": 63.23640441894531,
|
|
"learning_rate": 4.2052190435769554e-07,
|
|
"logits/chosen": -0.8851248621940613,
|
|
"logits/rejected": -0.8717115521430969,
|
|
"logps/chosen": -345.7448425292969,
|
|
"logps/ref_chosen": -310.4927978515625,
|
|
"logps/ref_rejected": -250.25347900390625,
|
|
"logps/rejected": -322.50146484375,
|
|
"loss": 4.5767,
|
|
"margin_dpo/margin_mean": 36.996002197265625,
|
|
"margin_dpo/margin_std": 60.53446960449219,
|
|
"step": 161
|
|
},
|
|
{
|
|
"epoch": 0.3392670157068063,
|
|
"fcm_dpo/beta": 0.009680384770035744,
|
|
"fcm_dpo/delta": 0.001748579554259777,
|
|
"fcm_dpo/margin": 34.674312591552734,
|
|
"fcm_dpo/q_t": 0.4228632152080536,
|
|
"grad_norm": 58.60459518432617,
|
|
"learning_rate": 4.1917855971495763e-07,
|
|
"logits/chosen": -0.8581404089927673,
|
|
"logits/rejected": -0.8511423468589783,
|
|
"logps/chosen": -327.4073791503906,
|
|
"logps/ref_chosen": -296.1105041503906,
|
|
"logps/ref_rejected": -253.4247589111328,
|
|
"logps/rejected": -319.3959655761719,
|
|
"loss": 4.6121,
|
|
"margin_dpo/margin_mean": 34.674312591552734,
|
|
"margin_dpo/margin_std": 54.75847625732422,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.3413612565445026,
|
|
"fcm_dpo/beta": 0.010051360353827477,
|
|
"fcm_dpo/delta": 0.02088327519595623,
|
|
"fcm_dpo/margin": 37.680484771728516,
|
|
"fcm_dpo/q_t": 0.41251957416534424,
|
|
"grad_norm": 72.17606353759766,
|
|
"learning_rate": 4.1782614253949255e-07,
|
|
"logits/chosen": -0.8856139779090881,
|
|
"logits/rejected": -0.8851478099822998,
|
|
"logps/chosen": -326.783935546875,
|
|
"logps/ref_chosen": -293.4999084472656,
|
|
"logps/ref_rejected": -266.7116394042969,
|
|
"logps/rejected": -337.67620849609375,
|
|
"loss": 4.4711,
|
|
"margin_dpo/margin_mean": 37.68048858642578,
|
|
"margin_dpo/margin_std": 53.57781982421875,
|
|
"step": 163
|
|
},
|
|
{
|
|
"epoch": 0.34345549738219894,
|
|
"fcm_dpo/beta": 0.010161195881664753,
|
|
"fcm_dpo/delta": 0.015082177706062794,
|
|
"fcm_dpo/margin": 34.78863525390625,
|
|
"fcm_dpo/q_t": 0.42124950885772705,
|
|
"grad_norm": 82.29058074951172,
|
|
"learning_rate": 4.164647253573289e-07,
|
|
"logits/chosen": -0.8554076552391052,
|
|
"logits/rejected": -0.8738577365875244,
|
|
"logps/chosen": -307.874755859375,
|
|
"logps/ref_chosen": -267.04949951171875,
|
|
"logps/ref_rejected": -215.9768829345703,
|
|
"logps/rejected": -291.59075927734375,
|
|
"loss": 4.6376,
|
|
"margin_dpo/margin_mean": 34.788639068603516,
|
|
"margin_dpo/margin_std": 61.51546859741211,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.34554973821989526,
|
|
"fcm_dpo/beta": 0.010130844078958035,
|
|
"fcm_dpo/delta": 0.030348509550094604,
|
|
"fcm_dpo/margin": 25.893747329711914,
|
|
"fcm_dpo/q_t": 0.44016268849372864,
|
|
"grad_norm": 80.56285858154297,
|
|
"learning_rate": 4.1509438117713863e-07,
|
|
"logits/chosen": -0.8746411204338074,
|
|
"logits/rejected": -0.8493326902389526,
|
|
"logps/chosen": -309.4532470703125,
|
|
"logps/ref_chosen": -278.06146240234375,
|
|
"logps/ref_rejected": -260.4288635253906,
|
|
"logps/rejected": -317.71435546875,
|
|
"loss": 4.862,
|
|
"margin_dpo/margin_mean": 25.893747329711914,
|
|
"margin_dpo/margin_std": 52.21025848388672,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.34764397905759165,
|
|
"fcm_dpo/beta": 0.010512001812458038,
|
|
"fcm_dpo/delta": 0.04555278643965721,
|
|
"fcm_dpo/margin": 31.762598037719727,
|
|
"fcm_dpo/q_t": 0.42573946714401245,
|
|
"grad_norm": 66.32711029052734,
|
|
"learning_rate": 4.137151834863213e-07,
|
|
"logits/chosen": -0.8395042419433594,
|
|
"logits/rejected": -0.8069786429405212,
|
|
"logps/chosen": -304.6323547363281,
|
|
"logps/ref_chosen": -275.9490661621094,
|
|
"logps/ref_rejected": -232.13473510742188,
|
|
"logps/rejected": -292.58062744140625,
|
|
"loss": 4.6734,
|
|
"margin_dpo/margin_mean": 31.762596130371094,
|
|
"margin_dpo/margin_std": 57.40185546875,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.34973821989528797,
|
|
"fcm_dpo/beta": 0.01087398361414671,
|
|
"fcm_dpo/delta": 0.006117178127169609,
|
|
"fcm_dpo/margin": 33.77849578857422,
|
|
"fcm_dpo/q_t": 0.4176112115383148,
|
|
"grad_norm": 79.66935729980469,
|
|
"learning_rate": 4.123272062470633e-07,
|
|
"logits/chosen": -0.8249646425247192,
|
|
"logits/rejected": -0.8123272061347961,
|
|
"logps/chosen": -314.0303955078125,
|
|
"logps/ref_chosen": -280.5514221191406,
|
|
"logps/ref_rejected": -255.2896728515625,
|
|
"logps/rejected": -322.5471496582031,
|
|
"loss": 4.6344,
|
|
"margin_dpo/margin_mean": 33.77849578857422,
|
|
"margin_dpo/margin_std": 60.363433837890625,
|
|
"step": 167
|
|
},
|
|
{
|
|
"epoch": 0.3518324607329843,
|
|
"fcm_dpo/beta": 0.01047597173601389,
|
|
"fcm_dpo/delta": -0.048771023750305176,
|
|
"fcm_dpo/margin": 38.110443115234375,
|
|
"fcm_dpo/q_t": 0.40823835134506226,
|
|
"grad_norm": 148.15753173828125,
|
|
"learning_rate": 4.1093052389237174e-07,
|
|
"logits/chosen": -0.80986487865448,
|
|
"logits/rejected": -0.7853665351867676,
|
|
"logps/chosen": -350.20916748046875,
|
|
"logps/ref_chosen": -315.7982177734375,
|
|
"logps/ref_rejected": -291.48406982421875,
|
|
"logps/rejected": -364.0054626464844,
|
|
"loss": 4.5115,
|
|
"margin_dpo/margin_mean": 38.110443115234375,
|
|
"margin_dpo/margin_std": 59.79719161987305,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.3539267015706806,
|
|
"fcm_dpo/beta": 0.009861285798251629,
|
|
"fcm_dpo/delta": -0.12296908348798752,
|
|
"fcm_dpo/margin": 52.32954406738281,
|
|
"fcm_dpo/q_t": 0.3826444447040558,
|
|
"grad_norm": 73.20549774169922,
|
|
"learning_rate": 4.0952521132208267e-07,
|
|
"logits/chosen": -0.8107226490974426,
|
|
"logits/rejected": -0.8243895173072815,
|
|
"logps/chosen": -291.12841796875,
|
|
"logps/ref_chosen": -261.06427001953125,
|
|
"logps/ref_rejected": -235.40663146972656,
|
|
"logps/rejected": -317.80035400390625,
|
|
"loss": 4.0596,
|
|
"margin_dpo/margin_mean": 52.32954406738281,
|
|
"margin_dpo/margin_std": 56.9708251953125,
|
|
"step": 169
|
|
},
|
|
{
|
|
"epoch": 0.35602094240837695,
|
|
"fcm_dpo/beta": 0.009434648789465427,
|
|
"fcm_dpo/delta": 0.022008666768670082,
|
|
"fcm_dpo/margin": 40.11225128173828,
|
|
"fcm_dpo/q_t": 0.41667625308036804,
|
|
"grad_norm": 67.7850570678711,
|
|
"learning_rate": 4.081113438988443e-07,
|
|
"logits/chosen": -0.7921425700187683,
|
|
"logits/rejected": -0.7939149141311646,
|
|
"logps/chosen": -343.00537109375,
|
|
"logps/ref_chosen": -308.96722412109375,
|
|
"logps/ref_rejected": -263.8466796875,
|
|
"logps/rejected": -337.9970703125,
|
|
"loss": 4.59,
|
|
"margin_dpo/margin_mean": 40.11225509643555,
|
|
"margin_dpo/margin_std": 70.82965850830078,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.3581151832460733,
|
|
"fcm_dpo/beta": 0.009155556559562683,
|
|
"fcm_dpo/delta": -0.0953391045331955,
|
|
"fcm_dpo/margin": 49.351959228515625,
|
|
"fcm_dpo/q_t": 0.3953308165073395,
|
|
"grad_norm": 91.96127319335938,
|
|
"learning_rate": 4.0668899744407567e-07,
|
|
"logits/chosen": -0.8171353936195374,
|
|
"logits/rejected": -0.8306329250335693,
|
|
"logps/chosen": -289.5184326171875,
|
|
"logps/ref_chosen": -258.8890380859375,
|
|
"logps/ref_rejected": -262.19140625,
|
|
"logps/rejected": -342.1727294921875,
|
|
"loss": 4.2024,
|
|
"margin_dpo/margin_mean": 49.351959228515625,
|
|
"margin_dpo/margin_std": 55.003509521484375,
|
|
"step": 171
|
|
},
|
|
{
|
|
"epoch": 0.36020942408376966,
|
|
"fcm_dpo/beta": 0.009115180931985378,
|
|
"fcm_dpo/delta": 0.04742582514882088,
|
|
"fcm_dpo/margin": 25.906282424926758,
|
|
"fcm_dpo/q_t": 0.44478750228881836,
|
|
"grad_norm": 70.82637786865234,
|
|
"learning_rate": 4.0525824823390043e-07,
|
|
"logits/chosen": -0.8440933227539062,
|
|
"logits/rejected": -0.8594982624053955,
|
|
"logps/chosen": -378.7886047363281,
|
|
"logps/ref_chosen": -339.0223388671875,
|
|
"logps/ref_rejected": -295.78759765625,
|
|
"logps/rejected": -361.46014404296875,
|
|
"loss": 4.9617,
|
|
"margin_dpo/margin_mean": 25.906280517578125,
|
|
"margin_dpo/margin_std": 60.43056106567383,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.362303664921466,
|
|
"fcm_dpo/beta": 0.009056363254785538,
|
|
"fcm_dpo/delta": -0.02586848847568035,
|
|
"fcm_dpo/margin": 36.4200553894043,
|
|
"fcm_dpo/q_t": 0.42598268389701843,
|
|
"grad_norm": 67.26022338867188,
|
|
"learning_rate": 4.0381917299505686e-07,
|
|
"logits/chosen": -0.8703533411026001,
|
|
"logits/rejected": -0.8670443296432495,
|
|
"logps/chosen": -341.75244140625,
|
|
"logps/ref_chosen": -300.1114501953125,
|
|
"logps/ref_rejected": -273.78460693359375,
|
|
"logps/rejected": -351.84564208984375,
|
|
"loss": 4.6727,
|
|
"margin_dpo/margin_mean": 36.4200553894043,
|
|
"margin_dpo/margin_std": 62.181610107421875,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.3643979057591623,
|
|
"fcm_dpo/beta": 0.009070999920368195,
|
|
"fcm_dpo/delta": 0.0008946415036916733,
|
|
"fcm_dpo/margin": 43.95625686645508,
|
|
"fcm_dpo/q_t": 0.4071962535381317,
|
|
"grad_norm": 80.60259246826172,
|
|
"learning_rate": 4.0237184890078243e-07,
|
|
"logits/chosen": -0.8544137477874756,
|
|
"logits/rejected": -0.8370189070701599,
|
|
"logps/chosen": -381.8381042480469,
|
|
"logps/ref_chosen": -335.0538635253906,
|
|
"logps/ref_rejected": -257.4646911621094,
|
|
"logps/rejected": -348.2052001953125,
|
|
"loss": 4.3964,
|
|
"margin_dpo/margin_mean": 43.95625686645508,
|
|
"margin_dpo/margin_std": 59.383609771728516,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.36649214659685864,
|
|
"fcm_dpo/beta": 0.00893239676952362,
|
|
"fcm_dpo/delta": -0.05215398222208023,
|
|
"fcm_dpo/margin": 39.2378044128418,
|
|
"fcm_dpo/q_t": 0.423565149307251,
|
|
"grad_norm": 96.69216918945312,
|
|
"learning_rate": 4.00916353566676e-07,
|
|
"logits/chosen": -0.8526287078857422,
|
|
"logits/rejected": -0.8526190519332886,
|
|
"logps/chosen": -346.5810241699219,
|
|
"logps/ref_chosen": -284.39556884765625,
|
|
"logps/ref_rejected": -283.3876647949219,
|
|
"logps/rejected": -384.81085205078125,
|
|
"loss": 4.6754,
|
|
"margin_dpo/margin_mean": 39.2378044128418,
|
|
"margin_dpo/margin_std": 69.70426177978516,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.36858638743455496,
|
|
"fcm_dpo/beta": 0.00879521295428276,
|
|
"fcm_dpo/delta": 0.0073929643258452415,
|
|
"fcm_dpo/margin": 32.00230026245117,
|
|
"fcm_dpo/q_t": 0.43624645471572876,
|
|
"grad_norm": 77.65434265136719,
|
|
"learning_rate": 3.994527650465352e-07,
|
|
"logits/chosen": -0.8234915137290955,
|
|
"logits/rejected": -0.8285030722618103,
|
|
"logps/chosen": -316.092529296875,
|
|
"logps/ref_chosen": -251.81280517578125,
|
|
"logps/ref_rejected": -242.05328369140625,
|
|
"logps/rejected": -338.3353271484375,
|
|
"loss": 4.9145,
|
|
"margin_dpo/margin_mean": 32.00230026245117,
|
|
"margin_dpo/margin_std": 72.6375961303711,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.3706806282722513,
|
|
"fcm_dpo/beta": 0.008641266264021397,
|
|
"fcm_dpo/delta": -0.004209747537970543,
|
|
"fcm_dpo/margin": 32.934574127197266,
|
|
"fcm_dpo/q_t": 0.43586111068725586,
|
|
"grad_norm": 73.62203216552734,
|
|
"learning_rate": 3.979811618281705e-07,
|
|
"logits/chosen": -0.9073963761329651,
|
|
"logits/rejected": -0.8772807717323303,
|
|
"logps/chosen": -370.17852783203125,
|
|
"logps/ref_chosen": -298.6463928222656,
|
|
"logps/ref_rejected": -295.66534423828125,
|
|
"logps/rejected": -400.1320495605469,
|
|
"loss": 4.931,
|
|
"margin_dpo/margin_mean": 32.93457794189453,
|
|
"margin_dpo/margin_std": 73.69795227050781,
|
|
"step": 177
|
|
},
|
|
{
|
|
"epoch": 0.37277486910994767,
|
|
"fcm_dpo/beta": 0.0086296908557415,
|
|
"fcm_dpo/delta": -0.004849469289183617,
|
|
"fcm_dpo/margin": 46.83763885498047,
|
|
"fcm_dpo/q_t": 0.4095942974090576,
|
|
"grad_norm": 68.19680786132812,
|
|
"learning_rate": 3.9650162282919654e-07,
|
|
"logits/chosen": -0.8406542539596558,
|
|
"logits/rejected": -0.8333706855773926,
|
|
"logps/chosen": -347.32861328125,
|
|
"logps/ref_chosen": -286.2576599121094,
|
|
"logps/ref_rejected": -243.97491455078125,
|
|
"logps/rejected": -351.88348388671875,
|
|
"loss": 4.4695,
|
|
"margin_dpo/margin_mean": 46.83763885498047,
|
|
"margin_dpo/margin_std": 71.97041320800781,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.374869109947644,
|
|
"fcm_dpo/beta": 0.008434956893324852,
|
|
"fcm_dpo/delta": -0.034470878541469574,
|
|
"fcm_dpo/margin": 42.3710823059082,
|
|
"fcm_dpo/q_t": 0.4181768000125885,
|
|
"grad_norm": 85.9342041015625,
|
|
"learning_rate": 3.9501422739279953e-07,
|
|
"logits/chosen": -0.8106435537338257,
|
|
"logits/rejected": -0.8113298416137695,
|
|
"logps/chosen": -324.47760009765625,
|
|
"logps/ref_chosen": -259.737060546875,
|
|
"logps/ref_rejected": -277.8813171386719,
|
|
"logps/rejected": -384.9929504394531,
|
|
"loss": 4.5927,
|
|
"margin_dpo/margin_mean": 42.3710823059082,
|
|
"margin_dpo/margin_std": 68.53620910644531,
|
|
"step": 179
|
|
},
|
|
{
|
|
"epoch": 0.3769633507853403,
|
|
"fcm_dpo/beta": 0.008497622795403004,
|
|
"fcm_dpo/delta": 0.006464208010584116,
|
|
"fcm_dpo/margin": 43.05263137817383,
|
|
"fcm_dpo/q_t": 0.41862213611602783,
|
|
"grad_norm": 76.22695922851562,
|
|
"learning_rate": 3.935190552834828e-07,
|
|
"logits/chosen": -0.8361107110977173,
|
|
"logits/rejected": -0.8606466054916382,
|
|
"logps/chosen": -335.56817626953125,
|
|
"logps/ref_chosen": -267.30889892578125,
|
|
"logps/ref_rejected": -230.4376983642578,
|
|
"logps/rejected": -341.74957275390625,
|
|
"loss": 4.5463,
|
|
"margin_dpo/margin_mean": 43.05263137817383,
|
|
"margin_dpo/margin_std": 68.31497192382812,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.37905759162303665,
|
|
"fcm_dpo/beta": 0.0083799809217453,
|
|
"fcm_dpo/delta": 0.008126728236675262,
|
|
"fcm_dpo/margin": 40.54942321777344,
|
|
"fcm_dpo/q_t": 0.42300912737846375,
|
|
"grad_norm": 69.96990203857422,
|
|
"learning_rate": 3.920161866827889e-07,
|
|
"logits/chosen": -0.850619375705719,
|
|
"logits/rejected": -0.8536400198936462,
|
|
"logps/chosen": -375.7586975097656,
|
|
"logps/ref_chosen": -300.49139404296875,
|
|
"logps/ref_rejected": -278.98284912109375,
|
|
"logps/rejected": -394.7995910644531,
|
|
"loss": 4.6624,
|
|
"margin_dpo/margin_mean": 40.54942321777344,
|
|
"margin_dpo/margin_std": 72.2303466796875,
|
|
"step": 181
|
|
},
|
|
{
|
|
"epoch": 0.381151832460733,
|
|
"fcm_dpo/beta": 0.00836803950369358,
|
|
"fcm_dpo/delta": -0.06589318066835403,
|
|
"fcm_dpo/margin": 52.413841247558594,
|
|
"fcm_dpo/q_t": 0.40031716227531433,
|
|
"grad_norm": 72.44232940673828,
|
|
"learning_rate": 3.90505702185e-07,
|
|
"logits/chosen": -0.8301101326942444,
|
|
"logits/rejected": -0.8559277653694153,
|
|
"logps/chosen": -355.3763427734375,
|
|
"logps/ref_chosen": -279.4981689453125,
|
|
"logps/ref_rejected": -263.6926574707031,
|
|
"logps/rejected": -391.984619140625,
|
|
"loss": 4.3269,
|
|
"margin_dpo/margin_mean": 52.413841247558594,
|
|
"margin_dpo/margin_std": 70.04794311523438,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.3832460732984293,
|
|
"fcm_dpo/beta": 0.00805988721549511,
|
|
"fcm_dpo/delta": 0.019830908626317978,
|
|
"fcm_dpo/margin": 47.256717681884766,
|
|
"fcm_dpo/q_t": 0.41498610377311707,
|
|
"grad_norm": 78.21089935302734,
|
|
"learning_rate": 3.889876827928156e-07,
|
|
"logits/chosen": -0.8640351295471191,
|
|
"logits/rejected": -0.8657877445220947,
|
|
"logps/chosen": -344.5203857421875,
|
|
"logps/ref_chosen": -271.2057189941406,
|
|
"logps/ref_rejected": -243.91549682617188,
|
|
"logps/rejected": -364.4868469238281,
|
|
"loss": 4.5346,
|
|
"margin_dpo/margin_mean": 47.256717681884766,
|
|
"margin_dpo/margin_std": 76.79619598388672,
|
|
"step": 183
|
|
},
|
|
{
|
|
"epoch": 0.38534031413612563,
|
|
"fcm_dpo/beta": 0.007635125890374184,
|
|
"fcm_dpo/delta": -0.10393635928630829,
|
|
"fcm_dpo/margin": 64.89164733886719,
|
|
"fcm_dpo/q_t": 0.3870086967945099,
|
|
"grad_norm": 56.21622085571289,
|
|
"learning_rate": 3.874622099130087e-07,
|
|
"logits/chosen": -0.9125643372535706,
|
|
"logits/rejected": -0.8902286291122437,
|
|
"logps/chosen": -386.58197021484375,
|
|
"logps/ref_chosen": -318.4457702636719,
|
|
"logps/ref_rejected": -266.640869140625,
|
|
"logps/rejected": -399.66864013671875,
|
|
"loss": 4.1687,
|
|
"margin_dpo/margin_mean": 64.89163970947266,
|
|
"margin_dpo/margin_std": 79.78623962402344,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.387434554973822,
|
|
"fcm_dpo/beta": 0.0075467368587851524,
|
|
"fcm_dpo/delta": 0.013490152545273304,
|
|
"fcm_dpo/margin": 47.568668365478516,
|
|
"fcm_dpo/q_t": 0.41884249448776245,
|
|
"grad_norm": 59.321022033691406,
|
|
"learning_rate": 3.859293653520604e-07,
|
|
"logits/chosen": -0.9010488986968994,
|
|
"logits/rejected": -0.8967990279197693,
|
|
"logps/chosen": -351.34808349609375,
|
|
"logps/ref_chosen": -274.308837890625,
|
|
"logps/ref_rejected": -260.7274169921875,
|
|
"logps/rejected": -385.3353271484375,
|
|
"loss": 4.5723,
|
|
"margin_dpo/margin_mean": 47.568668365478516,
|
|
"margin_dpo/margin_std": 76.99652862548828,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.38952879581151834,
|
|
"fcm_dpo/beta": 0.0076223099604249,
|
|
"fcm_dpo/delta": 0.00433752965182066,
|
|
"fcm_dpo/margin": 46.567955017089844,
|
|
"fcm_dpo/q_t": 0.418188214302063,
|
|
"grad_norm": 67.55210876464844,
|
|
"learning_rate": 3.8438923131177237e-07,
|
|
"logits/chosen": -0.8974340558052063,
|
|
"logits/rejected": -0.8985111713409424,
|
|
"logps/chosen": -374.4447937011719,
|
|
"logps/ref_chosen": -299.00537109375,
|
|
"logps/ref_rejected": -274.4014587402344,
|
|
"logps/rejected": -396.4088134765625,
|
|
"loss": 4.5636,
|
|
"margin_dpo/margin_mean": 46.567955017089844,
|
|
"margin_dpo/margin_std": 71.06839752197266,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.39162303664921466,
|
|
"fcm_dpo/beta": 0.007788485381752253,
|
|
"fcm_dpo/delta": 0.023075303062796593,
|
|
"fcm_dpo/margin": 42.33740997314453,
|
|
"fcm_dpo/q_t": 0.4230867028236389,
|
|
"grad_norm": 82.19164276123047,
|
|
"learning_rate": 3.828418903848593e-07,
|
|
"logits/chosen": -0.8443529605865479,
|
|
"logits/rejected": -0.8297170400619507,
|
|
"logps/chosen": -410.2223205566406,
|
|
"logps/ref_chosen": -329.8253173828125,
|
|
"logps/ref_rejected": -263.73175048828125,
|
|
"logps/rejected": -386.4662170410156,
|
|
"loss": 4.7667,
|
|
"margin_dpo/margin_mean": 42.33740997314453,
|
|
"margin_dpo/margin_std": 83.36969757080078,
|
|
"step": 187
|
|
},
|
|
{
|
|
"epoch": 0.393717277486911,
|
|
"fcm_dpo/beta": 0.007786841131746769,
|
|
"fcm_dpo/delta": 0.005178054794669151,
|
|
"fcm_dpo/margin": 48.28009033203125,
|
|
"fcm_dpo/q_t": 0.41652175784111023,
|
|
"grad_norm": 68.4957504272461,
|
|
"learning_rate": 3.812874255505191e-07,
|
|
"logits/chosen": -0.8509972095489502,
|
|
"logits/rejected": -0.8415097594261169,
|
|
"logps/chosen": -337.3232727050781,
|
|
"logps/ref_chosen": -263.005615234375,
|
|
"logps/ref_rejected": -247.08668518066406,
|
|
"logps/rejected": -369.68438720703125,
|
|
"loss": 4.6187,
|
|
"margin_dpo/margin_mean": 48.280086517333984,
|
|
"margin_dpo/margin_std": 84.95806121826172,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.3958115183246073,
|
|
"fcm_dpo/beta": 0.0074915410950779915,
|
|
"fcm_dpo/delta": -0.07813064754009247,
|
|
"fcm_dpo/margin": 55.84880065917969,
|
|
"fcm_dpo/q_t": 0.40445610880851746,
|
|
"grad_norm": 74.38936614990234,
|
|
"learning_rate": 3.797259201699833e-07,
|
|
"logits/chosen": -0.8831048011779785,
|
|
"logits/rejected": -0.8907921314239502,
|
|
"logps/chosen": -337.052490234375,
|
|
"logps/ref_chosen": -272.96038818359375,
|
|
"logps/ref_rejected": -275.13238525390625,
|
|
"logps/rejected": -395.07330322265625,
|
|
"loss": 4.3464,
|
|
"margin_dpo/margin_mean": 55.84880065917969,
|
|
"margin_dpo/margin_std": 72.70423126220703,
|
|
"step": 189
|
|
},
|
|
{
|
|
"epoch": 0.39790575916230364,
|
|
"fcm_dpo/beta": 0.007398220710456371,
|
|
"fcm_dpo/delta": 0.0288606658577919,
|
|
"fcm_dpo/margin": 50.26428985595703,
|
|
"fcm_dpo/q_t": 0.4147360920906067,
|
|
"grad_norm": 70.62735748291016,
|
|
"learning_rate": 3.781574579820464e-07,
|
|
"logits/chosen": -0.8793290853500366,
|
|
"logits/rejected": -0.8391152024269104,
|
|
"logps/chosen": -323.90863037109375,
|
|
"logps/ref_chosen": -257.79754638671875,
|
|
"logps/ref_rejected": -225.2164306640625,
|
|
"logps/rejected": -341.591796875,
|
|
"loss": 4.4976,
|
|
"margin_dpo/margin_mean": 50.26428985595703,
|
|
"margin_dpo/margin_std": 73.94573974609375,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.4,
|
|
"fcm_dpo/beta": 0.007520074490457773,
|
|
"fcm_dpo/delta": -0.0024281367659568787,
|
|
"fcm_dpo/margin": 50.59221649169922,
|
|
"fcm_dpo/q_t": 0.41343408823013306,
|
|
"grad_norm": 70.87568664550781,
|
|
"learning_rate": 3.765821230985757e-07,
|
|
"logits/chosen": -0.9008585214614868,
|
|
"logits/rejected": -0.9023429155349731,
|
|
"logps/chosen": -303.3919982910156,
|
|
"logps/ref_chosen": -243.8585205078125,
|
|
"logps/ref_rejected": -245.12136840820312,
|
|
"logps/rejected": -355.2470703125,
|
|
"loss": 4.5217,
|
|
"margin_dpo/margin_mean": 50.59221267700195,
|
|
"margin_dpo/margin_std": 78.84461212158203,
|
|
"step": 191
|
|
},
|
|
{
|
|
"epoch": 0.40209424083769635,
|
|
"fcm_dpo/beta": 0.007562983315438032,
|
|
"fcm_dpo/delta": 0.018396003171801567,
|
|
"fcm_dpo/margin": 40.44701385498047,
|
|
"fcm_dpo/q_t": 0.4299015998840332,
|
|
"grad_norm": 53.1352653503418,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -0.8452181816101074,
|
|
"logits/rejected": -0.8295493125915527,
|
|
"logps/chosen": -338.7875671386719,
|
|
"logps/ref_chosen": -266.9799499511719,
|
|
"logps/ref_rejected": -260.1697082519531,
|
|
"logps/rejected": -372.42437744140625,
|
|
"loss": 4.7355,
|
|
"margin_dpo/margin_mean": 40.447021484375,
|
|
"margin_dpo/margin_std": 72.97218322753906,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.4041884816753927,
|
|
"fcm_dpo/beta": 0.007469853386282921,
|
|
"fcm_dpo/delta": -0.0201749037951231,
|
|
"fcm_dpo/margin": 49.940818786621094,
|
|
"fcm_dpo/q_t": 0.4158126711845398,
|
|
"grad_norm": 71.28538513183594,
|
|
"learning_rate": 3.734111735307796e-07,
|
|
"logits/chosen": -0.8989495038986206,
|
|
"logits/rejected": -0.8711603879928589,
|
|
"logps/chosen": -363.9617919921875,
|
|
"logps/ref_chosen": -280.25323486328125,
|
|
"logps/ref_rejected": -291.0348815917969,
|
|
"logps/rejected": -424.6842956542969,
|
|
"loss": 4.5614,
|
|
"margin_dpo/margin_mean": 49.940818786621094,
|
|
"margin_dpo/margin_std": 80.18729400634766,
|
|
"step": 193
|
|
},
|
|
{
|
|
"epoch": 0.406282722513089,
|
|
"fcm_dpo/beta": 0.007623044308274984,
|
|
"fcm_dpo/delta": 0.0396825447678566,
|
|
"fcm_dpo/margin": 38.62468719482422,
|
|
"fcm_dpo/q_t": 0.4337855279445648,
|
|
"grad_norm": 79.49999237060547,
|
|
"learning_rate": 3.7181572889485623e-07,
|
|
"logits/chosen": -0.8728510737419128,
|
|
"logits/rejected": -0.8639151453971863,
|
|
"logps/chosen": -375.9040222167969,
|
|
"logps/ref_chosen": -288.13946533203125,
|
|
"logps/ref_rejected": -251.31529235839844,
|
|
"logps/rejected": -377.70452880859375,
|
|
"loss": 4.7918,
|
|
"margin_dpo/margin_mean": 38.62468719482422,
|
|
"margin_dpo/margin_std": 75.4052505493164,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.4083769633507853,
|
|
"fcm_dpo/beta": 0.007884586229920387,
|
|
"fcm_dpo/delta": 0.040129128843545914,
|
|
"fcm_dpo/margin": 37.130008697509766,
|
|
"fcm_dpo/q_t": 0.43522733449935913,
|
|
"grad_norm": 89.50051879882812,
|
|
"learning_rate": 3.7021375165108377e-07,
|
|
"logits/chosen": -0.859967827796936,
|
|
"logits/rejected": -0.8635672926902771,
|
|
"logps/chosen": -366.3628234863281,
|
|
"logps/ref_chosen": -274.0006408691406,
|
|
"logps/ref_rejected": -280.22723388671875,
|
|
"logps/rejected": -409.71942138671875,
|
|
"loss": 4.8251,
|
|
"margin_dpo/margin_mean": 37.130008697509766,
|
|
"margin_dpo/margin_std": 76.77651977539062,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.41047120418848165,
|
|
"fcm_dpo/beta": 0.007902967743575573,
|
|
"fcm_dpo/delta": -0.017355965450406075,
|
|
"fcm_dpo/margin": 52.63897705078125,
|
|
"fcm_dpo/q_t": 0.40730372071266174,
|
|
"grad_norm": 91.81265258789062,
|
|
"learning_rate": 3.6860532770864005e-07,
|
|
"logits/chosen": -0.8587782979011536,
|
|
"logits/rejected": -0.8644378185272217,
|
|
"logps/chosen": -354.6200256347656,
|
|
"logps/ref_chosen": -274.90069580078125,
|
|
"logps/ref_rejected": -248.7281951904297,
|
|
"logps/rejected": -381.0865478515625,
|
|
"loss": 4.4737,
|
|
"margin_dpo/margin_mean": 52.63897705078125,
|
|
"margin_dpo/margin_std": 84.32215881347656,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.41256544502617803,
|
|
"fcm_dpo/beta": 0.00753373745828867,
|
|
"fcm_dpo/delta": -0.10619766265153885,
|
|
"fcm_dpo/margin": 58.62234115600586,
|
|
"fcm_dpo/q_t": 0.40042629837989807,
|
|
"grad_norm": 108.23560333251953,
|
|
"learning_rate": 3.6699054332241985e-07,
|
|
"logits/chosen": -0.8893225193023682,
|
|
"logits/rejected": -0.874314546585083,
|
|
"logps/chosen": -397.1566162109375,
|
|
"logps/ref_chosen": -309.5348205566406,
|
|
"logps/ref_rejected": -264.3179931640625,
|
|
"logps/rejected": -410.5621032714844,
|
|
"loss": 4.3535,
|
|
"margin_dpo/margin_mean": 58.62234115600586,
|
|
"margin_dpo/margin_std": 79.52429962158203,
|
|
"step": 197
|
|
},
|
|
{
|
|
"epoch": 0.41465968586387436,
|
|
"fcm_dpo/beta": 0.00704828929156065,
|
|
"fcm_dpo/delta": -0.030336569994688034,
|
|
"fcm_dpo/margin": 60.71779251098633,
|
|
"fcm_dpo/q_t": 0.4027983844280243,
|
|
"grad_norm": 77.68168640136719,
|
|
"learning_rate": 3.653694850884091e-07,
|
|
"logits/chosen": -0.8886722922325134,
|
|
"logits/rejected": -0.8632485866546631,
|
|
"logps/chosen": -382.4344482421875,
|
|
"logps/ref_chosen": -301.0134582519531,
|
|
"logps/ref_rejected": -292.84185791015625,
|
|
"logps/rejected": -434.98065185546875,
|
|
"loss": 4.3485,
|
|
"margin_dpo/margin_mean": 60.71779251098633,
|
|
"margin_dpo/margin_std": 82.84037780761719,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.4167539267015707,
|
|
"fcm_dpo/beta": 0.007051732391119003,
|
|
"fcm_dpo/delta": -0.012272719293832779,
|
|
"fcm_dpo/margin": 53.73326110839844,
|
|
"fcm_dpo/q_t": 0.4135727882385254,
|
|
"grad_norm": 73.13438415527344,
|
|
"learning_rate": 3.6374223993904124e-07,
|
|
"logits/chosen": -0.8650781512260437,
|
|
"logits/rejected": -0.8219231367111206,
|
|
"logps/chosen": -353.032470703125,
|
|
"logps/ref_chosen": -264.6058654785156,
|
|
"logps/ref_rejected": -214.9014892578125,
|
|
"logps/rejected": -357.0614013671875,
|
|
"loss": 4.4976,
|
|
"margin_dpo/margin_mean": 53.73326110839844,
|
|
"margin_dpo/margin_std": 81.83422088623047,
|
|
"step": 199
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"fcm_dpo/beta": 0.007016919087618589,
|
|
"fcm_dpo/delta": 0.007905922830104828,
|
|
"fcm_dpo/margin": 44.08357238769531,
|
|
"fcm_dpo/q_t": 0.43098166584968567,
|
|
"grad_norm": 82.60724639892578,
|
|
"learning_rate": 3.621088951385353e-07,
|
|
"logits/chosen": -0.9180527925491333,
|
|
"logits/rejected": -0.8939597010612488,
|
|
"logps/chosen": -419.4546813964844,
|
|
"logps/ref_chosen": -324.1588134765625,
|
|
"logps/ref_rejected": -277.80218505859375,
|
|
"logps/rejected": -417.181640625,
|
|
"loss": 4.8634,
|
|
"margin_dpo/margin_mean": 44.08357238769531,
|
|
"margin_dpo/margin_std": 95.21434020996094,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"eval_fcm_dpo/beta": 0.007041628006845713,
|
|
"eval_logits/chosen": -0.8935934901237488,
|
|
"eval_logits/rejected": -0.8767105340957642,
|
|
"eval_logps/chosen": -385.81689453125,
|
|
"eval_logps/ref_chosen": -287.8268127441406,
|
|
"eval_logps/ref_rejected": -266.9300231933594,
|
|
"eval_logps/rejected": -415.6835021972656,
|
|
"eval_loss": 0.5780735015869141,
|
|
"eval_margin_dpo/margin_mean": 50.763328552246094,
|
|
"eval_margin_dpo/margin_std": 82.26912689208984,
|
|
"eval_runtime": 79.4075,
|
|
"eval_samples_per_second": 25.187,
|
|
"eval_steps_per_second": 3.148,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.42094240837696334,
|
|
"fcm_dpo/beta": 0.007051797583699226,
|
|
"fcm_dpo/delta": -0.013565017841756344,
|
|
"fcm_dpo/margin": 52.115760803222656,
|
|
"fcm_dpo/q_t": 0.41692644357681274,
|
|
"grad_norm": 118.92222595214844,
|
|
"learning_rate": 3.604695382782159e-07,
|
|
"logits/chosen": -0.8829444050788879,
|
|
"logits/rejected": -0.8692610263824463,
|
|
"logps/chosen": -370.8045349121094,
|
|
"logps/ref_chosen": -271.49566650390625,
|
|
"logps/ref_rejected": -245.71414184570312,
|
|
"logps/rejected": -397.1387939453125,
|
|
"loss": 4.6069,
|
|
"margin_dpo/margin_mean": 52.115760803222656,
|
|
"margin_dpo/margin_std": 88.78590393066406,
|
|
"step": 201
|
|
},
|
|
{
|
|
"epoch": 0.42303664921465967,
|
|
"fcm_dpo/beta": 0.006848223973065615,
|
|
"fcm_dpo/delta": -0.02687113918364048,
|
|
"fcm_dpo/margin": 52.322425842285156,
|
|
"fcm_dpo/q_t": 0.4185172915458679,
|
|
"grad_norm": 76.39278411865234,
|
|
"learning_rate": 3.588242572718162e-07,
|
|
"logits/chosen": -0.8918448090553284,
|
|
"logits/rejected": -0.8787520527839661,
|
|
"logps/chosen": -366.652587890625,
|
|
"logps/ref_chosen": -272.0979309082031,
|
|
"logps/ref_rejected": -235.94805908203125,
|
|
"logps/rejected": -382.82513427734375,
|
|
"loss": 4.6214,
|
|
"margin_dpo/margin_mean": 52.322425842285156,
|
|
"margin_dpo/margin_std": 89.93933868408203,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.42513089005235605,
|
|
"fcm_dpo/beta": 0.0069344537332654,
|
|
"fcm_dpo/delta": 0.04031352326273918,
|
|
"fcm_dpo/margin": 40.129302978515625,
|
|
"fcm_dpo/q_t": 0.43582794070243835,
|
|
"grad_norm": 95.53482055664062,
|
|
"learning_rate": 3.571731403507635e-07,
|
|
"logits/chosen": -0.8573673963546753,
|
|
"logits/rejected": -0.8636147975921631,
|
|
"logps/chosen": -379.1627197265625,
|
|
"logps/ref_chosen": -280.2221374511719,
|
|
"logps/ref_rejected": -251.79798889160156,
|
|
"logps/rejected": -390.86785888671875,
|
|
"loss": 4.8197,
|
|
"margin_dpo/margin_mean": 40.129302978515625,
|
|
"margin_dpo/margin_std": 78.97547149658203,
|
|
"step": 203
|
|
},
|
|
{
|
|
"epoch": 0.4272251308900524,
|
|
"fcm_dpo/beta": 0.006983473431318998,
|
|
"fcm_dpo/delta": 0.0020564058795571327,
|
|
"fcm_dpo/margin": 56.94648361206055,
|
|
"fcm_dpo/q_t": 0.4088953733444214,
|
|
"grad_norm": 83.27347564697266,
|
|
"learning_rate": 3.5551627605944746e-07,
|
|
"logits/chosen": -0.9110021591186523,
|
|
"logits/rejected": -0.8834875226020813,
|
|
"logps/chosen": -407.1368408203125,
|
|
"logps/ref_chosen": -318.7960510253906,
|
|
"logps/ref_rejected": -269.69921875,
|
|
"logps/rejected": -414.98651123046875,
|
|
"loss": 4.4147,
|
|
"margin_dpo/margin_mean": 56.94648361206055,
|
|
"margin_dpo/margin_std": 79.84234619140625,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.4293193717277487,
|
|
"fcm_dpo/beta": 0.006971952971071005,
|
|
"fcm_dpo/delta": -0.04280940815806389,
|
|
"fcm_dpo/margin": 63.21536636352539,
|
|
"fcm_dpo/q_t": 0.402166485786438,
|
|
"grad_norm": 75.73958587646484,
|
|
"learning_rate": 3.5385375325047163e-07,
|
|
"logits/chosen": -0.8463097810745239,
|
|
"logits/rejected": -0.8146329522132874,
|
|
"logps/chosen": -370.4803161621094,
|
|
"logps/ref_chosen": -283.7620544433594,
|
|
"logps/ref_rejected": -297.69439697265625,
|
|
"logps/rejected": -447.6280212402344,
|
|
"loss": 4.3512,
|
|
"margin_dpo/margin_mean": 63.21536636352539,
|
|
"margin_dpo/margin_std": 89.52297973632812,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.431413612565445,
|
|
"fcm_dpo/beta": 0.006694549694657326,
|
|
"fcm_dpo/delta": -0.02712525613605976,
|
|
"fcm_dpo/margin": 49.10927963256836,
|
|
"fcm_dpo/q_t": 0.42545533180236816,
|
|
"grad_norm": 70.90359497070312,
|
|
"learning_rate": 3.5218566107988867e-07,
|
|
"logits/chosen": -0.8882994055747986,
|
|
"logits/rejected": -0.909928560256958,
|
|
"logps/chosen": -385.359619140625,
|
|
"logps/ref_chosen": -293.66387939453125,
|
|
"logps/ref_rejected": -291.3056640625,
|
|
"logps/rejected": -432.1107177734375,
|
|
"loss": 4.7352,
|
|
"margin_dpo/margin_mean": 49.10927963256836,
|
|
"margin_dpo/margin_std": 92.05156707763672,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.43350785340314135,
|
|
"fcm_dpo/beta": 0.006603881251066923,
|
|
"fcm_dpo/delta": -0.01149575226008892,
|
|
"fcm_dpo/margin": 49.813499450683594,
|
|
"fcm_dpo/q_t": 0.4254821538925171,
|
|
"grad_norm": 85.93264770507812,
|
|
"learning_rate": 3.505120890024195e-07,
|
|
"logits/chosen": -0.8319991827011108,
|
|
"logits/rejected": -0.8400675058364868,
|
|
"logps/chosen": -354.9524230957031,
|
|
"logps/ref_chosen": -270.5350646972656,
|
|
"logps/ref_rejected": -278.7747497558594,
|
|
"logps/rejected": -413.005615234375,
|
|
"loss": 4.7927,
|
|
"margin_dpo/margin_mean": 49.81349563598633,
|
|
"margin_dpo/margin_std": 102.63397216796875,
|
|
"step": 207
|
|
},
|
|
{
|
|
"epoch": 0.4356020942408377,
|
|
"fcm_dpo/beta": 0.006531773135066032,
|
|
"fcm_dpo/delta": -0.045300256460905075,
|
|
"fcm_dpo/margin": 64.10903930664062,
|
|
"fcm_dpo/q_t": 0.4059593081474304,
|
|
"grad_norm": 69.88282012939453,
|
|
"learning_rate": 3.4883312676665534e-07,
|
|
"logits/chosen": -0.8945930004119873,
|
|
"logits/rejected": -0.8459950685501099,
|
|
"logps/chosen": -368.4111633300781,
|
|
"logps/ref_chosen": -279.582763671875,
|
|
"logps/ref_rejected": -290.041015625,
|
|
"logps/rejected": -442.978515625,
|
|
"loss": 4.368,
|
|
"margin_dpo/margin_mean": 64.10904693603516,
|
|
"margin_dpo/margin_std": 88.38412475585938,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.437696335078534,
|
|
"fcm_dpo/beta": 0.006543098483234644,
|
|
"fcm_dpo/delta": 0.0669432207942009,
|
|
"fcm_dpo/margin": 40.375823974609375,
|
|
"fcm_dpo/q_t": 0.4394206702709198,
|
|
"grad_norm": 94.81318664550781,
|
|
"learning_rate": 3.4714886441024573e-07,
|
|
"logits/chosen": -0.8037338852882385,
|
|
"logits/rejected": -0.8001971244812012,
|
|
"logps/chosen": -417.86273193359375,
|
|
"logps/ref_chosen": -318.8725280761719,
|
|
"logps/ref_rejected": -270.64324951171875,
|
|
"logps/rejected": -410.0093078613281,
|
|
"loss": 4.9026,
|
|
"margin_dpo/margin_mean": 40.37582778930664,
|
|
"margin_dpo/margin_std": 88.71975708007812,
|
|
"step": 209
|
|
},
|
|
{
|
|
"epoch": 0.4397905759162304,
|
|
"fcm_dpo/beta": 0.006677450146526098,
|
|
"fcm_dpo/delta": 0.031221631914377213,
|
|
"fcm_dpo/margin": 55.327545166015625,
|
|
"fcm_dpo/q_t": 0.4165865480899811,
|
|
"grad_norm": 75.42611694335938,
|
|
"learning_rate": 3.454593922550693e-07,
|
|
"logits/chosen": -0.8339765667915344,
|
|
"logits/rejected": -0.8234052062034607,
|
|
"logps/chosen": -369.9593505859375,
|
|
"logps/ref_chosen": -283.14031982421875,
|
|
"logps/ref_rejected": -287.2986755371094,
|
|
"logps/rejected": -429.44525146484375,
|
|
"loss": 4.5568,
|
|
"margin_dpo/margin_mean": 55.327545166015625,
|
|
"margin_dpo/margin_std": 88.78600311279297,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.4418848167539267,
|
|
"fcm_dpo/beta": 0.006575165782123804,
|
|
"fcm_dpo/delta": -0.08240824937820435,
|
|
"fcm_dpo/margin": 66.77010345458984,
|
|
"fcm_dpo/q_t": 0.39799728989601135,
|
|
"grad_norm": 68.85022735595703,
|
|
"learning_rate": 3.4376480090239047e-07,
|
|
"logits/chosen": -0.8611884117126465,
|
|
"logits/rejected": -0.8390584588050842,
|
|
"logps/chosen": -354.44036865234375,
|
|
"logps/ref_chosen": -276.4228515625,
|
|
"logps/ref_rejected": -252.40603637695312,
|
|
"logps/rejected": -397.193603515625,
|
|
"loss": 4.2563,
|
|
"margin_dpo/margin_mean": 66.77011108398438,
|
|
"margin_dpo/margin_std": 76.85353088378906,
|
|
"step": 211
|
|
},
|
|
{
|
|
"epoch": 0.44397905759162304,
|
|
"fcm_dpo/beta": 0.006527946330606937,
|
|
"fcm_dpo/delta": 0.03064761869609356,
|
|
"fcm_dpo/margin": 41.976829528808594,
|
|
"fcm_dpo/q_t": 0.43568912148475647,
|
|
"grad_norm": 106.75749969482422,
|
|
"learning_rate": 3.4206518122800055e-07,
|
|
"logits/chosen": -0.8498879075050354,
|
|
"logits/rejected": -0.8480284214019775,
|
|
"logps/chosen": -362.7585144042969,
|
|
"logps/ref_chosen": -271.7055358886719,
|
|
"logps/ref_rejected": -241.18511962890625,
|
|
"logps/rejected": -374.2149658203125,
|
|
"loss": 4.8563,
|
|
"margin_dpo/margin_mean": 41.976829528808594,
|
|
"margin_dpo/margin_std": 84.9686279296875,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.44607329842931936,
|
|
"fcm_dpo/beta": 0.006641289219260216,
|
|
"fcm_dpo/delta": 0.0012545166537165642,
|
|
"fcm_dpo/margin": 50.06464385986328,
|
|
"fcm_dpo/q_t": 0.425908625125885,
|
|
"grad_norm": 103.77131652832031,
|
|
"learning_rate": 3.403606243773448e-07,
|
|
"logits/chosen": -0.8439345955848694,
|
|
"logits/rejected": -0.8587431311607361,
|
|
"logps/chosen": -396.8846435546875,
|
|
"logps/ref_chosen": -302.2976379394531,
|
|
"logps/ref_rejected": -303.6202087402344,
|
|
"logps/rejected": -448.2718200683594,
|
|
"loss": 4.7044,
|
|
"margin_dpo/margin_mean": 50.06464385986328,
|
|
"margin_dpo/margin_std": 93.97140502929688,
|
|
"step": 213
|
|
},
|
|
{
|
|
"epoch": 0.4481675392670157,
|
|
"fcm_dpo/beta": 0.006577081512659788,
|
|
"fcm_dpo/delta": -0.015572590753436089,
|
|
"fcm_dpo/margin": 62.8525390625,
|
|
"fcm_dpo/q_t": 0.405367910861969,
|
|
"grad_norm": 92.35052490234375,
|
|
"learning_rate": 3.3865122176063385e-07,
|
|
"logits/chosen": -0.8551336526870728,
|
|
"logits/rejected": -0.8550840616226196,
|
|
"logps/chosen": -380.4796447753906,
|
|
"logps/ref_chosen": -272.13262939453125,
|
|
"logps/ref_rejected": -294.82354736328125,
|
|
"logps/rejected": -466.0230712890625,
|
|
"loss": 4.3617,
|
|
"margin_dpo/margin_mean": 62.8525390625,
|
|
"margin_dpo/margin_std": 80.1742935180664,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.450261780104712,
|
|
"fcm_dpo/beta": 0.006535133346915245,
|
|
"fcm_dpo/delta": -0.00995543785393238,
|
|
"fcm_dpo/margin": 42.939754486083984,
|
|
"fcm_dpo/q_t": 0.43834444880485535,
|
|
"grad_norm": 89.51997375488281,
|
|
"learning_rate": 3.3693706504794243e-07,
|
|
"logits/chosen": -0.8979097604751587,
|
|
"logits/rejected": -0.8771700263023376,
|
|
"logps/chosen": -402.0087585449219,
|
|
"logps/ref_chosen": -291.3782958984375,
|
|
"logps/ref_rejected": -261.05792236328125,
|
|
"logps/rejected": -414.62811279296875,
|
|
"loss": 4.8962,
|
|
"margin_dpo/margin_mean": 42.939754486083984,
|
|
"margin_dpo/margin_std": 94.85189819335938,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.4523560209424084,
|
|
"fcm_dpo/beta": 0.006419910583645105,
|
|
"fcm_dpo/delta": 0.00330403633415699,
|
|
"fcm_dpo/margin": 56.20130920410156,
|
|
"fcm_dpo/q_t": 0.418740838766098,
|
|
"grad_norm": 73.42216491699219,
|
|
"learning_rate": 3.3521824616429284e-07,
|
|
"logits/chosen": -0.922217607498169,
|
|
"logits/rejected": -0.9141356945037842,
|
|
"logps/chosen": -444.15765380859375,
|
|
"logps/ref_chosen": -338.50543212890625,
|
|
"logps/ref_rejected": -305.76104736328125,
|
|
"logps/rejected": -467.61456298828125,
|
|
"loss": 4.6374,
|
|
"margin_dpo/margin_mean": 56.20130920410156,
|
|
"margin_dpo/margin_std": 99.16251373291016,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.4544502617801047,
|
|
"fcm_dpo/beta": 0.006329713389277458,
|
|
"fcm_dpo/delta": -0.04338594153523445,
|
|
"fcm_dpo/margin": 69.61051940917969,
|
|
"fcm_dpo/q_t": 0.4017544090747833,
|
|
"grad_norm": 74.04085540771484,
|
|
"learning_rate": 3.334948572847253e-07,
|
|
"logits/chosen": -0.8338743448257446,
|
|
"logits/rejected": -0.7945307493209839,
|
|
"logps/chosen": -406.1719665527344,
|
|
"logps/ref_chosen": -293.5498046875,
|
|
"logps/ref_rejected": -256.7830810546875,
|
|
"logps/rejected": -439.01580810546875,
|
|
"loss": 4.3716,
|
|
"margin_dpo/margin_mean": 69.61051940917969,
|
|
"margin_dpo/margin_std": 101.19683074951172,
|
|
"step": 217
|
|
},
|
|
{
|
|
"epoch": 0.45654450261780105,
|
|
"fcm_dpo/beta": 0.006193040404468775,
|
|
"fcm_dpo/delta": -0.006936352234333754,
|
|
"fcm_dpo/margin": 65.53926086425781,
|
|
"fcm_dpo/q_t": 0.40807247161865234,
|
|
"grad_norm": 74.45264434814453,
|
|
"learning_rate": 3.317669908293554e-07,
|
|
"logits/chosen": -0.8653545379638672,
|
|
"logits/rejected": -0.8810456991195679,
|
|
"logps/chosen": -432.00384521484375,
|
|
"logps/ref_chosen": -320.579345703125,
|
|
"logps/ref_rejected": -294.0381164550781,
|
|
"logps/rejected": -471.00189208984375,
|
|
"loss": 4.421,
|
|
"margin_dpo/margin_mean": 65.53926086425781,
|
|
"margin_dpo/margin_std": 95.39942932128906,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.4586387434554974,
|
|
"fcm_dpo/beta": 0.006144728511571884,
|
|
"fcm_dpo/delta": -0.03086962178349495,
|
|
"fcm_dpo/margin": 63.95124816894531,
|
|
"fcm_dpo/q_t": 0.41116613149642944,
|
|
"grad_norm": 76.28233337402344,
|
|
"learning_rate": 3.300347394584172e-07,
|
|
"logits/chosen": -0.8417205214500427,
|
|
"logits/rejected": -0.8665354251861572,
|
|
"logps/chosen": -379.6574401855469,
|
|
"logps/ref_chosen": -268.4186096191406,
|
|
"logps/ref_rejected": -265.7808837890625,
|
|
"logps/rejected": -440.970947265625,
|
|
"loss": 4.5161,
|
|
"margin_dpo/margin_mean": 63.95124816894531,
|
|
"margin_dpo/margin_std": 100.1285629272461,
|
|
"step": 219
|
|
},
|
|
{
|
|
"epoch": 0.4607329842931937,
|
|
"fcm_dpo/beta": 0.006020966451615095,
|
|
"fcm_dpo/delta": -0.006018012762069702,
|
|
"fcm_dpo/margin": 67.32181549072266,
|
|
"fcm_dpo/q_t": 0.4066336750984192,
|
|
"grad_norm": 67.24308776855469,
|
|
"learning_rate": 3.2829819606729477e-07,
|
|
"logits/chosen": -0.9152698516845703,
|
|
"logits/rejected": -0.8838323354721069,
|
|
"logps/chosen": -423.98956298828125,
|
|
"logps/ref_chosen": -312.8864440917969,
|
|
"logps/ref_rejected": -259.5191955566406,
|
|
"logps/rejected": -437.9441223144531,
|
|
"loss": 4.4003,
|
|
"margin_dpo/margin_mean": 67.32180786132812,
|
|
"margin_dpo/margin_std": 93.29956817626953,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.46282722513089003,
|
|
"fcm_dpo/beta": 0.006161336787045002,
|
|
"fcm_dpo/delta": -0.010663069784641266,
|
|
"fcm_dpo/margin": 55.39729309082031,
|
|
"fcm_dpo/q_t": 0.4242016077041626,
|
|
"grad_norm": 75.08380889892578,
|
|
"learning_rate": 3.265574537815398e-07,
|
|
"logits/chosen": -0.8223947286605835,
|
|
"logits/rejected": -0.8319114446640015,
|
|
"logps/chosen": -420.740966796875,
|
|
"logps/ref_chosen": -300.32586669921875,
|
|
"logps/ref_rejected": -286.312255859375,
|
|
"logps/rejected": -462.12469482421875,
|
|
"loss": 4.7053,
|
|
"margin_dpo/margin_mean": 55.39729309082031,
|
|
"margin_dpo/margin_std": 102.78092193603516,
|
|
"step": 221
|
|
},
|
|
{
|
|
"epoch": 0.4649214659685864,
|
|
"fcm_dpo/beta": 0.006012233905494213,
|
|
"fcm_dpo/delta": 0.020973335951566696,
|
|
"fcm_dpo/margin": 58.58289337158203,
|
|
"fcm_dpo/q_t": 0.4185751974582672,
|
|
"grad_norm": 64.78793334960938,
|
|
"learning_rate": 3.248126059518784e-07,
|
|
"logits/chosen": -0.9194785952568054,
|
|
"logits/rejected": -0.8871119022369385,
|
|
"logps/chosen": -414.5599060058594,
|
|
"logps/ref_chosen": -297.1113586425781,
|
|
"logps/ref_rejected": -235.53146362304688,
|
|
"logps/rejected": -411.56292724609375,
|
|
"loss": 4.5659,
|
|
"margin_dpo/margin_mean": 58.5828971862793,
|
|
"margin_dpo/margin_std": 90.73377990722656,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.46701570680628274,
|
|
"fcm_dpo/beta": 0.0061044651083648205,
|
|
"fcm_dpo/delta": 0.009785637259483337,
|
|
"fcm_dpo/margin": 60.01387023925781,
|
|
"fcm_dpo/q_t": 0.4157088100910187,
|
|
"grad_norm": 87.03768157958984,
|
|
"learning_rate": 3.230637461492043e-07,
|
|
"logits/chosen": -0.8523328304290771,
|
|
"logits/rejected": -0.816593587398529,
|
|
"logps/chosen": -409.787841796875,
|
|
"logps/ref_chosen": -286.41510009765625,
|
|
"logps/ref_rejected": -241.1181640625,
|
|
"logps/rejected": -424.5048522949219,
|
|
"loss": 4.5608,
|
|
"margin_dpo/margin_mean": 60.01387023925781,
|
|
"margin_dpo/margin_std": 96.16004180908203,
|
|
"step": 223
|
|
},
|
|
{
|
|
"epoch": 0.46910994764397906,
|
|
"fcm_dpo/beta": 0.006154531612992287,
|
|
"fcm_dpo/delta": -0.03359140083193779,
|
|
"fcm_dpo/margin": 64.03076171875,
|
|
"fcm_dpo/q_t": 0.41083890199661255,
|
|
"grad_norm": 90.42691040039062,
|
|
"learning_rate": 3.213109681595612e-07,
|
|
"logits/chosen": -0.8257457613945007,
|
|
"logits/rejected": -0.8410820960998535,
|
|
"logps/chosen": -365.0633544921875,
|
|
"logps/ref_chosen": -249.49234008789062,
|
|
"logps/ref_rejected": -233.10752868652344,
|
|
"logps/rejected": -412.70928955078125,
|
|
"loss": 4.4432,
|
|
"margin_dpo/margin_mean": 64.03076171875,
|
|
"margin_dpo/margin_std": 90.88447570800781,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.4712041884816754,
|
|
"fcm_dpo/beta": 0.005969415418803692,
|
|
"fcm_dpo/delta": 0.012547025457024574,
|
|
"fcm_dpo/margin": 59.763431549072266,
|
|
"fcm_dpo/q_t": 0.4206285774707794,
|
|
"grad_norm": 93.77128601074219,
|
|
"learning_rate": 3.1955436597911315e-07,
|
|
"logits/chosen": -0.8661923408508301,
|
|
"logits/rejected": -0.839612603187561,
|
|
"logps/chosen": -444.10955810546875,
|
|
"logps/ref_chosen": -311.8583679199219,
|
|
"logps/ref_rejected": -336.8523864746094,
|
|
"logps/rejected": -528.8670043945312,
|
|
"loss": 4.6451,
|
|
"margin_dpo/margin_mean": 59.763431549072266,
|
|
"margin_dpo/margin_std": 106.94706726074219,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.4732984293193717,
|
|
"fcm_dpo/beta": 0.006135667208582163,
|
|
"fcm_dpo/delta": 0.021072357892990112,
|
|
"fcm_dpo/margin": 50.27009582519531,
|
|
"fcm_dpo/q_t": 0.4313587546348572,
|
|
"grad_norm": 90.91165924072266,
|
|
"learning_rate": 3.1779403380910425e-07,
|
|
"logits/chosen": -0.8851912021636963,
|
|
"logits/rejected": -0.8732432723045349,
|
|
"logps/chosen": -371.9580993652344,
|
|
"logps/ref_chosen": -252.20123291015625,
|
|
"logps/ref_rejected": -254.41162109375,
|
|
"logps/rejected": -424.4385681152344,
|
|
"loss": 4.7914,
|
|
"margin_dpo/margin_mean": 50.27009201049805,
|
|
"margin_dpo/margin_std": 102.74089050292969,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.47539267015706804,
|
|
"fcm_dpo/beta": 0.006114313844591379,
|
|
"fcm_dpo/delta": -0.011526936665177345,
|
|
"fcm_dpo/margin": 67.09516143798828,
|
|
"fcm_dpo/q_t": 0.4065951108932495,
|
|
"grad_norm": 83.9355239868164,
|
|
"learning_rate": 3.160300660508064e-07,
|
|
"logits/chosen": -0.8550764918327332,
|
|
"logits/rejected": -0.8507223129272461,
|
|
"logps/chosen": -400.2503967285156,
|
|
"logps/ref_chosen": -285.25946044921875,
|
|
"logps/ref_rejected": -261.3220520019531,
|
|
"logps/rejected": -443.40814208984375,
|
|
"loss": 4.4568,
|
|
"margin_dpo/margin_mean": 67.09516143798828,
|
|
"margin_dpo/margin_std": 102.07121276855469,
|
|
"step": 227
|
|
},
|
|
{
|
|
"epoch": 0.4774869109947644,
|
|
"fcm_dpo/beta": 0.006071330048143864,
|
|
"fcm_dpo/delta": -0.03209439292550087,
|
|
"fcm_dpo/margin": 67.11500549316406,
|
|
"fcm_dpo/q_t": 0.407455712556839,
|
|
"grad_norm": 74.55585479736328,
|
|
"learning_rate": 3.1426255730045695e-07,
|
|
"logits/chosen": -0.8666589856147766,
|
|
"logits/rejected": -0.8316813707351685,
|
|
"logps/chosen": -417.8719482421875,
|
|
"logps/ref_chosen": -313.81878662109375,
|
|
"logps/ref_rejected": -258.07061767578125,
|
|
"logps/rejected": -429.23876953125,
|
|
"loss": 4.41,
|
|
"margin_dpo/margin_mean": 67.11499786376953,
|
|
"margin_dpo/margin_std": 92.93700408935547,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.47958115183246075,
|
|
"fcm_dpo/beta": 0.005753038451075554,
|
|
"fcm_dpo/delta": -0.07461543381214142,
|
|
"fcm_dpo/margin": 77.0120849609375,
|
|
"fcm_dpo/q_t": 0.3978939652442932,
|
|
"grad_norm": 78.69764709472656,
|
|
"learning_rate": 3.1249160234418644e-07,
|
|
"logits/chosen": -0.8438678979873657,
|
|
"logits/rejected": -0.8469147086143494,
|
|
"logps/chosen": -410.0827331542969,
|
|
"logps/ref_chosen": -291.9707946777344,
|
|
"logps/ref_rejected": -263.42059326171875,
|
|
"logps/rejected": -458.5446472167969,
|
|
"loss": 4.2955,
|
|
"margin_dpo/margin_mean": 77.0120849609375,
|
|
"margin_dpo/margin_std": 98.03457641601562,
|
|
"step": 229
|
|
},
|
|
{
|
|
"epoch": 0.4816753926701571,
|
|
"fcm_dpo/beta": 0.005542149767279625,
|
|
"fcm_dpo/delta": -0.007225902751088142,
|
|
"fcm_dpo/margin": 67.66149139404297,
|
|
"fcm_dpo/q_t": 0.41533219814300537,
|
|
"grad_norm": 66.38292694091797,
|
|
"learning_rate": 3.1071729615293424e-07,
|
|
"logits/chosen": -0.8894914984703064,
|
|
"logits/rejected": -0.8882502913475037,
|
|
"logps/chosen": -343.53985595703125,
|
|
"logps/ref_chosen": -233.2601318359375,
|
|
"logps/ref_rejected": -238.922119140625,
|
|
"logps/rejected": -416.86334228515625,
|
|
"loss": 4.5255,
|
|
"margin_dpo/margin_mean": 67.66149139404297,
|
|
"margin_dpo/margin_std": 105.4851303100586,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.4837696335078534,
|
|
"fcm_dpo/beta": 0.005650937091559172,
|
|
"fcm_dpo/delta": 0.033986907452344894,
|
|
"fcm_dpo/margin": 53.576927185058594,
|
|
"fcm_dpo/q_t": 0.4296777844429016,
|
|
"grad_norm": 108.66732788085938,
|
|
"learning_rate": 3.0893973387735683e-07,
|
|
"logits/chosen": -0.872734010219574,
|
|
"logits/rejected": -0.8595795035362244,
|
|
"logps/chosen": -443.4432067871094,
|
|
"logps/ref_chosen": -322.1551818847656,
|
|
"logps/ref_rejected": -280.97613525390625,
|
|
"logps/rejected": -455.8411560058594,
|
|
"loss": 4.7128,
|
|
"margin_dpo/margin_mean": 53.57693099975586,
|
|
"margin_dpo/margin_std": 93.04562377929688,
|
|
"step": 231
|
|
},
|
|
{
|
|
"epoch": 0.48586387434554973,
|
|
"fcm_dpo/beta": 0.005723518319427967,
|
|
"fcm_dpo/delta": -0.00418502651154995,
|
|
"fcm_dpo/margin": 59.888526916503906,
|
|
"fcm_dpo/q_t": 0.4218381643295288,
|
|
"grad_norm": 60.41381072998047,
|
|
"learning_rate": 3.071590108427243e-07,
|
|
"logits/chosen": -0.8330118656158447,
|
|
"logits/rejected": -0.8127278089523315,
|
|
"logps/chosen": -386.4776306152344,
|
|
"logps/ref_chosen": -271.7437744140625,
|
|
"logps/ref_rejected": -249.94981384277344,
|
|
"logps/rejected": -424.57220458984375,
|
|
"loss": 4.6566,
|
|
"margin_dpo/margin_mean": 59.888526916503906,
|
|
"margin_dpo/margin_std": 101.21884155273438,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.48795811518324606,
|
|
"fcm_dpo/beta": 0.005629006307572126,
|
|
"fcm_dpo/delta": -0.04465636610984802,
|
|
"fcm_dpo/margin": 66.1546401977539,
|
|
"fcm_dpo/q_t": 0.4142661988735199,
|
|
"grad_norm": 70.80461120605469,
|
|
"learning_rate": 3.05375222543809e-07,
|
|
"logits/chosen": -0.888208270072937,
|
|
"logits/rejected": -0.8782560229301453,
|
|
"logps/chosen": -396.4659423828125,
|
|
"logps/ref_chosen": -285.3423156738281,
|
|
"logps/ref_rejected": -266.34320068359375,
|
|
"logps/rejected": -443.6214599609375,
|
|
"loss": 4.5031,
|
|
"margin_dpo/margin_mean": 66.1546401977539,
|
|
"margin_dpo/margin_std": 94.81285858154297,
|
|
"step": 233
|
|
},
|
|
{
|
|
"epoch": 0.4900523560209424,
|
|
"fcm_dpo/beta": 0.005589499603956938,
|
|
"fcm_dpo/delta": 0.039450354874134064,
|
|
"fcm_dpo/margin": 60.44734191894531,
|
|
"fcm_dpo/q_t": 0.42315778136253357,
|
|
"grad_norm": 74.92431640625,
|
|
"learning_rate": 3.035884646397637e-07,
|
|
"logits/chosen": -0.8505942821502686,
|
|
"logits/rejected": -0.8354034423828125,
|
|
"logps/chosen": -406.40936279296875,
|
|
"logps/ref_chosen": -294.9057312011719,
|
|
"logps/ref_rejected": -299.37054443359375,
|
|
"logps/rejected": -471.3215637207031,
|
|
"loss": 4.673,
|
|
"margin_dpo/margin_mean": 60.44734191894531,
|
|
"margin_dpo/margin_std": 108.0342025756836,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.49214659685863876,
|
|
"fcm_dpo/beta": 0.005842198152095079,
|
|
"fcm_dpo/delta": 0.028937824070453644,
|
|
"fcm_dpo/margin": 63.570350646972656,
|
|
"fcm_dpo/q_t": 0.41493111848831177,
|
|
"grad_norm": 84.25675964355469,
|
|
"learning_rate": 3.017988329489923e-07,
|
|
"logits/chosen": -0.8750848770141602,
|
|
"logits/rejected": -0.8671593070030212,
|
|
"logps/chosen": -400.6502380371094,
|
|
"logps/ref_chosen": -289.49755859375,
|
|
"logps/ref_rejected": -247.55076599121094,
|
|
"logps/rejected": -422.2738037109375,
|
|
"loss": 4.5306,
|
|
"margin_dpo/margin_mean": 63.57035827636719,
|
|
"margin_dpo/margin_std": 100.27571105957031,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.4942408376963351,
|
|
"fcm_dpo/beta": 0.005945481359958649,
|
|
"fcm_dpo/delta": 0.008337687700986862,
|
|
"fcm_dpo/margin": 61.09648895263672,
|
|
"fcm_dpo/q_t": 0.4176371097564697,
|
|
"grad_norm": 99.53137969970703,
|
|
"learning_rate": 3.000064234440111e-07,
|
|
"logits/chosen": -0.8861789107322693,
|
|
"logits/rejected": -0.886303722858429,
|
|
"logps/chosen": -398.58074951171875,
|
|
"logps/ref_chosen": -288.8846435546875,
|
|
"logps/ref_rejected": -242.0452880859375,
|
|
"logps/rejected": -412.837890625,
|
|
"loss": 4.5815,
|
|
"margin_dpo/margin_mean": 61.09648895263672,
|
|
"margin_dpo/margin_std": 100.48046875,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.4963350785340314,
|
|
"fcm_dpo/beta": 0.005861309356987476,
|
|
"fcm_dpo/delta": -0.013880519196391106,
|
|
"fcm_dpo/margin": 64.32054901123047,
|
|
"fcm_dpo/q_t": 0.4137817323207855,
|
|
"grad_norm": 74.3822250366211,
|
|
"learning_rate": 2.9821133224630223e-07,
|
|
"logits/chosen": -0.8518946170806885,
|
|
"logits/rejected": -0.8302955627441406,
|
|
"logps/chosen": -378.9283447265625,
|
|
"logps/ref_chosen": -265.47869873046875,
|
|
"logps/ref_rejected": -267.9891357421875,
|
|
"logps/rejected": -445.7593078613281,
|
|
"loss": 4.4939,
|
|
"margin_dpo/margin_mean": 64.320556640625,
|
|
"margin_dpo/margin_std": 96.50254821777344,
|
|
"step": 237
|
|
},
|
|
{
|
|
"epoch": 0.49842931937172774,
|
|
"fcm_dpo/beta": 0.005778386257588863,
|
|
"fcm_dpo/delta": -0.012118183076381683,
|
|
"fcm_dpo/margin": 63.46300506591797,
|
|
"fcm_dpo/q_t": 0.4175938665866852,
|
|
"grad_norm": 70.04883575439453,
|
|
"learning_rate": 2.964136556211588e-07,
|
|
"logits/chosen": -0.8639597296714783,
|
|
"logits/rejected": -0.8326221704483032,
|
|
"logps/chosen": -430.5649108886719,
|
|
"logps/ref_chosen": -312.0026550292969,
|
|
"logps/ref_rejected": -270.0257263183594,
|
|
"logps/rejected": -452.0509338378906,
|
|
"loss": 4.5471,
|
|
"margin_dpo/margin_mean": 63.46300506591797,
|
|
"margin_dpo/margin_std": 100.95140075683594,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.5005235602094241,
|
|
"fcm_dpo/beta": 0.005817565601319075,
|
|
"fcm_dpo/delta": 0.051502831280231476,
|
|
"fcm_dpo/margin": 54.329078674316406,
|
|
"fcm_dpo/q_t": 0.4292038083076477,
|
|
"grad_norm": 98.75479888916016,
|
|
"learning_rate": 2.946134899725226e-07,
|
|
"logits/chosen": -0.8470821380615234,
|
|
"logits/rejected": -0.8864820003509521,
|
|
"logps/chosen": -374.77459716796875,
|
|
"logps/ref_chosen": -267.167236328125,
|
|
"logps/ref_rejected": -275.99468994140625,
|
|
"logps/rejected": -437.93115234375,
|
|
"loss": 4.81,
|
|
"margin_dpo/margin_mean": 54.32908248901367,
|
|
"margin_dpo/margin_std": 114.33778381347656,
|
|
"step": 239
|
|
},
|
|
{
|
|
"epoch": 0.5026178010471204,
|
|
"fcm_dpo/beta": 0.0058992416597902775,
|
|
"fcm_dpo/delta": -0.005943944677710533,
|
|
"fcm_dpo/margin": 68.67879486083984,
|
|
"fcm_dpo/q_t": 0.40951305627822876,
|
|
"grad_norm": 67.22561645507812,
|
|
"learning_rate": 2.9281093183781403e-07,
|
|
"logits/chosen": -0.9137483835220337,
|
|
"logits/rejected": -0.907131552696228,
|
|
"logps/chosen": -390.2958984375,
|
|
"logps/ref_chosen": -285.9796142578125,
|
|
"logps/ref_rejected": -256.8258056640625,
|
|
"logps/rejected": -429.8209228515625,
|
|
"loss": 4.4476,
|
|
"margin_dpo/margin_mean": 68.67879486083984,
|
|
"margin_dpo/margin_std": 104.33981323242188,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5047120418848168,
|
|
"fcm_dpo/beta": 0.005862614139914513,
|
|
"fcm_dpo/delta": -0.021802593022584915,
|
|
"fcm_dpo/margin": 54.68340301513672,
|
|
"fcm_dpo/q_t": 0.4280877113342285,
|
|
"grad_norm": 72.86209106445312,
|
|
"learning_rate": 2.910060778827554e-07,
|
|
"logits/chosen": -0.8242596983909607,
|
|
"logits/rejected": -0.8007845878601074,
|
|
"logps/chosen": -376.6900634765625,
|
|
"logps/ref_chosen": -261.516845703125,
|
|
"logps/ref_rejected": -250.2250518798828,
|
|
"logps/rejected": -420.0816650390625,
|
|
"loss": 4.7379,
|
|
"margin_dpo/margin_mean": 54.68340301513672,
|
|
"margin_dpo/margin_std": 101.14353942871094,
|
|
"step": 241
|
|
},
|
|
{
|
|
"epoch": 0.506806282722513,
|
|
"fcm_dpo/beta": 0.005866930354386568,
|
|
"fcm_dpo/delta": 0.0024305330589413643,
|
|
"fcm_dpo/margin": 62.140159606933594,
|
|
"fcm_dpo/q_t": 0.4178650677204132,
|
|
"grad_norm": 91.07866668701172,
|
|
"learning_rate": 2.891990248961871e-07,
|
|
"logits/chosen": -0.8852311968803406,
|
|
"logits/rejected": -0.8625715374946594,
|
|
"logps/chosen": -377.75555419921875,
|
|
"logps/ref_chosen": -270.51397705078125,
|
|
"logps/ref_rejected": -244.8560791015625,
|
|
"logps/rejected": -414.23779296875,
|
|
"loss": 4.5198,
|
|
"margin_dpo/margin_mean": 62.140159606933594,
|
|
"margin_dpo/margin_std": 93.75469970703125,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.5089005235602094,
|
|
"fcm_dpo/beta": 0.005890816915780306,
|
|
"fcm_dpo/delta": -0.03509378433227539,
|
|
"fcm_dpo/margin": 69.48873138427734,
|
|
"fcm_dpo/q_t": 0.41002142429351807,
|
|
"grad_norm": 100.6084213256836,
|
|
"learning_rate": 2.873898697848762e-07,
|
|
"logits/chosen": -0.8823165893554688,
|
|
"logits/rejected": -0.8675477504730225,
|
|
"logps/chosen": -425.2785949707031,
|
|
"logps/ref_chosen": -324.68206787109375,
|
|
"logps/ref_rejected": -307.1111755371094,
|
|
"logps/rejected": -477.1964111328125,
|
|
"loss": 4.4415,
|
|
"margin_dpo/margin_mean": 69.48873138427734,
|
|
"margin_dpo/margin_std": 104.60400390625,
|
|
"step": 243
|
|
},
|
|
{
|
|
"epoch": 0.5109947643979058,
|
|
"fcm_dpo/beta": 0.005591516848653555,
|
|
"fcm_dpo/delta": -0.020960122346878052,
|
|
"fcm_dpo/margin": 70.47373962402344,
|
|
"fcm_dpo/q_t": 0.4083375036716461,
|
|
"grad_norm": 69.3757095336914,
|
|
"learning_rate": 2.8557870956832133e-07,
|
|
"logits/chosen": -0.8660691380500793,
|
|
"logits/rejected": -0.8196066617965698,
|
|
"logps/chosen": -420.827880859375,
|
|
"logps/ref_chosen": -318.979248046875,
|
|
"logps/ref_rejected": -269.67572021484375,
|
|
"logps/rejected": -441.9980773925781,
|
|
"loss": 4.421,
|
|
"margin_dpo/margin_mean": 70.47373962402344,
|
|
"margin_dpo/margin_std": 94.018310546875,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.5130890052356021,
|
|
"fcm_dpo/beta": 0.005576140247285366,
|
|
"fcm_dpo/delta": -0.03441760316491127,
|
|
"fcm_dpo/margin": 69.78692626953125,
|
|
"fcm_dpo/q_t": 0.40951162576675415,
|
|
"grad_norm": 68.46469116210938,
|
|
"learning_rate": 2.837656413735479e-07,
|
|
"logits/chosen": -0.8726673126220703,
|
|
"logits/rejected": -0.8780596852302551,
|
|
"logps/chosen": -396.1797790527344,
|
|
"logps/ref_chosen": -294.8980712890625,
|
|
"logps/ref_rejected": -239.8111114501953,
|
|
"logps/rejected": -410.8797302246094,
|
|
"loss": 4.4389,
|
|
"margin_dpo/margin_mean": 69.78692626953125,
|
|
"margin_dpo/margin_std": 94.53988647460938,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.5151832460732985,
|
|
"fcm_dpo/beta": 0.0054697575978934765,
|
|
"fcm_dpo/delta": 0.019709015265107155,
|
|
"fcm_dpo/margin": 50.34794616699219,
|
|
"fcm_dpo/q_t": 0.4375728964805603,
|
|
"grad_norm": 76.81562042236328,
|
|
"learning_rate": 2.8195076242990116e-07,
|
|
"logits/chosen": -0.8578512072563171,
|
|
"logits/rejected": -0.8632481098175049,
|
|
"logps/chosen": -399.8997497558594,
|
|
"logps/ref_chosen": -280.6854248046875,
|
|
"logps/ref_rejected": -253.65382385253906,
|
|
"logps/rejected": -423.2160949707031,
|
|
"loss": 4.8526,
|
|
"margin_dpo/margin_mean": 50.34794616699219,
|
|
"margin_dpo/margin_std": 104.99066925048828,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.5172774869109947,
|
|
"fcm_dpo/beta": 0.005524596199393272,
|
|
"fcm_dpo/delta": -0.014701386913657188,
|
|
"fcm_dpo/margin": 62.018592834472656,
|
|
"fcm_dpo/q_t": 0.42184895277023315,
|
|
"grad_norm": 54.281822204589844,
|
|
"learning_rate": 2.801341700638307e-07,
|
|
"logits/chosen": -0.8541011214256287,
|
|
"logits/rejected": -0.8522176742553711,
|
|
"logps/chosen": -391.5700988769531,
|
|
"logps/ref_chosen": -281.1091003417969,
|
|
"logps/ref_rejected": -260.3700866699219,
|
|
"logps/rejected": -432.8497009277344,
|
|
"loss": 4.594,
|
|
"margin_dpo/margin_mean": 62.018592834472656,
|
|
"margin_dpo/margin_std": 97.36813354492188,
|
|
"step": 247
|
|
},
|
|
{
|
|
"epoch": 0.5193717277486911,
|
|
"fcm_dpo/beta": 0.0054681808687746525,
|
|
"fcm_dpo/delta": 0.015473026782274246,
|
|
"fcm_dpo/margin": 64.1961669921875,
|
|
"fcm_dpo/q_t": 0.4186955392360687,
|
|
"grad_norm": 63.69669723510742,
|
|
"learning_rate": 2.7831596169367227e-07,
|
|
"logits/chosen": -0.8172490000724792,
|
|
"logits/rejected": -0.8243724703788757,
|
|
"logps/chosen": -383.0730895996094,
|
|
"logps/ref_chosen": -270.318359375,
|
|
"logps/ref_rejected": -233.46778869628906,
|
|
"logps/rejected": -410.4187316894531,
|
|
"loss": 4.5768,
|
|
"margin_dpo/margin_mean": 64.1961669921875,
|
|
"margin_dpo/margin_std": 99.34171295166016,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.5214659685863874,
|
|
"fcm_dpo/beta": 0.005608071107417345,
|
|
"fcm_dpo/delta": 0.03930206224322319,
|
|
"fcm_dpo/margin": 53.0928955078125,
|
|
"fcm_dpo/q_t": 0.43159425258636475,
|
|
"grad_norm": 73.9835205078125,
|
|
"learning_rate": 2.7649623482442274e-07,
|
|
"logits/chosen": -0.8559899926185608,
|
|
"logits/rejected": -0.8310372233390808,
|
|
"logps/chosen": -408.27587890625,
|
|
"logps/ref_chosen": -275.8088684082031,
|
|
"logps/ref_rejected": -243.45138549804688,
|
|
"logps/rejected": -429.0112609863281,
|
|
"loss": 4.8237,
|
|
"margin_dpo/margin_mean": 53.092891693115234,
|
|
"margin_dpo/margin_std": 110.56341552734375,
|
|
"step": 249
|
|
},
|
|
{
|
|
"epoch": 0.5235602094240838,
|
|
"fcm_dpo/beta": 0.005487566348165274,
|
|
"fcm_dpo/delta": -0.07008317857980728,
|
|
"fcm_dpo/margin": 72.46844482421875,
|
|
"fcm_dpo/q_t": 0.4099717140197754,
|
|
"grad_norm": 65.68206787109375,
|
|
"learning_rate": 2.7467508704251135e-07,
|
|
"logits/chosen": -0.8580400943756104,
|
|
"logits/rejected": -0.8534699082374573,
|
|
"logps/chosen": -428.248779296875,
|
|
"logps/ref_chosen": -292.4945373535156,
|
|
"logps/ref_rejected": -284.2869567871094,
|
|
"logps/rejected": -492.5096740722656,
|
|
"loss": 4.5088,
|
|
"margin_dpo/margin_mean": 72.46844482421875,
|
|
"margin_dpo/margin_std": 115.16688537597656,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.5256544502617801,
|
|
"fcm_dpo/beta": 0.005311680492013693,
|
|
"fcm_dpo/delta": -0.06994134932756424,
|
|
"fcm_dpo/margin": 68.04769134521484,
|
|
"fcm_dpo/q_t": 0.4215785264968872,
|
|
"grad_norm": 83.41246032714844,
|
|
"learning_rate": 2.7285261601056697e-07,
|
|
"logits/chosen": -0.8579660654067993,
|
|
"logits/rejected": -0.8418058753013611,
|
|
"logps/chosen": -408.4231872558594,
|
|
"logps/ref_chosen": -281.736572265625,
|
|
"logps/ref_rejected": -255.9419708251953,
|
|
"logps/rejected": -450.6763000488281,
|
|
"loss": 4.6009,
|
|
"margin_dpo/margin_mean": 68.04769134521484,
|
|
"margin_dpo/margin_std": 112.29561614990234,
|
|
"step": 251
|
|
},
|
|
{
|
|
"epoch": 0.5277486910994764,
|
|
"fcm_dpo/beta": 0.005035778041929007,
|
|
"fcm_dpo/delta": -0.015283557586371899,
|
|
"fcm_dpo/margin": 65.64408874511719,
|
|
"fcm_dpo/q_t": 0.422878623008728,
|
|
"grad_norm": 59.68517303466797,
|
|
"learning_rate": 2.7102891946217994e-07,
|
|
"logits/chosen": -0.900702178478241,
|
|
"logits/rejected": -0.8740664124488831,
|
|
"logps/chosen": -434.17852783203125,
|
|
"logps/ref_chosen": -295.9674072265625,
|
|
"logps/ref_rejected": -280.111572265625,
|
|
"logps/rejected": -483.966796875,
|
|
"loss": 4.6299,
|
|
"margin_dpo/margin_mean": 65.64408111572266,
|
|
"margin_dpo/margin_std": 106.49784088134766,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.5298429319371728,
|
|
"fcm_dpo/beta": 0.005060987081378698,
|
|
"fcm_dpo/delta": 0.01637944206595421,
|
|
"fcm_dpo/margin": 60.1594123840332,
|
|
"fcm_dpo/q_t": 0.4316590428352356,
|
|
"grad_norm": 71.8646240234375,
|
|
"learning_rate": 2.692040951966617e-07,
|
|
"logits/chosen": -0.8741543292999268,
|
|
"logits/rejected": -0.8654627799987793,
|
|
"logps/chosen": -426.32373046875,
|
|
"logps/ref_chosen": -277.072265625,
|
|
"logps/ref_rejected": -247.31643676757812,
|
|
"logps/rejected": -456.7273254394531,
|
|
"loss": 4.7849,
|
|
"margin_dpo/margin_mean": 60.15940856933594,
|
|
"margin_dpo/margin_std": 116.84982299804688,
|
|
"step": 253
|
|
},
|
|
{
|
|
"epoch": 0.5319371727748691,
|
|
"fcm_dpo/beta": 0.005230544600635767,
|
|
"fcm_dpo/delta": 0.0011180378496646881,
|
|
"fcm_dpo/margin": 66.23080444335938,
|
|
"fcm_dpo/q_t": 0.4211539626121521,
|
|
"grad_norm": 74.46015167236328,
|
|
"learning_rate": 2.6737824107379947e-07,
|
|
"logits/chosen": -0.8116303086280823,
|
|
"logits/rejected": -0.7934176921844482,
|
|
"logps/chosen": -410.2343444824219,
|
|
"logps/ref_chosen": -269.9478454589844,
|
|
"logps/ref_rejected": -249.45005798339844,
|
|
"logps/rejected": -455.96734619140625,
|
|
"loss": 4.5907,
|
|
"margin_dpo/margin_mean": 66.23080444335938,
|
|
"margin_dpo/margin_std": 105.38709259033203,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.5340314136125655,
|
|
"fcm_dpo/beta": 0.005045011173933744,
|
|
"fcm_dpo/delta": -0.041239336133003235,
|
|
"fcm_dpo/margin": 81.40687561035156,
|
|
"fcm_dpo/q_t": 0.4063635468482971,
|
|
"grad_norm": 63.57786560058594,
|
|
"learning_rate": 2.655514550086086e-07,
|
|
"logits/chosen": -0.8293032050132751,
|
|
"logits/rejected": -0.7931342720985413,
|
|
"logps/chosen": -439.56988525390625,
|
|
"logps/ref_chosen": -306.6552734375,
|
|
"logps/ref_rejected": -254.47528076171875,
|
|
"logps/rejected": -468.7967529296875,
|
|
"loss": 4.4478,
|
|
"margin_dpo/margin_mean": 81.40687561035156,
|
|
"margin_dpo/margin_std": 123.06184387207031,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.5361256544502618,
|
|
"fcm_dpo/beta": 0.004904068075120449,
|
|
"fcm_dpo/delta": -0.02955251932144165,
|
|
"fcm_dpo/margin": 66.60382843017578,
|
|
"fcm_dpo/q_t": 0.4222066402435303,
|
|
"grad_norm": 98.1750259399414,
|
|
"learning_rate": 2.6372383496608186e-07,
|
|
"logits/chosen": -0.8569645285606384,
|
|
"logits/rejected": -0.8438127040863037,
|
|
"logps/chosen": -463.7706604003906,
|
|
"logps/ref_chosen": -323.7181701660156,
|
|
"logps/ref_rejected": -254.1871337890625,
|
|
"logps/rejected": -460.8434143066406,
|
|
"loss": 4.7499,
|
|
"margin_dpo/margin_mean": 66.60383605957031,
|
|
"margin_dpo/margin_std": 121.21151733398438,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.5382198952879581,
|
|
"fcm_dpo/beta": 0.004877617582678795,
|
|
"fcm_dpo/delta": 0.029186341911554337,
|
|
"fcm_dpo/margin": 76.22327423095703,
|
|
"fcm_dpo/q_t": 0.4146016538143158,
|
|
"grad_norm": 75.77265167236328,
|
|
"learning_rate": 2.618954789559356e-07,
|
|
"logits/chosen": -0.8500054478645325,
|
|
"logits/rejected": -0.8413119316101074,
|
|
"logps/chosen": -405.16949462890625,
|
|
"logps/ref_chosen": -267.21209716796875,
|
|
"logps/ref_rejected": -249.12579345703125,
|
|
"logps/rejected": -463.3064880371094,
|
|
"loss": 4.485,
|
|
"margin_dpo/margin_mean": 76.22327423095703,
|
|
"margin_dpo/margin_std": 111.8670425415039,
|
|
"step": 257
|
|
},
|
|
{
|
|
"epoch": 0.5403141361256545,
|
|
"fcm_dpo/beta": 0.004769755993038416,
|
|
"fcm_dpo/delta": -0.0641399696469307,
|
|
"fcm_dpo/margin": 77.75953674316406,
|
|
"fcm_dpo/q_t": 0.4129870533943176,
|
|
"grad_norm": 69.70844268798828,
|
|
"learning_rate": 2.600664850273538e-07,
|
|
"logits/chosen": -0.8756992816925049,
|
|
"logits/rejected": -0.8453248143196106,
|
|
"logps/chosen": -424.1638488769531,
|
|
"logps/ref_chosen": -277.6827392578125,
|
|
"logps/ref_rejected": -250.73385620117188,
|
|
"logps/rejected": -474.9745178222656,
|
|
"loss": 4.4652,
|
|
"margin_dpo/margin_mean": 77.75953674316406,
|
|
"margin_dpo/margin_std": 105.55914306640625,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.5424083769633508,
|
|
"fcm_dpo/beta": 0.004843279719352722,
|
|
"fcm_dpo/delta": 0.04182092845439911,
|
|
"fcm_dpo/margin": 74.08509826660156,
|
|
"fcm_dpo/q_t": 0.4186086654663086,
|
|
"grad_norm": 64.96932983398438,
|
|
"learning_rate": 2.582369512637302e-07,
|
|
"logits/chosen": -0.8835079073905945,
|
|
"logits/rejected": -0.8777540326118469,
|
|
"logps/chosen": -430.7030029296875,
|
|
"logps/ref_chosen": -294.6099853515625,
|
|
"logps/ref_rejected": -272.2725830078125,
|
|
"logps/rejected": -482.45074462890625,
|
|
"loss": 4.5272,
|
|
"margin_dpo/margin_mean": 74.0851058959961,
|
|
"margin_dpo/margin_std": 110.6324234008789,
|
|
"step": 259
|
|
},
|
|
{
|
|
"epoch": 0.5445026178010471,
|
|
"fcm_dpo/beta": 0.005027398467063904,
|
|
"fcm_dpo/delta": 0.04363645985722542,
|
|
"fcm_dpo/margin": 30.95121192932129,
|
|
"fcm_dpo/q_t": 0.464776873588562,
|
|
"grad_norm": 92.6754379272461,
|
|
"learning_rate": 2.5640697577740815e-07,
|
|
"logits/chosen": -0.8735372424125671,
|
|
"logits/rejected": -0.8678438663482666,
|
|
"logps/chosen": -436.0877380371094,
|
|
"logps/ref_chosen": -290.85711669921875,
|
|
"logps/ref_rejected": -277.5970153808594,
|
|
"logps/rejected": -453.77886962890625,
|
|
"loss": 5.227,
|
|
"margin_dpo/margin_mean": 30.951213836669922,
|
|
"margin_dpo/margin_std": 104.45787048339844,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.5465968586387434,
|
|
"fcm_dpo/beta": 0.005115479230880737,
|
|
"fcm_dpo/delta": 0.01615825854241848,
|
|
"fcm_dpo/margin": 57.5634765625,
|
|
"fcm_dpo/q_t": 0.4337337613105774,
|
|
"grad_norm": 66.06670379638672,
|
|
"learning_rate": 2.5457665670441937e-07,
|
|
"logits/chosen": -0.7693032026290894,
|
|
"logits/rejected": -0.779098391532898,
|
|
"logps/chosen": -405.8778991699219,
|
|
"logps/ref_chosen": -251.13223266601562,
|
|
"logps/ref_rejected": -244.76016235351562,
|
|
"logps/rejected": -457.0693054199219,
|
|
"loss": 4.8364,
|
|
"margin_dpo/margin_mean": 57.5634765625,
|
|
"margin_dpo/margin_std": 120.9645767211914,
|
|
"step": 261
|
|
},
|
|
{
|
|
"epoch": 0.5486910994764398,
|
|
"fcm_dpo/beta": 0.005165139213204384,
|
|
"fcm_dpo/delta": -0.0015618130564689636,
|
|
"fcm_dpo/margin": 77.5936279296875,
|
|
"fcm_dpo/q_t": 0.4082508683204651,
|
|
"grad_norm": 67.6512680053711,
|
|
"learning_rate": 2.527460921992209e-07,
|
|
"logits/chosen": -0.802527666091919,
|
|
"logits/rejected": -0.7902448177337646,
|
|
"logps/chosen": -441.3621520996094,
|
|
"logps/ref_chosen": -299.7217712402344,
|
|
"logps/ref_rejected": -277.0969543457031,
|
|
"logps/rejected": -496.3309326171875,
|
|
"loss": 4.4112,
|
|
"margin_dpo/margin_mean": 77.5936279296875,
|
|
"margin_dpo/margin_std": 107.57256317138672,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 0.5507853403141362,
|
|
"fcm_dpo/beta": 0.0051011075265705585,
|
|
"fcm_dpo/delta": -0.00584130734205246,
|
|
"fcm_dpo/margin": 68.14926147460938,
|
|
"fcm_dpo/q_t": 0.4196454882621765,
|
|
"grad_norm": 53.193485260009766,
|
|
"learning_rate": 2.509153804294318e-07,
|
|
"logits/chosen": -0.8066005706787109,
|
|
"logits/rejected": -0.7877044677734375,
|
|
"logps/chosen": -427.0610656738281,
|
|
"logps/ref_chosen": -279.95257568359375,
|
|
"logps/ref_rejected": -256.5327453613281,
|
|
"logps/rejected": -471.7904357910156,
|
|
"loss": 4.6222,
|
|
"margin_dpo/margin_mean": 68.14926147460938,
|
|
"margin_dpo/margin_std": 111.96087646484375,
|
|
"step": 263
|
|
},
|
|
{
|
|
"epoch": 0.5528795811518324,
|
|
"fcm_dpo/beta": 0.004925370682030916,
|
|
"fcm_dpo/delta": -0.04882174730300903,
|
|
"fcm_dpo/margin": 83.56140899658203,
|
|
"fcm_dpo/q_t": 0.40587475895881653,
|
|
"grad_norm": 90.4492416381836,
|
|
"learning_rate": 2.4908461957056825e-07,
|
|
"logits/chosen": -0.833106517791748,
|
|
"logits/rejected": -0.8265419602394104,
|
|
"logps/chosen": -398.6038513183594,
|
|
"logps/ref_chosen": -260.53509521484375,
|
|
"logps/ref_rejected": -255.53799438476562,
|
|
"logps/rejected": -477.16815185546875,
|
|
"loss": 4.3584,
|
|
"margin_dpo/margin_mean": 83.56140899658203,
|
|
"margin_dpo/margin_std": 109.89978790283203,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 0.5549738219895288,
|
|
"fcm_dpo/beta": 0.0048619420267641544,
|
|
"fcm_dpo/delta": -0.008534754626452923,
|
|
"fcm_dpo/margin": 83.80038452148438,
|
|
"fcm_dpo/q_t": 0.40826499462127686,
|
|
"grad_norm": 55.20663833618164,
|
|
"learning_rate": 2.4725390780077905e-07,
|
|
"logits/chosen": -0.8974971771240234,
|
|
"logits/rejected": -0.8996181488037109,
|
|
"logps/chosen": -429.3133239746094,
|
|
"logps/ref_chosen": -283.7130432128906,
|
|
"logps/ref_rejected": -270.3209533691406,
|
|
"logps/rejected": -499.7215881347656,
|
|
"loss": 4.456,
|
|
"margin_dpo/margin_mean": 83.80038452148438,
|
|
"margin_dpo/margin_std": 126.92974853515625,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.5570680628272251,
|
|
"fcm_dpo/beta": 0.00486271595582366,
|
|
"fcm_dpo/delta": -0.01186143234372139,
|
|
"fcm_dpo/margin": 84.45220184326172,
|
|
"fcm_dpo/q_t": 0.4058952331542969,
|
|
"grad_norm": 56.53108596801758,
|
|
"learning_rate": 2.454233432955807e-07,
|
|
"logits/chosen": -0.9156976342201233,
|
|
"logits/rejected": -0.8858764171600342,
|
|
"logps/chosen": -408.8074645996094,
|
|
"logps/ref_chosen": -278.09930419921875,
|
|
"logps/ref_rejected": -260.6734619140625,
|
|
"logps/rejected": -475.8338623046875,
|
|
"loss": 4.3442,
|
|
"margin_dpo/margin_mean": 84.45219421386719,
|
|
"margin_dpo/margin_std": 107.75846862792969,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 0.5591623036649215,
|
|
"fcm_dpo/beta": 0.004877687431871891,
|
|
"fcm_dpo/delta": 0.04254019632935524,
|
|
"fcm_dpo/margin": 63.55180740356445,
|
|
"fcm_dpo/q_t": 0.42915400862693787,
|
|
"grad_norm": 75.33948516845703,
|
|
"learning_rate": 2.435930242225919e-07,
|
|
"logits/chosen": -0.8628742694854736,
|
|
"logits/rejected": -0.8702367544174194,
|
|
"logps/chosen": -437.90972900390625,
|
|
"logps/ref_chosen": -280.33319091796875,
|
|
"logps/ref_rejected": -247.78099060058594,
|
|
"logps/rejected": -468.9093322753906,
|
|
"loss": 4.7191,
|
|
"margin_dpo/margin_mean": 63.55180740356445,
|
|
"margin_dpo/margin_std": 111.55426025390625,
|
|
"step": 267
|
|
},
|
|
{
|
|
"epoch": 0.5612565445026177,
|
|
"fcm_dpo/beta": 0.005009549204260111,
|
|
"fcm_dpo/delta": -0.00161817017942667,
|
|
"fcm_dpo/margin": 80.10919189453125,
|
|
"fcm_dpo/q_t": 0.4091428220272064,
|
|
"grad_norm": 85.9771728515625,
|
|
"learning_rate": 2.4176304873626984e-07,
|
|
"logits/chosen": -0.8233053684234619,
|
|
"logits/rejected": -0.8017244338989258,
|
|
"logps/chosen": -450.60400390625,
|
|
"logps/ref_chosen": -304.1787109375,
|
|
"logps/ref_rejected": -272.80316162109375,
|
|
"logps/rejected": -499.337646484375,
|
|
"loss": 4.4485,
|
|
"margin_dpo/margin_mean": 80.10919189453125,
|
|
"margin_dpo/margin_std": 118.68637084960938,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 0.5633507853403141,
|
|
"fcm_dpo/beta": 0.005120859481394291,
|
|
"fcm_dpo/delta": 0.07810668647289276,
|
|
"fcm_dpo/margin": 56.62480926513672,
|
|
"fcm_dpo/q_t": 0.4334860146045685,
|
|
"grad_norm": 95.90203857421875,
|
|
"learning_rate": 2.399335149726463e-07,
|
|
"logits/chosen": -0.8469225168228149,
|
|
"logits/rejected": -0.8406363725662231,
|
|
"logps/chosen": -397.451171875,
|
|
"logps/ref_chosen": -249.84512329101562,
|
|
"logps/ref_rejected": -223.37356567382812,
|
|
"logps/rejected": -427.6044616699219,
|
|
"loss": 4.8125,
|
|
"margin_dpo/margin_mean": 56.62481689453125,
|
|
"margin_dpo/margin_std": 115.53034973144531,
|
|
"step": 269
|
|
},
|
|
{
|
|
"epoch": 0.5654450261780105,
|
|
"fcm_dpo/beta": 0.005242231767624617,
|
|
"fcm_dpo/delta": -0.004636672325432301,
|
|
"fcm_dpo/margin": 72.5596923828125,
|
|
"fcm_dpo/q_t": 0.41470980644226074,
|
|
"grad_norm": 76.87252807617188,
|
|
"learning_rate": 2.381045210440644e-07,
|
|
"logits/chosen": -0.9306774735450745,
|
|
"logits/rejected": -0.9363957047462463,
|
|
"logps/chosen": -470.66033935546875,
|
|
"logps/ref_chosen": -318.5623779296875,
|
|
"logps/ref_rejected": -281.1880798339844,
|
|
"logps/rejected": -505.8457336425781,
|
|
"loss": 4.5897,
|
|
"margin_dpo/margin_mean": 72.5596923828125,
|
|
"margin_dpo/margin_std": 127.58087158203125,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.5675392670157068,
|
|
"fcm_dpo/beta": 0.005433530081063509,
|
|
"fcm_dpo/delta": 0.04082181677222252,
|
|
"fcm_dpo/margin": 66.16189575195312,
|
|
"fcm_dpo/q_t": 0.41947028040885925,
|
|
"grad_norm": 74.05042266845703,
|
|
"learning_rate": 2.3627616503391812e-07,
|
|
"logits/chosen": -0.7789531946182251,
|
|
"logits/rejected": -0.7725875377655029,
|
|
"logps/chosen": -430.7342224121094,
|
|
"logps/ref_chosen": -284.104736328125,
|
|
"logps/ref_rejected": -253.9580535888672,
|
|
"logps/rejected": -466.74945068359375,
|
|
"loss": 4.5904,
|
|
"margin_dpo/margin_mean": 66.16189575195312,
|
|
"margin_dpo/margin_std": 110.10599517822266,
|
|
"step": 271
|
|
},
|
|
{
|
|
"epoch": 0.5696335078534032,
|
|
"fcm_dpo/beta": 0.005346586927771568,
|
|
"fcm_dpo/delta": -0.024095682427287102,
|
|
"fcm_dpo/margin": 72.80128479003906,
|
|
"fcm_dpo/q_t": 0.41350868344306946,
|
|
"grad_norm": 74.77401733398438,
|
|
"learning_rate": 2.344485449913914e-07,
|
|
"logits/chosen": -0.8851820826530457,
|
|
"logits/rejected": -0.8685218691825867,
|
|
"logps/chosen": -435.2021789550781,
|
|
"logps/ref_chosen": -297.3590087890625,
|
|
"logps/ref_rejected": -279.20196533203125,
|
|
"logps/rejected": -489.84637451171875,
|
|
"loss": 4.5746,
|
|
"margin_dpo/margin_mean": 72.80128479003906,
|
|
"margin_dpo/margin_std": 126.85377502441406,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 0.5717277486910994,
|
|
"fcm_dpo/beta": 0.005236013326793909,
|
|
"fcm_dpo/delta": -0.0287264883518219,
|
|
"fcm_dpo/margin": 77.18766021728516,
|
|
"fcm_dpo/q_t": 0.4100334346294403,
|
|
"grad_norm": 94.14215850830078,
|
|
"learning_rate": 2.3262175892620062e-07,
|
|
"logits/chosen": -0.8636338710784912,
|
|
"logits/rejected": -0.8754411935806274,
|
|
"logps/chosen": -429.3293151855469,
|
|
"logps/ref_chosen": -293.20574951171875,
|
|
"logps/ref_rejected": -274.7646789550781,
|
|
"logps/rejected": -488.0759582519531,
|
|
"loss": 4.4805,
|
|
"margin_dpo/margin_mean": 77.18766021728516,
|
|
"margin_dpo/margin_std": 121.6794204711914,
|
|
"step": 273
|
|
},
|
|
{
|
|
"epoch": 0.5738219895287958,
|
|
"fcm_dpo/beta": 0.005103556904941797,
|
|
"fcm_dpo/delta": -0.03355909138917923,
|
|
"fcm_dpo/margin": 84.57637023925781,
|
|
"fcm_dpo/q_t": 0.4024480879306793,
|
|
"grad_norm": 73.24378204345703,
|
|
"learning_rate": 2.3079590480333827e-07,
|
|
"logits/chosen": -0.8207792043685913,
|
|
"logits/rejected": -0.7887733578681946,
|
|
"logps/chosen": -406.8528747558594,
|
|
"logps/ref_chosen": -270.55865478515625,
|
|
"logps/ref_rejected": -239.47048950195312,
|
|
"logps/rejected": -460.3410339355469,
|
|
"loss": 4.3589,
|
|
"margin_dpo/margin_mean": 84.57637023925781,
|
|
"margin_dpo/margin_std": 119.9758529663086,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 0.5759162303664922,
|
|
"fcm_dpo/beta": 0.004932451993227005,
|
|
"fcm_dpo/delta": -0.0627283900976181,
|
|
"fcm_dpo/margin": 93.08427429199219,
|
|
"fcm_dpo/q_t": 0.3963705599308014,
|
|
"grad_norm": 54.85517883300781,
|
|
"learning_rate": 2.2897108053782e-07,
|
|
"logits/chosen": -0.8735396265983582,
|
|
"logits/rejected": -0.8545541167259216,
|
|
"logps/chosen": -372.95513916015625,
|
|
"logps/ref_chosen": -250.31922912597656,
|
|
"logps/ref_rejected": -249.3187255859375,
|
|
"logps/rejected": -465.03887939453125,
|
|
"loss": 4.2232,
|
|
"margin_dpo/margin_mean": 93.08427429199219,
|
|
"margin_dpo/margin_std": 114.38727569580078,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.5780104712041885,
|
|
"fcm_dpo/beta": 0.004693637136369944,
|
|
"fcm_dpo/delta": -0.02087191678583622,
|
|
"fcm_dpo/margin": 84.33247375488281,
|
|
"fcm_dpo/q_t": 0.4087947607040405,
|
|
"grad_norm": 56.812870025634766,
|
|
"learning_rate": 2.2714738398943308e-07,
|
|
"logits/chosen": -0.9385483264923096,
|
|
"logits/rejected": -0.9158673286437988,
|
|
"logps/chosen": -425.16796875,
|
|
"logps/ref_chosen": -297.6310729980469,
|
|
"logps/ref_rejected": -295.225830078125,
|
|
"logps/rejected": -507.0952453613281,
|
|
"loss": 4.4153,
|
|
"margin_dpo/margin_mean": 84.33247375488281,
|
|
"margin_dpo/margin_std": 113.92533111572266,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 0.5801047120418849,
|
|
"fcm_dpo/beta": 0.004713152069598436,
|
|
"fcm_dpo/delta": -0.0023143887519836426,
|
|
"fcm_dpo/margin": 60.579532623291016,
|
|
"fcm_dpo/q_t": 0.435024619102478,
|
|
"grad_norm": 88.57623291015625,
|
|
"learning_rate": 2.2532491295748865e-07,
|
|
"logits/chosen": -0.8644924163818359,
|
|
"logits/rejected": -0.862896203994751,
|
|
"logps/chosen": -413.9727478027344,
|
|
"logps/ref_chosen": -266.3604736328125,
|
|
"logps/ref_rejected": -253.36767578125,
|
|
"logps/rejected": -461.5594787597656,
|
|
"loss": 4.8088,
|
|
"margin_dpo/margin_mean": 60.57952880859375,
|
|
"margin_dpo/margin_std": 119.62329864501953,
|
|
"step": 277
|
|
},
|
|
{
|
|
"epoch": 0.5821989528795811,
|
|
"fcm_dpo/beta": 0.004758753813803196,
|
|
"fcm_dpo/delta": 0.0039313724264502525,
|
|
"fcm_dpo/margin": 48.2127685546875,
|
|
"fcm_dpo/q_t": 0.4520694613456726,
|
|
"grad_norm": 71.64497375488281,
|
|
"learning_rate": 2.2350376517557726e-07,
|
|
"logits/chosen": -0.8992569446563721,
|
|
"logits/rejected": -0.8604874610900879,
|
|
"logps/chosen": -429.3371887207031,
|
|
"logps/ref_chosen": -267.40728759765625,
|
|
"logps/ref_rejected": -229.5758514404297,
|
|
"logps/rejected": -439.71844482421875,
|
|
"loss": 5.129,
|
|
"margin_dpo/margin_mean": 48.2127685546875,
|
|
"margin_dpo/margin_std": 137.95343017578125,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 0.5842931937172775,
|
|
"fcm_dpo/beta": 0.004630140028893948,
|
|
"fcm_dpo/delta": -0.053754448890686035,
|
|
"fcm_dpo/margin": 88.50892639160156,
|
|
"fcm_dpo/q_t": 0.4081512689590454,
|
|
"grad_norm": 72.25919342041016,
|
|
"learning_rate": 2.2168403830632769e-07,
|
|
"logits/chosen": -0.8258615732192993,
|
|
"logits/rejected": -0.8100024461746216,
|
|
"logps/chosen": -460.78607177734375,
|
|
"logps/ref_chosen": -313.3677978515625,
|
|
"logps/ref_rejected": -299.1744384765625,
|
|
"logps/rejected": -535.1016235351562,
|
|
"loss": 4.433,
|
|
"margin_dpo/margin_mean": 88.50892639160156,
|
|
"margin_dpo/margin_std": 131.84158325195312,
|
|
"step": 279
|
|
},
|
|
{
|
|
"epoch": 0.5863874345549738,
|
|
"fcm_dpo/beta": 0.00471831439062953,
|
|
"fcm_dpo/delta": 0.07752120494842529,
|
|
"fcm_dpo/margin": 68.82743072509766,
|
|
"fcm_dpo/q_t": 0.4256266951560974,
|
|
"grad_norm": 99.79914855957031,
|
|
"learning_rate": 2.1986582993616925e-07,
|
|
"logits/chosen": -0.8818544745445251,
|
|
"logits/rejected": -0.8929405212402344,
|
|
"logps/chosen": -402.27252197265625,
|
|
"logps/ref_chosen": -265.5558166503906,
|
|
"logps/ref_rejected": -247.1573944091797,
|
|
"logps/rejected": -452.7015075683594,
|
|
"loss": 4.6549,
|
|
"margin_dpo/margin_mean": 68.82743072509766,
|
|
"margin_dpo/margin_std": 118.33948516845703,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.5884816753926702,
|
|
"fcm_dpo/beta": 0.004735568072646856,
|
|
"fcm_dpo/delta": -0.01806649938225746,
|
|
"fcm_dpo/margin": 82.60304260253906,
|
|
"fcm_dpo/q_t": 0.4123595356941223,
|
|
"grad_norm": 80.9848403930664,
|
|
"learning_rate": 2.1804923757009882e-07,
|
|
"logits/chosen": -0.8654816746711731,
|
|
"logits/rejected": -0.8740391731262207,
|
|
"logps/chosen": -450.2287292480469,
|
|
"logps/ref_chosen": -295.2995910644531,
|
|
"logps/ref_rejected": -293.80877685546875,
|
|
"logps/rejected": -531.3409423828125,
|
|
"loss": 4.449,
|
|
"margin_dpo/margin_mean": 82.60304260253906,
|
|
"margin_dpo/margin_std": 119.16482543945312,
|
|
"step": 281
|
|
},
|
|
{
|
|
"epoch": 0.5905759162303665,
|
|
"fcm_dpo/beta": 0.004867776297032833,
|
|
"fcm_dpo/delta": 0.05005919188261032,
|
|
"fcm_dpo/margin": 72.23600769042969,
|
|
"fcm_dpo/q_t": 0.42044195532798767,
|
|
"grad_norm": 58.8224983215332,
|
|
"learning_rate": 2.1623435862645205e-07,
|
|
"logits/chosen": -0.8650054335594177,
|
|
"logits/rejected": -0.8629118204116821,
|
|
"logps/chosen": -474.85797119140625,
|
|
"logps/ref_chosen": -318.63714599609375,
|
|
"logps/ref_rejected": -273.5943603515625,
|
|
"logps/rejected": -502.0511474609375,
|
|
"loss": 4.6176,
|
|
"margin_dpo/margin_mean": 72.23600769042969,
|
|
"margin_dpo/margin_std": 123.56523895263672,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 0.5926701570680628,
|
|
"fcm_dpo/beta": 0.005018922034651041,
|
|
"fcm_dpo/delta": -0.012126617133617401,
|
|
"fcm_dpo/margin": 71.27740478515625,
|
|
"fcm_dpo/q_t": 0.4213730990886688,
|
|
"grad_norm": 69.60215759277344,
|
|
"learning_rate": 2.1442129043167873e-07,
|
|
"logits/chosen": -0.8856289386749268,
|
|
"logits/rejected": -0.8754106163978577,
|
|
"logps/chosen": -412.56719970703125,
|
|
"logps/ref_chosen": -254.66053771972656,
|
|
"logps/ref_rejected": -236.8627166748047,
|
|
"logps/rejected": -466.04681396484375,
|
|
"loss": 4.6072,
|
|
"margin_dpo/margin_mean": 71.27740478515625,
|
|
"margin_dpo/margin_std": 120.48731994628906,
|
|
"step": 283
|
|
},
|
|
{
|
|
"epoch": 0.5947643979057592,
|
|
"fcm_dpo/beta": 0.004824446514248848,
|
|
"fcm_dpo/delta": -0.02942648157477379,
|
|
"fcm_dpo/margin": 76.38724517822266,
|
|
"fcm_dpo/q_t": 0.417685866355896,
|
|
"grad_norm": 83.93893432617188,
|
|
"learning_rate": 2.1261013021512378e-07,
|
|
"logits/chosen": -0.8511925935745239,
|
|
"logits/rejected": -0.8199682235717773,
|
|
"logps/chosen": -439.28143310546875,
|
|
"logps/ref_chosen": -273.355224609375,
|
|
"logps/ref_rejected": -259.84759521484375,
|
|
"logps/rejected": -502.1610107421875,
|
|
"loss": 4.6521,
|
|
"margin_dpo/margin_mean": 76.38724517822266,
|
|
"margin_dpo/margin_std": 137.0977020263672,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 0.5968586387434555,
|
|
"fcm_dpo/beta": 0.004868158604949713,
|
|
"fcm_dpo/delta": 0.020393716171383858,
|
|
"fcm_dpo/margin": 51.72495651245117,
|
|
"fcm_dpo/q_t": 0.4441811740398407,
|
|
"grad_norm": 100.43576049804688,
|
|
"learning_rate": 2.1080097510381294e-07,
|
|
"logits/chosen": -0.857195258140564,
|
|
"logits/rejected": -0.8506641983985901,
|
|
"logps/chosen": -479.5204162597656,
|
|
"logps/ref_chosen": -309.8022155761719,
|
|
"logps/ref_rejected": -279.11846923828125,
|
|
"logps/rejected": -500.56158447265625,
|
|
"loss": 5.0083,
|
|
"margin_dpo/margin_mean": 51.72496032714844,
|
|
"margin_dpo/margin_std": 129.5350341796875,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.5989528795811518,
|
|
"fcm_dpo/beta": 0.004937829915434122,
|
|
"fcm_dpo/delta": 0.030296623706817627,
|
|
"fcm_dpo/margin": 75.0483169555664,
|
|
"fcm_dpo/q_t": 0.418967604637146,
|
|
"grad_norm": 80.72938537597656,
|
|
"learning_rate": 2.089939221172446e-07,
|
|
"logits/chosen": -0.8420528769493103,
|
|
"logits/rejected": -0.8334404230117798,
|
|
"logps/chosen": -428.751708984375,
|
|
"logps/ref_chosen": -271.4655456542969,
|
|
"logps/ref_rejected": -279.531494140625,
|
|
"logps/rejected": -511.86590576171875,
|
|
"loss": 4.5885,
|
|
"margin_dpo/margin_mean": 75.0483169555664,
|
|
"margin_dpo/margin_std": 129.76441955566406,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 0.6010471204188481,
|
|
"fcm_dpo/beta": 0.005080940201878548,
|
|
"fcm_dpo/delta": 0.021761678159236908,
|
|
"fcm_dpo/margin": 69.6319808959961,
|
|
"fcm_dpo/q_t": 0.4205819368362427,
|
|
"grad_norm": 79.26040649414062,
|
|
"learning_rate": 2.0718906816218595e-07,
|
|
"logits/chosen": -0.862068772315979,
|
|
"logits/rejected": -0.8453592658042908,
|
|
"logps/chosen": -439.6429138183594,
|
|
"logps/ref_chosen": -277.0932312011719,
|
|
"logps/ref_rejected": -233.55599975585938,
|
|
"logps/rejected": -465.7376708984375,
|
|
"loss": 4.6931,
|
|
"margin_dpo/margin_mean": 69.63197326660156,
|
|
"margin_dpo/margin_std": 132.2081298828125,
|
|
"step": 287
|
|
},
|
|
{
|
|
"epoch": 0.6031413612565445,
|
|
"fcm_dpo/beta": 0.0051753930747509,
|
|
"fcm_dpo/delta": 0.0042931921780109406,
|
|
"fcm_dpo/margin": 69.9853286743164,
|
|
"fcm_dpo/q_t": 0.41905421018600464,
|
|
"grad_norm": 87.4442367553711,
|
|
"learning_rate": 2.053865100274774e-07,
|
|
"logits/chosen": -0.8746588230133057,
|
|
"logits/rejected": -0.8808341026306152,
|
|
"logps/chosen": -454.9645690917969,
|
|
"logps/ref_chosen": -293.1681823730469,
|
|
"logps/ref_rejected": -263.4059143066406,
|
|
"logps/rejected": -495.1876525878906,
|
|
"loss": 4.6358,
|
|
"margin_dpo/margin_mean": 69.98532104492188,
|
|
"margin_dpo/margin_std": 125.97311401367188,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 0.6052356020942409,
|
|
"fcm_dpo/beta": 0.0051370663568377495,
|
|
"fcm_dpo/delta": 0.02070062793791294,
|
|
"fcm_dpo/margin": 44.1146240234375,
|
|
"fcm_dpo/q_t": 0.44912248849868774,
|
|
"grad_norm": 95.3306655883789,
|
|
"learning_rate": 2.035863443788411e-07,
|
|
"logits/chosen": -0.8595668077468872,
|
|
"logits/rejected": -0.8414284586906433,
|
|
"logps/chosen": -511.45074462890625,
|
|
"logps/ref_chosen": -329.9574279785156,
|
|
"logps/ref_rejected": -276.7565002441406,
|
|
"logps/rejected": -502.36444091796875,
|
|
"loss": 5.0894,
|
|
"margin_dpo/margin_mean": 44.11461639404297,
|
|
"margin_dpo/margin_std": 121.32159423828125,
|
|
"step": 289
|
|
},
|
|
{
|
|
"epoch": 0.6073298429319371,
|
|
"fcm_dpo/beta": 0.005096154753118753,
|
|
"fcm_dpo/delta": -7.194560021162033e-05,
|
|
"fcm_dpo/margin": 67.14459991455078,
|
|
"fcm_dpo/q_t": 0.42460641264915466,
|
|
"grad_norm": 65.90011596679688,
|
|
"learning_rate": 2.0178866775369774e-07,
|
|
"logits/chosen": -0.870934247970581,
|
|
"logits/rejected": -0.8062535524368286,
|
|
"logps/chosen": -490.2457275390625,
|
|
"logps/ref_chosen": -324.6690673828125,
|
|
"logps/ref_rejected": -311.8439636230469,
|
|
"logps/rejected": -544.565185546875,
|
|
"loss": 4.7519,
|
|
"margin_dpo/margin_mean": 67.14459991455078,
|
|
"margin_dpo/margin_std": 129.93505859375,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6094240837696335,
|
|
"fcm_dpo/beta": 0.00493369298055768,
|
|
"fcm_dpo/delta": -0.10477574914693832,
|
|
"fcm_dpo/margin": 93.4791259765625,
|
|
"fcm_dpo/q_t": 0.3963235020637512,
|
|
"grad_norm": 67.77316284179688,
|
|
"learning_rate": 1.9999357655598891e-07,
|
|
"logits/chosen": -0.843751847743988,
|
|
"logits/rejected": -0.8341413736343384,
|
|
"logps/chosen": -427.7283020019531,
|
|
"logps/ref_chosen": -274.1440734863281,
|
|
"logps/ref_rejected": -278.07208251953125,
|
|
"logps/rejected": -525.1354370117188,
|
|
"loss": 4.2918,
|
|
"margin_dpo/margin_mean": 93.4791259765625,
|
|
"margin_dpo/margin_std": 124.0925064086914,
|
|
"step": 291
|
|
},
|
|
{
|
|
"epoch": 0.6115183246073298,
|
|
"fcm_dpo/beta": 0.004825775045901537,
|
|
"fcm_dpo/delta": 0.0411670058965683,
|
|
"fcm_dpo/margin": 69.62248229980469,
|
|
"fcm_dpo/q_t": 0.4242509603500366,
|
|
"grad_norm": 80.3649673461914,
|
|
"learning_rate": 1.9820116705100775e-07,
|
|
"logits/chosen": -0.8350319266319275,
|
|
"logits/rejected": -0.8311899304389954,
|
|
"logps/chosen": -408.04681396484375,
|
|
"logps/ref_chosen": -259.3636779785156,
|
|
"logps/ref_rejected": -279.30218505859375,
|
|
"logps/rejected": -497.6078186035156,
|
|
"loss": 4.6933,
|
|
"margin_dpo/margin_mean": 69.62248229980469,
|
|
"margin_dpo/margin_std": 126.6540756225586,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 0.6136125654450262,
|
|
"fcm_dpo/beta": 0.004759517032653093,
|
|
"fcm_dpo/delta": -0.059455305337905884,
|
|
"fcm_dpo/margin": 83.2960205078125,
|
|
"fcm_dpo/q_t": 0.4100151062011719,
|
|
"grad_norm": 87.87876892089844,
|
|
"learning_rate": 1.9641153536023642e-07,
|
|
"logits/chosen": -0.9336636066436768,
|
|
"logits/rejected": -0.8930791020393372,
|
|
"logps/chosen": -467.6336975097656,
|
|
"logps/ref_chosen": -303.77081298828125,
|
|
"logps/ref_rejected": -270.07513427734375,
|
|
"logps/rejected": -517.2340087890625,
|
|
"loss": 4.4479,
|
|
"margin_dpo/margin_mean": 83.29600524902344,
|
|
"margin_dpo/margin_std": 118.2719497680664,
|
|
"step": 293
|
|
},
|
|
{
|
|
"epoch": 0.6157068062827226,
|
|
"fcm_dpo/beta": 0.004677574150264263,
|
|
"fcm_dpo/delta": -0.016423970460891724,
|
|
"fcm_dpo/margin": 69.96250915527344,
|
|
"fcm_dpo/q_t": 0.4272857904434204,
|
|
"grad_norm": 78.82388305664062,
|
|
"learning_rate": 1.9462477745619106e-07,
|
|
"logits/chosen": -0.8205356597900391,
|
|
"logits/rejected": -0.8255838751792908,
|
|
"logps/chosen": -395.3521423339844,
|
|
"logps/ref_chosen": -240.23831176757812,
|
|
"logps/ref_rejected": -229.187744140625,
|
|
"logps/rejected": -454.2640686035156,
|
|
"loss": 4.7076,
|
|
"margin_dpo/margin_mean": 69.96250915527344,
|
|
"margin_dpo/margin_std": 129.4488525390625,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 0.6178010471204188,
|
|
"fcm_dpo/beta": 0.0048176939599215984,
|
|
"fcm_dpo/delta": 0.07201889902353287,
|
|
"fcm_dpo/margin": 68.49911499023438,
|
|
"fcm_dpo/q_t": 0.4257529377937317,
|
|
"grad_norm": 70.18067169189453,
|
|
"learning_rate": 1.928409891572757e-07,
|
|
"logits/chosen": -0.8293942809104919,
|
|
"logits/rejected": -0.8370305299758911,
|
|
"logps/chosen": -405.40020751953125,
|
|
"logps/ref_chosen": -251.00970458984375,
|
|
"logps/ref_rejected": -244.15142822265625,
|
|
"logps/rejected": -467.04107666015625,
|
|
"loss": 4.6407,
|
|
"margin_dpo/margin_mean": 68.49911499023438,
|
|
"margin_dpo/margin_std": 115.05201721191406,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.6198952879581152,
|
|
"fcm_dpo/beta": 0.004698417615145445,
|
|
"fcm_dpo/delta": -0.10293805599212646,
|
|
"fcm_dpo/margin": 94.19628143310547,
|
|
"fcm_dpo/q_t": 0.39925792813301086,
|
|
"grad_norm": 77.33747100830078,
|
|
"learning_rate": 1.9106026612264315e-07,
|
|
"logits/chosen": -0.8193361163139343,
|
|
"logits/rejected": -0.7859427332878113,
|
|
"logps/chosen": -453.1386413574219,
|
|
"logps/ref_chosen": -293.880615234375,
|
|
"logps/ref_rejected": -283.4175720214844,
|
|
"logps/rejected": -536.871826171875,
|
|
"loss": 4.3803,
|
|
"margin_dpo/margin_mean": 94.19628143310547,
|
|
"margin_dpo/margin_std": 136.64402770996094,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 0.6219895287958115,
|
|
"fcm_dpo/beta": 0.004520534072071314,
|
|
"fcm_dpo/delta": 0.020609168335795403,
|
|
"fcm_dpo/margin": 70.18128967285156,
|
|
"fcm_dpo/q_t": 0.4274723529815674,
|
|
"grad_norm": 74.41461181640625,
|
|
"learning_rate": 1.8928270384706582e-07,
|
|
"logits/chosen": -0.9127456545829773,
|
|
"logits/rejected": -0.9068336486816406,
|
|
"logps/chosen": -445.99737548828125,
|
|
"logps/ref_chosen": -289.4600830078125,
|
|
"logps/ref_rejected": -283.69110107421875,
|
|
"logps/rejected": -510.4096374511719,
|
|
"loss": 4.6623,
|
|
"margin_dpo/margin_mean": 70.18128204345703,
|
|
"margin_dpo/margin_std": 116.06803894042969,
|
|
"step": 297
|
|
},
|
|
{
|
|
"epoch": 0.6240837696335079,
|
|
"fcm_dpo/beta": 0.004513401072472334,
|
|
"fcm_dpo/delta": -0.054587312042713165,
|
|
"fcm_dpo/margin": 85.99492645263672,
|
|
"fcm_dpo/q_t": 0.41201794147491455,
|
|
"grad_norm": 96.58114624023438,
|
|
"learning_rate": 1.875083976558136e-07,
|
|
"logits/chosen": -0.8198727369308472,
|
|
"logits/rejected": -0.806681752204895,
|
|
"logps/chosen": -451.9382629394531,
|
|
"logps/ref_chosen": -306.5150146484375,
|
|
"logps/ref_rejected": -280.6969909667969,
|
|
"logps/rejected": -512.1151123046875,
|
|
"loss": 4.4997,
|
|
"margin_dpo/margin_mean": 85.99492645263672,
|
|
"margin_dpo/margin_std": 131.11622619628906,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 0.6261780104712041,
|
|
"fcm_dpo/beta": 0.004364268854260445,
|
|
"fcm_dpo/delta": 0.0030471403151750565,
|
|
"fcm_dpo/margin": 65.27977752685547,
|
|
"fcm_dpo/q_t": 0.434037983417511,
|
|
"grad_norm": 108.02179718017578,
|
|
"learning_rate": 1.8573744269954297e-07,
|
|
"logits/chosen": -0.8087353706359863,
|
|
"logits/rejected": -0.7938067317008972,
|
|
"logps/chosen": -452.2359619140625,
|
|
"logps/ref_chosen": -281.36376953125,
|
|
"logps/ref_rejected": -270.39508056640625,
|
|
"logps/rejected": -506.5469970703125,
|
|
"loss": 4.7327,
|
|
"margin_dpo/margin_mean": 65.27976989746094,
|
|
"margin_dpo/margin_std": 106.35151672363281,
|
|
"step": 299
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"fcm_dpo/beta": 0.004509190563112497,
|
|
"fcm_dpo/delta": 0.03436897695064545,
|
|
"fcm_dpo/margin": 64.97026062011719,
|
|
"fcm_dpo/q_t": 0.4333421587944031,
|
|
"grad_norm": 89.92118072509766,
|
|
"learning_rate": 1.839699339491937e-07,
|
|
"logits/chosen": -0.8405961394309998,
|
|
"logits/rejected": -0.8179261684417725,
|
|
"logps/chosen": -488.3516845703125,
|
|
"logps/ref_chosen": -314.83575439453125,
|
|
"logps/ref_rejected": -269.1154479980469,
|
|
"logps/rejected": -507.60162353515625,
|
|
"loss": 4.8012,
|
|
"margin_dpo/margin_mean": 64.97026062011719,
|
|
"margin_dpo/margin_std": 130.7782745361328,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6303664921465969,
|
|
"fcm_dpo/beta": 0.004641157109290361,
|
|
"fcm_dpo/delta": 0.02008698135614395,
|
|
"fcm_dpo/margin": 65.45753479003906,
|
|
"fcm_dpo/q_t": 0.4307628273963928,
|
|
"grad_norm": 92.52803039550781,
|
|
"learning_rate": 1.8220596619089573e-07,
|
|
"logits/chosen": -0.8634936809539795,
|
|
"logits/rejected": -0.8615702986717224,
|
|
"logps/chosen": -440.056396484375,
|
|
"logps/ref_chosen": -279.89453125,
|
|
"logps/ref_rejected": -271.6694641113281,
|
|
"logps/rejected": -497.28887939453125,
|
|
"loss": 4.7366,
|
|
"margin_dpo/margin_mean": 65.45753479003906,
|
|
"margin_dpo/margin_std": 119.31248474121094,
|
|
"step": 301
|
|
},
|
|
{
|
|
"epoch": 0.6324607329842932,
|
|
"fcm_dpo/beta": 0.004585019312798977,
|
|
"fcm_dpo/delta": -0.02878117375075817,
|
|
"fcm_dpo/margin": 86.58937072753906,
|
|
"fcm_dpo/q_t": 0.41024476289749146,
|
|
"grad_norm": 90.13982391357422,
|
|
"learning_rate": 1.8044563402088682e-07,
|
|
"logits/chosen": -0.8196691274642944,
|
|
"logits/rejected": -0.8060354590415955,
|
|
"logps/chosen": -425.8458557128906,
|
|
"logps/ref_chosen": -271.3318176269531,
|
|
"logps/ref_rejected": -256.5587158203125,
|
|
"logps/rejected": -497.662109375,
|
|
"loss": 4.4549,
|
|
"margin_dpo/margin_mean": 86.58937072753906,
|
|
"margin_dpo/margin_std": 128.49935913085938,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 0.6345549738219896,
|
|
"fcm_dpo/beta": 0.004555375315248966,
|
|
"fcm_dpo/delta": 0.001330256462097168,
|
|
"fcm_dpo/margin": 75.77513122558594,
|
|
"fcm_dpo/q_t": 0.4200977385044098,
|
|
"grad_norm": 67.80012512207031,
|
|
"learning_rate": 1.7868903184043885e-07,
|
|
"logits/chosen": -0.8044949769973755,
|
|
"logits/rejected": -0.7818641662597656,
|
|
"logps/chosen": -465.8041687011719,
|
|
"logps/ref_chosen": -304.88104248046875,
|
|
"logps/ref_rejected": -269.063720703125,
|
|
"logps/rejected": -505.7619934082031,
|
|
"loss": 4.5828,
|
|
"margin_dpo/margin_mean": 75.77513122558594,
|
|
"margin_dpo/margin_std": 117.83062744140625,
|
|
"step": 303
|
|
},
|
|
{
|
|
"epoch": 0.6366492146596858,
|
|
"fcm_dpo/beta": 0.00458510359749198,
|
|
"fcm_dpo/delta": 0.032261330634355545,
|
|
"fcm_dpo/margin": 80.3938980102539,
|
|
"fcm_dpo/q_t": 0.4192785322666168,
|
|
"grad_norm": 106.86233520507812,
|
|
"learning_rate": 1.7693625385079574e-07,
|
|
"logits/chosen": -0.8235783576965332,
|
|
"logits/rejected": -0.8320932984352112,
|
|
"logps/chosen": -467.0224914550781,
|
|
"logps/ref_chosen": -290.7109680175781,
|
|
"logps/ref_rejected": -237.6885986328125,
|
|
"logps/rejected": -494.3940124511719,
|
|
"loss": 4.578,
|
|
"margin_dpo/margin_mean": 80.39390563964844,
|
|
"margin_dpo/margin_std": 138.0482940673828,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 0.6387434554973822,
|
|
"fcm_dpo/beta": 0.004341554827988148,
|
|
"fcm_dpo/delta": -0.15256354212760925,
|
|
"fcm_dpo/margin": 125.14611053466797,
|
|
"fcm_dpo/q_t": 0.376392126083374,
|
|
"grad_norm": 80.46356964111328,
|
|
"learning_rate": 1.7518739404812155e-07,
|
|
"logits/chosen": -0.8713183999061584,
|
|
"logits/rejected": -0.8404445052146912,
|
|
"logps/chosen": -404.1719970703125,
|
|
"logps/ref_chosen": -256.4839782714844,
|
|
"logps/ref_rejected": -266.4063415527344,
|
|
"logps/rejected": -539.2404174804688,
|
|
"loss": 4.0074,
|
|
"margin_dpo/margin_mean": 125.14610290527344,
|
|
"margin_dpo/margin_std": 134.68260192871094,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.6408376963350786,
|
|
"fcm_dpo/beta": 0.004074703436344862,
|
|
"fcm_dpo/delta": -0.005278175696730614,
|
|
"fcm_dpo/margin": 75.07460021972656,
|
|
"fcm_dpo/q_t": 0.42913058400154114,
|
|
"grad_norm": 89.13674926757812,
|
|
"learning_rate": 1.7344254621846017e-07,
|
|
"logits/chosen": -0.8748356103897095,
|
|
"logits/rejected": -0.85565185546875,
|
|
"logps/chosen": -481.8687438964844,
|
|
"logps/ref_chosen": -320.6492004394531,
|
|
"logps/ref_rejected": -273.36773681640625,
|
|
"logps/rejected": -509.661865234375,
|
|
"loss": 4.6633,
|
|
"margin_dpo/margin_mean": 75.07460021972656,
|
|
"margin_dpo/margin_std": 118.12226867675781,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 0.6429319371727749,
|
|
"fcm_dpo/beta": 0.00412240345031023,
|
|
"fcm_dpo/delta": 0.012996518984436989,
|
|
"fcm_dpo/margin": 80.91729736328125,
|
|
"fcm_dpo/q_t": 0.4214695692062378,
|
|
"grad_norm": 82.34172821044922,
|
|
"learning_rate": 1.717018039327053e-07,
|
|
"logits/chosen": -0.797187328338623,
|
|
"logits/rejected": -0.8197383880615234,
|
|
"logps/chosen": -467.67266845703125,
|
|
"logps/ref_chosen": -279.4541931152344,
|
|
"logps/ref_rejected": -240.3796844482422,
|
|
"logps/rejected": -509.5154724121094,
|
|
"loss": 4.5522,
|
|
"margin_dpo/margin_mean": 80.91729736328125,
|
|
"margin_dpo/margin_std": 112.15681457519531,
|
|
"step": 307
|
|
},
|
|
{
|
|
"epoch": 0.6450261780104712,
|
|
"fcm_dpo/beta": 0.004232144448906183,
|
|
"fcm_dpo/delta": 0.035529762506484985,
|
|
"fcm_dpo/margin": 54.43116760253906,
|
|
"fcm_dpo/q_t": 0.44700539112091064,
|
|
"grad_norm": 83.30457305908203,
|
|
"learning_rate": 1.699652605415828e-07,
|
|
"logits/chosen": -0.8497411608695984,
|
|
"logits/rejected": -0.861391007900238,
|
|
"logps/chosen": -498.0782775878906,
|
|
"logps/ref_chosen": -296.598388671875,
|
|
"logps/ref_rejected": -258.6953430175781,
|
|
"logps/rejected": -514.6063232421875,
|
|
"loss": 4.9775,
|
|
"margin_dpo/margin_mean": 54.43116760253906,
|
|
"margin_dpo/margin_std": 127.53193664550781,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 0.6471204188481675,
|
|
"fcm_dpo/beta": 0.004292917437851429,
|
|
"fcm_dpo/delta": -0.0011705029755830765,
|
|
"fcm_dpo/margin": 93.36857604980469,
|
|
"fcm_dpo/q_t": 0.4098639190196991,
|
|
"grad_norm": 85.22145080566406,
|
|
"learning_rate": 1.6823300917064458e-07,
|
|
"logits/chosen": -0.8477848768234253,
|
|
"logits/rejected": -0.8559066653251648,
|
|
"logps/chosen": -473.5204772949219,
|
|
"logps/ref_chosen": -281.3881530761719,
|
|
"logps/ref_rejected": -262.458740234375,
|
|
"logps/rejected": -547.9596557617188,
|
|
"loss": 4.4233,
|
|
"margin_dpo/margin_mean": 93.36858367919922,
|
|
"margin_dpo/margin_std": 136.71563720703125,
|
|
"step": 309
|
|
},
|
|
{
|
|
"epoch": 0.6492146596858639,
|
|
"fcm_dpo/beta": 0.004227439407259226,
|
|
"fcm_dpo/delta": -0.016845714300870895,
|
|
"fcm_dpo/margin": 92.73661041259766,
|
|
"fcm_dpo/q_t": 0.4104473292827606,
|
|
"grad_norm": 60.97142791748047,
|
|
"learning_rate": 1.6650514271527465e-07,
|
|
"logits/chosen": -0.8435422778129578,
|
|
"logits/rejected": -0.8177253603935242,
|
|
"logps/chosen": -460.1365966796875,
|
|
"logps/ref_chosen": -279.1872863769531,
|
|
"logps/ref_rejected": -261.8279724121094,
|
|
"logps/rejected": -535.5139770507812,
|
|
"loss": 4.4209,
|
|
"margin_dpo/margin_mean": 92.73661804199219,
|
|
"margin_dpo/margin_std": 128.3031768798828,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.6513089005235602,
|
|
"fcm_dpo/beta": 0.004221054259687662,
|
|
"fcm_dpo/delta": 0.017837712541222572,
|
|
"fcm_dpo/margin": 90.44427490234375,
|
|
"fcm_dpo/q_t": 0.4134417176246643,
|
|
"grad_norm": 96.84386444091797,
|
|
"learning_rate": 1.647817538357072e-07,
|
|
"logits/chosen": -0.8366580009460449,
|
|
"logits/rejected": -0.8164286017417908,
|
|
"logps/chosen": -461.29248046875,
|
|
"logps/ref_chosen": -271.39813232421875,
|
|
"logps/ref_rejected": -266.12701416015625,
|
|
"logps/rejected": -546.4656372070312,
|
|
"loss": 4.533,
|
|
"margin_dpo/margin_mean": 90.44427490234375,
|
|
"margin_dpo/margin_std": 142.28948974609375,
|
|
"step": 311
|
|
},
|
|
{
|
|
"epoch": 0.6534031413612565,
|
|
"fcm_dpo/beta": 0.004198572598397732,
|
|
"fcm_dpo/delta": -0.027397790923714638,
|
|
"fcm_dpo/margin": 80.31361389160156,
|
|
"fcm_dpo/q_t": 0.42590564489364624,
|
|
"grad_norm": 79.09515380859375,
|
|
"learning_rate": 1.6306293495205755e-07,
|
|
"logits/chosen": -0.854910135269165,
|
|
"logits/rejected": -0.8275889158248901,
|
|
"logps/chosen": -471.61541748046875,
|
|
"logps/ref_chosen": -282.3850402832031,
|
|
"logps/ref_rejected": -246.35389709472656,
|
|
"logps/rejected": -515.89794921875,
|
|
"loss": 4.7836,
|
|
"margin_dpo/margin_mean": 80.31360626220703,
|
|
"margin_dpo/margin_std": 162.17025756835938,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 0.6554973821989529,
|
|
"fcm_dpo/beta": 0.00422461424022913,
|
|
"fcm_dpo/delta": -0.010405285283923149,
|
|
"fcm_dpo/margin": 79.79673767089844,
|
|
"fcm_dpo/q_t": 0.42779484391212463,
|
|
"grad_norm": 71.66043090820312,
|
|
"learning_rate": 1.6134877823936607e-07,
|
|
"logits/chosen": -0.8918551802635193,
|
|
"logits/rejected": -0.8759722113609314,
|
|
"logps/chosen": -499.627685546875,
|
|
"logps/ref_chosen": -303.630859375,
|
|
"logps/ref_rejected": -273.1156921386719,
|
|
"logps/rejected": -548.9091796875,
|
|
"loss": 4.7486,
|
|
"margin_dpo/margin_mean": 79.79674530029297,
|
|
"margin_dpo/margin_std": 153.3138427734375,
|
|
"step": 313
|
|
},
|
|
{
|
|
"epoch": 0.6575916230366492,
|
|
"fcm_dpo/beta": 0.004204154014587402,
|
|
"fcm_dpo/delta": 0.028585905209183693,
|
|
"fcm_dpo/margin": 82.11723327636719,
|
|
"fcm_dpo/q_t": 0.4200282096862793,
|
|
"grad_norm": 86.5287094116211,
|
|
"learning_rate": 1.5963937562265522e-07,
|
|
"logits/chosen": -0.9091111421585083,
|
|
"logits/rejected": -0.8985500931739807,
|
|
"logps/chosen": -488.7541809082031,
|
|
"logps/ref_chosen": -302.3042907714844,
|
|
"logps/ref_rejected": -273.6416015625,
|
|
"logps/rejected": -542.208740234375,
|
|
"loss": 4.5815,
|
|
"margin_dpo/margin_mean": 82.11723327636719,
|
|
"margin_dpo/margin_std": 129.3647918701172,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 0.6596858638743456,
|
|
"fcm_dpo/beta": 0.004223410505801439,
|
|
"fcm_dpo/delta": -0.0216530729085207,
|
|
"fcm_dpo/margin": 99.56622314453125,
|
|
"fcm_dpo/q_t": 0.4038830101490021,
|
|
"grad_norm": 83.03425598144531,
|
|
"learning_rate": 1.5793481877199943e-07,
|
|
"logits/chosen": -0.8853978514671326,
|
|
"logits/rejected": -0.8687517642974854,
|
|
"logps/chosen": -488.8519287109375,
|
|
"logps/ref_chosen": -302.729248046875,
|
|
"logps/ref_rejected": -270.26910400390625,
|
|
"logps/rejected": -555.9579467773438,
|
|
"loss": 4.3519,
|
|
"margin_dpo/margin_mean": 99.56622314453125,
|
|
"margin_dpo/margin_std": 133.00856018066406,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.6617801047120419,
|
|
"fcm_dpo/beta": 0.00421603349968791,
|
|
"fcm_dpo/delta": 0.030252397060394287,
|
|
"fcm_dpo/margin": 87.81375885009766,
|
|
"fcm_dpo/q_t": 0.41594982147216797,
|
|
"grad_norm": 83.00886535644531,
|
|
"learning_rate": 1.562351990976095e-07,
|
|
"logits/chosen": -0.8991329669952393,
|
|
"logits/rejected": -0.8829232454299927,
|
|
"logps/chosen": -503.4185791015625,
|
|
"logps/ref_chosen": -310.5706481933594,
|
|
"logps/ref_rejected": -272.9354553222656,
|
|
"logps/rejected": -553.5971069335938,
|
|
"loss": 4.5283,
|
|
"margin_dpo/margin_mean": 87.81375885009766,
|
|
"margin_dpo/margin_std": 137.16415405273438,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 0.6638743455497382,
|
|
"fcm_dpo/beta": 0.0043156505562365055,
|
|
"fcm_dpo/delta": 0.03792955353856087,
|
|
"fcm_dpo/margin": 84.17947387695312,
|
|
"fcm_dpo/q_t": 0.41570180654525757,
|
|
"grad_norm": 84.58760070800781,
|
|
"learning_rate": 1.5454060774493065e-07,
|
|
"logits/chosen": -0.8874872326850891,
|
|
"logits/rejected": -0.856697142124176,
|
|
"logps/chosen": -421.66094970703125,
|
|
"logps/ref_chosen": -253.90036010742188,
|
|
"logps/ref_rejected": -218.74078369140625,
|
|
"logps/rejected": -470.6808166503906,
|
|
"loss": 4.4808,
|
|
"margin_dpo/margin_mean": 84.17947387695312,
|
|
"margin_dpo/margin_std": 114.98719787597656,
|
|
"step": 317
|
|
},
|
|
{
|
|
"epoch": 0.6659685863874345,
|
|
"fcm_dpo/beta": 0.004230810329318047,
|
|
"fcm_dpo/delta": -0.027981776744127274,
|
|
"fcm_dpo/margin": 100.45878601074219,
|
|
"fcm_dpo/q_t": 0.40261608362197876,
|
|
"grad_norm": 77.26625061035156,
|
|
"learning_rate": 1.5285113558975427e-07,
|
|
"logits/chosen": -0.8962690234184265,
|
|
"logits/rejected": -0.8660792112350464,
|
|
"logps/chosen": -450.1083068847656,
|
|
"logps/ref_chosen": -270.8228759765625,
|
|
"logps/ref_rejected": -255.30972290039062,
|
|
"logps/rejected": -535.053955078125,
|
|
"loss": 4.3339,
|
|
"margin_dpo/margin_mean": 100.45878601074219,
|
|
"margin_dpo/margin_std": 129.95045471191406,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 0.6680628272251309,
|
|
"fcm_dpo/beta": 0.004204513505101204,
|
|
"fcm_dpo/delta": -0.003102468326687813,
|
|
"fcm_dpo/margin": 95.72822570800781,
|
|
"fcm_dpo/q_t": 0.4059513807296753,
|
|
"grad_norm": 78.08094787597656,
|
|
"learning_rate": 1.5116687323334464e-07,
|
|
"logits/chosen": -0.8939340710639954,
|
|
"logits/rejected": -0.8638850450515747,
|
|
"logps/chosen": -490.3048095703125,
|
|
"logps/ref_chosen": -301.0028076171875,
|
|
"logps/ref_rejected": -242.39002990722656,
|
|
"logps/rejected": -527.4202270507812,
|
|
"loss": 4.3517,
|
|
"margin_dpo/margin_mean": 95.72822570800781,
|
|
"margin_dpo/margin_std": 119.91444396972656,
|
|
"step": 319
|
|
},
|
|
{
|
|
"epoch": 0.6701570680628273,
|
|
"fcm_dpo/beta": 0.004289536736905575,
|
|
"fcm_dpo/delta": 0.027375973761081696,
|
|
"fcm_dpo/margin": 86.9979248046875,
|
|
"fcm_dpo/q_t": 0.4167506992816925,
|
|
"grad_norm": 90.90167999267578,
|
|
"learning_rate": 1.4948791099758052e-07,
|
|
"logits/chosen": -0.8552721738815308,
|
|
"logits/rejected": -0.847703218460083,
|
|
"logps/chosen": -487.06640625,
|
|
"logps/ref_chosen": -303.6225891113281,
|
|
"logps/ref_rejected": -280.85174560546875,
|
|
"logps/rejected": -551.29345703125,
|
|
"loss": 4.5643,
|
|
"margin_dpo/margin_mean": 86.99790954589844,
|
|
"margin_dpo/margin_std": 145.2067108154297,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.6722513089005235,
|
|
"fcm_dpo/beta": 0.004387743771076202,
|
|
"fcm_dpo/delta": 0.007859650999307632,
|
|
"fcm_dpo/margin": 63.733009338378906,
|
|
"fcm_dpo/q_t": 0.43765002489089966,
|
|
"grad_norm": 98.84996795654297,
|
|
"learning_rate": 1.478143389201113e-07,
|
|
"logits/chosen": -0.8751094341278076,
|
|
"logits/rejected": -0.839246392250061,
|
|
"logps/chosen": -492.1627197265625,
|
|
"logps/ref_chosen": -288.98583984375,
|
|
"logps/ref_rejected": -241.1822052001953,
|
|
"logps/rejected": -508.09210205078125,
|
|
"loss": 4.8733,
|
|
"margin_dpo/margin_mean": 63.733009338378906,
|
|
"margin_dpo/margin_std": 136.8006134033203,
|
|
"step": 321
|
|
},
|
|
{
|
|
"epoch": 0.6743455497382199,
|
|
"fcm_dpo/beta": 0.004408278036862612,
|
|
"fcm_dpo/delta": 0.0005717193707823753,
|
|
"fcm_dpo/margin": 78.75841522216797,
|
|
"fcm_dpo/q_t": 0.4216492176055908,
|
|
"grad_norm": 109.37677001953125,
|
|
"learning_rate": 1.461462467495284e-07,
|
|
"logits/chosen": -0.9387675523757935,
|
|
"logits/rejected": -0.9010107517242432,
|
|
"logps/chosen": -514.61083984375,
|
|
"logps/ref_chosen": -308.54345703125,
|
|
"logps/ref_rejected": -269.7995910644531,
|
|
"logps/rejected": -554.6253662109375,
|
|
"loss": 4.6688,
|
|
"margin_dpo/margin_mean": 78.75841522216797,
|
|
"margin_dpo/margin_std": 143.8833770751953,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 0.6764397905759162,
|
|
"fcm_dpo/beta": 0.00443174596875906,
|
|
"fcm_dpo/delta": 0.02427692338824272,
|
|
"fcm_dpo/margin": 56.93408203125,
|
|
"fcm_dpo/q_t": 0.4440670907497406,
|
|
"grad_norm": 74.94538879394531,
|
|
"learning_rate": 1.4448372394055246e-07,
|
|
"logits/chosen": -0.8896764516830444,
|
|
"logits/rejected": -0.8818888664245605,
|
|
"logps/chosen": -475.91375732421875,
|
|
"logps/ref_chosen": -282.49365234375,
|
|
"logps/ref_rejected": -227.7105255126953,
|
|
"logps/rejected": -478.0647277832031,
|
|
"loss": 4.9551,
|
|
"margin_dpo/margin_mean": 56.934078216552734,
|
|
"margin_dpo/margin_std": 135.64027404785156,
|
|
"step": 323
|
|
},
|
|
{
|
|
"epoch": 0.6785340314136126,
|
|
"fcm_dpo/beta": 0.004333406686782837,
|
|
"fcm_dpo/delta": -0.036704014986753464,
|
|
"fcm_dpo/margin": 94.2815170288086,
|
|
"fcm_dpo/q_t": 0.4057313799858093,
|
|
"grad_norm": 85.12026977539062,
|
|
"learning_rate": 1.428268596492364e-07,
|
|
"logits/chosen": -0.8165361881256104,
|
|
"logits/rejected": -0.8111166954040527,
|
|
"logps/chosen": -414.974609375,
|
|
"logps/ref_chosen": -239.33836364746094,
|
|
"logps/ref_rejected": -230.53775024414062,
|
|
"logps/rejected": -500.4554748535156,
|
|
"loss": 4.3573,
|
|
"margin_dpo/margin_mean": 94.2815170288086,
|
|
"margin_dpo/margin_std": 120.80596923828125,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 0.680628272251309,
|
|
"fcm_dpo/beta": 0.0042426129803061485,
|
|
"fcm_dpo/delta": -0.04353080317378044,
|
|
"fcm_dpo/margin": 80.39572143554688,
|
|
"fcm_dpo/q_t": 0.42427390813827515,
|
|
"grad_norm": 99.24666595458984,
|
|
"learning_rate": 1.4117574272818386e-07,
|
|
"logits/chosen": -0.8341665267944336,
|
|
"logits/rejected": -0.8182295560836792,
|
|
"logps/chosen": -474.089599609375,
|
|
"logps/ref_chosen": -280.62896728515625,
|
|
"logps/ref_rejected": -270.5085754394531,
|
|
"logps/rejected": -544.3649291992188,
|
|
"loss": 4.7072,
|
|
"margin_dpo/margin_mean": 80.39572143554688,
|
|
"margin_dpo/margin_std": 150.9606475830078,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.6827225130890052,
|
|
"fcm_dpo/beta": 0.004248365759849548,
|
|
"fcm_dpo/delta": 0.007518507540225983,
|
|
"fcm_dpo/margin": 86.27885437011719,
|
|
"fcm_dpo/q_t": 0.41677290201187134,
|
|
"grad_norm": 89.02911376953125,
|
|
"learning_rate": 1.3953046172178413e-07,
|
|
"logits/chosen": -0.9490206837654114,
|
|
"logits/rejected": -0.9398881793022156,
|
|
"logps/chosen": -419.48565673828125,
|
|
"logps/ref_chosen": -240.9871368408203,
|
|
"logps/ref_rejected": -261.0238342285156,
|
|
"logps/rejected": -525.8012084960938,
|
|
"loss": 4.5545,
|
|
"margin_dpo/margin_mean": 86.27885437011719,
|
|
"margin_dpo/margin_std": 138.67755126953125,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 0.6848167539267016,
|
|
"fcm_dpo/beta": 0.0041707661002874374,
|
|
"fcm_dpo/delta": -0.025365371257066727,
|
|
"fcm_dpo/margin": 95.02404022216797,
|
|
"fcm_dpo/q_t": 0.4098770022392273,
|
|
"grad_norm": 87.0390625,
|
|
"learning_rate": 1.3789110486146468e-07,
|
|
"logits/chosen": -0.8910970091819763,
|
|
"logits/rejected": -0.8668678402900696,
|
|
"logps/chosen": -449.1541748046875,
|
|
"logps/ref_chosen": -279.52001953125,
|
|
"logps/ref_rejected": -269.51824951171875,
|
|
"logps/rejected": -534.1763916015625,
|
|
"loss": 4.4215,
|
|
"margin_dpo/margin_mean": 95.02404022216797,
|
|
"margin_dpo/margin_std": 131.46246337890625,
|
|
"step": 327
|
|
},
|
|
{
|
|
"epoch": 0.6869109947643979,
|
|
"fcm_dpo/beta": 0.00401464756578207,
|
|
"fcm_dpo/delta": -0.02622944489121437,
|
|
"fcm_dpo/margin": 85.4066162109375,
|
|
"fcm_dpo/q_t": 0.4205254912376404,
|
|
"grad_norm": 76.4749984741211,
|
|
"learning_rate": 1.362577600609588e-07,
|
|
"logits/chosen": -0.8552443981170654,
|
|
"logits/rejected": -0.8537087440490723,
|
|
"logps/chosen": -487.69732666015625,
|
|
"logps/ref_chosen": -301.033447265625,
|
|
"logps/ref_rejected": -284.2101135253906,
|
|
"logps/rejected": -556.2805786132812,
|
|
"loss": 4.5527,
|
|
"margin_dpo/margin_mean": 85.4066162109375,
|
|
"margin_dpo/margin_std": 124.48458099365234,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 0.6890052356020943,
|
|
"fcm_dpo/beta": 0.003990216180682182,
|
|
"fcm_dpo/delta": -0.024558117613196373,
|
|
"fcm_dpo/margin": 92.10006713867188,
|
|
"fcm_dpo/q_t": 0.42077887058258057,
|
|
"grad_norm": 75.36544799804688,
|
|
"learning_rate": 1.3463051491159093e-07,
|
|
"logits/chosen": -0.8669880628585815,
|
|
"logits/rejected": -0.8379704356193542,
|
|
"logps/chosen": -517.63623046875,
|
|
"logps/ref_chosen": -319.9888610839844,
|
|
"logps/ref_rejected": -307.5588684082031,
|
|
"logps/rejected": -597.3063354492188,
|
|
"loss": 4.6236,
|
|
"margin_dpo/margin_mean": 92.10005950927734,
|
|
"margin_dpo/margin_std": 161.97157287597656,
|
|
"step": 329
|
|
},
|
|
{
|
|
"epoch": 0.6910994764397905,
|
|
"fcm_dpo/beta": 0.003911246545612812,
|
|
"fcm_dpo/delta": -0.030769605189561844,
|
|
"fcm_dpo/margin": 88.48474884033203,
|
|
"fcm_dpo/q_t": 0.42029914259910583,
|
|
"grad_norm": 71.30492401123047,
|
|
"learning_rate": 1.3300945667758012e-07,
|
|
"logits/chosen": -0.8565258979797363,
|
|
"logits/rejected": -0.8653894066810608,
|
|
"logps/chosen": -496.8501281738281,
|
|
"logps/ref_chosen": -301.11474609375,
|
|
"logps/ref_rejected": -299.673095703125,
|
|
"logps/rejected": -583.8932495117188,
|
|
"loss": 4.5507,
|
|
"margin_dpo/margin_mean": 88.48474884033203,
|
|
"margin_dpo/margin_std": 128.97657775878906,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.6931937172774869,
|
|
"fcm_dpo/beta": 0.003975651692599058,
|
|
"fcm_dpo/delta": 0.05870046094059944,
|
|
"fcm_dpo/margin": 86.33970642089844,
|
|
"fcm_dpo/q_t": 0.4234520494937897,
|
|
"grad_norm": 80.85090637207031,
|
|
"learning_rate": 1.3139467229135998e-07,
|
|
"logits/chosen": -0.8877269625663757,
|
|
"logits/rejected": -0.8683705925941467,
|
|
"logps/chosen": -460.4813537597656,
|
|
"logps/ref_chosen": -277.59149169921875,
|
|
"logps/ref_rejected": -256.025634765625,
|
|
"logps/rejected": -525.2552490234375,
|
|
"loss": 4.6499,
|
|
"margin_dpo/margin_mean": 86.33970642089844,
|
|
"margin_dpo/margin_std": 154.41885375976562,
|
|
"step": 331
|
|
},
|
|
{
|
|
"epoch": 0.6952879581151833,
|
|
"fcm_dpo/beta": 0.004023251123726368,
|
|
"fcm_dpo/delta": 0.006525871343910694,
|
|
"fcm_dpo/margin": 86.42638397216797,
|
|
"fcm_dpo/q_t": 0.4220367670059204,
|
|
"grad_norm": 101.5770492553711,
|
|
"learning_rate": 1.2978624834891626e-07,
|
|
"logits/chosen": -0.8897654414176941,
|
|
"logits/rejected": -0.8640623688697815,
|
|
"logps/chosen": -462.71600341796875,
|
|
"logps/ref_chosen": -269.97369384765625,
|
|
"logps/ref_rejected": -235.03164672851562,
|
|
"logps/rejected": -514.2003173828125,
|
|
"loss": 4.6419,
|
|
"margin_dpo/margin_mean": 86.4263916015625,
|
|
"margin_dpo/margin_std": 151.5164794921875,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 0.6973821989528796,
|
|
"fcm_dpo/beta": 0.004221638664603233,
|
|
"fcm_dpo/delta": 0.05486953258514404,
|
|
"fcm_dpo/margin": 73.84952545166016,
|
|
"fcm_dpo/q_t": 0.4293982982635498,
|
|
"grad_norm": 72.1107177734375,
|
|
"learning_rate": 1.281842711051438e-07,
|
|
"logits/chosen": -0.962734043598175,
|
|
"logits/rejected": -0.9257638454437256,
|
|
"logps/chosen": -493.1941833496094,
|
|
"logps/ref_chosen": -296.76300048828125,
|
|
"logps/ref_rejected": -265.97991943359375,
|
|
"logps/rejected": -536.2606201171875,
|
|
"loss": 4.7055,
|
|
"margin_dpo/margin_mean": 73.84952545166016,
|
|
"margin_dpo/margin_std": 133.64422607421875,
|
|
"step": 333
|
|
},
|
|
{
|
|
"epoch": 0.6994764397905759,
|
|
"fcm_dpo/beta": 0.0043148864060640335,
|
|
"fcm_dpo/delta": -0.00039426563307642937,
|
|
"fcm_dpo/margin": 82.25745391845703,
|
|
"fcm_dpo/q_t": 0.4192145764827728,
|
|
"grad_norm": 142.5056610107422,
|
|
"learning_rate": 1.2658882646922033e-07,
|
|
"logits/chosen": -0.8579068183898926,
|
|
"logits/rejected": -0.8308162093162537,
|
|
"logps/chosen": -490.0419616699219,
|
|
"logps/ref_chosen": -301.0367431640625,
|
|
"logps/ref_rejected": -268.87652587890625,
|
|
"logps/rejected": -540.1392211914062,
|
|
"loss": 4.6052,
|
|
"margin_dpo/margin_mean": 82.2574462890625,
|
|
"margin_dpo/margin_std": 134.9955596923828,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 0.7015706806282722,
|
|
"fcm_dpo/beta": 0.0042117261327803135,
|
|
"fcm_dpo/delta": -0.005903298035264015,
|
|
"fcm_dpo/margin": 88.66685485839844,
|
|
"fcm_dpo/q_t": 0.4157736599445343,
|
|
"grad_norm": 82.71585845947266,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": -0.8564417958259583,
|
|
"logits/rejected": -0.850709080696106,
|
|
"logps/chosen": -481.53729248046875,
|
|
"logps/ref_chosen": -276.13275146484375,
|
|
"logps/ref_rejected": -243.44203186035156,
|
|
"logps/rejected": -537.5133666992188,
|
|
"loss": 4.551,
|
|
"margin_dpo/margin_mean": 88.66685485839844,
|
|
"margin_dpo/margin_std": 140.4600372314453,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.7036649214659686,
|
|
"fcm_dpo/beta": 0.004215937573462725,
|
|
"fcm_dpo/delta": -0.017655007541179657,
|
|
"fcm_dpo/margin": 89.4686508178711,
|
|
"fcm_dpo/q_t": 0.4191427230834961,
|
|
"grad_norm": 69.75729370117188,
|
|
"learning_rate": 1.2341787690142435e-07,
|
|
"logits/chosen": -0.8736502528190613,
|
|
"logits/rejected": -0.812055230140686,
|
|
"logps/chosen": -450.0522155761719,
|
|
"logps/ref_chosen": -246.2626495361328,
|
|
"logps/ref_rejected": -261.0617980957031,
|
|
"logps/rejected": -554.3200073242188,
|
|
"loss": 4.574,
|
|
"margin_dpo/margin_mean": 89.4686508178711,
|
|
"margin_dpo/margin_std": 151.58094787597656,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 0.7057591623036649,
|
|
"fcm_dpo/beta": 0.0042139277793467045,
|
|
"fcm_dpo/delta": -0.02299882099032402,
|
|
"fcm_dpo/margin": 93.4886474609375,
|
|
"fcm_dpo/q_t": 0.41079553961753845,
|
|
"grad_norm": 79.24539184570312,
|
|
"learning_rate": 1.2184254201795363e-07,
|
|
"logits/chosen": -0.8735207915306091,
|
|
"logits/rejected": -0.8454840779304504,
|
|
"logps/chosen": -458.2684631347656,
|
|
"logps/ref_chosen": -266.9937744140625,
|
|
"logps/ref_rejected": -253.015625,
|
|
"logps/rejected": -537.7789916992188,
|
|
"loss": 4.4457,
|
|
"margin_dpo/margin_mean": 93.48865509033203,
|
|
"margin_dpo/margin_std": 134.67636108398438,
|
|
"step": 337
|
|
},
|
|
{
|
|
"epoch": 0.7078534031413612,
|
|
"fcm_dpo/beta": 0.0041390592232346535,
|
|
"fcm_dpo/delta": 0.013191011734306812,
|
|
"fcm_dpo/margin": 93.37725830078125,
|
|
"fcm_dpo/q_t": 0.4126090407371521,
|
|
"grad_norm": 120.32525634765625,
|
|
"learning_rate": 1.202740798300168e-07,
|
|
"logits/chosen": -0.8974190354347229,
|
|
"logits/rejected": -0.8797302842140198,
|
|
"logps/chosen": -459.82012939453125,
|
|
"logps/ref_chosen": -276.5925598144531,
|
|
"logps/ref_rejected": -233.979248046875,
|
|
"logps/rejected": -510.5840759277344,
|
|
"loss": 4.4602,
|
|
"margin_dpo/margin_mean": 93.37725830078125,
|
|
"margin_dpo/margin_std": 135.0716552734375,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 0.7099476439790576,
|
|
"fcm_dpo/beta": 0.0041606188751757145,
|
|
"fcm_dpo/delta": -0.020034300163388252,
|
|
"fcm_dpo/margin": 100.59808349609375,
|
|
"fcm_dpo/q_t": 0.40568268299102783,
|
|
"grad_norm": 83.7204818725586,
|
|
"learning_rate": 1.1871257444948096e-07,
|
|
"logits/chosen": -0.9190041422843933,
|
|
"logits/rejected": -0.906645655632019,
|
|
"logps/chosen": -496.95233154296875,
|
|
"logps/ref_chosen": -303.5277404785156,
|
|
"logps/ref_rejected": -283.11676025390625,
|
|
"logps/rejected": -577.1394653320312,
|
|
"loss": 4.4169,
|
|
"margin_dpo/margin_mean": 100.59807586669922,
|
|
"margin_dpo/margin_std": 148.40968322753906,
|
|
"step": 339
|
|
},
|
|
{
|
|
"epoch": 0.7120418848167539,
|
|
"fcm_dpo/beta": 0.004161160439252853,
|
|
"fcm_dpo/delta": 0.032229986041784286,
|
|
"fcm_dpo/margin": 82.14170837402344,
|
|
"fcm_dpo/q_t": 0.42351680994033813,
|
|
"grad_norm": 130.79080200195312,
|
|
"learning_rate": 1.1715810961514072e-07,
|
|
"logits/chosen": -0.8522177338600159,
|
|
"logits/rejected": -0.8474718928337097,
|
|
"logps/chosen": -462.0082092285156,
|
|
"logps/ref_chosen": -261.5257568359375,
|
|
"logps/ref_rejected": -259.39862060546875,
|
|
"logps/rejected": -542.0227661132812,
|
|
"loss": 4.7617,
|
|
"margin_dpo/margin_mean": 82.14170837402344,
|
|
"margin_dpo/margin_std": 163.79534912109375,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7141361256544503,
|
|
"fcm_dpo/beta": 0.004220739006996155,
|
|
"fcm_dpo/delta": 0.013660956174135208,
|
|
"fcm_dpo/margin": 72.86450958251953,
|
|
"fcm_dpo/q_t": 0.4320552945137024,
|
|
"grad_norm": 74.50018310546875,
|
|
"learning_rate": 1.1561076868822755e-07,
|
|
"logits/chosen": -0.8715128302574158,
|
|
"logits/rejected": -0.8439801335334778,
|
|
"logps/chosen": -547.073974609375,
|
|
"logps/ref_chosen": -315.903564453125,
|
|
"logps/ref_rejected": -308.02392578125,
|
|
"logps/rejected": -612.058837890625,
|
|
"loss": 4.9256,
|
|
"margin_dpo/margin_mean": 72.86450958251953,
|
|
"margin_dpo/margin_std": 168.4367218017578,
|
|
"step": 341
|
|
},
|
|
{
|
|
"epoch": 0.7162303664921466,
|
|
"fcm_dpo/beta": 0.0043666851706802845,
|
|
"fcm_dpo/delta": 0.02423066645860672,
|
|
"fcm_dpo/margin": 86.04833984375,
|
|
"fcm_dpo/q_t": 0.4149947166442871,
|
|
"grad_norm": 124.93977355957031,
|
|
"learning_rate": 1.1407063464793965e-07,
|
|
"logits/chosen": -0.872001051902771,
|
|
"logits/rejected": -0.8675014972686768,
|
|
"logps/chosen": -462.8236389160156,
|
|
"logps/ref_chosen": -269.17864990234375,
|
|
"logps/ref_rejected": -260.8977355957031,
|
|
"logps/rejected": -540.591064453125,
|
|
"loss": 4.5223,
|
|
"margin_dpo/margin_mean": 86.04833984375,
|
|
"margin_dpo/margin_std": 132.913330078125,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 0.7183246073298429,
|
|
"fcm_dpo/beta": 0.004478834103792906,
|
|
"fcm_dpo/delta": 0.07679037004709244,
|
|
"fcm_dpo/margin": 72.70214080810547,
|
|
"fcm_dpo/q_t": 0.4263514280319214,
|
|
"grad_norm": 76.69132232666016,
|
|
"learning_rate": 1.125377900869913e-07,
|
|
"logits/chosen": -0.8687958121299744,
|
|
"logits/rejected": -0.8440370559692383,
|
|
"logps/chosen": -512.5940551757812,
|
|
"logps/ref_chosen": -310.719970703125,
|
|
"logps/ref_rejected": -263.5224914550781,
|
|
"logps/rejected": -538.0987548828125,
|
|
"loss": 4.6727,
|
|
"margin_dpo/margin_mean": 72.70214080810547,
|
|
"margin_dpo/margin_std": 129.76275634765625,
|
|
"step": 343
|
|
},
|
|
{
|
|
"epoch": 0.7204188481675393,
|
|
"fcm_dpo/beta": 0.004755112808197737,
|
|
"fcm_dpo/delta": 0.020119212567806244,
|
|
"fcm_dpo/margin": 74.89362335205078,
|
|
"fcm_dpo/q_t": 0.41980746388435364,
|
|
"grad_norm": 83.10517883300781,
|
|
"learning_rate": 1.110123172071844e-07,
|
|
"logits/chosen": -0.8663875460624695,
|
|
"logits/rejected": -0.84798264503479,
|
|
"logps/chosen": -502.5235595703125,
|
|
"logps/ref_chosen": -301.7999267578125,
|
|
"logps/ref_rejected": -257.9061584472656,
|
|
"logps/rejected": -533.5234375,
|
|
"loss": 4.6603,
|
|
"margin_dpo/margin_mean": 74.89362335205078,
|
|
"margin_dpo/margin_std": 134.99688720703125,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 0.7225130890052356,
|
|
"fcm_dpo/beta": 0.0046288310550153255,
|
|
"fcm_dpo/delta": -0.04487127810716629,
|
|
"fcm_dpo/margin": 70.49630737304688,
|
|
"fcm_dpo/q_t": 0.4266197681427002,
|
|
"grad_norm": 96.98583221435547,
|
|
"learning_rate": 1.09494297815e-07,
|
|
"logits/chosen": -0.8665734529495239,
|
|
"logits/rejected": -0.8673686385154724,
|
|
"logps/chosen": -476.5813293457031,
|
|
"logps/ref_chosen": -283.0184326171875,
|
|
"logps/ref_rejected": -266.8457336425781,
|
|
"logps/rejected": -530.9049072265625,
|
|
"loss": 4.6982,
|
|
"margin_dpo/margin_mean": 70.49630737304688,
|
|
"margin_dpo/margin_std": 119.51622009277344,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.724607329842932,
|
|
"fcm_dpo/beta": 0.004324314650148153,
|
|
"fcm_dpo/delta": -0.08264245837926865,
|
|
"fcm_dpo/margin": 94.54742431640625,
|
|
"fcm_dpo/q_t": 0.4078022241592407,
|
|
"grad_norm": 105.37242889404297,
|
|
"learning_rate": 1.0798381331721107e-07,
|
|
"logits/chosen": -0.9467877149581909,
|
|
"logits/rejected": -0.8949044942855835,
|
|
"logps/chosen": -467.9733581542969,
|
|
"logps/ref_chosen": -268.44122314453125,
|
|
"logps/ref_rejected": -227.8225860595703,
|
|
"logps/rejected": -521.9021606445312,
|
|
"loss": 4.4796,
|
|
"margin_dpo/margin_mean": 94.54740905761719,
|
|
"margin_dpo/margin_std": 140.02549743652344,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 0.7267015706806282,
|
|
"fcm_dpo/beta": 0.004258990287780762,
|
|
"fcm_dpo/delta": 0.015542431734502316,
|
|
"fcm_dpo/margin": 73.12641906738281,
|
|
"fcm_dpo/q_t": 0.4280158281326294,
|
|
"grad_norm": 129.21615600585938,
|
|
"learning_rate": 1.0648094471651722e-07,
|
|
"logits/chosen": -0.7985144853591919,
|
|
"logits/rejected": -0.8250570297241211,
|
|
"logps/chosen": -464.026123046875,
|
|
"logps/ref_chosen": -273.70355224609375,
|
|
"logps/ref_rejected": -243.65521240234375,
|
|
"logps/rejected": -507.10418701171875,
|
|
"loss": 4.6989,
|
|
"margin_dpo/margin_mean": 73.12641906738281,
|
|
"margin_dpo/margin_std": 124.44351196289062,
|
|
"step": 347
|
|
},
|
|
{
|
|
"epoch": 0.7287958115183246,
|
|
"fcm_dpo/beta": 0.004324248060584068,
|
|
"fcm_dpo/delta": 0.028727039694786072,
|
|
"fcm_dpo/margin": 64.96218872070312,
|
|
"fcm_dpo/q_t": 0.4387781620025635,
|
|
"grad_norm": 88.1794204711914,
|
|
"learning_rate": 1.0498577260720048e-07,
|
|
"logits/chosen": -0.9022852182388306,
|
|
"logits/rejected": -0.8873810172080994,
|
|
"logps/chosen": -476.2867431640625,
|
|
"logps/ref_chosen": -285.64141845703125,
|
|
"logps/ref_rejected": -265.6270446777344,
|
|
"logps/rejected": -521.2345581054688,
|
|
"loss": 4.856,
|
|
"margin_dpo/margin_mean": 64.96218872070312,
|
|
"margin_dpo/margin_std": 137.919677734375,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 0.7308900523560209,
|
|
"fcm_dpo/beta": 0.004297360777854919,
|
|
"fcm_dpo/delta": -0.05822340026497841,
|
|
"fcm_dpo/margin": 99.03752899169922,
|
|
"fcm_dpo/q_t": 0.40331995487213135,
|
|
"grad_norm": 105.6890869140625,
|
|
"learning_rate": 1.0349837717080347e-07,
|
|
"logits/chosen": -0.8452585935592651,
|
|
"logits/rejected": -0.8333654999732971,
|
|
"logps/chosen": -520.8684692382812,
|
|
"logps/ref_chosen": -328.3175048828125,
|
|
"logps/ref_rejected": -292.37872314453125,
|
|
"logps/rejected": -583.96728515625,
|
|
"loss": 4.4394,
|
|
"margin_dpo/margin_mean": 99.03752899169922,
|
|
"margin_dpo/margin_std": 149.55874633789062,
|
|
"step": 349
|
|
},
|
|
{
|
|
"epoch": 0.7329842931937173,
|
|
"fcm_dpo/beta": 0.004243547562509775,
|
|
"fcm_dpo/delta": 0.00749508710578084,
|
|
"fcm_dpo/margin": 84.4142837524414,
|
|
"fcm_dpo/q_t": 0.41955479979515076,
|
|
"grad_norm": 84.40665435791016,
|
|
"learning_rate": 1.0201883817182949e-07,
|
|
"logits/chosen": -0.837745189666748,
|
|
"logits/rejected": -0.8492467403411865,
|
|
"logps/chosen": -495.3204345703125,
|
|
"logps/ref_chosen": -292.8046569824219,
|
|
"logps/ref_rejected": -250.35504150390625,
|
|
"logps/rejected": -537.2850341796875,
|
|
"loss": 4.6258,
|
|
"margin_dpo/margin_mean": 84.41429138183594,
|
|
"margin_dpo/margin_std": 147.89923095703125,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.7350785340314137,
|
|
"fcm_dpo/beta": 0.0042998939752578735,
|
|
"fcm_dpo/delta": 0.04794508218765259,
|
|
"fcm_dpo/margin": 55.46940231323242,
|
|
"fcm_dpo/q_t": 0.4465586841106415,
|
|
"grad_norm": 104.56925964355469,
|
|
"learning_rate": 1.0054723495346482e-07,
|
|
"logits/chosen": -0.9081228971481323,
|
|
"logits/rejected": -0.890868604183197,
|
|
"logps/chosen": -506.27545166015625,
|
|
"logps/ref_chosen": -311.8890380859375,
|
|
"logps/ref_rejected": -263.59033203125,
|
|
"logps/rejected": -513.4461669921875,
|
|
"loss": 5.0328,
|
|
"margin_dpo/margin_mean": 55.46940231323242,
|
|
"margin_dpo/margin_std": 144.48912048339844,
|
|
"step": 351
|
|
},
|
|
{
|
|
"epoch": 0.7371727748691099,
|
|
"fcm_dpo/beta": 0.004258439410477877,
|
|
"fcm_dpo/delta": -0.04971547797322273,
|
|
"fcm_dpo/margin": 104.7568359375,
|
|
"fcm_dpo/q_t": 0.3977693021297455,
|
|
"grad_norm": 80.12105560302734,
|
|
"learning_rate": 9.908364643332398e-08,
|
|
"logits/chosen": -0.8247711658477783,
|
|
"logits/rejected": -0.7942376136779785,
|
|
"logps/chosen": -434.2840270996094,
|
|
"logps/ref_chosen": -254.9078826904297,
|
|
"logps/ref_rejected": -257.1688232421875,
|
|
"logps/rejected": -541.3017578125,
|
|
"loss": 4.3164,
|
|
"margin_dpo/margin_mean": 104.75682830810547,
|
|
"margin_dpo/margin_std": 138.1365203857422,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 0.7392670157068063,
|
|
"fcm_dpo/beta": 0.0041551426984369755,
|
|
"fcm_dpo/delta": -0.007299358025193214,
|
|
"fcm_dpo/margin": 88.50493621826172,
|
|
"fcm_dpo/q_t": 0.41657987236976624,
|
|
"grad_norm": 94.97777557373047,
|
|
"learning_rate": 9.76281510992176e-08,
|
|
"logits/chosen": -0.8463056087493896,
|
|
"logits/rejected": -0.8403018712997437,
|
|
"logps/chosen": -462.415771484375,
|
|
"logps/ref_chosen": -270.3760681152344,
|
|
"logps/ref_rejected": -264.65234375,
|
|
"logps/rejected": -545.197021484375,
|
|
"loss": 4.5474,
|
|
"margin_dpo/margin_mean": 88.50493621826172,
|
|
"margin_dpo/margin_std": 137.2867431640625,
|
|
"step": 353
|
|
},
|
|
{
|
|
"epoch": 0.7413612565445026,
|
|
"fcm_dpo/beta": 0.0042567066848278046,
|
|
"fcm_dpo/delta": 0.03705097734928131,
|
|
"fcm_dpo/margin": 65.5347671508789,
|
|
"fcm_dpo/q_t": 0.43766477704048157,
|
|
"grad_norm": 72.41242980957031,
|
|
"learning_rate": 9.618082700494318e-08,
|
|
"logits/chosen": -0.8418868780136108,
|
|
"logits/rejected": -0.8751367330551147,
|
|
"logps/chosen": -450.8121337890625,
|
|
"logps/ref_chosen": -257.6485595703125,
|
|
"logps/ref_rejected": -246.94203186035156,
|
|
"logps/rejected": -505.6404113769531,
|
|
"loss": 4.8655,
|
|
"margin_dpo/margin_mean": 65.53477478027344,
|
|
"margin_dpo/margin_std": 140.56112670898438,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 0.743455497382199,
|
|
"fcm_dpo/beta": 0.004172952845692635,
|
|
"fcm_dpo/delta": -0.03083830326795578,
|
|
"fcm_dpo/margin": 95.28348541259766,
|
|
"fcm_dpo/q_t": 0.41133761405944824,
|
|
"grad_norm": 79.00312042236328,
|
|
"learning_rate": 9.474175176609956e-08,
|
|
"logits/chosen": -0.8952876925468445,
|
|
"logits/rejected": -0.8920347690582275,
|
|
"logps/chosen": -486.91064453125,
|
|
"logps/ref_chosen": -293.35333251953125,
|
|
"logps/ref_rejected": -275.6051940917969,
|
|
"logps/rejected": -564.4459838867188,
|
|
"loss": 4.5619,
|
|
"margin_dpo/margin_mean": 95.28348541259766,
|
|
"margin_dpo/margin_std": 158.88365173339844,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.7455497382198953,
|
|
"fcm_dpo/beta": 0.004256485030055046,
|
|
"fcm_dpo/delta": 0.039695486426353455,
|
|
"fcm_dpo/margin": 67.3565444946289,
|
|
"fcm_dpo/q_t": 0.43414920568466187,
|
|
"grad_norm": 107.32857513427734,
|
|
"learning_rate": 9.331100255592436e-08,
|
|
"logits/chosen": -0.8071290850639343,
|
|
"logits/rejected": -0.834446132183075,
|
|
"logps/chosen": -386.90557861328125,
|
|
"logps/ref_chosen": -204.25550842285156,
|
|
"logps/ref_rejected": -213.467529296875,
|
|
"logps/rejected": -463.4741516113281,
|
|
"loss": 4.7445,
|
|
"margin_dpo/margin_mean": 67.35653686523438,
|
|
"margin_dpo/margin_std": 116.29612731933594,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 0.7476439790575916,
|
|
"fcm_dpo/beta": 0.004231618717312813,
|
|
"fcm_dpo/delta": -0.04037679359316826,
|
|
"fcm_dpo/margin": 89.51058959960938,
|
|
"fcm_dpo/q_t": 0.4154285788536072,
|
|
"grad_norm": 89.29653930664062,
|
|
"learning_rate": 9.18886561011557e-08,
|
|
"logits/chosen": -0.7836157083511353,
|
|
"logits/rejected": -0.7811770439147949,
|
|
"logps/chosen": -463.0835876464844,
|
|
"logps/ref_chosen": -266.3705749511719,
|
|
"logps/ref_rejected": -239.04490661621094,
|
|
"logps/rejected": -525.2685546875,
|
|
"loss": 4.5513,
|
|
"margin_dpo/margin_mean": 89.51058197021484,
|
|
"margin_dpo/margin_std": 143.0915985107422,
|
|
"step": 357
|
|
},
|
|
{
|
|
"epoch": 0.749738219895288,
|
|
"fcm_dpo/beta": 0.004202042240649462,
|
|
"fcm_dpo/delta": -0.018734265118837357,
|
|
"fcm_dpo/margin": 99.36197662353516,
|
|
"fcm_dpo/q_t": 0.40445786714553833,
|
|
"grad_norm": 67.8469467163086,
|
|
"learning_rate": 9.047478867791731e-08,
|
|
"logits/chosen": -0.8765335083007812,
|
|
"logits/rejected": -0.8540104627609253,
|
|
"logps/chosen": -477.45489501953125,
|
|
"logps/ref_chosen": -299.1474609375,
|
|
"logps/ref_rejected": -257.2531433105469,
|
|
"logps/rejected": -534.9225463867188,
|
|
"loss": 4.4071,
|
|
"margin_dpo/margin_mean": 99.36197662353516,
|
|
"margin_dpo/margin_std": 142.21287536621094,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 0.7518324607329843,
|
|
"fcm_dpo/beta": 0.004150255583226681,
|
|
"fcm_dpo/delta": -0.015420145355165005,
|
|
"fcm_dpo/margin": 84.46359252929688,
|
|
"fcm_dpo/q_t": 0.42003896832466125,
|
|
"grad_norm": 105.28828430175781,
|
|
"learning_rate": 8.906947610762825e-08,
|
|
"logits/chosen": -0.8406848907470703,
|
|
"logits/rejected": -0.8504139184951782,
|
|
"logps/chosen": -489.37860107421875,
|
|
"logps/ref_chosen": -302.99786376953125,
|
|
"logps/ref_rejected": -260.4137268066406,
|
|
"logps/rejected": -531.258056640625,
|
|
"loss": 4.5874,
|
|
"margin_dpo/margin_mean": 84.46359252929688,
|
|
"margin_dpo/margin_std": 134.04608154296875,
|
|
"step": 359
|
|
},
|
|
{
|
|
"epoch": 0.7539267015706806,
|
|
"fcm_dpo/beta": 0.004011007957160473,
|
|
"fcm_dpo/delta": -0.024799039587378502,
|
|
"fcm_dpo/margin": 78.62114715576172,
|
|
"fcm_dpo/q_t": 0.4280991852283478,
|
|
"grad_norm": 78.52674102783203,
|
|
"learning_rate": 8.76727937529367e-08,
|
|
"logits/chosen": -0.8800378441810608,
|
|
"logits/rejected": -0.8672605752944946,
|
|
"logps/chosen": -503.25250244140625,
|
|
"logps/ref_chosen": -309.6114501953125,
|
|
"logps/ref_rejected": -256.64031982421875,
|
|
"logps/rejected": -528.9025268554688,
|
|
"loss": 4.7802,
|
|
"margin_dpo/margin_mean": 78.62113952636719,
|
|
"margin_dpo/margin_std": 147.0281982421875,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.7560209424083769,
|
|
"fcm_dpo/beta": 0.004003939218819141,
|
|
"fcm_dpo/delta": -0.013433972373604774,
|
|
"fcm_dpo/margin": 103.06330871582031,
|
|
"fcm_dpo/q_t": 0.4082551598548889,
|
|
"grad_norm": 89.6968765258789,
|
|
"learning_rate": 8.628481651367875e-08,
|
|
"logits/chosen": -0.8125383853912354,
|
|
"logits/rejected": -0.7936655282974243,
|
|
"logps/chosen": -427.3260498046875,
|
|
"logps/ref_chosen": -263.3797607421875,
|
|
"logps/ref_rejected": -271.18157958984375,
|
|
"logps/rejected": -538.191162109375,
|
|
"loss": 4.4476,
|
|
"margin_dpo/margin_mean": 103.06332397460938,
|
|
"margin_dpo/margin_std": 158.7577362060547,
|
|
"step": 361
|
|
},
|
|
{
|
|
"epoch": 0.7581151832460733,
|
|
"fcm_dpo/beta": 0.004012004937976599,
|
|
"fcm_dpo/delta": 0.03470272570848465,
|
|
"fcm_dpo/margin": 84.82909393310547,
|
|
"fcm_dpo/q_t": 0.4218509793281555,
|
|
"grad_norm": 105.21653747558594,
|
|
"learning_rate": 8.490561882286135e-08,
|
|
"logits/chosen": -0.8308958411216736,
|
|
"logits/rejected": -0.8169477581977844,
|
|
"logps/chosen": -480.77783203125,
|
|
"logps/ref_chosen": -303.2583923339844,
|
|
"logps/ref_rejected": -243.22891235351562,
|
|
"logps/rejected": -505.57745361328125,
|
|
"loss": 4.5377,
|
|
"margin_dpo/margin_mean": 84.82909393310547,
|
|
"margin_dpo/margin_std": 120.54029083251953,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 0.7602094240837697,
|
|
"fcm_dpo/beta": 0.004087929613888264,
|
|
"fcm_dpo/delta": 0.005159515887498856,
|
|
"fcm_dpo/margin": 88.3600082397461,
|
|
"fcm_dpo/q_t": 0.419447124004364,
|
|
"grad_norm": 70.80838012695312,
|
|
"learning_rate": 8.353527464267104e-08,
|
|
"logits/chosen": -0.8576685190200806,
|
|
"logits/rejected": -0.8124222755432129,
|
|
"logps/chosen": -484.9991760253906,
|
|
"logps/ref_chosen": -303.34722900390625,
|
|
"logps/ref_rejected": -262.05419921875,
|
|
"logps/rejected": -532.066162109375,
|
|
"loss": 4.6004,
|
|
"margin_dpo/margin_mean": 88.36001586914062,
|
|
"margin_dpo/margin_std": 147.58880615234375,
|
|
"step": 363
|
|
},
|
|
{
|
|
"epoch": 0.762303664921466,
|
|
"fcm_dpo/beta": 0.0041374522261321545,
|
|
"fcm_dpo/delta": 0.045890823006629944,
|
|
"fcm_dpo/margin": 81.01228332519531,
|
|
"fcm_dpo/q_t": 0.42662695050239563,
|
|
"grad_norm": 78.44388580322266,
|
|
"learning_rate": 8.217385746050742e-08,
|
|
"logits/chosen": -0.829826831817627,
|
|
"logits/rejected": -0.8344470858573914,
|
|
"logps/chosen": -491.9500427246094,
|
|
"logps/ref_chosen": -285.54376220703125,
|
|
"logps/ref_rejected": -284.84619140625,
|
|
"logps/rejected": -572.2648315429688,
|
|
"loss": 4.727,
|
|
"margin_dpo/margin_mean": 81.01229095458984,
|
|
"margin_dpo/margin_std": 156.3982391357422,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 0.7643979057591623,
|
|
"fcm_dpo/beta": 0.004236162174493074,
|
|
"fcm_dpo/delta": -0.0016637099906802177,
|
|
"fcm_dpo/margin": 78.55353546142578,
|
|
"fcm_dpo/q_t": 0.42524176836013794,
|
|
"grad_norm": 84.3416748046875,
|
|
"learning_rate": 8.082144028504231e-08,
|
|
"logits/chosen": -0.8499488830566406,
|
|
"logits/rejected": -0.8522325158119202,
|
|
"logps/chosen": -469.8583984375,
|
|
"logps/ref_chosen": -274.7878112792969,
|
|
"logps/ref_rejected": -256.5738220214844,
|
|
"logps/rejected": -530.1979370117188,
|
|
"loss": 4.6929,
|
|
"margin_dpo/margin_mean": 78.55352783203125,
|
|
"margin_dpo/margin_std": 141.90133666992188,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.7664921465968586,
|
|
"fcm_dpo/beta": 0.0043019927106797695,
|
|
"fcm_dpo/delta": 0.025872234255075455,
|
|
"fcm_dpo/margin": 87.0827407836914,
|
|
"fcm_dpo/q_t": 0.4149934649467468,
|
|
"grad_norm": 73.31763458251953,
|
|
"learning_rate": 7.947809564230445e-08,
|
|
"logits/chosen": -0.802319347858429,
|
|
"logits/rejected": -0.8103886842727661,
|
|
"logps/chosen": -465.9633483886719,
|
|
"logps/ref_chosen": -286.6496276855469,
|
|
"logps/ref_rejected": -251.97140502929688,
|
|
"logps/rejected": -518.367919921875,
|
|
"loss": 4.548,
|
|
"margin_dpo/margin_mean": 87.08274841308594,
|
|
"margin_dpo/margin_std": 142.1220703125,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 0.768586387434555,
|
|
"fcm_dpo/beta": 0.004179948940873146,
|
|
"fcm_dpo/delta": -0.04910705238580704,
|
|
"fcm_dpo/margin": 94.82267761230469,
|
|
"fcm_dpo/q_t": 0.41003602743148804,
|
|
"grad_norm": 79.67829895019531,
|
|
"learning_rate": 7.814389557179016e-08,
|
|
"logits/chosen": -0.8363237977027893,
|
|
"logits/rejected": -0.812557578086853,
|
|
"logps/chosen": -481.86407470703125,
|
|
"logps/ref_chosen": -301.9449768066406,
|
|
"logps/ref_rejected": -265.5677185058594,
|
|
"logps/rejected": -540.3095092773438,
|
|
"loss": 4.4263,
|
|
"margin_dpo/margin_mean": 94.82267761230469,
|
|
"margin_dpo/margin_std": 134.9529266357422,
|
|
"step": 367
|
|
},
|
|
{
|
|
"epoch": 0.7706806282722513,
|
|
"fcm_dpo/beta": 0.004044831730425358,
|
|
"fcm_dpo/delta": -0.05517401173710823,
|
|
"fcm_dpo/margin": 111.77228546142578,
|
|
"fcm_dpo/q_t": 0.39542829990386963,
|
|
"grad_norm": 81.13077545166016,
|
|
"learning_rate": 7.681891162260015e-08,
|
|
"logits/chosen": -0.8041467666625977,
|
|
"logits/rejected": -0.8103511333465576,
|
|
"logps/chosen": -464.62017822265625,
|
|
"logps/ref_chosen": -294.62652587890625,
|
|
"logps/ref_rejected": -258.7628479003906,
|
|
"logps/rejected": -540.5286865234375,
|
|
"loss": 4.2009,
|
|
"margin_dpo/margin_mean": 111.77228546142578,
|
|
"margin_dpo/margin_std": 126.04940795898438,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 0.7727748691099476,
|
|
"fcm_dpo/beta": 0.004040499217808247,
|
|
"fcm_dpo/delta": 0.0448799803853035,
|
|
"fcm_dpo/margin": 80.99122619628906,
|
|
"fcm_dpo/q_t": 0.4241292178630829,
|
|
"grad_norm": 75.39326477050781,
|
|
"learning_rate": 7.550321484960251e-08,
|
|
"logits/chosen": -0.8849146962165833,
|
|
"logits/rejected": -0.8683555126190186,
|
|
"logps/chosen": -463.06689453125,
|
|
"logps/ref_chosen": -282.5057373046875,
|
|
"logps/ref_rejected": -266.41607666015625,
|
|
"logps/rejected": -527.968505859375,
|
|
"loss": 4.5871,
|
|
"margin_dpo/margin_mean": 80.9912338256836,
|
|
"margin_dpo/margin_std": 121.13188171386719,
|
|
"step": 369
|
|
},
|
|
{
|
|
"epoch": 0.774869109947644,
|
|
"fcm_dpo/beta": 0.004057829733937979,
|
|
"fcm_dpo/delta": -0.008466588333249092,
|
|
"fcm_dpo/margin": 100.35714721679688,
|
|
"fcm_dpo/q_t": 0.4076663553714752,
|
|
"grad_norm": 73.64390563964844,
|
|
"learning_rate": 7.419687580962222e-08,
|
|
"logits/chosen": -0.8662645816802979,
|
|
"logits/rejected": -0.8904479146003723,
|
|
"logps/chosen": -421.3550720214844,
|
|
"logps/ref_chosen": -251.00640869140625,
|
|
"logps/ref_rejected": -238.12542724609375,
|
|
"logps/rejected": -508.8311767578125,
|
|
"loss": 4.453,
|
|
"margin_dpo/margin_mean": 100.3571548461914,
|
|
"margin_dpo/margin_std": 152.1621856689453,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.7769633507853403,
|
|
"fcm_dpo/beta": 0.004208661150187254,
|
|
"fcm_dpo/delta": 0.04080788791179657,
|
|
"fcm_dpo/margin": 66.90896606445312,
|
|
"fcm_dpo/q_t": 0.4354203939437866,
|
|
"grad_norm": 99.27238464355469,
|
|
"learning_rate": 7.289996455765748e-08,
|
|
"logits/chosen": -0.8164863586425781,
|
|
"logits/rejected": -0.8000683188438416,
|
|
"logps/chosen": -491.5625915527344,
|
|
"logps/ref_chosen": -296.6591491699219,
|
|
"logps/ref_rejected": -251.14675903320312,
|
|
"logps/rejected": -512.9591064453125,
|
|
"loss": 4.8201,
|
|
"margin_dpo/margin_mean": 66.90897369384766,
|
|
"margin_dpo/margin_std": 131.07626342773438,
|
|
"step": 371
|
|
},
|
|
{
|
|
"epoch": 0.7790575916230367,
|
|
"fcm_dpo/beta": 0.004100184887647629,
|
|
"fcm_dpo/delta": -0.043359022587537766,
|
|
"fcm_dpo/margin": 100.76567840576172,
|
|
"fcm_dpo/q_t": 0.40735262632369995,
|
|
"grad_norm": 83.19894409179688,
|
|
"learning_rate": 7.161255064312283e-08,
|
|
"logits/chosen": -0.7977643013000488,
|
|
"logits/rejected": -0.7871427536010742,
|
|
"logps/chosen": -520.5551147460938,
|
|
"logps/ref_chosen": -331.3714599609375,
|
|
"logps/ref_rejected": -285.56805419921875,
|
|
"logps/rejected": -575.5173950195312,
|
|
"loss": 4.449,
|
|
"margin_dpo/margin_mean": 100.76567840576172,
|
|
"margin_dpo/margin_std": 149.07765197753906,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 0.7811518324607329,
|
|
"fcm_dpo/beta": 0.004079765640199184,
|
|
"fcm_dpo/delta": 0.007463835179805756,
|
|
"fcm_dpo/margin": 90.10908508300781,
|
|
"fcm_dpo/q_t": 0.41459009051322937,
|
|
"grad_norm": 77.88961791992188,
|
|
"learning_rate": 7.033470310611945e-08,
|
|
"logits/chosen": -0.8931885361671448,
|
|
"logits/rejected": -0.8614373207092285,
|
|
"logps/chosen": -496.9786376953125,
|
|
"logps/ref_chosen": -321.9429931640625,
|
|
"logps/ref_rejected": -271.2288513183594,
|
|
"logps/rejected": -536.37353515625,
|
|
"loss": 4.4999,
|
|
"margin_dpo/margin_mean": 90.10908508300781,
|
|
"margin_dpo/margin_std": 128.7992401123047,
|
|
"step": 373
|
|
},
|
|
{
|
|
"epoch": 0.7832460732984293,
|
|
"fcm_dpo/beta": 0.0040543461218476295,
|
|
"fcm_dpo/delta": -0.009488560259342194,
|
|
"fcm_dpo/margin": 81.82614135742188,
|
|
"fcm_dpo/q_t": 0.4247649908065796,
|
|
"grad_norm": 60.28491973876953,
|
|
"learning_rate": 6.906649047373245e-08,
|
|
"logits/chosen": -0.8894190192222595,
|
|
"logits/rejected": -0.8857640027999878,
|
|
"logps/chosen": -503.9471435546875,
|
|
"logps/ref_chosen": -319.1685485839844,
|
|
"logps/ref_rejected": -284.6263732910156,
|
|
"logps/rejected": -551.2310791015625,
|
|
"loss": 4.6405,
|
|
"margin_dpo/margin_mean": 81.82613372802734,
|
|
"margin_dpo/margin_std": 135.69937133789062,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 0.7853403141361257,
|
|
"fcm_dpo/beta": 0.0040982929058372974,
|
|
"fcm_dpo/delta": 0.019265731796622276,
|
|
"fcm_dpo/margin": 69.89092254638672,
|
|
"fcm_dpo/q_t": 0.4342796802520752,
|
|
"grad_norm": 83.22428894042969,
|
|
"learning_rate": 6.780798075635675e-08,
|
|
"logits/chosen": -0.8811032176017761,
|
|
"logits/rejected": -0.860395073890686,
|
|
"logps/chosen": -508.0401611328125,
|
|
"logps/ref_chosen": -314.87579345703125,
|
|
"logps/ref_rejected": -259.1965026855469,
|
|
"logps/rejected": -522.2518310546875,
|
|
"loss": 4.7725,
|
|
"margin_dpo/margin_mean": 69.89092254638672,
|
|
"margin_dpo/margin_std": 129.87033081054688,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.787434554973822,
|
|
"fcm_dpo/beta": 0.0042695943266153336,
|
|
"fcm_dpo/delta": 0.045676298439502716,
|
|
"fcm_dpo/margin": 83.08172607421875,
|
|
"fcm_dpo/q_t": 0.42078420519828796,
|
|
"grad_norm": 97.34649658203125,
|
|
"learning_rate": 6.655924144404906e-08,
|
|
"logits/chosen": -0.8564417958259583,
|
|
"logits/rejected": -0.8596733212471008,
|
|
"logps/chosen": -483.13177490234375,
|
|
"logps/ref_chosen": -287.6732482910156,
|
|
"logps/ref_rejected": -256.6697082519531,
|
|
"logps/rejected": -535.2098999023438,
|
|
"loss": 4.6391,
|
|
"margin_dpo/margin_mean": 83.08173370361328,
|
|
"margin_dpo/margin_std": 145.4075164794922,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 0.7895287958115184,
|
|
"fcm_dpo/beta": 0.004419329576194286,
|
|
"fcm_dpo/delta": 0.04526427388191223,
|
|
"fcm_dpo/margin": 62.52194595336914,
|
|
"fcm_dpo/q_t": 0.43734949827194214,
|
|
"grad_norm": 94.04608154296875,
|
|
"learning_rate": 6.532033950290885e-08,
|
|
"logits/chosen": -0.8489782214164734,
|
|
"logits/rejected": -0.846108615398407,
|
|
"logps/chosen": -511.7943115234375,
|
|
"logps/ref_chosen": -305.261474609375,
|
|
"logps/ref_rejected": -271.8887023925781,
|
|
"logps/rejected": -540.9434814453125,
|
|
"loss": 4.9284,
|
|
"margin_dpo/margin_mean": 62.52194595336914,
|
|
"margin_dpo/margin_std": 146.0769500732422,
|
|
"step": 377
|
|
},
|
|
{
|
|
"epoch": 0.7916230366492146,
|
|
"fcm_dpo/beta": 0.004514284431934357,
|
|
"fcm_dpo/delta": 0.037094950675964355,
|
|
"fcm_dpo/margin": 74.94515991210938,
|
|
"fcm_dpo/q_t": 0.4244396984577179,
|
|
"grad_norm": 101.25901794433594,
|
|
"learning_rate": 6.409134137148736e-08,
|
|
"logits/chosen": -0.8392292857170105,
|
|
"logits/rejected": -0.82718425989151,
|
|
"logps/chosen": -479.11871337890625,
|
|
"logps/ref_chosen": -281.5295715332031,
|
|
"logps/ref_rejected": -296.980224609375,
|
|
"logps/rejected": -569.5145874023438,
|
|
"loss": 4.6926,
|
|
"margin_dpo/margin_mean": 74.94515991210938,
|
|
"margin_dpo/margin_std": 139.2892608642578,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 0.793717277486911,
|
|
"fcm_dpo/beta": 0.004631263203918934,
|
|
"fcm_dpo/delta": 0.02148812636733055,
|
|
"fcm_dpo/margin": 75.75040435791016,
|
|
"fcm_dpo/q_t": 0.4205975830554962,
|
|
"grad_norm": 86.93419647216797,
|
|
"learning_rate": 6.28723129572247e-08,
|
|
"logits/chosen": -0.9042958617210388,
|
|
"logits/rejected": -0.8856334686279297,
|
|
"logps/chosen": -448.8710632324219,
|
|
"logps/ref_chosen": -265.0807800292969,
|
|
"logps/ref_rejected": -230.58932495117188,
|
|
"logps/rejected": -490.1300048828125,
|
|
"loss": 4.6479,
|
|
"margin_dpo/margin_mean": 75.75040435791016,
|
|
"margin_dpo/margin_std": 135.121337890625,
|
|
"step": 379
|
|
},
|
|
{
|
|
"epoch": 0.7958115183246073,
|
|
"fcm_dpo/beta": 0.004544341005384922,
|
|
"fcm_dpo/delta": -0.061347946524620056,
|
|
"fcm_dpo/margin": 84.75240325927734,
|
|
"fcm_dpo/q_t": 0.41487401723861694,
|
|
"grad_norm": 89.89930725097656,
|
|
"learning_rate": 6.166331963291519e-08,
|
|
"logits/chosen": -0.8618828058242798,
|
|
"logits/rejected": -0.839131772518158,
|
|
"logps/chosen": -501.7915954589844,
|
|
"logps/ref_chosen": -305.90838623046875,
|
|
"logps/ref_rejected": -286.5906677246094,
|
|
"logps/rejected": -567.226318359375,
|
|
"loss": 4.5946,
|
|
"margin_dpo/margin_mean": 84.75240325927734,
|
|
"margin_dpo/margin_std": 142.7577667236328,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.7979057591623037,
|
|
"fcm_dpo/beta": 0.00437317555770278,
|
|
"fcm_dpo/delta": -0.05076461285352707,
|
|
"fcm_dpo/margin": 95.79244995117188,
|
|
"fcm_dpo/q_t": 0.4083157181739807,
|
|
"grad_norm": 88.44004821777344,
|
|
"learning_rate": 6.046442623320145e-08,
|
|
"logits/chosen": -0.8316866159439087,
|
|
"logits/rejected": -0.7976770997047424,
|
|
"logps/chosen": -444.54351806640625,
|
|
"logps/ref_chosen": -252.87066650390625,
|
|
"logps/ref_rejected": -261.1927490234375,
|
|
"logps/rejected": -548.6580200195312,
|
|
"loss": 4.461,
|
|
"margin_dpo/margin_mean": 95.79244995117188,
|
|
"margin_dpo/margin_std": 149.4072723388672,
|
|
"step": 381
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"fcm_dpo/beta": 0.004201515112072229,
|
|
"fcm_dpo/delta": -0.05036533996462822,
|
|
"fcm_dpo/margin": 106.53213500976562,
|
|
"fcm_dpo/q_t": 0.39790698885917664,
|
|
"grad_norm": 85.8750991821289,
|
|
"learning_rate": 5.9275697051098275e-08,
|
|
"logits/chosen": -0.8765906095504761,
|
|
"logits/rejected": -0.8718726634979248,
|
|
"logps/chosen": -472.893310546875,
|
|
"logps/ref_chosen": -289.2114562988281,
|
|
"logps/ref_rejected": -278.45751953125,
|
|
"logps/rejected": -568.6714477539062,
|
|
"loss": 4.2905,
|
|
"margin_dpo/margin_mean": 106.53213500976562,
|
|
"margin_dpo/margin_std": 138.6427001953125,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 0.8020942408376963,
|
|
"fcm_dpo/beta": 0.0041129072196781635,
|
|
"fcm_dpo/delta": -0.022012613713741302,
|
|
"fcm_dpo/margin": 95.4480972290039,
|
|
"fcm_dpo/q_t": 0.4109508991241455,
|
|
"grad_norm": 95.30079650878906,
|
|
"learning_rate": 5.809719583454414e-08,
|
|
"logits/chosen": -0.8609268665313721,
|
|
"logits/rejected": -0.8443728685379028,
|
|
"logps/chosen": -453.16119384765625,
|
|
"logps/ref_chosen": -273.630859375,
|
|
"logps/ref_rejected": -261.44024658203125,
|
|
"logps/rejected": -536.418701171875,
|
|
"loss": 4.4769,
|
|
"margin_dpo/margin_mean": 95.4480972290039,
|
|
"margin_dpo/margin_std": 144.06008911132812,
|
|
"step": 383
|
|
},
|
|
{
|
|
"epoch": 0.8041884816753927,
|
|
"fcm_dpo/beta": 0.004073744174093008,
|
|
"fcm_dpo/delta": 0.0247461199760437,
|
|
"fcm_dpo/margin": 83.81874084472656,
|
|
"fcm_dpo/q_t": 0.4218423664569855,
|
|
"grad_norm": 79.8589859008789,
|
|
"learning_rate": 5.6928985782982524e-08,
|
|
"logits/chosen": -0.8856214880943298,
|
|
"logits/rejected": -0.8819814324378967,
|
|
"logps/chosen": -463.6858825683594,
|
|
"logps/ref_chosen": -274.5699462890625,
|
|
"logps/ref_rejected": -285.8253479003906,
|
|
"logps/rejected": -558.7599487304688,
|
|
"loss": 4.5903,
|
|
"margin_dpo/margin_mean": 83.81874084472656,
|
|
"margin_dpo/margin_std": 133.73779296875,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 0.806282722513089,
|
|
"fcm_dpo/beta": 0.00418306328356266,
|
|
"fcm_dpo/delta": 0.019975785166025162,
|
|
"fcm_dpo/margin": 74.35530853271484,
|
|
"fcm_dpo/q_t": 0.42889341711997986,
|
|
"grad_norm": 95.49700164794922,
|
|
"learning_rate": 5.57711295439732e-08,
|
|
"logits/chosen": -0.8372036218643188,
|
|
"logits/rejected": -0.8347649574279785,
|
|
"logps/chosen": -479.953857421875,
|
|
"logps/ref_chosen": -284.150634765625,
|
|
"logps/ref_rejected": -244.87921142578125,
|
|
"logps/rejected": -515.037841796875,
|
|
"loss": 4.6754,
|
|
"margin_dpo/margin_mean": 74.35530853271484,
|
|
"margin_dpo/margin_std": 125.24661254882812,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.8083769633507853,
|
|
"fcm_dpo/beta": 0.003999017644673586,
|
|
"fcm_dpo/delta": -0.0784250870347023,
|
|
"fcm_dpo/margin": 106.15267944335938,
|
|
"fcm_dpo/q_t": 0.403393030166626,
|
|
"grad_norm": 86.98873138427734,
|
|
"learning_rate": 5.4623689209832484e-08,
|
|
"logits/chosen": -0.8149560689926147,
|
|
"logits/rejected": -0.813643217086792,
|
|
"logps/chosen": -503.624755859375,
|
|
"logps/ref_chosen": -320.1762390136719,
|
|
"logps/ref_rejected": -302.05023193359375,
|
|
"logps/rejected": -591.6513671875,
|
|
"loss": 4.3475,
|
|
"margin_dpo/margin_mean": 106.15266418457031,
|
|
"margin_dpo/margin_std": 134.6205596923828,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 0.8104712041884817,
|
|
"fcm_dpo/beta": 0.0039867255836725235,
|
|
"fcm_dpo/delta": 0.02052573300898075,
|
|
"fcm_dpo/margin": 82.58099365234375,
|
|
"fcm_dpo/q_t": 0.4234527051448822,
|
|
"grad_norm": 85.00679016113281,
|
|
"learning_rate": 5.3486726314303175e-08,
|
|
"logits/chosen": -0.8490534424781799,
|
|
"logits/rejected": -0.8536210656166077,
|
|
"logps/chosen": -472.10137939453125,
|
|
"logps/ref_chosen": -272.2801513671875,
|
|
"logps/ref_rejected": -265.1615905761719,
|
|
"logps/rejected": -547.5638427734375,
|
|
"loss": 4.6743,
|
|
"margin_dpo/margin_mean": 82.58099365234375,
|
|
"margin_dpo/margin_std": 141.04251098632812,
|
|
"step": 387
|
|
},
|
|
{
|
|
"epoch": 0.812565445026178,
|
|
"fcm_dpo/beta": 0.00392739474773407,
|
|
"fcm_dpo/delta": -0.014465522021055222,
|
|
"fcm_dpo/margin": 58.15325927734375,
|
|
"fcm_dpo/q_t": 0.4480987787246704,
|
|
"grad_norm": 101.83181762695312,
|
|
"learning_rate": 5.2360301829254745e-08,
|
|
"logits/chosen": -0.8298582434654236,
|
|
"logits/rejected": -0.8175155520439148,
|
|
"logps/chosen": -483.71356201171875,
|
|
"logps/ref_chosen": -272.5313415527344,
|
|
"logps/ref_rejected": -239.55735778808594,
|
|
"logps/rejected": -508.89288330078125,
|
|
"loss": 5.0353,
|
|
"margin_dpo/margin_mean": 58.153263092041016,
|
|
"margin_dpo/margin_std": 146.24984741210938,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 0.8146596858638744,
|
|
"fcm_dpo/beta": 0.003913066349923611,
|
|
"fcm_dpo/delta": -0.007896999828517437,
|
|
"fcm_dpo/margin": 82.79248809814453,
|
|
"fcm_dpo/q_t": 0.42776596546173096,
|
|
"grad_norm": 78.87464141845703,
|
|
"learning_rate": 5.1244476161413806e-08,
|
|
"logits/chosen": -0.8589950203895569,
|
|
"logits/rejected": -0.8571079969406128,
|
|
"logps/chosen": -489.0984802246094,
|
|
"logps/ref_chosen": -281.0892639160156,
|
|
"logps/ref_rejected": -246.50045776367188,
|
|
"logps/rejected": -537.3021240234375,
|
|
"loss": 4.7148,
|
|
"margin_dpo/margin_mean": 82.79248809814453,
|
|
"margin_dpo/margin_std": 149.3322296142578,
|
|
"step": 389
|
|
},
|
|
{
|
|
"epoch": 0.8167539267015707,
|
|
"fcm_dpo/beta": 0.003945834934711456,
|
|
"fcm_dpo/delta": -0.002285042777657509,
|
|
"fcm_dpo/margin": 89.30833435058594,
|
|
"fcm_dpo/q_t": 0.41917407512664795,
|
|
"grad_norm": 66.9520034790039,
|
|
"learning_rate": 5.013930914912476e-08,
|
|
"logits/chosen": -0.8890639543533325,
|
|
"logits/rejected": -0.8983543515205383,
|
|
"logps/chosen": -483.90277099609375,
|
|
"logps/ref_chosen": -283.98748779296875,
|
|
"logps/ref_rejected": -283.465087890625,
|
|
"logps/rejected": -572.688720703125,
|
|
"loss": 4.5633,
|
|
"margin_dpo/margin_mean": 89.3083267211914,
|
|
"margin_dpo/margin_std": 138.32273864746094,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.818848167539267,
|
|
"fcm_dpo/beta": 0.0038222242146730423,
|
|
"fcm_dpo/delta": -0.038259271532297134,
|
|
"fcm_dpo/margin": 95.59449768066406,
|
|
"fcm_dpo/q_t": 0.4170438051223755,
|
|
"grad_norm": 66.74435424804688,
|
|
"learning_rate": 4.904486005914027e-08,
|
|
"logits/chosen": -0.8340939879417419,
|
|
"logits/rejected": -0.81984543800354,
|
|
"logps/chosen": -498.55877685546875,
|
|
"logps/ref_chosen": -283.86138916015625,
|
|
"logps/ref_rejected": -263.5093688964844,
|
|
"logps/rejected": -573.8013305664062,
|
|
"loss": 4.5437,
|
|
"margin_dpo/margin_mean": 95.59449768066406,
|
|
"margin_dpo/margin_std": 147.83688354492188,
|
|
"step": 391
|
|
},
|
|
{
|
|
"epoch": 0.8209424083769633,
|
|
"fcm_dpo/beta": 0.0036374391056597233,
|
|
"fcm_dpo/delta": -0.05550744757056236,
|
|
"fcm_dpo/margin": 108.41232299804688,
|
|
"fcm_dpo/q_t": 0.4071521759033203,
|
|
"grad_norm": 60.498085021972656,
|
|
"learning_rate": 4.796118758344353e-08,
|
|
"logits/chosen": -0.8120825886726379,
|
|
"logits/rejected": -0.8258223533630371,
|
|
"logps/chosen": -504.80535888671875,
|
|
"logps/ref_chosen": -310.070068359375,
|
|
"logps/ref_rejected": -252.89817810058594,
|
|
"logps/rejected": -556.0457153320312,
|
|
"loss": 4.3798,
|
|
"margin_dpo/margin_mean": 108.41232299804688,
|
|
"margin_dpo/margin_std": 132.855712890625,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 0.8230366492146597,
|
|
"fcm_dpo/beta": 0.003737508552148938,
|
|
"fcm_dpo/delta": 0.06319691240787506,
|
|
"fcm_dpo/margin": 80.97676849365234,
|
|
"fcm_dpo/q_t": 0.43021535873413086,
|
|
"grad_norm": 58.454811096191406,
|
|
"learning_rate": 4.688834983610082e-08,
|
|
"logits/chosen": -0.8596187829971313,
|
|
"logits/rejected": -0.8425779342651367,
|
|
"logps/chosen": -482.13433837890625,
|
|
"logps/ref_chosen": -286.7156677246094,
|
|
"logps/ref_rejected": -230.00357055664062,
|
|
"logps/rejected": -506.39910888671875,
|
|
"loss": 4.6788,
|
|
"margin_dpo/margin_mean": 80.97676849365234,
|
|
"margin_dpo/margin_std": 135.61444091796875,
|
|
"step": 393
|
|
},
|
|
{
|
|
"epoch": 0.8251308900523561,
|
|
"fcm_dpo/beta": 0.0038439815398305655,
|
|
"fcm_dpo/delta": 0.020402822643518448,
|
|
"fcm_dpo/margin": 80.80021667480469,
|
|
"fcm_dpo/q_t": 0.4315696358680725,
|
|
"grad_norm": 80.42355346679688,
|
|
"learning_rate": 4.582640435014459e-08,
|
|
"logits/chosen": -0.8997487425804138,
|
|
"logits/rejected": -0.8925558924674988,
|
|
"logps/chosen": -524.9955444335938,
|
|
"logps/ref_chosen": -325.9934387207031,
|
|
"logps/ref_rejected": -317.42706298828125,
|
|
"logps/rejected": -597.2293701171875,
|
|
"loss": 4.7637,
|
|
"margin_dpo/margin_mean": 80.80021667480469,
|
|
"margin_dpo/margin_std": 155.62696838378906,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 0.8272251308900523,
|
|
"fcm_dpo/beta": 0.0038858535699546337,
|
|
"fcm_dpo/delta": -0.007227412424981594,
|
|
"fcm_dpo/margin": 104.61137390136719,
|
|
"fcm_dpo/q_t": 0.41017356514930725,
|
|
"grad_norm": 86.83416748046875,
|
|
"learning_rate": 4.477540807448832e-08,
|
|
"logits/chosen": -0.8377059698104858,
|
|
"logits/rejected": -0.8466890454292297,
|
|
"logps/chosen": -462.9100036621094,
|
|
"logps/ref_chosen": -268.90081787109375,
|
|
"logps/ref_rejected": -272.85809326171875,
|
|
"logps/rejected": -571.4786376953125,
|
|
"loss": 4.4153,
|
|
"margin_dpo/margin_mean": 104.61136627197266,
|
|
"margin_dpo/margin_std": 150.77389526367188,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.8293193717277487,
|
|
"fcm_dpo/beta": 0.0039118435233831406,
|
|
"fcm_dpo/delta": 0.03104759007692337,
|
|
"fcm_dpo/margin": 82.23450469970703,
|
|
"fcm_dpo/q_t": 0.42661815881729126,
|
|
"grad_norm": 90.65218353271484,
|
|
"learning_rate": 4.373541737087263e-08,
|
|
"logits/chosen": -0.8444753885269165,
|
|
"logits/rejected": -0.8268455862998962,
|
|
"logps/chosen": -496.99774169921875,
|
|
"logps/ref_chosen": -291.19830322265625,
|
|
"logps/ref_rejected": -253.2803955078125,
|
|
"logps/rejected": -541.3143310546875,
|
|
"loss": 4.6647,
|
|
"margin_dpo/margin_mean": 82.2344970703125,
|
|
"margin_dpo/margin_std": 138.35423278808594,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 0.831413612565445,
|
|
"fcm_dpo/beta": 0.0039418223313987255,
|
|
"fcm_dpo/delta": -0.02278411015868187,
|
|
"fcm_dpo/margin": 79.29293823242188,
|
|
"fcm_dpo/q_t": 0.4295212924480438,
|
|
"grad_norm": 88.5928726196289,
|
|
"learning_rate": 4.270648801084295e-08,
|
|
"logits/chosen": -0.8564522862434387,
|
|
"logits/rejected": -0.8358593583106995,
|
|
"logps/chosen": -506.43353271484375,
|
|
"logps/ref_chosen": -309.8224182128906,
|
|
"logps/ref_rejected": -291.9057922363281,
|
|
"logps/rejected": -567.809814453125,
|
|
"loss": 4.7629,
|
|
"margin_dpo/margin_mean": 79.29293823242188,
|
|
"margin_dpo/margin_std": 146.21200561523438,
|
|
"step": 397
|
|
},
|
|
{
|
|
"epoch": 0.8335078534031414,
|
|
"fcm_dpo/beta": 0.0038612037897109985,
|
|
"fcm_dpo/delta": 0.01422278955578804,
|
|
"fcm_dpo/margin": 67.4949722290039,
|
|
"fcm_dpo/q_t": 0.4390268921852112,
|
|
"grad_norm": 99.15848541259766,
|
|
"learning_rate": 4.168867517275806e-08,
|
|
"logits/chosen": -0.76848304271698,
|
|
"logits/rejected": -0.8057135343551636,
|
|
"logps/chosen": -519.5203247070312,
|
|
"logps/ref_chosen": -297.8135070800781,
|
|
"logps/ref_rejected": -270.5025634765625,
|
|
"logps/rejected": -559.704345703125,
|
|
"loss": 5.0489,
|
|
"margin_dpo/margin_mean": 67.4949722290039,
|
|
"margin_dpo/margin_std": 172.52947998046875,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 0.8356020942408376,
|
|
"fcm_dpo/beta": 0.0039389231242239475,
|
|
"fcm_dpo/delta": 0.028249990195035934,
|
|
"fcm_dpo/margin": 84.23013305664062,
|
|
"fcm_dpo/q_t": 0.4258180558681488,
|
|
"grad_norm": 118.06265258789062,
|
|
"learning_rate": 4.0682033438831584e-08,
|
|
"logits/chosen": -0.8657823204994202,
|
|
"logits/rejected": -0.8276122808456421,
|
|
"logps/chosen": -506.72381591796875,
|
|
"logps/ref_chosen": -292.8467712402344,
|
|
"logps/ref_rejected": -268.3638916015625,
|
|
"logps/rejected": -566.4711303710938,
|
|
"loss": 4.6836,
|
|
"margin_dpo/margin_mean": 84.23013305664062,
|
|
"margin_dpo/margin_std": 150.7490234375,
|
|
"step": 399
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"fcm_dpo/beta": 0.0039605312049388885,
|
|
"fcm_dpo/delta": -0.0016172388568520546,
|
|
"fcm_dpo/margin": 82.38078308105469,
|
|
"fcm_dpo/q_t": 0.4250349998474121,
|
|
"grad_norm": 81.15113830566406,
|
|
"learning_rate": 3.968661679220467e-08,
|
|
"logits/chosen": -0.9057132601737976,
|
|
"logits/rejected": -0.9061620831489563,
|
|
"logps/chosen": -469.662841796875,
|
|
"logps/ref_chosen": -263.6763916015625,
|
|
"logps/ref_rejected": -258.67266845703125,
|
|
"logps/rejected": -547.0399169921875,
|
|
"loss": 4.6885,
|
|
"margin_dpo/margin_mean": 82.38077545166016,
|
|
"margin_dpo/margin_std": 143.50645446777344,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"eval_fcm_dpo/beta": 0.004001657944172621,
|
|
"eval_logits/chosen": -0.8691577911376953,
|
|
"eval_logits/rejected": -0.8528784513473511,
|
|
"eval_logps/chosen": -496.1020812988281,
|
|
"eval_logps/ref_chosen": -287.8268127441406,
|
|
"eval_logps/ref_rejected": -266.9300231933594,
|
|
"eval_logps/rejected": -563.8033447265625,
|
|
"eval_loss": 0.5783530473709106,
|
|
"eval_margin_dpo/margin_mean": 88.59801483154297,
|
|
"eval_margin_dpo/margin_std": 143.06631469726562,
|
|
"eval_runtime": 78.6395,
|
|
"eval_samples_per_second": 25.433,
|
|
"eval_steps_per_second": 3.179,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8397905759162304,
|
|
"fcm_dpo/beta": 0.003983091097325087,
|
|
"fcm_dpo/delta": -0.011052620597183704,
|
|
"fcm_dpo/margin": 89.36437225341797,
|
|
"fcm_dpo/q_t": 0.419382244348526,
|
|
"grad_norm": 93.5476303100586,
|
|
"learning_rate": 3.8702478614051345e-08,
|
|
"logits/chosen": -0.8399940729141235,
|
|
"logits/rejected": -0.8391348719596863,
|
|
"logps/chosen": -523.5870361328125,
|
|
"logps/ref_chosen": -318.2853088378906,
|
|
"logps/ref_rejected": -293.75225830078125,
|
|
"logps/rejected": -588.4183349609375,
|
|
"loss": 4.5995,
|
|
"margin_dpo/margin_mean": 89.36437225341797,
|
|
"margin_dpo/margin_std": 148.24929809570312,
|
|
"step": 401
|
|
},
|
|
{
|
|
"epoch": 0.8418848167539267,
|
|
"fcm_dpo/beta": 0.004000155255198479,
|
|
"fcm_dpo/delta": 0.028147714212536812,
|
|
"fcm_dpo/margin": 87.35748291015625,
|
|
"fcm_dpo/q_t": 0.4228289723396301,
|
|
"grad_norm": 73.04254150390625,
|
|
"learning_rate": 3.772967168071517e-08,
|
|
"logits/chosen": -0.8962114453315735,
|
|
"logits/rejected": -0.8734457492828369,
|
|
"logps/chosen": -506.75689697265625,
|
|
"logps/ref_chosen": -309.4278564453125,
|
|
"logps/ref_rejected": -282.0279846191406,
|
|
"logps/rejected": -566.7145385742188,
|
|
"loss": 4.6107,
|
|
"margin_dpo/margin_mean": 87.35748291015625,
|
|
"margin_dpo/margin_std": 151.18124389648438,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 0.8439790575916231,
|
|
"fcm_dpo/beta": 0.003969723358750343,
|
|
"fcm_dpo/delta": -0.08291508257389069,
|
|
"fcm_dpo/margin": 120.73661804199219,
|
|
"fcm_dpo/q_t": 0.39133724570274353,
|
|
"grad_norm": 61.04887390136719,
|
|
"learning_rate": 3.676824816087978e-08,
|
|
"logits/chosen": -0.8727239966392517,
|
|
"logits/rejected": -0.8497612476348877,
|
|
"logps/chosen": -505.31695556640625,
|
|
"logps/ref_chosen": -309.0284729003906,
|
|
"logps/ref_rejected": -272.9622497558594,
|
|
"logps/rejected": -589.9873046875,
|
|
"loss": 4.1603,
|
|
"margin_dpo/margin_mean": 120.73661804199219,
|
|
"margin_dpo/margin_std": 138.56192016601562,
|
|
"step": 403
|
|
},
|
|
{
|
|
"epoch": 0.8460732984293193,
|
|
"fcm_dpo/beta": 0.0038393645081669092,
|
|
"fcm_dpo/delta": 0.025593645870685577,
|
|
"fcm_dpo/margin": 90.4571533203125,
|
|
"fcm_dpo/q_t": 0.42148569226264954,
|
|
"grad_norm": 75.76798248291016,
|
|
"learning_rate": 3.581825961277074e-08,
|
|
"logits/chosen": -0.9005187749862671,
|
|
"logits/rejected": -0.876252293586731,
|
|
"logps/chosen": -509.41229248046875,
|
|
"logps/ref_chosen": -297.2837219238281,
|
|
"logps/ref_rejected": -256.99041748046875,
|
|
"logps/rejected": -559.5761108398438,
|
|
"loss": 4.6459,
|
|
"margin_dpo/margin_mean": 90.45716094970703,
|
|
"margin_dpo/margin_std": 158.7395477294922,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 0.8481675392670157,
|
|
"fcm_dpo/beta": 0.003894682740792632,
|
|
"fcm_dpo/delta": 0.0062955510802567005,
|
|
"fcm_dpo/margin": 95.108154296875,
|
|
"fcm_dpo/q_t": 0.41595664620399475,
|
|
"grad_norm": 55.920188903808594,
|
|
"learning_rate": 3.487975698139084e-08,
|
|
"logits/chosen": -0.8308718800544739,
|
|
"logits/rejected": -0.839844822883606,
|
|
"logps/chosen": -462.46856689453125,
|
|
"logps/ref_chosen": -257.96533203125,
|
|
"logps/ref_rejected": -255.811279296875,
|
|
"logps/rejected": -555.4227294921875,
|
|
"loss": 4.5156,
|
|
"margin_dpo/margin_mean": 95.108154296875,
|
|
"margin_dpo/margin_std": 146.5327911376953,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.8502617801047121,
|
|
"fcm_dpo/beta": 0.003938999027013779,
|
|
"fcm_dpo/delta": 0.024059396237134933,
|
|
"fcm_dpo/margin": 70.01969909667969,
|
|
"fcm_dpo/q_t": 0.43556931614875793,
|
|
"grad_norm": 102.48121643066406,
|
|
"learning_rate": 3.3952790595787986e-08,
|
|
"logits/chosen": -0.8572216033935547,
|
|
"logits/rejected": -0.8341256380081177,
|
|
"logps/chosen": -500.6681823730469,
|
|
"logps/ref_chosen": -285.1810607910156,
|
|
"logps/ref_rejected": -264.41351318359375,
|
|
"logps/rejected": -549.9203491210938,
|
|
"loss": 4.7904,
|
|
"margin_dpo/margin_mean": 70.01969909667969,
|
|
"margin_dpo/margin_std": 130.00796508789062,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 0.8523560209424084,
|
|
"fcm_dpo/beta": 0.003916039131581783,
|
|
"fcm_dpo/delta": -0.02865220047533512,
|
|
"fcm_dpo/margin": 95.51104736328125,
|
|
"fcm_dpo/q_t": 0.4164567291736603,
|
|
"grad_norm": 60.025779724121094,
|
|
"learning_rate": 3.303741016635614e-08,
|
|
"logits/chosen": -0.8459261655807495,
|
|
"logits/rejected": -0.8647856712341309,
|
|
"logps/chosen": -484.513427734375,
|
|
"logps/ref_chosen": -265.23809814453125,
|
|
"logps/ref_rejected": -219.0631561279297,
|
|
"logps/rejected": -533.849609375,
|
|
"loss": 4.563,
|
|
"margin_dpo/margin_mean": 95.51104736328125,
|
|
"margin_dpo/margin_std": 157.53103637695312,
|
|
"step": 407
|
|
},
|
|
{
|
|
"epoch": 0.8544502617801047,
|
|
"fcm_dpo/beta": 0.003969256300479174,
|
|
"fcm_dpo/delta": -0.00904204323887825,
|
|
"fcm_dpo/margin": 96.21162414550781,
|
|
"fcm_dpo/q_t": 0.41344723105430603,
|
|
"grad_norm": 95.24205017089844,
|
|
"learning_rate": 3.2133664782169944e-08,
|
|
"logits/chosen": -0.8817875981330872,
|
|
"logits/rejected": -0.8740147352218628,
|
|
"logps/chosen": -494.42425537109375,
|
|
"logps/ref_chosen": -296.9726257324219,
|
|
"logps/ref_rejected": -295.4786376953125,
|
|
"logps/rejected": -589.1419067382812,
|
|
"loss": 4.5008,
|
|
"margin_dpo/margin_mean": 96.21162414550781,
|
|
"margin_dpo/margin_std": 141.05984497070312,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 0.856544502617801,
|
|
"fcm_dpo/beta": 0.0038298959843814373,
|
|
"fcm_dpo/delta": -0.018867127597332,
|
|
"fcm_dpo/margin": 99.35574340820312,
|
|
"fcm_dpo/q_t": 0.4180399775505066,
|
|
"grad_norm": 59.48664093017578,
|
|
"learning_rate": 3.12416029083514e-08,
|
|
"logits/chosen": -0.8571524024009705,
|
|
"logits/rejected": -0.8414179086685181,
|
|
"logps/chosen": -496.5021057128906,
|
|
"logps/ref_chosen": -287.37933349609375,
|
|
"logps/ref_rejected": -275.80291748046875,
|
|
"logps/rejected": -584.2814331054688,
|
|
"loss": 4.6107,
|
|
"margin_dpo/margin_mean": 99.35574340820312,
|
|
"margin_dpo/margin_std": 180.15744018554688,
|
|
"step": 409
|
|
},
|
|
{
|
|
"epoch": 0.8586387434554974,
|
|
"fcm_dpo/beta": 0.003783642081543803,
|
|
"fcm_dpo/delta": 0.0071961102075874805,
|
|
"fcm_dpo/margin": 87.63327026367188,
|
|
"fcm_dpo/q_t": 0.4247705340385437,
|
|
"grad_norm": 72.14511108398438,
|
|
"learning_rate": 3.036127238347164e-08,
|
|
"logits/chosen": -0.8766495585441589,
|
|
"logits/rejected": -0.8835284113883972,
|
|
"logps/chosen": -481.9991455078125,
|
|
"logps/ref_chosen": -281.7801818847656,
|
|
"logps/ref_rejected": -266.7550354003906,
|
|
"logps/rejected": -554.6072998046875,
|
|
"loss": 4.6469,
|
|
"margin_dpo/margin_mean": 87.63327026367188,
|
|
"margin_dpo/margin_std": 149.70281982421875,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.8607329842931937,
|
|
"fcm_dpo/beta": 0.00382496346719563,
|
|
"fcm_dpo/delta": -0.014396967366337776,
|
|
"fcm_dpo/margin": 101.3914794921875,
|
|
"fcm_dpo/q_t": 0.41269832849502563,
|
|
"grad_norm": 72.0369644165039,
|
|
"learning_rate": 2.9492720416985e-08,
|
|
"logits/chosen": -0.8845562934875488,
|
|
"logits/rejected": -0.8408842086791992,
|
|
"logps/chosen": -481.53240966796875,
|
|
"logps/ref_chosen": -281.5872497558594,
|
|
"logps/ref_rejected": -254.78916931152344,
|
|
"logps/rejected": -556.1257934570312,
|
|
"loss": 4.4243,
|
|
"margin_dpo/margin_mean": 101.3914794921875,
|
|
"margin_dpo/margin_std": 137.44744873046875,
|
|
"step": 411
|
|
},
|
|
{
|
|
"epoch": 0.86282722513089,
|
|
"fcm_dpo/beta": 0.003669111290946603,
|
|
"fcm_dpo/delta": -0.013924513012170792,
|
|
"fcm_dpo/margin": 79.34294128417969,
|
|
"fcm_dpo/q_t": 0.43426990509033203,
|
|
"grad_norm": 120.3460922241211,
|
|
"learning_rate": 2.863599358669755e-08,
|
|
"logits/chosen": -0.8453624248504639,
|
|
"logits/rejected": -0.85740065574646,
|
|
"logps/chosen": -497.730224609375,
|
|
"logps/ref_chosen": -276.5341796875,
|
|
"logps/ref_rejected": -273.8751220703125,
|
|
"logps/rejected": -574.4141235351562,
|
|
"loss": 4.7748,
|
|
"margin_dpo/margin_mean": 79.34294128417969,
|
|
"margin_dpo/margin_std": 146.27236938476562,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 0.8649214659685864,
|
|
"fcm_dpo/beta": 0.0038752357941120863,
|
|
"fcm_dpo/delta": 0.08048185706138611,
|
|
"fcm_dpo/margin": 76.79857635498047,
|
|
"fcm_dpo/q_t": 0.43248826265335083,
|
|
"grad_norm": 86.02536010742188,
|
|
"learning_rate": 2.7791137836269158e-08,
|
|
"logits/chosen": -0.8663272261619568,
|
|
"logits/rejected": -0.8638010621070862,
|
|
"logps/chosen": -479.69171142578125,
|
|
"logps/ref_chosen": -271.2745666503906,
|
|
"logps/ref_rejected": -270.16912841796875,
|
|
"logps/rejected": -555.3848876953125,
|
|
"loss": 4.7365,
|
|
"margin_dpo/margin_mean": 76.79857635498047,
|
|
"margin_dpo/margin_std": 141.86328125,
|
|
"step": 413
|
|
},
|
|
{
|
|
"epoch": 0.8670157068062827,
|
|
"fcm_dpo/beta": 0.004013517405837774,
|
|
"fcm_dpo/delta": -0.019187791272997856,
|
|
"fcm_dpo/margin": 97.65512084960938,
|
|
"fcm_dpo/q_t": 0.412454217672348,
|
|
"grad_norm": 71.61355590820312,
|
|
"learning_rate": 2.6958198472749717e-08,
|
|
"logits/chosen": -0.9010047316551208,
|
|
"logits/rejected": -0.8984125852584839,
|
|
"logps/chosen": -504.4794616699219,
|
|
"logps/ref_chosen": -297.11505126953125,
|
|
"logps/ref_rejected": -271.7034606933594,
|
|
"logps/rejected": -576.7229614257812,
|
|
"loss": 4.4714,
|
|
"margin_dpo/margin_mean": 97.65512084960938,
|
|
"margin_dpo/margin_std": 143.94935607910156,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 0.8691099476439791,
|
|
"fcm_dpo/beta": 0.003955254331231117,
|
|
"fcm_dpo/delta": 0.011070379987359047,
|
|
"fcm_dpo/margin": 91.44115447998047,
|
|
"fcm_dpo/q_t": 0.4168679714202881,
|
|
"grad_norm": 71.55652618408203,
|
|
"learning_rate": 2.613722016414943e-08,
|
|
"logits/chosen": -0.8896200060844421,
|
|
"logits/rejected": -0.875502347946167,
|
|
"logps/chosen": -498.7607116699219,
|
|
"logps/ref_chosen": -297.6926574707031,
|
|
"logps/ref_rejected": -279.0503234863281,
|
|
"logps/rejected": -571.5595703125,
|
|
"loss": 4.5043,
|
|
"margin_dpo/margin_mean": 91.44115447998047,
|
|
"margin_dpo/margin_std": 134.12338256835938,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.8712041884816754,
|
|
"fcm_dpo/beta": 0.0038398101460188627,
|
|
"fcm_dpo/delta": -0.04415631666779518,
|
|
"fcm_dpo/margin": 108.07146453857422,
|
|
"fcm_dpo/q_t": 0.404560923576355,
|
|
"grad_norm": 62.76100158691406,
|
|
"learning_rate": 2.5328246937043525e-08,
|
|
"logits/chosen": -0.8924229145050049,
|
|
"logits/rejected": -0.8925044536590576,
|
|
"logps/chosen": -505.1080627441406,
|
|
"logps/ref_chosen": -311.8255615234375,
|
|
"logps/ref_rejected": -268.6170654296875,
|
|
"logps/rejected": -569.9710693359375,
|
|
"loss": 4.3709,
|
|
"margin_dpo/margin_mean": 108.07145690917969,
|
|
"margin_dpo/margin_std": 146.09751892089844,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 0.8732984293193717,
|
|
"fcm_dpo/beta": 0.0037508481182157993,
|
|
"fcm_dpo/delta": -0.03793646767735481,
|
|
"fcm_dpo/margin": 87.92816162109375,
|
|
"fcm_dpo/q_t": 0.42599597573280334,
|
|
"grad_norm": 88.90306091308594,
|
|
"learning_rate": 2.4531322174210973e-08,
|
|
"logits/chosen": -0.8363179564476013,
|
|
"logits/rejected": -0.8270238041877747,
|
|
"logps/chosen": -521.0562744140625,
|
|
"logps/ref_chosen": -310.43682861328125,
|
|
"logps/ref_rejected": -277.15283203125,
|
|
"logps/rejected": -575.700439453125,
|
|
"loss": 4.67,
|
|
"margin_dpo/margin_mean": 87.92816162109375,
|
|
"margin_dpo/margin_std": 148.5767822265625,
|
|
"step": 417
|
|
},
|
|
{
|
|
"epoch": 0.875392670157068,
|
|
"fcm_dpo/beta": 0.003646118799224496,
|
|
"fcm_dpo/delta": -0.036676209419965744,
|
|
"fcm_dpo/margin": 90.21328735351562,
|
|
"fcm_dpo/q_t": 0.425502210855484,
|
|
"grad_norm": 65.71238708496094,
|
|
"learning_rate": 2.3746488612308295e-08,
|
|
"logits/chosen": -0.8376297354698181,
|
|
"logits/rejected": -0.8152227401733398,
|
|
"logps/chosen": -498.4665222167969,
|
|
"logps/ref_chosen": -278.49591064453125,
|
|
"logps/ref_rejected": -276.56671142578125,
|
|
"logps/rejected": -586.7506103515625,
|
|
"loss": 4.674,
|
|
"margin_dpo/margin_mean": 90.21328735351562,
|
|
"margin_dpo/margin_std": 156.35293579101562,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 0.8774869109947644,
|
|
"fcm_dpo/beta": 0.003632680047303438,
|
|
"fcm_dpo/delta": 0.0225251205265522,
|
|
"fcm_dpo/margin": 93.54913330078125,
|
|
"fcm_dpo/q_t": 0.4209168255329132,
|
|
"grad_norm": 73.43122100830078,
|
|
"learning_rate": 2.297378833957761e-08,
|
|
"logits/chosen": -0.879563570022583,
|
|
"logits/rejected": -0.8511139750480652,
|
|
"logps/chosen": -518.3619384765625,
|
|
"logps/ref_chosen": -298.9002380371094,
|
|
"logps/ref_rejected": -246.1540985107422,
|
|
"logps/rejected": -559.1648559570312,
|
|
"loss": 4.6436,
|
|
"margin_dpo/margin_mean": 93.54913330078125,
|
|
"margin_dpo/margin_std": 161.06155395507812,
|
|
"step": 419
|
|
},
|
|
{
|
|
"epoch": 0.8795811518324608,
|
|
"fcm_dpo/beta": 0.0036652141716331244,
|
|
"fcm_dpo/delta": 0.0008689370006322861,
|
|
"fcm_dpo/margin": 108.68309020996094,
|
|
"fcm_dpo/q_t": 0.4112725853919983,
|
|
"grad_norm": 93.50495147705078,
|
|
"learning_rate": 2.2213262793589482e-08,
|
|
"logits/chosen": -0.8324740529060364,
|
|
"logits/rejected": -0.8063269853591919,
|
|
"logps/chosen": -477.1441345214844,
|
|
"logps/ref_chosen": -264.5608825683594,
|
|
"logps/ref_rejected": -245.67031860351562,
|
|
"logps/rejected": -566.9367065429688,
|
|
"loss": 4.5493,
|
|
"margin_dpo/margin_mean": 108.68309020996094,
|
|
"margin_dpo/margin_std": 184.1924285888672,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.881675392670157,
|
|
"fcm_dpo/beta": 0.0037031802348792553,
|
|
"fcm_dpo/delta": 0.020971816033124924,
|
|
"fcm_dpo/margin": 92.56175994873047,
|
|
"fcm_dpo/q_t": 0.41996240615844727,
|
|
"grad_norm": 95.28728485107422,
|
|
"learning_rate": 2.1464952759020856e-08,
|
|
"logits/chosen": -0.9097946882247925,
|
|
"logits/rejected": -0.8875160217285156,
|
|
"logps/chosen": -492.8577880859375,
|
|
"logps/ref_chosen": -297.70501708984375,
|
|
"logps/ref_rejected": -243.74771118164062,
|
|
"logps/rejected": -531.4622192382812,
|
|
"loss": 4.5551,
|
|
"margin_dpo/margin_mean": 92.56175994873047,
|
|
"margin_dpo/margin_std": 136.10009765625,
|
|
"step": 421
|
|
},
|
|
{
|
|
"epoch": 0.8837696335078534,
|
|
"fcm_dpo/beta": 0.0038025444373488426,
|
|
"fcm_dpo/delta": 0.02667798101902008,
|
|
"fcm_dpo/margin": 92.04246520996094,
|
|
"fcm_dpo/q_t": 0.42156127095222473,
|
|
"grad_norm": 65.87984466552734,
|
|
"learning_rate": 2.07288983654679e-08,
|
|
"logits/chosen": -0.7622085809707642,
|
|
"logits/rejected": -0.8034301996231079,
|
|
"logps/chosen": -495.5381164550781,
|
|
"logps/ref_chosen": -288.3587646484375,
|
|
"logps/ref_rejected": -256.4377746582031,
|
|
"logps/rejected": -555.6596069335938,
|
|
"loss": 4.6091,
|
|
"margin_dpo/margin_mean": 92.04246520996094,
|
|
"margin_dpo/margin_std": 156.10035705566406,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 0.8858638743455497,
|
|
"fcm_dpo/beta": 0.0037769617047160864,
|
|
"fcm_dpo/delta": -0.0050736647099256516,
|
|
"fcm_dpo/margin": 99.29490661621094,
|
|
"fcm_dpo/q_t": 0.41419917345046997,
|
|
"grad_norm": 78.68079376220703,
|
|
"learning_rate": 2.0005139085293942e-08,
|
|
"logits/chosen": -0.8863624334335327,
|
|
"logits/rejected": -0.8650214076042175,
|
|
"logps/chosen": -506.8820495605469,
|
|
"logps/ref_chosen": -296.00701904296875,
|
|
"logps/ref_rejected": -261.3480529785156,
|
|
"logps/rejected": -571.5179443359375,
|
|
"loss": 4.5105,
|
|
"margin_dpo/margin_mean": 99.29490661621094,
|
|
"margin_dpo/margin_std": 152.7718963623047,
|
|
"step": 423
|
|
},
|
|
{
|
|
"epoch": 0.8879581151832461,
|
|
"fcm_dpo/beta": 0.0038537802174687386,
|
|
"fcm_dpo/delta": 0.03373270109295845,
|
|
"fcm_dpo/margin": 95.16132354736328,
|
|
"fcm_dpo/q_t": 0.4156450033187866,
|
|
"grad_norm": 85.12425231933594,
|
|
"learning_rate": 1.9293713731512673e-08,
|
|
"logits/chosen": -0.8723393678665161,
|
|
"logits/rejected": -0.86158686876297,
|
|
"logps/chosen": -502.749755859375,
|
|
"logps/ref_chosen": -309.421875,
|
|
"logps/ref_rejected": -249.14886474609375,
|
|
"logps/rejected": -537.6380615234375,
|
|
"loss": 4.4869,
|
|
"margin_dpo/margin_mean": 95.16132354736328,
|
|
"margin_dpo/margin_std": 134.9690704345703,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 0.8900523560209425,
|
|
"fcm_dpo/beta": 0.0038695167750120163,
|
|
"fcm_dpo/delta": -0.025678057223558426,
|
|
"fcm_dpo/margin": 84.49606323242188,
|
|
"fcm_dpo/q_t": 0.42579007148742676,
|
|
"grad_norm": 82.99080657958984,
|
|
"learning_rate": 1.8594660455706763e-08,
|
|
"logits/chosen": -0.849087655544281,
|
|
"logits/rejected": -0.8525911569595337,
|
|
"logps/chosen": -484.4912109375,
|
|
"logps/ref_chosen": -280.50909423828125,
|
|
"logps/ref_rejected": -276.8252258300781,
|
|
"logps/rejected": -565.3033447265625,
|
|
"loss": 4.6773,
|
|
"margin_dpo/margin_mean": 84.49606323242188,
|
|
"margin_dpo/margin_std": 146.6011199951172,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.8921465968586387,
|
|
"fcm_dpo/beta": 0.003897757502272725,
|
|
"fcm_dpo/delta": 0.036198608577251434,
|
|
"fcm_dpo/margin": 93.49591064453125,
|
|
"fcm_dpo/q_t": 0.41721320152282715,
|
|
"grad_norm": 73.07560729980469,
|
|
"learning_rate": 1.7908016745981856e-08,
|
|
"logits/chosen": -0.8812398314476013,
|
|
"logits/rejected": -0.8626936674118042,
|
|
"logps/chosen": -500.7666931152344,
|
|
"logps/ref_chosen": -292.78521728515625,
|
|
"logps/ref_rejected": -255.62698364257812,
|
|
"logps/rejected": -557.1043701171875,
|
|
"loss": 4.5514,
|
|
"margin_dpo/margin_mean": 93.49591064453125,
|
|
"margin_dpo/margin_std": 146.1980743408203,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 0.8942408376963351,
|
|
"fcm_dpo/beta": 0.003795379539951682,
|
|
"fcm_dpo/delta": -0.13308368623256683,
|
|
"fcm_dpo/margin": 130.365966796875,
|
|
"fcm_dpo/q_t": 0.38931721448898315,
|
|
"grad_norm": 111.64500427246094,
|
|
"learning_rate": 1.7233819424956247e-08,
|
|
"logits/chosen": -0.8529506921768188,
|
|
"logits/rejected": -0.817730724811554,
|
|
"logps/chosen": -492.5892333984375,
|
|
"logps/ref_chosen": -288.7687072753906,
|
|
"logps/ref_rejected": -268.4986572265625,
|
|
"logps/rejected": -602.6851806640625,
|
|
"loss": 4.2511,
|
|
"margin_dpo/margin_mean": 130.36595153808594,
|
|
"margin_dpo/margin_std": 172.0804901123047,
|
|
"step": 427
|
|
},
|
|
{
|
|
"epoch": 0.8963350785340314,
|
|
"fcm_dpo/beta": 0.0035148044116795063,
|
|
"fcm_dpo/delta": -0.02863123267889023,
|
|
"fcm_dpo/margin": 112.14768981933594,
|
|
"fcm_dpo/q_t": 0.40907180309295654,
|
|
"grad_norm": 72.58002471923828,
|
|
"learning_rate": 1.6572104647786245e-08,
|
|
"logits/chosen": -0.8079947233200073,
|
|
"logits/rejected": -0.837670624256134,
|
|
"logps/chosen": -522.7776489257812,
|
|
"logps/ref_chosen": -295.5209655761719,
|
|
"logps/ref_rejected": -275.71026611328125,
|
|
"logps/rejected": -615.1146850585938,
|
|
"loss": 4.4223,
|
|
"margin_dpo/margin_mean": 112.14769744873047,
|
|
"margin_dpo/margin_std": 153.65895080566406,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 0.8984293193717278,
|
|
"fcm_dpo/beta": 0.0034794604871422052,
|
|
"fcm_dpo/delta": 0.012085347436368465,
|
|
"fcm_dpo/margin": 96.80742645263672,
|
|
"fcm_dpo/q_t": 0.4216752350330353,
|
|
"grad_norm": 73.65662384033203,
|
|
"learning_rate": 1.5922907900227017e-08,
|
|
"logits/chosen": -0.8177259564399719,
|
|
"logits/rejected": -0.8231886625289917,
|
|
"logps/chosen": -485.8576965332031,
|
|
"logps/ref_chosen": -274.392333984375,
|
|
"logps/ref_rejected": -258.574462890625,
|
|
"logps/rejected": -566.8472900390625,
|
|
"loss": 4.6748,
|
|
"margin_dpo/margin_mean": 96.80743408203125,
|
|
"margin_dpo/margin_std": 170.624267578125,
|
|
"step": 429
|
|
},
|
|
{
|
|
"epoch": 0.900523560209424,
|
|
"fcm_dpo/beta": 0.003550901310518384,
|
|
"fcm_dpo/delta": 0.013876799494028091,
|
|
"fcm_dpo/margin": 88.0494613647461,
|
|
"fcm_dpo/q_t": 0.4287679195404053,
|
|
"grad_norm": 69.73419952392578,
|
|
"learning_rate": 1.5286263996730026e-08,
|
|
"logits/chosen": -0.8986079096794128,
|
|
"logits/rejected": -0.869090735912323,
|
|
"logps/chosen": -497.7119140625,
|
|
"logps/ref_chosen": -288.7391357421875,
|
|
"logps/ref_rejected": -268.6106262207031,
|
|
"logps/rejected": -565.6328125,
|
|
"loss": 4.6721,
|
|
"margin_dpo/margin_mean": 88.0494613647461,
|
|
"margin_dpo/margin_std": 147.50392150878906,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9026178010471204,
|
|
"fcm_dpo/beta": 0.0035896864719688892,
|
|
"fcm_dpo/delta": 0.020418308675289154,
|
|
"fcm_dpo/margin": 72.26271057128906,
|
|
"fcm_dpo/q_t": 0.4400396943092346,
|
|
"grad_norm": 72.33751678466797,
|
|
"learning_rate": 1.4662207078575684e-08,
|
|
"logits/chosen": -0.8635554909706116,
|
|
"logits/rejected": -0.8318578004837036,
|
|
"logps/chosen": -481.857177734375,
|
|
"logps/ref_chosen": -275.7247314453125,
|
|
"logps/ref_rejected": -268.91729736328125,
|
|
"logps/rejected": -547.3124389648438,
|
|
"loss": 4.8212,
|
|
"margin_dpo/margin_mean": 72.26271057128906,
|
|
"margin_dpo/margin_std": 134.55499267578125,
|
|
"step": 431
|
|
},
|
|
{
|
|
"epoch": 0.9047120418848168,
|
|
"fcm_dpo/beta": 0.003594242036342621,
|
|
"fcm_dpo/delta": 0.006677491124719381,
|
|
"fcm_dpo/margin": 99.71732330322266,
|
|
"fcm_dpo/q_t": 0.4194951355457306,
|
|
"grad_norm": 72.6844482421875,
|
|
"learning_rate": 1.40507706120426e-08,
|
|
"logits/chosen": -0.9026849269866943,
|
|
"logits/rejected": -0.8854382038116455,
|
|
"logps/chosen": -486.9830017089844,
|
|
"logps/ref_chosen": -291.42010498046875,
|
|
"logps/ref_rejected": -255.48202514648438,
|
|
"logps/rejected": -550.7622680664062,
|
|
"loss": 4.558,
|
|
"margin_dpo/margin_mean": 99.71731567382812,
|
|
"margin_dpo/margin_std": 157.44149780273438,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 0.9068062827225131,
|
|
"fcm_dpo/beta": 0.003687588730826974,
|
|
"fcm_dpo/delta": 0.06109008565545082,
|
|
"fcm_dpo/margin": 85.34246063232422,
|
|
"fcm_dpo/q_t": 0.42769885063171387,
|
|
"grad_norm": 72.68798065185547,
|
|
"learning_rate": 1.345198738661285e-08,
|
|
"logits/chosen": -0.8471591472625732,
|
|
"logits/rejected": -0.8419995307922363,
|
|
"logps/chosen": -460.63702392578125,
|
|
"logps/ref_chosen": -246.2268829345703,
|
|
"logps/ref_rejected": -253.65924072265625,
|
|
"logps/rejected": -553.4118041992188,
|
|
"loss": 4.6434,
|
|
"margin_dpo/margin_mean": 85.34246063232422,
|
|
"margin_dpo/margin_std": 140.0343017578125,
|
|
"step": 433
|
|
},
|
|
{
|
|
"epoch": 0.9089005235602095,
|
|
"fcm_dpo/beta": 0.0038499152287840843,
|
|
"fcm_dpo/delta": 0.029952505603432655,
|
|
"fcm_dpo/margin": 85.55077362060547,
|
|
"fcm_dpo/q_t": 0.4248325228691101,
|
|
"grad_norm": 72.78632354736328,
|
|
"learning_rate": 1.2865889513213628e-08,
|
|
"logits/chosen": -0.8568861484527588,
|
|
"logits/rejected": -0.8612566590309143,
|
|
"logps/chosen": -520.3125,
|
|
"logps/ref_chosen": -295.4618225097656,
|
|
"logps/ref_rejected": -256.2254333496094,
|
|
"logps/rejected": -566.6268920898438,
|
|
"loss": 4.6448,
|
|
"margin_dpo/margin_mean": 85.55077362060547,
|
|
"margin_dpo/margin_std": 144.971435546875,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 0.9109947643979057,
|
|
"fcm_dpo/beta": 0.0038717712741345167,
|
|
"fcm_dpo/delta": -0.01573629304766655,
|
|
"fcm_dpo/margin": 99.90605163574219,
|
|
"fcm_dpo/q_t": 0.4140922725200653,
|
|
"grad_norm": 112.00686645507812,
|
|
"learning_rate": 1.2292508422495157e-08,
|
|
"logits/chosen": -0.8434274196624756,
|
|
"logits/rejected": -0.831263542175293,
|
|
"logps/chosen": -464.3940124511719,
|
|
"logps/ref_chosen": -260.7384033203125,
|
|
"logps/ref_rejected": -248.5688018798828,
|
|
"logps/rejected": -552.1304321289062,
|
|
"loss": 4.4539,
|
|
"margin_dpo/margin_mean": 99.90605163574219,
|
|
"margin_dpo/margin_std": 141.65713500976562,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.9130890052356021,
|
|
"fcm_dpo/beta": 0.0039107538759708405,
|
|
"fcm_dpo/delta": 0.03949038311839104,
|
|
"fcm_dpo/margin": 78.44125366210938,
|
|
"fcm_dpo/q_t": 0.4342585504055023,
|
|
"grad_norm": 115.6869888305664,
|
|
"learning_rate": 1.1731874863145142e-08,
|
|
"logits/chosen": -0.8294715285301208,
|
|
"logits/rejected": -0.8246615529060364,
|
|
"logps/chosen": -540.929443359375,
|
|
"logps/ref_chosen": -319.3224792480469,
|
|
"logps/ref_rejected": -299.30322265625,
|
|
"logps/rejected": -599.3515014648438,
|
|
"loss": 4.8039,
|
|
"margin_dpo/margin_mean": 78.44124603271484,
|
|
"margin_dpo/margin_std": 165.59027099609375,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 0.9151832460732985,
|
|
"fcm_dpo/beta": 0.0038340180180966854,
|
|
"fcm_dpo/delta": -0.055059827864170074,
|
|
"fcm_dpo/margin": 108.45451354980469,
|
|
"fcm_dpo/q_t": 0.4066217243671417,
|
|
"grad_norm": 66.90118408203125,
|
|
"learning_rate": 1.118401890024001e-08,
|
|
"logits/chosen": -0.8663026094436646,
|
|
"logits/rejected": -0.8491877913475037,
|
|
"logps/chosen": -483.2367248535156,
|
|
"logps/ref_chosen": -278.82879638671875,
|
|
"logps/ref_rejected": -272.55303955078125,
|
|
"logps/rejected": -585.41552734375,
|
|
"loss": 4.4401,
|
|
"margin_dpo/margin_mean": 108.45451354980469,
|
|
"margin_dpo/margin_std": 160.95172119140625,
|
|
"step": 437
|
|
},
|
|
{
|
|
"epoch": 0.9172774869109948,
|
|
"fcm_dpo/beta": 0.003806520253419876,
|
|
"fcm_dpo/delta": 0.009153133258223534,
|
|
"fcm_dpo/margin": 50.68528747558594,
|
|
"fcm_dpo/q_t": 0.4563140869140625,
|
|
"grad_norm": 79.08209991455078,
|
|
"learning_rate": 1.06489699136324e-08,
|
|
"logits/chosen": -0.8532021045684814,
|
|
"logits/rejected": -0.8665780425071716,
|
|
"logps/chosen": -475.7437438964844,
|
|
"logps/ref_chosen": -259.31903076171875,
|
|
"logps/ref_rejected": -240.99581909179688,
|
|
"logps/rejected": -508.10577392578125,
|
|
"loss": 5.1451,
|
|
"margin_dpo/margin_mean": 50.6852912902832,
|
|
"margin_dpo/margin_std": 148.322021484375,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 0.9193717277486911,
|
|
"fcm_dpo/beta": 0.0038379242178052664,
|
|
"fcm_dpo/delta": 0.033137980848550797,
|
|
"fcm_dpo/margin": 95.86003112792969,
|
|
"fcm_dpo/q_t": 0.41530516743659973,
|
|
"grad_norm": 98.0456771850586,
|
|
"learning_rate": 1.0126756596375685e-08,
|
|
"logits/chosen": -0.831727147102356,
|
|
"logits/rejected": -0.8308047652244568,
|
|
"logps/chosen": -466.54345703125,
|
|
"logps/ref_chosen": -257.1243896484375,
|
|
"logps/ref_rejected": -243.20416259765625,
|
|
"logps/rejected": -548.4832763671875,
|
|
"loss": 4.4882,
|
|
"margin_dpo/margin_mean": 95.86003112792969,
|
|
"margin_dpo/margin_std": 138.56790161132812,
|
|
"step": 439
|
|
},
|
|
{
|
|
"epoch": 0.9214659685863874,
|
|
"fcm_dpo/beta": 0.00401148060336709,
|
|
"fcm_dpo/delta": 0.05375323444604874,
|
|
"fcm_dpo/margin": 69.56246185302734,
|
|
"fcm_dpo/q_t": 0.4353058934211731,
|
|
"grad_norm": 70.47123718261719,
|
|
"learning_rate": 9.617406953185136e-09,
|
|
"logits/chosen": -0.8858019113540649,
|
|
"logits/rejected": -0.876878023147583,
|
|
"logps/chosen": -533.7391967773438,
|
|
"logps/ref_chosen": -307.5315246582031,
|
|
"logps/ref_rejected": -264.3540954589844,
|
|
"logps/rejected": -560.124267578125,
|
|
"loss": 4.796,
|
|
"margin_dpo/margin_mean": 69.56246185302734,
|
|
"margin_dpo/margin_std": 130.65516662597656,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.9235602094240838,
|
|
"fcm_dpo/beta": 0.004070944152772427,
|
|
"fcm_dpo/delta": -0.028871532529592514,
|
|
"fcm_dpo/margin": 104.92424774169922,
|
|
"fcm_dpo/q_t": 0.402651846408844,
|
|
"grad_norm": 99.01258850097656,
|
|
"learning_rate": 9.12094829893642e-09,
|
|
"logits/chosen": -0.8418979048728943,
|
|
"logits/rejected": -0.8321986794471741,
|
|
"logps/chosen": -520.474365234375,
|
|
"logps/ref_chosen": -309.9819641113281,
|
|
"logps/ref_rejected": -297.4968566894531,
|
|
"logps/rejected": -612.9135131835938,
|
|
"loss": 4.3406,
|
|
"margin_dpo/margin_mean": 104.92424774169922,
|
|
"margin_dpo/margin_std": 137.64004516601562,
|
|
"step": 441
|
|
},
|
|
{
|
|
"epoch": 0.9256544502617801,
|
|
"fcm_dpo/beta": 0.004000660963356495,
|
|
"fcm_dpo/delta": 0.02609052136540413,
|
|
"fcm_dpo/margin": 78.4242172241211,
|
|
"fcm_dpo/q_t": 0.42918646335601807,
|
|
"grad_norm": 61.02368927001953,
|
|
"learning_rate": 8.637407257200496e-09,
|
|
"logits/chosen": -0.9113872647285461,
|
|
"logits/rejected": -0.8679369688034058,
|
|
"logps/chosen": -509.2906494140625,
|
|
"logps/ref_chosen": -278.9791564941406,
|
|
"logps/ref_rejected": -242.87310791015625,
|
|
"logps/rejected": -551.6088256835938,
|
|
"loss": 4.7697,
|
|
"margin_dpo/margin_mean": 78.42420959472656,
|
|
"margin_dpo/margin_std": 155.16122436523438,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 0.9277486910994764,
|
|
"fcm_dpo/beta": 0.0041396538726985455,
|
|
"fcm_dpo/delta": 0.014572596177458763,
|
|
"fcm_dpo/margin": 85.65485382080078,
|
|
"fcm_dpo/q_t": 0.41839903593063354,
|
|
"grad_norm": 95.5350570678711,
|
|
"learning_rate": 8.166809758815895e-09,
|
|
"logits/chosen": -0.8070948123931885,
|
|
"logits/rejected": -0.827476441860199,
|
|
"logps/chosen": -486.9132080078125,
|
|
"logps/ref_chosen": -273.5590515136719,
|
|
"logps/ref_rejected": -264.0199279785156,
|
|
"logps/rejected": -563.0289306640625,
|
|
"loss": 4.6045,
|
|
"margin_dpo/margin_mean": 85.65485382080078,
|
|
"margin_dpo/margin_std": 139.71743774414062,
|
|
"step": 443
|
|
},
|
|
{
|
|
"epoch": 0.9298429319371728,
|
|
"fcm_dpo/beta": 0.0040481919422745705,
|
|
"fcm_dpo/delta": -0.02957877703011036,
|
|
"fcm_dpo/margin": 99.84343719482422,
|
|
"fcm_dpo/q_t": 0.41018837690353394,
|
|
"grad_norm": 85.15692901611328,
|
|
"learning_rate": 7.709181040498253e-09,
|
|
"logits/chosen": -0.8221174478530884,
|
|
"logits/rejected": -0.8085671663284302,
|
|
"logps/chosen": -508.9268798828125,
|
|
"logps/ref_chosen": -298.1441955566406,
|
|
"logps/ref_rejected": -268.0572814941406,
|
|
"logps/rejected": -578.6834106445312,
|
|
"loss": 4.487,
|
|
"margin_dpo/margin_mean": 99.84344482421875,
|
|
"margin_dpo/margin_std": 157.79037475585938,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 0.9319371727748691,
|
|
"fcm_dpo/beta": 0.003955287858843803,
|
|
"fcm_dpo/delta": -0.06290034204721451,
|
|
"fcm_dpo/margin": 85.2093276977539,
|
|
"fcm_dpo/q_t": 0.4250834882259369,
|
|
"grad_norm": 72.96340942382812,
|
|
"learning_rate": 7.2645456434869965e-09,
|
|
"logits/chosen": -0.8938105702400208,
|
|
"logits/rejected": -0.9017377495765686,
|
|
"logps/chosen": -467.8274230957031,
|
|
"logps/ref_chosen": -254.54067993164062,
|
|
"logps/ref_rejected": -264.2445983886719,
|
|
"logps/rejected": -562.7406616210938,
|
|
"loss": 4.7053,
|
|
"margin_dpo/margin_mean": 85.2093276977539,
|
|
"margin_dpo/margin_std": 150.60330200195312,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.9340314136125655,
|
|
"fcm_dpo/beta": 0.003790568793192506,
|
|
"fcm_dpo/delta": 0.013937651179730892,
|
|
"fcm_dpo/margin": 94.15351867675781,
|
|
"fcm_dpo/q_t": 0.4192274510860443,
|
|
"grad_norm": 73.7974853515625,
|
|
"learning_rate": 6.832927412229017e-09,
|
|
"logits/chosen": -0.8159938454627991,
|
|
"logits/rejected": -0.818867564201355,
|
|
"logps/chosen": -512.8572387695312,
|
|
"logps/ref_chosen": -306.72247314453125,
|
|
"logps/ref_rejected": -266.3735656738281,
|
|
"logps/rejected": -566.661865234375,
|
|
"loss": 4.5708,
|
|
"margin_dpo/margin_mean": 94.15351867675781,
|
|
"margin_dpo/margin_std": 150.21368408203125,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 0.9361256544502617,
|
|
"fcm_dpo/beta": 0.0036977354902774096,
|
|
"fcm_dpo/delta": -0.06611842662096024,
|
|
"fcm_dpo/margin": 105.1883316040039,
|
|
"fcm_dpo/q_t": 0.4103267788887024,
|
|
"grad_norm": 62.978233337402344,
|
|
"learning_rate": 6.414349493100129e-09,
|
|
"logits/chosen": -0.8234304189682007,
|
|
"logits/rejected": -0.8186841607093811,
|
|
"logps/chosen": -459.4782409667969,
|
|
"logps/ref_chosen": -260.51727294921875,
|
|
"logps/ref_rejected": -236.47061157226562,
|
|
"logps/rejected": -540.619873046875,
|
|
"loss": 4.4225,
|
|
"margin_dpo/margin_mean": 105.18832397460938,
|
|
"margin_dpo/margin_std": 141.39520263671875,
|
|
"step": 447
|
|
},
|
|
{
|
|
"epoch": 0.9382198952879581,
|
|
"fcm_dpo/beta": 0.003641946241259575,
|
|
"fcm_dpo/delta": 0.012333293445408344,
|
|
"fcm_dpo/margin": 97.73287963867188,
|
|
"fcm_dpo/q_t": 0.4186139404773712,
|
|
"grad_norm": 81.14934539794922,
|
|
"learning_rate": 6.0088343331638756e-09,
|
|
"logits/chosen": -0.8353531360626221,
|
|
"logits/rejected": -0.8291007280349731,
|
|
"logps/chosen": -484.96905517578125,
|
|
"logps/ref_chosen": -268.78704833984375,
|
|
"logps/ref_rejected": -262.1703796386719,
|
|
"logps/rejected": -576.0852661132812,
|
|
"loss": 4.5206,
|
|
"margin_dpo/margin_mean": 97.73287963867188,
|
|
"margin_dpo/margin_std": 139.15956115722656,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 0.9403141361256544,
|
|
"fcm_dpo/beta": 0.0036854774225503206,
|
|
"fcm_dpo/delta": -0.009766047820448875,
|
|
"fcm_dpo/margin": 96.53168487548828,
|
|
"fcm_dpo/q_t": 0.4160364866256714,
|
|
"grad_norm": 130.65623474121094,
|
|
"learning_rate": 5.616403678967624e-09,
|
|
"logits/chosen": -0.9169028997421265,
|
|
"logits/rejected": -0.9006680250167847,
|
|
"logps/chosen": -533.5242919921875,
|
|
"logps/ref_chosen": -330.9514465332031,
|
|
"logps/ref_rejected": -239.76974487304688,
|
|
"logps/rejected": -538.8742065429688,
|
|
"loss": 4.5593,
|
|
"margin_dpo/margin_mean": 96.53167724609375,
|
|
"margin_dpo/margin_std": 148.29200744628906,
|
|
"step": 449
|
|
},
|
|
{
|
|
"epoch": 0.9424083769633508,
|
|
"fcm_dpo/beta": 0.0036835160572081804,
|
|
"fcm_dpo/delta": 0.015206391923129559,
|
|
"fcm_dpo/margin": 87.07230377197266,
|
|
"fcm_dpo/q_t": 0.42582279443740845,
|
|
"grad_norm": 80.95256042480469,
|
|
"learning_rate": 5.2370785753763356e-09,
|
|
"logits/chosen": -0.8152453303337097,
|
|
"logits/rejected": -0.8101465702056885,
|
|
"logps/chosen": -507.47833251953125,
|
|
"logps/ref_chosen": -284.26544189453125,
|
|
"logps/ref_rejected": -250.5401611328125,
|
|
"logps/rejected": -560.825439453125,
|
|
"loss": 4.6198,
|
|
"margin_dpo/margin_mean": 87.07230377197266,
|
|
"margin_dpo/margin_std": 132.29653930664062,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.9445026178010472,
|
|
"fcm_dpo/beta": 0.0036616751458495855,
|
|
"fcm_dpo/delta": -0.018945183604955673,
|
|
"fcm_dpo/margin": 79.61962890625,
|
|
"fcm_dpo/q_t": 0.43234091997146606,
|
|
"grad_norm": 58.6148567199707,
|
|
"learning_rate": 4.8708793644441086e-09,
|
|
"logits/chosen": -0.8456954956054688,
|
|
"logits/rejected": -0.8164753913879395,
|
|
"logps/chosen": -531.6286010742188,
|
|
"logps/ref_chosen": -302.3209228515625,
|
|
"logps/ref_rejected": -254.09747314453125,
|
|
"logps/rejected": -563.0247802734375,
|
|
"loss": 4.7455,
|
|
"margin_dpo/margin_mean": 79.61962890625,
|
|
"margin_dpo/margin_std": 140.28172302246094,
|
|
"step": 451
|
|
},
|
|
{
|
|
"epoch": 0.9465968586387434,
|
|
"fcm_dpo/beta": 0.003525385633111,
|
|
"fcm_dpo/delta": -0.05114733427762985,
|
|
"fcm_dpo/margin": 109.65728759765625,
|
|
"fcm_dpo/q_t": 0.41492602229118347,
|
|
"grad_norm": 70.34674835205078,
|
|
"learning_rate": 4.517825684323323e-09,
|
|
"logits/chosen": -0.8856893181800842,
|
|
"logits/rejected": -0.8606870770454407,
|
|
"logps/chosen": -508.6758117675781,
|
|
"logps/ref_chosen": -299.39215087890625,
|
|
"logps/ref_rejected": -284.3475036621094,
|
|
"logps/rejected": -603.2885131835938,
|
|
"loss": 4.5145,
|
|
"margin_dpo/margin_mean": 109.65728759765625,
|
|
"margin_dpo/margin_std": 172.53543090820312,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 0.9486910994764398,
|
|
"fcm_dpo/beta": 0.0035962536931037903,
|
|
"fcm_dpo/delta": 0.05969720333814621,
|
|
"fcm_dpo/margin": 95.231201171875,
|
|
"fcm_dpo/q_t": 0.42202088236808777,
|
|
"grad_norm": 62.857933044433594,
|
|
"learning_rate": 4.1779364682113794e-09,
|
|
"logits/chosen": -0.8256725072860718,
|
|
"logits/rejected": -0.8185856938362122,
|
|
"logps/chosen": -548.9927978515625,
|
|
"logps/ref_chosen": -324.6517028808594,
|
|
"logps/ref_rejected": -304.1527099609375,
|
|
"logps/rejected": -623.7249755859375,
|
|
"loss": 4.5858,
|
|
"margin_dpo/margin_mean": 95.23121643066406,
|
|
"margin_dpo/margin_std": 154.75979614257812,
|
|
"step": 453
|
|
},
|
|
{
|
|
"epoch": 0.9507853403141361,
|
|
"fcm_dpo/beta": 0.0036729713901877403,
|
|
"fcm_dpo/delta": -0.00931104552000761,
|
|
"fcm_dpo/margin": 111.07328796386719,
|
|
"fcm_dpo/q_t": 0.407396137714386,
|
|
"grad_norm": 63.60319137573242,
|
|
"learning_rate": 3.851229943335393e-09,
|
|
"logits/chosen": -0.8640636801719666,
|
|
"logits/rejected": -0.8737033009529114,
|
|
"logps/chosen": -515.0318603515625,
|
|
"logps/ref_chosen": -299.6117248535156,
|
|
"logps/ref_rejected": -303.74224853515625,
|
|
"logps/rejected": -630.2357177734375,
|
|
"loss": 4.4341,
|
|
"margin_dpo/margin_mean": 111.07328796386719,
|
|
"margin_dpo/margin_std": 162.49261474609375,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 0.9528795811518325,
|
|
"fcm_dpo/beta": 0.003721519373357296,
|
|
"fcm_dpo/delta": 0.018941761925816536,
|
|
"fcm_dpo/margin": 76.07414245605469,
|
|
"fcm_dpo/q_t": 0.4374096989631653,
|
|
"grad_norm": 75.65412139892578,
|
|
"learning_rate": 3.5377236299748147e-09,
|
|
"logits/chosen": -0.8347422480583191,
|
|
"logits/rejected": -0.8438669443130493,
|
|
"logps/chosen": -480.7205810546875,
|
|
"logps/ref_chosen": -273.6116943359375,
|
|
"logps/ref_rejected": -274.4293518066406,
|
|
"logps/rejected": -557.6124267578125,
|
|
"loss": 4.8186,
|
|
"margin_dpo/margin_mean": 76.07414245605469,
|
|
"margin_dpo/margin_std": 152.4740753173828,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.9549738219895288,
|
|
"fcm_dpo/beta": 0.0037363064475357533,
|
|
"fcm_dpo/delta": 0.0011149700731039047,
|
|
"fcm_dpo/margin": 100.02578735351562,
|
|
"fcm_dpo/q_t": 0.419757217168808,
|
|
"grad_norm": 85.6431884765625,
|
|
"learning_rate": 3.2374343405217884e-09,
|
|
"logits/chosen": -0.7592871785163879,
|
|
"logits/rejected": -0.7727906107902527,
|
|
"logps/chosen": -556.2762451171875,
|
|
"logps/ref_chosen": -322.17193603515625,
|
|
"logps/ref_rejected": -294.54461669921875,
|
|
"logps/rejected": -628.6747436523438,
|
|
"loss": 4.6092,
|
|
"margin_dpo/margin_mean": 100.02578735351562,
|
|
"margin_dpo/margin_std": 177.65438842773438,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 0.9570680628272251,
|
|
"fcm_dpo/beta": 0.0036805581767112017,
|
|
"fcm_dpo/delta": -0.012985389679670334,
|
|
"fcm_dpo/margin": 104.80935668945312,
|
|
"fcm_dpo/q_t": 0.4097025394439697,
|
|
"grad_norm": 78.89384460449219,
|
|
"learning_rate": 2.9503781785795713e-09,
|
|
"logits/chosen": -0.8332885503768921,
|
|
"logits/rejected": -0.8361387848854065,
|
|
"logps/chosen": -535.7222290039062,
|
|
"logps/ref_chosen": -307.7962341308594,
|
|
"logps/ref_rejected": -274.5501403808594,
|
|
"logps/rejected": -607.2855224609375,
|
|
"loss": 4.549,
|
|
"margin_dpo/margin_mean": 104.80935668945312,
|
|
"margin_dpo/margin_std": 170.84185791015625,
|
|
"step": 457
|
|
},
|
|
{
|
|
"epoch": 0.9591623036649215,
|
|
"fcm_dpo/beta": 0.0037839512806385756,
|
|
"fcm_dpo/delta": 0.04983433336019516,
|
|
"fcm_dpo/margin": 92.8138427734375,
|
|
"fcm_dpo/q_t": 0.4212765693664551,
|
|
"grad_norm": 72.37201690673828,
|
|
"learning_rate": 2.6765705380989432e-09,
|
|
"logits/chosen": -0.8502821922302246,
|
|
"logits/rejected": -0.8296154737472534,
|
|
"logps/chosen": -515.9027709960938,
|
|
"logps/ref_chosen": -297.0316467285156,
|
|
"logps/ref_rejected": -276.1112365722656,
|
|
"logps/rejected": -587.796142578125,
|
|
"loss": 4.6273,
|
|
"margin_dpo/margin_mean": 92.8138427734375,
|
|
"margin_dpo/margin_std": 158.17807006835938,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 0.9612565445026178,
|
|
"fcm_dpo/beta": 0.003876004833728075,
|
|
"fcm_dpo/delta": 0.018467195332050323,
|
|
"fcm_dpo/margin": 78.23777770996094,
|
|
"fcm_dpo/q_t": 0.4319803714752197,
|
|
"grad_norm": 74.26271057128906,
|
|
"learning_rate": 2.416026102552732e-09,
|
|
"logits/chosen": -0.9194495677947998,
|
|
"logits/rejected": -0.9117699861526489,
|
|
"logps/chosen": -500.1540222167969,
|
|
"logps/ref_chosen": -293.5252990722656,
|
|
"logps/ref_rejected": -289.30126953125,
|
|
"logps/rejected": -574.167724609375,
|
|
"loss": 4.7467,
|
|
"margin_dpo/margin_mean": 78.23778533935547,
|
|
"margin_dpo/margin_std": 145.30636596679688,
|
|
"step": 459
|
|
},
|
|
{
|
|
"epoch": 0.9633507853403142,
|
|
"fcm_dpo/beta": 0.003939047455787659,
|
|
"fcm_dpo/delta": 0.0354711078107357,
|
|
"fcm_dpo/margin": 78.7183837890625,
|
|
"fcm_dpo/q_t": 0.42859572172164917,
|
|
"grad_norm": 85.72238159179688,
|
|
"learning_rate": 2.168758844148272e-09,
|
|
"logits/chosen": -0.8774435520172119,
|
|
"logits/rejected": -0.8768860101699829,
|
|
"logps/chosen": -532.5010986328125,
|
|
"logps/ref_chosen": -318.7803649902344,
|
|
"logps/ref_rejected": -258.7906799316406,
|
|
"logps/rejected": -551.229736328125,
|
|
"loss": 4.7105,
|
|
"margin_dpo/margin_mean": 78.71837615966797,
|
|
"margin_dpo/margin_std": 142.5164794921875,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.9654450261780104,
|
|
"fcm_dpo/beta": 0.004013788886368275,
|
|
"fcm_dpo/delta": -0.008083513006567955,
|
|
"fcm_dpo/margin": 87.99292755126953,
|
|
"fcm_dpo/q_t": 0.4219253361225128,
|
|
"grad_norm": 99.12641143798828,
|
|
"learning_rate": 1.9347820230782295e-09,
|
|
"logits/chosen": -0.8341564536094666,
|
|
"logits/rejected": -0.8578203320503235,
|
|
"logps/chosen": -459.1229248046875,
|
|
"logps/ref_chosen": -243.9099884033203,
|
|
"logps/ref_rejected": -232.6382293701172,
|
|
"logps/rejected": -535.8441162109375,
|
|
"loss": 4.6733,
|
|
"margin_dpo/margin_mean": 87.99293518066406,
|
|
"margin_dpo/margin_std": 161.89053344726562,
|
|
"step": 461
|
|
},
|
|
{
|
|
"epoch": 0.9675392670157068,
|
|
"fcm_dpo/beta": 0.0038696322590112686,
|
|
"fcm_dpo/delta": -0.05201539024710655,
|
|
"fcm_dpo/margin": 108.68580627441406,
|
|
"fcm_dpo/q_t": 0.40523770451545715,
|
|
"grad_norm": 98.13914489746094,
|
|
"learning_rate": 1.7141081868094209e-09,
|
|
"logits/chosen": -0.8697165250778198,
|
|
"logits/rejected": -0.8183139562606812,
|
|
"logps/chosen": -559.72705078125,
|
|
"logps/ref_chosen": -344.09100341796875,
|
|
"logps/ref_rejected": -252.45037841796875,
|
|
"logps/rejected": -576.7722778320312,
|
|
"loss": 4.3988,
|
|
"margin_dpo/margin_mean": 108.68580627441406,
|
|
"margin_dpo/margin_std": 156.15455627441406,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 0.9696335078534032,
|
|
"fcm_dpo/beta": 0.003800945356488228,
|
|
"fcm_dpo/delta": -0.012423641048371792,
|
|
"fcm_dpo/margin": 81.46054077148438,
|
|
"fcm_dpo/q_t": 0.4303792119026184,
|
|
"grad_norm": 74.05850219726562,
|
|
"learning_rate": 1.5067491694100153e-09,
|
|
"logits/chosen": -0.8569101095199585,
|
|
"logits/rejected": -0.8149942755699158,
|
|
"logps/chosen": -511.5487976074219,
|
|
"logps/ref_chosen": -297.1424560546875,
|
|
"logps/ref_rejected": -234.0208282470703,
|
|
"logps/rejected": -529.8877563476562,
|
|
"loss": 4.7769,
|
|
"margin_dpo/margin_mean": 81.46053314208984,
|
|
"margin_dpo/margin_std": 161.78460693359375,
|
|
"step": 463
|
|
},
|
|
{
|
|
"epoch": 0.9717277486910995,
|
|
"fcm_dpo/beta": 0.003825580468401313,
|
|
"fcm_dpo/delta": 0.033597130328416824,
|
|
"fcm_dpo/margin": 81.79236602783203,
|
|
"fcm_dpo/q_t": 0.4282078742980957,
|
|
"grad_norm": 81.68612670898438,
|
|
"learning_rate": 1.3127160909147672e-09,
|
|
"logits/chosen": -0.8513908386230469,
|
|
"logits/rejected": -0.8705244064331055,
|
|
"logps/chosen": -494.5838623046875,
|
|
"logps/ref_chosen": -265.71075439453125,
|
|
"logps/ref_rejected": -256.4108581542969,
|
|
"logps/rejected": -567.0762939453125,
|
|
"loss": 4.7027,
|
|
"margin_dpo/margin_mean": 81.7923583984375,
|
|
"margin_dpo/margin_std": 145.36318969726562,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 0.9738219895287958,
|
|
"fcm_dpo/beta": 0.003732735523954034,
|
|
"fcm_dpo/delta": -0.07069091498851776,
|
|
"fcm_dpo/margin": 111.66655731201172,
|
|
"fcm_dpo/q_t": 0.4052296280860901,
|
|
"grad_norm": 71.67578125,
|
|
"learning_rate": 1.1320193567288527e-09,
|
|
"logits/chosen": -0.9181026816368103,
|
|
"logits/rejected": -0.8817926049232483,
|
|
"logps/chosen": -494.74371337890625,
|
|
"logps/ref_chosen": -293.1527404785156,
|
|
"logps/ref_rejected": -293.70947265625,
|
|
"logps/rejected": -606.967041015625,
|
|
"loss": 4.3887,
|
|
"margin_dpo/margin_mean": 111.66655731201172,
|
|
"margin_dpo/margin_std": 149.8941650390625,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.9759162303664921,
|
|
"fcm_dpo/beta": 0.0035918080247938633,
|
|
"fcm_dpo/delta": -0.013611567206680775,
|
|
"fcm_dpo/margin": 114.79224395751953,
|
|
"fcm_dpo/q_t": 0.4061741828918457,
|
|
"grad_norm": 64.40978240966797,
|
|
"learning_rate": 9.64668657069706e-10,
|
|
"logits/chosen": -0.8348801136016846,
|
|
"logits/rejected": -0.7807033061981201,
|
|
"logps/chosen": -456.18829345703125,
|
|
"logps/ref_chosen": -261.4775695800781,
|
|
"logps/ref_rejected": -248.36282348632812,
|
|
"logps/rejected": -557.8658447265625,
|
|
"loss": 4.3506,
|
|
"margin_dpo/margin_mean": 114.79224395751953,
|
|
"margin_dpo/margin_std": 149.470703125,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 0.9780104712041885,
|
|
"fcm_dpo/beta": 0.0036084747407585382,
|
|
"fcm_dpo/delta": 0.014053348451852798,
|
|
"fcm_dpo/margin": 83.56181335449219,
|
|
"fcm_dpo/q_t": 0.43221747875213623,
|
|
"grad_norm": 71.90433502197266,
|
|
"learning_rate": 8.106729664475176e-10,
|
|
"logits/chosen": -0.807646632194519,
|
|
"logits/rejected": -0.8021789789199829,
|
|
"logps/chosen": -482.59112548828125,
|
|
"logps/ref_chosen": -266.354248046875,
|
|
"logps/ref_rejected": -277.76324462890625,
|
|
"logps/rejected": -577.5619506835938,
|
|
"loss": 4.7741,
|
|
"margin_dpo/margin_mean": 83.56181335449219,
|
|
"margin_dpo/margin_std": 162.9016571044922,
|
|
"step": 467
|
|
},
|
|
{
|
|
"epoch": 0.9801047120418848,
|
|
"fcm_dpo/beta": 0.0037187100388109684,
|
|
"fcm_dpo/delta": 0.021228276193141937,
|
|
"fcm_dpo/margin": 73.15608978271484,
|
|
"fcm_dpo/q_t": 0.43574419617652893,
|
|
"grad_norm": 87.2890853881836,
|
|
"learning_rate": 6.700405431837585e-10,
|
|
"logits/chosen": -0.9090844988822937,
|
|
"logits/rejected": -0.8785009980201721,
|
|
"logps/chosen": -532.4305419921875,
|
|
"logps/ref_chosen": -317.9631652832031,
|
|
"logps/ref_rejected": -261.8744201660156,
|
|
"logps/rejected": -549.4978637695312,
|
|
"loss": 4.877,
|
|
"margin_dpo/margin_mean": 73.15608978271484,
|
|
"margin_dpo/margin_std": 154.40957641601562,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 0.9821989528795811,
|
|
"fcm_dpo/beta": 0.0036812869366258383,
|
|
"fcm_dpo/delta": 0.003829888068139553,
|
|
"fcm_dpo/margin": 107.45279693603516,
|
|
"fcm_dpo/q_t": 0.4098895788192749,
|
|
"grad_norm": 60.7659912109375,
|
|
"learning_rate": 5.427789289685347e-10,
|
|
"logits/chosen": -0.8316172957420349,
|
|
"logits/rejected": -0.8154798746109009,
|
|
"logps/chosen": -528.8984985351562,
|
|
"logps/ref_chosen": -324.8868103027344,
|
|
"logps/ref_rejected": -264.0421447753906,
|
|
"logps/rejected": -575.5067138671875,
|
|
"loss": 4.4592,
|
|
"margin_dpo/margin_mean": 107.45278930664062,
|
|
"margin_dpo/margin_std": 159.00772094726562,
|
|
"step": 469
|
|
},
|
|
{
|
|
"epoch": 0.9842931937172775,
|
|
"fcm_dpo/beta": 0.0037065560463815928,
|
|
"fcm_dpo/delta": -0.03380197659134865,
|
|
"fcm_dpo/margin": 101.50714111328125,
|
|
"fcm_dpo/q_t": 0.4139646291732788,
|
|
"grad_norm": 65.40029907226562,
|
|
"learning_rate": 4.288949484559934e-10,
|
|
"logits/chosen": -0.8253117799758911,
|
|
"logits/rejected": -0.8232603669166565,
|
|
"logps/chosen": -519.7571411132812,
|
|
"logps/ref_chosen": -314.7042541503906,
|
|
"logps/ref_rejected": -259.2276611328125,
|
|
"logps/rejected": -565.7877197265625,
|
|
"loss": 4.4905,
|
|
"margin_dpo/margin_mean": 101.50714111328125,
|
|
"margin_dpo/margin_std": 147.7560577392578,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.9863874345549738,
|
|
"fcm_dpo/beta": 0.0036814697086811066,
|
|
"fcm_dpo/delta": -0.013820381835103035,
|
|
"fcm_dpo/margin": 97.27108001708984,
|
|
"fcm_dpo/q_t": 0.4229424297809601,
|
|
"grad_norm": 81.09204864501953,
|
|
"learning_rate": 3.2839470889836627e-10,
|
|
"logits/chosen": -0.8735456466674805,
|
|
"logits/rejected": -0.8611137270927429,
|
|
"logps/chosen": -511.9947814941406,
|
|
"logps/ref_chosen": -292.5748291015625,
|
|
"logps/ref_rejected": -298.7525329589844,
|
|
"logps/rejected": -615.4434814453125,
|
|
"loss": 4.6116,
|
|
"margin_dpo/margin_mean": 97.27108001708984,
|
|
"margin_dpo/margin_std": 168.31797790527344,
|
|
"step": 471
|
|
},
|
|
{
|
|
"epoch": 0.9884816753926702,
|
|
"fcm_dpo/beta": 0.0036161583848297596,
|
|
"fcm_dpo/delta": 0.009807462804019451,
|
|
"fcm_dpo/margin": 107.87297058105469,
|
|
"fcm_dpo/q_t": 0.41221684217453003,
|
|
"grad_norm": 71.03485870361328,
|
|
"learning_rate": 2.412835998185092e-10,
|
|
"logits/chosen": -0.8705576658248901,
|
|
"logits/rejected": -0.8849227428436279,
|
|
"logps/chosen": -442.5689697265625,
|
|
"logps/ref_chosen": -243.37380981445312,
|
|
"logps/ref_rejected": -251.12109375,
|
|
"logps/rejected": -558.189208984375,
|
|
"loss": 4.4378,
|
|
"margin_dpo/margin_mean": 107.87297058105469,
|
|
"margin_dpo/margin_std": 152.4186248779297,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 0.9905759162303664,
|
|
"fcm_dpo/beta": 0.0036868297029286623,
|
|
"fcm_dpo/delta": 0.006859378889203072,
|
|
"fcm_dpo/margin": 106.62423706054688,
|
|
"fcm_dpo/q_t": 0.408770352602005,
|
|
"grad_norm": 72.31365203857422,
|
|
"learning_rate": 1.6756629272085544e-10,
|
|
"logits/chosen": -0.8231157064437866,
|
|
"logits/rejected": -0.822382390499115,
|
|
"logps/chosen": -495.9771728515625,
|
|
"logps/ref_chosen": -286.3286437988281,
|
|
"logps/ref_rejected": -258.6535339355469,
|
|
"logps/rejected": -574.9263916015625,
|
|
"loss": 4.3838,
|
|
"margin_dpo/margin_mean": 106.62422180175781,
|
|
"margin_dpo/margin_std": 135.7769012451172,
|
|
"step": 473
|
|
},
|
|
{
|
|
"epoch": 0.9926701570680628,
|
|
"fcm_dpo/beta": 0.003645974677056074,
|
|
"fcm_dpo/delta": 0.02563265711069107,
|
|
"fcm_dpo/margin": 90.80755615234375,
|
|
"fcm_dpo/q_t": 0.42361754179000854,
|
|
"grad_norm": 84.59915161132812,
|
|
"learning_rate": 1.072467408408384e-10,
|
|
"logits/chosen": -0.8701086640357971,
|
|
"logits/rejected": -0.8666247129440308,
|
|
"logps/chosen": -503.46722412109375,
|
|
"logps/ref_chosen": -288.08966064453125,
|
|
"logps/ref_rejected": -266.69696044921875,
|
|
"logps/rejected": -572.8821411132812,
|
|
"loss": 4.5952,
|
|
"margin_dpo/margin_mean": 90.80755615234375,
|
|
"margin_dpo/margin_std": 136.625,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 0.9947643979057592,
|
|
"fcm_dpo/beta": 0.003674778388813138,
|
|
"fcm_dpo/delta": -0.003959197551012039,
|
|
"fcm_dpo/margin": 81.83253479003906,
|
|
"fcm_dpo/q_t": 0.43231987953186035,
|
|
"grad_norm": 84.51021575927734,
|
|
"learning_rate": 6.032817893297793e-11,
|
|
"logits/chosen": -0.8477621078491211,
|
|
"logits/rejected": -0.860824704170227,
|
|
"logps/chosen": -456.145751953125,
|
|
"logps/ref_chosen": -256.0030517578125,
|
|
"logps/ref_rejected": -244.50660705566406,
|
|
"logps/rejected": -526.4818115234375,
|
|
"loss": 4.7206,
|
|
"margin_dpo/margin_mean": 81.83253479003906,
|
|
"margin_dpo/margin_std": 143.8907012939453,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.9968586387434555,
|
|
"fcm_dpo/beta": 0.00364405638538301,
|
|
"fcm_dpo/delta": -0.0139269158244133,
|
|
"fcm_dpo/margin": 90.71614074707031,
|
|
"fcm_dpo/q_t": 0.4268645644187927,
|
|
"grad_norm": 66.6181640625,
|
|
"learning_rate": 2.6813123097352287e-11,
|
|
"logits/chosen": -0.9170237183570862,
|
|
"logits/rejected": -0.8723958730697632,
|
|
"logps/chosen": -522.1641845703125,
|
|
"logps/ref_chosen": -321.467529296875,
|
|
"logps/ref_rejected": -295.0592956542969,
|
|
"logps/rejected": -586.4722290039062,
|
|
"loss": 4.6752,
|
|
"margin_dpo/margin_mean": 90.71612548828125,
|
|
"margin_dpo/margin_std": 153.20860290527344,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"fcm_dpo/beta": 0.0037430531810969114,
|
|
"fcm_dpo/delta": -0.004374518990516663,
|
|
"fcm_dpo/margin": 91.00071716308594,
|
|
"fcm_dpo/q_t": 0.42345336079597473,
|
|
"grad_norm": 73.29693603515625,
|
|
"learning_rate": 6.7033706447061635e-12,
|
|
"logits/chosen": -0.795288622379303,
|
|
"logits/rejected": -0.8026962876319885,
|
|
"logps/chosen": -497.5155029296875,
|
|
"logps/ref_chosen": -276.7939758300781,
|
|
"logps/ref_rejected": -244.82919311523438,
|
|
"logps/rejected": -556.551513671875,
|
|
"loss": 4.7313,
|
|
"margin_dpo/margin_mean": 91.00071716308594,
|
|
"margin_dpo/margin_std": 175.25076293945312,
|
|
"step": 477
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"step": 477,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4.781781088631108,
|
|
"train_runtime": 6847.9581,
|
|
"train_samples_per_second": 8.927,
|
|
"train_steps_per_second": 0.07
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 477,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|