{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.05443763732910156, "KL/mean": -0.04046645760536194, "KL/rejected_KL_mean": -0.026498794555664062, "KL/std": 0.47091516852378845, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02793481945991516, "fcm_dpo/q_t": 0.500069797039032, "grad_norm": 28.59173011779785, "learning_rate": 0.0, "logits/chosen": -0.5898098945617676, "logits/rejected": -0.604260265827179, "logps/chosen": -275.28570556640625, "logps/ref_chosen": -275.2312927246094, "logps/ref_rejected": -222.9380340576172, "logps/rejected": -222.96453857421875, "loss": 5.5463, "margin_dpo/margin_mean": -0.02793477475643158, "margin_dpo/margin_std": 0.5724214911460876, "step": 1 }, { "KL/chosen_KL_mean": 0.04458808898925781, "KL/mean": 0.0374295711517334, "KL/rejected_KL_mean": 0.030269622802734375, "KL/std": 0.49036121368408203, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014312177896499634, "fcm_dpo/q_t": 0.4999642074108124, "grad_norm": 27.88316535949707, "learning_rate": 1.0416666666666666e-08, "logits/chosen": -0.6574729681015015, "logits/rejected": -0.6464410424232483, "logps/chosen": -264.7165222167969, "logps/ref_chosen": -264.7611083984375, "logps/ref_rejected": -242.5597686767578, "logps/rejected": -242.52951049804688, "loss": 5.5446, "margin_dpo/margin_mean": 0.014312252402305603, "margin_dpo/margin_std": 0.6423971652984619, "step": 2 }, { "KL/chosen_KL_mean": 0.01624298095703125, "KL/mean": -0.02929476648569107, "KL/rejected_KL_mean": -0.0748291015625, "KL/std": 0.480156272649765, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09107272326946259, "fcm_dpo/q_t": 0.499772310256958, "grad_norm": 25.9517879486084, "learning_rate": 2.083333333333333e-08, "logits/chosen": -0.6839807033538818, "logits/rejected": -0.735178530216217, "logps/chosen": -274.0856628417969, "logps/ref_chosen": -274.1018981933594, "logps/ref_rejected": -286.5882568359375, "logps/rejected": -286.6631164550781, "loss": 5.5416, "margin_dpo/margin_mean": 0.09107272326946259, "margin_dpo/margin_std": 0.6260531544685364, "step": 3 }, { "KL/chosen_KL_mean": -0.030345916748046875, "KL/mean": -0.059315308928489685, "KL/rejected_KL_mean": -0.08828353881835938, "KL/std": 0.6341561079025269, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05793502926826477, "fcm_dpo/q_t": 0.4998551607131958, "grad_norm": 31.600723266601562, "learning_rate": 3.125e-08, "logits/chosen": -0.6173092126846313, "logits/rejected": -0.6115694642066956, "logps/chosen": -329.8685607910156, "logps/ref_chosen": -329.8382568359375, "logps/ref_rejected": -303.2850646972656, "logps/rejected": -303.3733215332031, "loss": 5.5429, "margin_dpo/margin_mean": 0.05793498456478119, "margin_dpo/margin_std": 0.8304102420806885, "step": 4 }, { "KL/chosen_KL_mean": -0.03276824951171875, "KL/mean": 0.0054293423891067505, "KL/rejected_KL_mean": 0.04363250732421875, "KL/std": 0.567704975605011, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07639628648757935, "fcm_dpo/q_t": 0.5001909732818604, "grad_norm": 29.59075927734375, "learning_rate": 4.166666666666666e-08, "logits/chosen": -0.57039475440979, "logits/rejected": -0.5864973068237305, "logps/chosen": -301.771728515625, "logps/ref_chosen": -301.7389221191406, "logps/ref_rejected": -274.7654724121094, "logps/rejected": -274.72186279296875, "loss": 5.5483, "margin_dpo/margin_mean": -0.07639645040035248, "margin_dpo/margin_std": 0.8294250965118408, "step": 5 }, { "KL/chosen_KL_mean": 0.03812408447265625, "KL/mean": 0.029881909489631653, "KL/rejected_KL_mean": 0.021627426147460938, "KL/std": 0.5347965359687805, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016495585441589355, "fcm_dpo/q_t": 0.4999587833881378, "grad_norm": 28.209911346435547, "learning_rate": 5.208333333333333e-08, "logits/chosen": -0.6794368624687195, "logits/rejected": -0.6423019766807556, "logps/chosen": -285.65655517578125, "logps/ref_chosen": -285.6946716308594, "logps/ref_rejected": -245.8200225830078, "logps/rejected": -245.79840087890625, "loss": 5.5446, "margin_dpo/margin_mean": 0.016496747732162476, "margin_dpo/margin_std": 0.706045925617218, "step": 6 }, { "KL/chosen_KL_mean": -0.04505348205566406, "KL/mean": -0.026556827127933502, "KL/rejected_KL_mean": -0.008066177368164062, "KL/std": 0.46558207273483276, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.036994144320487976, "fcm_dpo/q_t": 0.5000925064086914, "grad_norm": 28.658977508544922, "learning_rate": 6.25e-08, "logits/chosen": -0.5635161399841309, "logits/rejected": -0.5949946045875549, "logps/chosen": -264.7004699707031, "logps/ref_chosen": -264.65545654296875, "logps/ref_rejected": -253.10305786132812, "logps/rejected": -253.11111450195312, "loss": 5.5467, "margin_dpo/margin_mean": -0.036993831396102905, "margin_dpo/margin_std": 0.6402279734611511, "step": 7 }, { "KL/chosen_KL_mean": 0.021518707275390625, "KL/mean": 0.013283878564834595, "KL/rejected_KL_mean": 0.0050487518310546875, "KL/std": 0.5391606092453003, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01646500825881958, "fcm_dpo/q_t": 0.4999588429927826, "grad_norm": 30.747331619262695, "learning_rate": 7.291666666666667e-08, "logits/chosen": -0.6774884462356567, "logits/rejected": -0.6843160390853882, "logps/chosen": -354.167236328125, "logps/ref_chosen": -354.1887512207031, "logps/ref_rejected": -282.9112243652344, "logps/rejected": -282.9061584472656, "loss": 5.5446, "margin_dpo/margin_mean": 0.016465887427330017, "margin_dpo/margin_std": 0.6959210634231567, "step": 8 }, { "KL/chosen_KL_mean": -0.06751441955566406, "KL/mean": -0.02276702970266342, "KL/rejected_KL_mean": 0.02196502685546875, "KL/std": 0.5027087330818176, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.08947370946407318, "fcm_dpo/q_t": 0.5002236366271973, "grad_norm": 27.86223030090332, "learning_rate": 8.333333333333333e-08, "logits/chosen": -0.6372052431106567, "logits/rejected": -0.6553710103034973, "logps/chosen": -285.61773681640625, "logps/ref_chosen": -285.5502014160156, "logps/ref_rejected": -267.99664306640625, "logps/rejected": -267.9747009277344, "loss": 5.5488, "margin_dpo/margin_mean": -0.08947360515594482, "margin_dpo/margin_std": 0.6783429980278015, "step": 9 }, { "KL/chosen_KL_mean": -0.0031414031982421875, "KL/mean": -0.010139897465705872, "KL/rejected_KL_mean": -0.017139434814453125, "KL/std": 0.4715445637702942, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014001935720443726, "fcm_dpo/q_t": 0.49996495246887207, "grad_norm": 26.741235733032227, "learning_rate": 9.375e-08, "logits/chosen": -0.6870542168617249, "logits/rejected": -0.6816274523735046, "logps/chosen": -251.91552734375, "logps/ref_chosen": -251.91238403320312, "logps/ref_rejected": -226.45260620117188, "logps/rejected": -226.46974182128906, "loss": 5.5447, "margin_dpo/margin_mean": 0.014002710580825806, "margin_dpo/margin_std": 0.6491466760635376, "step": 10 }, { "KL/chosen_KL_mean": 0.036834716796875, "KL/mean": 0.05126556009054184, "KL/rejected_KL_mean": 0.06568717956542969, "KL/std": 0.5492945313453674, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02885313332080841, "fcm_dpo/q_t": 0.5000721216201782, "grad_norm": 28.992164611816406, "learning_rate": 1.0416666666666667e-07, "logits/chosen": -0.5933151841163635, "logits/rejected": -0.648064374923706, "logps/chosen": -301.046630859375, "logps/ref_chosen": -301.08343505859375, "logps/ref_rejected": -259.546630859375, "logps/rejected": -259.4809265136719, "loss": 5.5464, "margin_dpo/margin_mean": -0.028853505849838257, "margin_dpo/margin_std": 0.7581360340118408, "step": 11 }, { "KL/chosen_KL_mean": 0.10894775390625, "KL/mean": 0.005401916801929474, "KL/rejected_KL_mean": -0.09813308715820312, "KL/std": 0.517360508441925, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20707155764102936, "fcm_dpo/q_t": 0.49948233366012573, "grad_norm": 30.014650344848633, "learning_rate": 1.1458333333333332e-07, "logits/chosen": -0.5822965502738953, "logits/rejected": -0.5422216653823853, "logps/chosen": -287.43914794921875, "logps/ref_chosen": -287.548095703125, "logps/ref_rejected": -277.37945556640625, "logps/rejected": -277.47760009765625, "loss": 5.537, "margin_dpo/margin_mean": 0.20707103610038757, "margin_dpo/margin_std": 0.7214613556861877, "step": 12 }, { "KL/chosen_KL_mean": 0.026136398315429688, "KL/mean": -0.010055795311927795, "KL/rejected_KL_mean": -0.04625511169433594, "KL/std": 0.46862512826919556, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07239121198654175, "fcm_dpo/q_t": 0.4998190402984619, "grad_norm": 27.232789993286133, "learning_rate": 1.25e-07, "logits/chosen": -0.6514136791229248, "logits/rejected": -0.658698558807373, "logps/chosen": -270.6402893066406, "logps/ref_chosen": -270.6664123535156, "logps/ref_rejected": -274.6546936035156, "logps/rejected": -274.7009582519531, "loss": 5.5423, "margin_dpo/margin_mean": 0.07239186763763428, "margin_dpo/margin_std": 0.6270844340324402, "step": 13 }, { "KL/chosen_KL_mean": 0.031122207641601562, "KL/mean": 0.017020471394062042, "KL/rejected_KL_mean": 0.002918243408203125, "KL/std": 0.5255962610244751, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.028207316994667053, "fcm_dpo/q_t": 0.4999294877052307, "grad_norm": 28.295730590820312, "learning_rate": 1.3541666666666666e-07, "logits/chosen": -0.620563268661499, "logits/rejected": -0.6489490866661072, "logps/chosen": -281.5621032714844, "logps/ref_chosen": -281.59320068359375, "logps/ref_rejected": -263.52215576171875, "logps/rejected": -263.51922607421875, "loss": 5.5441, "margin_dpo/margin_mean": 0.028206750750541687, "margin_dpo/margin_std": 0.6688634753227234, "step": 14 }, { "KL/chosen_KL_mean": 0.03985404968261719, "KL/mean": 0.014157146215438843, "KL/rejected_KL_mean": -0.011541366577148438, "KL/std": 0.4595259428024292, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05139949917793274, "fcm_dpo/q_t": 0.49987149238586426, "grad_norm": 30.344572067260742, "learning_rate": 1.4583333333333335e-07, "logits/chosen": -0.631891131401062, "logits/rejected": -0.6428329944610596, "logps/chosen": -298.41357421875, "logps/ref_chosen": -298.45343017578125, "logps/ref_rejected": -227.17832946777344, "logps/rejected": -227.18988037109375, "loss": 5.5432, "margin_dpo/margin_mean": 0.05139845609664917, "margin_dpo/margin_std": 0.6178590059280396, "step": 15 }, { "KL/chosen_KL_mean": 0.07450485229492188, "KL/mean": 0.021834172308444977, "KL/rejected_KL_mean": -0.030839920043945312, "KL/std": 0.539286732673645, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10533960163593292, "fcm_dpo/q_t": 0.4997366666793823, "grad_norm": 30.12870216369629, "learning_rate": 1.5624999999999999e-07, "logits/chosen": -0.5921044945716858, "logits/rejected": -0.588280439376831, "logps/chosen": -293.8921203613281, "logps/ref_chosen": -293.96661376953125, "logps/ref_rejected": -250.78443908691406, "logps/rejected": -250.81527709960938, "loss": 5.541, "margin_dpo/margin_mean": 0.10533848404884338, "margin_dpo/margin_std": 0.7177047729492188, "step": 16 }, { "KL/chosen_KL_mean": 0.03164863586425781, "KL/mean": 0.015181809663772583, "KL/rejected_KL_mean": -0.00128173828125, "KL/std": 0.49001258611679077, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03292769193649292, "fcm_dpo/q_t": 0.4999176859855652, "grad_norm": 27.68689727783203, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5722857117652893, "logits/rejected": -0.5986632704734802, "logps/chosen": -262.36236572265625, "logps/ref_chosen": -262.39398193359375, "logps/ref_rejected": -248.500244140625, "logps/rejected": -248.50152587890625, "loss": 5.5439, "margin_dpo/margin_mean": 0.03292873501777649, "margin_dpo/margin_std": 0.6998825073242188, "step": 17 }, { "KL/chosen_KL_mean": 0.013545989990234375, "KL/mean": -0.026699572801589966, "KL/rejected_KL_mean": -0.06694602966308594, "KL/std": 0.5444977879524231, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08047735691070557, "fcm_dpo/q_t": 0.49979880452156067, "grad_norm": 29.722179412841797, "learning_rate": 1.7708333333333334e-07, "logits/chosen": -0.6103276014328003, "logits/rejected": -0.6163787245750427, "logps/chosen": -293.6956787109375, "logps/ref_chosen": -293.709228515625, "logps/ref_rejected": -274.5875244140625, "logps/rejected": -274.65447998046875, "loss": 5.542, "margin_dpo/margin_mean": 0.08047764003276825, "margin_dpo/margin_std": 0.7309471368789673, "step": 18 }, { "KL/chosen_KL_mean": 0.04509544372558594, "KL/mean": 0.022539354860782623, "KL/rejected_KL_mean": -9.5367431640625e-06, "KL/std": 0.48370176553726196, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04510428011417389, "fcm_dpo/q_t": 0.49988722801208496, "grad_norm": 28.11159324645996, "learning_rate": 1.875e-07, "logits/chosen": -0.6241991519927979, "logits/rejected": -0.6202787756919861, "logps/chosen": -280.2205810546875, "logps/ref_chosen": -280.26568603515625, "logps/ref_rejected": -259.9742736816406, "logps/rejected": -259.9742736816406, "loss": 5.5434, "margin_dpo/margin_mean": 0.045103684067726135, "margin_dpo/margin_std": 0.6318709850311279, "step": 19 }, { "KL/chosen_KL_mean": 0.16654586791992188, "KL/mean": 0.10966788977384567, "KL/rejected_KL_mean": 0.052791595458984375, "KL/std": 0.5513770580291748, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11376507580280304, "fcm_dpo/q_t": 0.4997155964374542, "grad_norm": 29.670021057128906, "learning_rate": 1.9791666666666664e-07, "logits/chosen": -0.6223905086517334, "logits/rejected": -0.6546608805656433, "logps/chosen": -303.7288818359375, "logps/ref_chosen": -303.8954162597656, "logps/ref_rejected": -260.214599609375, "logps/rejected": -260.16180419921875, "loss": 5.5407, "margin_dpo/margin_mean": 0.11376452445983887, "margin_dpo/margin_std": 0.7740194797515869, "step": 20 }, { "KL/chosen_KL_mean": 0.11112022399902344, "KL/mean": 0.04954635351896286, "KL/rejected_KL_mean": -0.01203155517578125, "KL/std": 0.5387072563171387, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12316055595874786, "fcm_dpo/q_t": 0.4996921122074127, "grad_norm": 35.265602111816406, "learning_rate": 2.0833333333333333e-07, "logits/chosen": -0.6434299349784851, "logits/rejected": -0.6710826754570007, "logps/chosen": -301.4223937988281, "logps/ref_chosen": -301.5334777832031, "logps/ref_rejected": -280.28900146484375, "logps/rejected": -280.301025390625, "loss": 5.5403, "margin_dpo/margin_mean": 0.12316262722015381, "margin_dpo/margin_std": 0.7420811653137207, "step": 21 }, { "KL/chosen_KL_mean": 0.061832427978515625, "KL/mean": 0.07514448463916779, "KL/rejected_KL_mean": 0.08844947814941406, "KL/std": 0.5327686667442322, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.026612281799316406, "fcm_dpo/q_t": 0.5000665187835693, "grad_norm": 25.43146514892578, "learning_rate": 2.1875e-07, "logits/chosen": -0.6658810377120972, "logits/rejected": -0.6679869890213013, "logps/chosen": -259.93328857421875, "logps/ref_chosen": -259.9951477050781, "logps/ref_rejected": -243.0721435546875, "logps/rejected": -242.98370361328125, "loss": 5.5463, "margin_dpo/margin_mean": -0.026612013578414917, "margin_dpo/margin_std": 0.780571460723877, "step": 22 }, { "KL/chosen_KL_mean": 0.0414276123046875, "KL/mean": 0.010329000651836395, "KL/rejected_KL_mean": -0.020765304565429688, "KL/std": 0.5395455956459045, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06218160688877106, "fcm_dpo/q_t": 0.49984458088874817, "grad_norm": 27.855823516845703, "learning_rate": 2.2916666666666663e-07, "logits/chosen": -0.6225969791412354, "logits/rejected": -0.6568140387535095, "logps/chosen": -282.13934326171875, "logps/ref_chosen": -282.1807556152344, "logps/ref_rejected": -265.0758056640625, "logps/rejected": -265.0965576171875, "loss": 5.5428, "margin_dpo/margin_mean": 0.06218172609806061, "margin_dpo/margin_std": 0.7284041047096252, "step": 23 }, { "KL/chosen_KL_mean": 0.22072601318359375, "KL/mean": 0.10974334180355072, "KL/rejected_KL_mean": -0.0012378692626953125, "KL/std": 0.5583473443984985, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22195783257484436, "fcm_dpo/q_t": 0.4994451403617859, "grad_norm": 29.714969635009766, "learning_rate": 2.3958333333333335e-07, "logits/chosen": -0.650376558303833, "logits/rejected": -0.5625022053718567, "logps/chosen": -300.9588623046875, "logps/ref_chosen": -301.17962646484375, "logps/ref_rejected": -302.12786865234375, "logps/rejected": -302.12908935546875, "loss": 5.5364, "margin_dpo/margin_mean": 0.2219579666852951, "margin_dpo/margin_std": 0.722432017326355, "step": 24 }, { "KL/chosen_KL_mean": 0.1817455291748047, "KL/mean": 0.10572192817926407, "KL/rejected_KL_mean": 0.029699325561523438, "KL/std": 0.5168058276176453, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15205390751361847, "fcm_dpo/q_t": 0.49961987137794495, "grad_norm": 26.451797485351562, "learning_rate": 2.5e-07, "logits/chosen": -0.5981687307357788, "logits/rejected": -0.6063342690467834, "logps/chosen": -246.5647430419922, "logps/ref_chosen": -246.74649047851562, "logps/ref_rejected": -235.55638122558594, "logps/rejected": -235.52667236328125, "loss": 5.5392, "margin_dpo/margin_mean": 0.15205399692058563, "margin_dpo/margin_std": 0.7110834717750549, "step": 25 }, { "KL/chosen_KL_mean": 0.23089218139648438, "KL/mean": 0.15290102362632751, "KL/rejected_KL_mean": 0.07491302490234375, "KL/std": 0.5813014507293701, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1559784710407257, "fcm_dpo/q_t": 0.4996100664138794, "grad_norm": 28.624317169189453, "learning_rate": 2.604166666666667e-07, "logits/chosen": -0.6492102742195129, "logits/rejected": -0.6630299687385559, "logps/chosen": -281.9646911621094, "logps/ref_chosen": -282.1955871582031, "logps/ref_rejected": -235.3135528564453, "logps/rejected": -235.2386474609375, "loss": 5.539, "margin_dpo/margin_mean": 0.15597863495349884, "margin_dpo/margin_std": 0.8204578161239624, "step": 26 }, { "KL/chosen_KL_mean": 0.3009986877441406, "KL/mean": 0.2155596911907196, "KL/rejected_KL_mean": 0.13011741638183594, "KL/std": 0.6508041620254517, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17088854312896729, "fcm_dpo/q_t": 0.4995727837085724, "grad_norm": 27.787202835083008, "learning_rate": 2.708333333333333e-07, "logits/chosen": -0.6545995473861694, "logits/rejected": -0.674304723739624, "logps/chosen": -323.55535888671875, "logps/ref_chosen": -323.8563537597656, "logps/ref_rejected": -245.968017578125, "logps/rejected": -245.83790588378906, "loss": 5.5384, "margin_dpo/margin_mean": 0.17088766396045685, "margin_dpo/margin_std": 0.8634297847747803, "step": 27 }, { "KL/chosen_KL_mean": 0.2823295593261719, "KL/mean": 0.2152925282716751, "KL/rejected_KL_mean": 0.1482524871826172, "KL/std": 0.5848507881164551, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1340872049331665, "fcm_dpo/q_t": 0.4996648132801056, "grad_norm": 26.31330680847168, "learning_rate": 2.8125e-07, "logits/chosen": -0.6248371005058289, "logits/rejected": -0.6342379450798035, "logps/chosen": -247.9644012451172, "logps/ref_chosen": -248.24673461914062, "logps/ref_rejected": -240.0382080078125, "logps/rejected": -239.88995361328125, "loss": 5.5399, "margin_dpo/margin_mean": 0.13408710062503815, "margin_dpo/margin_std": 0.8620131015777588, "step": 28 }, { "KL/chosen_KL_mean": 0.3334083557128906, "KL/mean": 0.26686322689056396, "KL/rejected_KL_mean": 0.20032119750976562, "KL/std": 0.648971438407898, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1330825835466385, "fcm_dpo/q_t": 0.49966728687286377, "grad_norm": 29.749004364013672, "learning_rate": 2.916666666666667e-07, "logits/chosen": -0.590377688407898, "logits/rejected": -0.6123830676078796, "logps/chosen": -317.9230041503906, "logps/ref_chosen": -318.2564392089844, "logps/ref_rejected": -286.75848388671875, "logps/rejected": -286.55816650390625, "loss": 5.5399, "margin_dpo/margin_mean": 0.13308259844779968, "margin_dpo/margin_std": 0.8849895596504211, "step": 29 }, { "KL/chosen_KL_mean": 0.3189105987548828, "KL/mean": 0.20669472217559814, "KL/rejected_KL_mean": 0.09447669982910156, "KL/std": 0.6948302388191223, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22443243861198425, "fcm_dpo/q_t": 0.49943894147872925, "grad_norm": 28.941471099853516, "learning_rate": 3.020833333333333e-07, "logits/chosen": -0.5937674045562744, "logits/rejected": -0.6108086705207825, "logps/chosen": -252.730224609375, "logps/ref_chosen": -253.0491485595703, "logps/ref_rejected": -261.30029296875, "logps/rejected": -261.205810546875, "loss": 5.5363, "margin_dpo/margin_mean": 0.22443242371082306, "margin_dpo/margin_std": 0.9711648225784302, "step": 30 }, { "KL/chosen_KL_mean": 0.43929100036621094, "KL/mean": 0.26924771070480347, "KL/rejected_KL_mean": 0.09921455383300781, "KL/std": 0.7348856925964355, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3400823473930359, "fcm_dpo/q_t": 0.4991498291492462, "grad_norm": 24.970046997070312, "learning_rate": 3.1249999999999997e-07, "logits/chosen": -0.6570526361465454, "logits/rejected": -0.6919869184494019, "logps/chosen": -247.71372985839844, "logps/ref_chosen": -248.15301513671875, "logps/ref_rejected": -203.17703247070312, "logps/rejected": -203.0778350830078, "loss": 5.5317, "margin_dpo/margin_mean": 0.34008198976516724, "margin_dpo/margin_std": 0.9680950045585632, "step": 31 }, { "KL/chosen_KL_mean": 0.4526844024658203, "KL/mean": 0.3364194929599762, "KL/rejected_KL_mean": 0.22015380859375, "KL/std": 0.7210460901260376, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23253017663955688, "fcm_dpo/q_t": 0.4994186758995056, "grad_norm": 29.655418395996094, "learning_rate": 3.2291666666666666e-07, "logits/chosen": -0.6053009033203125, "logits/rejected": -0.609401524066925, "logps/chosen": -305.0872497558594, "logps/ref_chosen": -305.5399475097656, "logps/ref_rejected": -267.6527099609375, "logps/rejected": -267.4325256347656, "loss": 5.536, "margin_dpo/margin_mean": 0.23253048956394196, "margin_dpo/margin_std": 0.9508224725723267, "step": 32 }, { "KL/chosen_KL_mean": 0.6426219940185547, "KL/mean": 0.4425170421600342, "KL/rejected_KL_mean": 0.2424163818359375, "KL/std": 0.869756281375885, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4001980721950531, "fcm_dpo/q_t": 0.49899953603744507, "grad_norm": 28.3889102935791, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.6407966613769531, "logits/rejected": -0.6533186435699463, "logps/chosen": -285.59088134765625, "logps/ref_chosen": -286.2335205078125, "logps/ref_rejected": -255.38748168945312, "logps/rejected": -255.14508056640625, "loss": 5.5293, "margin_dpo/margin_mean": 0.40019840002059937, "margin_dpo/margin_std": 1.0171304941177368, "step": 33 }, { "KL/chosen_KL_mean": 0.7163200378417969, "KL/mean": 0.39098507165908813, "KL/rejected_KL_mean": 0.06563377380371094, "KL/std": 0.9822959899902344, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6506999135017395, "fcm_dpo/q_t": 0.4983733296394348, "grad_norm": 31.151697158813477, "learning_rate": 3.4375e-07, "logits/chosen": -0.6305243372917175, "logits/rejected": -0.6405848860740662, "logps/chosen": -340.87567138671875, "logps/ref_chosen": -341.5920104980469, "logps/ref_rejected": -278.8866882324219, "logps/rejected": -278.821044921875, "loss": 5.5193, "margin_dpo/margin_mean": 0.650700032711029, "margin_dpo/margin_std": 1.2110459804534912, "step": 34 }, { "KL/chosen_KL_mean": 0.695404052734375, "KL/mean": 0.49731701612472534, "KL/rejected_KL_mean": 0.299224853515625, "KL/std": 0.9898061752319336, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3961779773235321, "fcm_dpo/q_t": 0.4990096092224121, "grad_norm": 26.41559600830078, "learning_rate": 3.541666666666667e-07, "logits/chosen": -0.6202894449234009, "logits/rejected": -0.6397201418876648, "logps/chosen": -264.3841552734375, "logps/ref_chosen": -265.0795593261719, "logps/ref_rejected": -264.4876708984375, "logps/rejected": -264.1884460449219, "loss": 5.5295, "margin_dpo/margin_mean": 0.39617788791656494, "margin_dpo/margin_std": 1.3193421363830566, "step": 35 }, { "KL/chosen_KL_mean": 0.8081455230712891, "KL/mean": 0.492267370223999, "KL/rejected_KL_mean": 0.17638397216796875, "KL/std": 1.098358392715454, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6317663192749023, "fcm_dpo/q_t": 0.49842071533203125, "grad_norm": 31.96265411376953, "learning_rate": 3.645833333333333e-07, "logits/chosen": -0.6001757383346558, "logits/rejected": -0.6177087426185608, "logps/chosen": -296.5179748535156, "logps/ref_chosen": -297.3261413574219, "logps/ref_rejected": -282.09515380859375, "logps/rejected": -281.91876220703125, "loss": 5.5201, "margin_dpo/margin_mean": 0.6317658424377441, "margin_dpo/margin_std": 1.3704296350479126, "step": 36 }, { "KL/chosen_KL_mean": 0.6605033874511719, "KL/mean": 0.4618363380432129, "KL/rejected_KL_mean": 0.26319122314453125, "KL/std": 1.0892057418823242, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3973109722137451, "fcm_dpo/q_t": 0.49900680780410767, "grad_norm": 30.963905334472656, "learning_rate": 3.75e-07, "logits/chosen": -0.6110119223594666, "logits/rejected": -0.6263231039047241, "logps/chosen": -313.37353515625, "logps/ref_chosen": -314.0340270996094, "logps/ref_rejected": -299.3437805175781, "logps/rejected": -299.08056640625, "loss": 5.5295, "margin_dpo/margin_mean": 0.39730948209762573, "margin_dpo/margin_std": 1.4675891399383545, "step": 37 }, { "KL/chosen_KL_mean": 1.0695934295654297, "KL/mean": 0.758275032043457, "KL/rejected_KL_mean": 0.4469585418701172, "KL/std": 1.2175971269607544, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6226468086242676, "fcm_dpo/q_t": 0.49844348430633545, "grad_norm": 28.48445701599121, "learning_rate": 3.8541666666666665e-07, "logits/chosen": -0.6337790489196777, "logits/rejected": -0.6456969976425171, "logps/chosen": -281.4715881347656, "logps/ref_chosen": -282.54119873046875, "logps/ref_rejected": -269.7773132324219, "logps/rejected": -269.330322265625, "loss": 5.5206, "margin_dpo/margin_mean": 0.622646689414978, "margin_dpo/margin_std": 1.480731725692749, "step": 38 }, { "KL/chosen_KL_mean": 1.3233833312988281, "KL/mean": 0.7047661542892456, "KL/rejected_KL_mean": 0.08614921569824219, "KL/std": 1.4990904331207275, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2372335195541382, "fcm_dpo/q_t": 0.4969073534011841, "grad_norm": 29.540149688720703, "learning_rate": 3.958333333333333e-07, "logits/chosen": -0.6165052056312561, "logits/rejected": -0.6305378675460815, "logps/chosen": -275.44952392578125, "logps/ref_chosen": -276.7729187011719, "logps/ref_rejected": -249.95889282226562, "logps/rejected": -249.87274169921875, "loss": 5.4962, "margin_dpo/margin_mean": 1.2372331619262695, "margin_dpo/margin_std": 1.7841914892196655, "step": 39 }, { "KL/chosen_KL_mean": 1.01873779296875, "KL/mean": 0.6700364947319031, "KL/rejected_KL_mean": 0.32134056091308594, "KL/std": 1.3435927629470825, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6973839998245239, "fcm_dpo/q_t": 0.4982568323612213, "grad_norm": 27.431995391845703, "learning_rate": 4.0625e-07, "logits/chosen": -0.6209850907325745, "logits/rejected": -0.6562680006027222, "logps/chosen": -283.2883605957031, "logps/ref_chosen": -284.30706787109375, "logps/ref_rejected": -244.4459991455078, "logps/rejected": -244.1246337890625, "loss": 5.5177, "margin_dpo/margin_mean": 0.6973843574523926, "margin_dpo/margin_std": 1.787576675415039, "step": 40 }, { "KL/chosen_KL_mean": 1.0851669311523438, "KL/mean": 0.7243558168411255, "KL/rejected_KL_mean": 0.3635540008544922, "KL/std": 1.4870356321334839, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7216100096702576, "fcm_dpo/q_t": 0.4981961250305176, "grad_norm": 30.58069610595703, "learning_rate": 4.1666666666666667e-07, "logits/chosen": -0.6218698620796204, "logits/rejected": -0.646290123462677, "logps/chosen": -292.72998046875, "logps/ref_chosen": -293.8151550292969, "logps/ref_rejected": -252.16815185546875, "logps/rejected": -251.80459594726562, "loss": 5.5167, "margin_dpo/margin_mean": 0.7216103672981262, "margin_dpo/margin_std": 1.7212419509887695, "step": 41 }, { "KL/chosen_KL_mean": 1.3099021911621094, "KL/mean": 0.8193244934082031, "KL/rejected_KL_mean": 0.3287544250488281, "KL/std": 1.7772796154022217, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9811455011367798, "fcm_dpo/q_t": 0.4975475072860718, "grad_norm": 27.563005447387695, "learning_rate": 4.270833333333333e-07, "logits/chosen": -0.6424188613891602, "logits/rejected": -0.6603102684020996, "logps/chosen": -251.45033264160156, "logps/ref_chosen": -252.76023864746094, "logps/ref_rejected": -261.0414733886719, "logps/rejected": -260.7127380371094, "loss": 5.5066, "margin_dpo/margin_mean": 0.9811455607414246, "margin_dpo/margin_std": 2.2941291332244873, "step": 42 }, { "KL/chosen_KL_mean": 1.2875423431396484, "KL/mean": 0.6953762173652649, "KL/rejected_KL_mean": 0.10320854187011719, "KL/std": 1.8649598360061646, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.18433678150177, "fcm_dpo/q_t": 0.4970394968986511, "grad_norm": 29.913719177246094, "learning_rate": 4.375e-07, "logits/chosen": -0.5954654216766357, "logits/rejected": -0.611076295375824, "logps/chosen": -315.5472106933594, "logps/ref_chosen": -316.8347473144531, "logps/ref_rejected": -273.7649230957031, "logps/rejected": -273.66168212890625, "loss": 5.4985, "margin_dpo/margin_mean": 1.1843376159667969, "margin_dpo/margin_std": 2.194983720779419, "step": 43 }, { "KL/chosen_KL_mean": 1.4896392822265625, "KL/mean": 0.7002232670783997, "KL/rejected_KL_mean": -0.08919525146484375, "KL/std": 2.1647138595581055, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.578826665878296, "fcm_dpo/q_t": 0.49605363607406616, "grad_norm": 30.989070892333984, "learning_rate": 4.479166666666667e-07, "logits/chosen": -0.5941963791847229, "logits/rejected": -0.5926939845085144, "logps/chosen": -285.3860168457031, "logps/ref_chosen": -286.8757019042969, "logps/ref_rejected": -282.4681396484375, "logps/rejected": -282.5573425292969, "loss": 5.4832, "margin_dpo/margin_mean": 1.5788276195526123, "margin_dpo/margin_std": 2.984520435333252, "step": 44 }, { "KL/chosen_KL_mean": 1.6895427703857422, "KL/mean": 1.006917119026184, "KL/rejected_KL_mean": 0.3242931365966797, "KL/std": 2.26188063621521, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3652527332305908, "fcm_dpo/q_t": 0.49658796191215515, "grad_norm": 28.858821868896484, "learning_rate": 4.5833333333333327e-07, "logits/chosen": -0.6818274259567261, "logits/rejected": -0.7043554782867432, "logps/chosen": -322.5738525390625, "logps/ref_chosen": -324.2633972167969, "logps/ref_rejected": -293.09466552734375, "logps/rejected": -292.7703857421875, "loss": 5.4916, "margin_dpo/margin_mean": 1.36525297164917, "margin_dpo/margin_std": 2.8198041915893555, "step": 45 }, { "KL/chosen_KL_mean": 1.6807384490966797, "KL/mean": 0.9394608736038208, "KL/rejected_KL_mean": 0.19817733764648438, "KL/std": 2.450871467590332, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4825652837753296, "fcm_dpo/q_t": 0.4962944984436035, "grad_norm": 30.326108932495117, "learning_rate": 4.6874999999999996e-07, "logits/chosen": -0.6200026869773865, "logits/rejected": -0.633066713809967, "logps/chosen": -296.6549987792969, "logps/ref_chosen": -298.3357238769531, "logps/ref_rejected": -267.66204833984375, "logps/rejected": -267.4638366699219, "loss": 5.4869, "margin_dpo/margin_mean": 1.4825655221939087, "margin_dpo/margin_std": 2.687807321548462, "step": 46 }, { "KL/chosen_KL_mean": 1.51885986328125, "KL/mean": 1.0282460451126099, "KL/rejected_KL_mean": 0.5376300811767578, "KL/std": 2.3429102897644043, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9812304973602295, "fcm_dpo/q_t": 0.4975472688674927, "grad_norm": 26.336135864257812, "learning_rate": 4.791666666666667e-07, "logits/chosen": -0.5996535420417786, "logits/rejected": -0.6199891567230225, "logps/chosen": -261.0480651855469, "logps/ref_chosen": -262.5669250488281, "logps/ref_rejected": -258.70989990234375, "logps/rejected": -258.1722412109375, "loss": 5.5073, "margin_dpo/margin_mean": 0.9812304973602295, "margin_dpo/margin_std": 3.329333782196045, "step": 47 }, { "KL/chosen_KL_mean": 1.8202342987060547, "KL/mean": 1.0875205993652344, "KL/rejected_KL_mean": 0.3548126220703125, "KL/std": 2.496361494064331, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4654216766357422, "fcm_dpo/q_t": 0.49633827805519104, "grad_norm": 27.6088924407959, "learning_rate": 4.895833333333333e-07, "logits/chosen": -0.6158616542816162, "logits/rejected": -0.6411494016647339, "logps/chosen": -267.67303466796875, "logps/ref_chosen": -269.4932556152344, "logps/ref_rejected": -241.888916015625, "logps/rejected": -241.5341033935547, "loss": 5.4877, "margin_dpo/margin_mean": 1.465421199798584, "margin_dpo/margin_std": 2.838312864303589, "step": 48 }, { "KL/chosen_KL_mean": 2.259225845336914, "KL/mean": 1.3766483068466187, "KL/rejected_KL_mean": 0.4940757751464844, "KL/std": 2.617175817489624, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7651441097259521, "fcm_dpo/q_t": 0.4955885410308838, "grad_norm": 27.615480422973633, "learning_rate": 5e-07, "logits/chosen": -0.6648316383361816, "logits/rejected": -0.6493117809295654, "logps/chosen": -255.62527465820312, "logps/ref_chosen": -257.8844909667969, "logps/ref_rejected": -256.8912048339844, "logps/rejected": -256.3971252441406, "loss": 5.4761, "margin_dpo/margin_mean": 1.7651444673538208, "margin_dpo/margin_std": 3.3019487857818604, "step": 49 }, { "KL/chosen_KL_mean": 2.061922073364258, "KL/mean": 1.5143541097640991, "KL/rejected_KL_mean": 0.9667835235595703, "KL/std": 3.0319771766662598, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0951329469680786, "fcm_dpo/q_t": 0.4972635805606842, "grad_norm": 28.010818481445312, "learning_rate": 4.999932966293553e-07, "logits/chosen": -0.6277458667755127, "logits/rejected": -0.6493853330612183, "logps/chosen": -299.56695556640625, "logps/ref_chosen": -301.62884521484375, "logps/ref_rejected": -298.2716064453125, "logps/rejected": -297.3048400878906, "loss": 5.5031, "margin_dpo/margin_mean": 1.09513258934021, "margin_dpo/margin_std": 3.8171825408935547, "step": 50 }, { "KL/chosen_KL_mean": 2.1011962890625, "KL/mean": 1.1179943084716797, "KL/rejected_KL_mean": 0.1347827911376953, "KL/std": 3.6277670860290527, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.966414451599121, "fcm_dpo/q_t": 0.4950869083404541, "grad_norm": 29.124319076538086, "learning_rate": 4.999731868769026e-07, "logits/chosen": -0.5978573560714722, "logits/rejected": -0.5924739241600037, "logps/chosen": -267.27117919921875, "logps/ref_chosen": -269.37237548828125, "logps/ref_rejected": -297.0167541503906, "logps/rejected": -296.8819580078125, "loss": 5.4694, "margin_dpo/margin_mean": 1.9664149284362793, "margin_dpo/margin_std": 4.608149528503418, "step": 51 }, { "KL/chosen_KL_mean": 2.7861671447753906, "KL/mean": 1.2450693845748901, "KL/rejected_KL_mean": -0.2960319519042969, "KL/std": 3.817547082901001, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.0822057723999023, "fcm_dpo/q_t": 0.49230119585990906, "grad_norm": 30.227222442626953, "learning_rate": 4.99939671821067e-07, "logits/chosen": -0.6642853617668152, "logits/rejected": -0.6713843941688538, "logps/chosen": -304.1166076660156, "logps/ref_chosen": -306.9028015136719, "logps/ref_rejected": -281.24737548828125, "logps/rejected": -281.54339599609375, "loss": 5.4253, "margin_dpo/margin_mean": 3.082204580307007, "margin_dpo/margin_std": 4.780548095703125, "step": 52 }, { "KL/chosen_KL_mean": 2.8322067260742188, "KL/mean": 1.739880919456482, "KL/rejected_KL_mean": 0.647552490234375, "KL/std": 4.097768306732178, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.1846604347229004, "fcm_dpo/q_t": 0.49454259872436523, "grad_norm": 29.97759246826172, "learning_rate": 4.998927532591591e-07, "logits/chosen": -0.6540624499320984, "logits/rejected": -0.6947383284568787, "logps/chosen": -283.1437072753906, "logps/ref_chosen": -285.9759521484375, "logps/ref_rejected": -273.9073486328125, "logps/rejected": -273.2597961425781, "loss": 5.4613, "margin_dpo/margin_mean": 2.184659481048584, "margin_dpo/margin_std": 5.051357269287109, "step": 53 }, { "KL/chosen_KL_mean": 2.740182876586914, "KL/mean": 1.7801817655563354, "KL/rejected_KL_mean": 0.8201732635498047, "KL/std": 4.306924819946289, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9200044870376587, "fcm_dpo/q_t": 0.4952048659324646, "grad_norm": 26.368751525878906, "learning_rate": 4.998324337072792e-07, "logits/chosen": -0.6892499923706055, "logits/rejected": -0.6974684000015259, "logps/chosen": -303.7644348144531, "logps/ref_chosen": -306.504638671875, "logps/ref_rejected": -272.67431640625, "logps/rejected": -271.8541564941406, "loss": 5.4719, "margin_dpo/margin_mean": 1.9200036525726318, "margin_dpo/margin_std": 5.399931907653809, "step": 54 }, { "KL/chosen_KL_mean": 1.7780208587646484, "KL/mean": 0.6723177433013916, "KL/rejected_KL_mean": -0.4333839416503906, "KL/std": 3.7362210750579834, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.211399555206299, "fcm_dpo/q_t": 0.4944767355918884, "grad_norm": 24.839492797851562, "learning_rate": 4.997587164001815e-07, "logits/chosen": -0.6404720544815063, "logits/rejected": -0.6446375250816345, "logps/chosen": -220.5521240234375, "logps/ref_chosen": -222.33013916015625, "logps/ref_rejected": -206.59571838378906, "logps/rejected": -207.02911376953125, "loss": 5.4598, "margin_dpo/margin_mean": 2.211399555206299, "margin_dpo/margin_std": 4.992412567138672, "step": 55 }, { "KL/chosen_KL_mean": 2.8933238983154297, "KL/mean": 1.2889918088912964, "KL/rejected_KL_mean": -0.31532859802246094, "KL/std": 4.780613422393799, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.2086493968963623, "fcm_dpo/q_t": 0.49199068546295166, "grad_norm": 27.42078399658203, "learning_rate": 4.996716052911017e-07, "logits/chosen": -0.6034107804298401, "logits/rejected": -0.6173827648162842, "logps/chosen": -247.5848388671875, "logps/ref_chosen": -250.47816467285156, "logps/ref_rejected": -228.25848388671875, "logps/rejected": -228.57382202148438, "loss": 5.4221, "margin_dpo/margin_mean": 3.208649158477783, "margin_dpo/margin_std": 5.916287899017334, "step": 56 }, { "KL/chosen_KL_mean": 3.8232650756835938, "KL/mean": 1.7507920265197754, "KL/rejected_KL_mean": -0.3216896057128906, "KL/std": 4.765433311462402, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.14495849609375, "fcm_dpo/q_t": 0.48964935541152954, "grad_norm": 31.133817672729492, "learning_rate": 4.99571105051544e-07, "logits/chosen": -0.6981046199798584, "logits/rejected": -0.668836772441864, "logps/chosen": -311.2962646484375, "logps/ref_chosen": -315.1195373535156, "logps/ref_rejected": -272.755615234375, "logps/rejected": -273.0773010253906, "loss": 5.3845, "margin_dpo/margin_mean": 4.14495849609375, "margin_dpo/margin_std": 5.633490085601807, "step": 57 }, { "KL/chosen_KL_mean": 2.441774368286133, "KL/mean": 1.288618803024292, "KL/rejected_KL_mean": 0.13546371459960938, "KL/std": 4.413588047027588, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.30631685256958, "fcm_dpo/q_t": 0.4942378103733063, "grad_norm": 28.11114501953125, "learning_rate": 4.994572210710314e-07, "logits/chosen": -0.6136592030525208, "logits/rejected": -0.637704610824585, "logps/chosen": -262.7398986816406, "logps/ref_chosen": -265.1816711425781, "logps/ref_rejected": -268.2203369140625, "logps/rejected": -268.0848388671875, "loss": 5.457, "margin_dpo/margin_mean": 2.306316614151001, "margin_dpo/margin_std": 5.692234992980957, "step": 58 }, { "KL/chosen_KL_mean": 2.0896434783935547, "KL/mean": 1.329612374305725, "KL/rejected_KL_mean": 0.5695877075195312, "KL/std": 5.218595504760742, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5200635194778442, "fcm_dpo/q_t": 0.49620577692985535, "grad_norm": 29.61176109313965, "learning_rate": 4.993299594568162e-07, "logits/chosen": -0.6077237725257874, "logits/rejected": -0.601131796836853, "logps/chosen": -284.2643127441406, "logps/ref_chosen": -286.35394287109375, "logps/ref_rejected": -260.6757507324219, "logps/rejected": -260.106201171875, "loss": 5.4902, "margin_dpo/margin_mean": 1.5200631618499756, "margin_dpo/margin_std": 6.907461166381836, "step": 59 }, { "KL/chosen_KL_mean": 2.9706172943115234, "KL/mean": 1.2671046257019043, "KL/rejected_KL_mean": -0.43640899658203125, "KL/std": 5.212099075317383, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.407027244567871, "fcm_dpo/q_t": 0.49149319529533386, "grad_norm": 27.876686096191406, "learning_rate": 4.991893270335525e-07, "logits/chosen": -0.6783978343009949, "logits/rejected": -0.703014612197876, "logps/chosen": -255.7779998779297, "logps/ref_chosen": -258.74859619140625, "logps/ref_rejected": -255.04893493652344, "logps/rejected": -255.48533630371094, "loss": 5.4156, "margin_dpo/margin_mean": 3.4070284366607666, "margin_dpo/margin_std": 7.180816650390625, "step": 60 }, { "KL/chosen_KL_mean": 3.0242786407470703, "KL/mean": 1.413300633430481, "KL/rejected_KL_mean": -0.1976757049560547, "KL/std": 5.535090923309326, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.2219600677490234, "fcm_dpo/q_t": 0.4919591248035431, "grad_norm": 29.932416915893555, "learning_rate": 4.990353313429303e-07, "logits/chosen": -0.6423856616020203, "logits/rejected": -0.6588336229324341, "logps/chosen": -275.4436340332031, "logps/ref_chosen": -278.4678955078125, "logps/ref_rejected": -252.02720642089844, "logps/rejected": -252.22488403320312, "loss": 5.4239, "margin_dpo/margin_mean": 3.2219598293304443, "margin_dpo/margin_std": 7.598827838897705, "step": 61 }, { "KL/chosen_KL_mean": 4.0137786865234375, "KL/mean": 2.280641794204712, "KL/rejected_KL_mean": 0.5475139617919922, "KL/std": 5.737850189208984, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.466265916824341, "fcm_dpo/q_t": 0.4913550615310669, "grad_norm": 26.708263397216797, "learning_rate": 4.988679806432711e-07, "logits/chosen": -0.6178765296936035, "logits/rejected": -0.6604681611061096, "logps/chosen": -268.9105224609375, "logps/ref_chosen": -272.92431640625, "logps/ref_rejected": -260.7935485839844, "logps/rejected": -260.2460632324219, "loss": 5.4135, "margin_dpo/margin_mean": 3.46626615524292, "margin_dpo/margin_std": 7.4036478996276855, "step": 62 }, { "KL/chosen_KL_mean": 3.3519554138183594, "KL/mean": 1.6410435438156128, "KL/rejected_KL_mean": -0.06987190246582031, "KL/std": 5.6098198890686035, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4218201637268066, "fcm_dpo/q_t": 0.49145957827568054, "grad_norm": 28.320823669433594, "learning_rate": 4.986872839090852e-07, "logits/chosen": -0.6582809686660767, "logits/rejected": -0.6645787954330444, "logps/chosen": -273.7370300292969, "logps/ref_chosen": -277.0889892578125, "logps/ref_rejected": -273.3413391113281, "logps/rejected": -273.41119384765625, "loss": 5.4151, "margin_dpo/margin_mean": 3.4218192100524902, "margin_dpo/margin_std": 7.18504524230957, "step": 63 }, { "KL/chosen_KL_mean": 4.1611785888671875, "KL/mean": 2.0322928428649902, "KL/rejected_KL_mean": -0.09659004211425781, "KL/std": 6.245217800140381, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.2577714920043945, "fcm_dpo/q_t": 0.4893771708011627, "grad_norm": 28.30405616760254, "learning_rate": 4.9849325083059e-07, "logits/chosen": -0.6383385062217712, "logits/rejected": -0.6341136693954468, "logps/chosen": -279.6632995605469, "logps/ref_chosen": -283.8244934082031, "logps/ref_rejected": -263.29351806640625, "logps/rejected": -263.39007568359375, "loss": 5.3841, "margin_dpo/margin_mean": 4.257771015167236, "margin_dpo/margin_std": 8.265096664428711, "step": 64 }, { "KL/chosen_KL_mean": 3.4199390411376953, "KL/mean": 1.7757415771484375, "KL/rejected_KL_mean": 0.1315479278564453, "KL/std": 5.751835823059082, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.2883832454681396, "fcm_dpo/q_t": 0.4917920231819153, "grad_norm": 27.830177307128906, "learning_rate": 4.982858918131906e-07, "logits/chosen": -0.705287754535675, "logits/rejected": -0.6732350587844849, "logps/chosen": -261.45001220703125, "logps/ref_chosen": -264.8699645996094, "logps/ref_rejected": -268.5076904296875, "logps/rejected": -268.37615966796875, "loss": 5.4205, "margin_dpo/margin_mean": 3.2883825302124023, "margin_dpo/margin_std": 7.373363971710205, "step": 65 }, { "KL/chosen_KL_mean": 3.138427734375, "KL/mean": 1.04404616355896, "KL/rejected_KL_mean": -1.0503253936767578, "KL/std": 7.417426109313965, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.188744068145752, "fcm_dpo/q_t": 0.4895643889904022, "grad_norm": 27.398483276367188, "learning_rate": 4.980652179769217e-07, "logits/chosen": -0.6739647388458252, "logits/rejected": -0.6936634182929993, "logps/chosen": -269.7898864746094, "logps/ref_chosen": -272.9283142089844, "logps/ref_rejected": -280.94696044921875, "logps/rejected": -281.9972839355469, "loss": 5.3898, "margin_dpo/margin_mean": 4.188744068145752, "margin_dpo/margin_std": 10.030872344970703, "step": 66 }, { "KL/chosen_KL_mean": 4.055376052856445, "KL/mean": 2.244265079498291, "KL/rejected_KL_mean": 0.43314552307128906, "KL/std": 7.153616428375244, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.6222338676452637, "fcm_dpo/q_t": 0.4909643232822418, "grad_norm": 25.641693115234375, "learning_rate": 4.978312411558517e-07, "logits/chosen": -0.7017156481742859, "logits/rejected": -0.7357592582702637, "logps/chosen": -262.1315612792969, "logps/ref_chosen": -266.18695068359375, "logps/ref_rejected": -250.17405700683594, "logps/rejected": -249.74093627929688, "loss": 5.41, "margin_dpo/margin_mean": 3.6222333908081055, "margin_dpo/margin_std": 8.869529724121094, "step": 67 }, { "KL/chosen_KL_mean": 3.0623416900634766, "KL/mean": 0.7446590065956116, "KL/rejected_KL_mean": -1.5730171203613281, "KL/std": 8.063435554504395, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.635357856750488, "fcm_dpo/q_t": 0.48847031593322754, "grad_norm": 27.99388313293457, "learning_rate": 4.975839738974473e-07, "logits/chosen": -0.6817157864570618, "logits/rejected": -0.6947129368782043, "logps/chosen": -294.8762512207031, "logps/ref_chosen": -297.9385986328125, "logps/ref_rejected": -261.5141296386719, "logps/rejected": -263.0871276855469, "loss": 5.374, "margin_dpo/margin_mean": 4.635357856750488, "margin_dpo/margin_std": 10.582465171813965, "step": 68 }, { "KL/chosen_KL_mean": 3.916715621948242, "KL/mean": 0.8802259564399719, "KL/rejected_KL_mean": -2.156261444091797, "KL/std": 7.877801418304443, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.072974681854248, "fcm_dpo/q_t": 0.4848654866218567, "grad_norm": 28.498083114624023, "learning_rate": 4.97323429461901e-07, "logits/chosen": -0.6778756380081177, "logits/rejected": -0.7077374458312988, "logps/chosen": -261.7008056640625, "logps/ref_chosen": -265.6175231933594, "logps/ref_rejected": -236.8287353515625, "logps/rejected": -238.9849853515625, "loss": 5.3158, "margin_dpo/margin_mean": 6.0729756355285645, "margin_dpo/margin_std": 9.63194465637207, "step": 69 }, { "KL/chosen_KL_mean": 4.352378845214844, "KL/mean": 1.3984824419021606, "KL/rejected_KL_mean": -1.5554084777832031, "KL/std": 8.641536712646484, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.907784461975098, "fcm_dpo/q_t": 0.4852859377861023, "grad_norm": 28.619237899780273, "learning_rate": 4.970496218214204e-07, "logits/chosen": -0.6764180064201355, "logits/rejected": -0.7098894119262695, "logps/chosen": -291.87353515625, "logps/ref_chosen": -296.2259216308594, "logps/ref_rejected": -254.68496704101562, "logps/rejected": -256.2403564453125, "loss": 5.3252, "margin_dpo/margin_mean": 5.907782554626465, "margin_dpo/margin_std": 10.884395599365234, "step": 70 }, { "KL/chosen_KL_mean": 5.073089599609375, "KL/mean": 2.5308048725128174, "KL/rejected_KL_mean": -0.011476516723632812, "KL/std": 9.272811889648438, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.084573268890381, "fcm_dpo/q_t": 0.48732662200927734, "grad_norm": 28.492225646972656, "learning_rate": 4.967625656594781e-07, "logits/chosen": -0.6502672433853149, "logits/rejected": -0.6391031742095947, "logps/chosen": -283.8541564941406, "logps/ref_chosen": -288.92724609375, "logps/ref_rejected": -278.6405334472656, "logps/rejected": -278.6519775390625, "loss": 5.3608, "margin_dpo/margin_mean": 5.084572792053223, "margin_dpo/margin_std": 12.225592613220215, "step": 71 }, { "KL/chosen_KL_mean": 3.342805862426758, "KL/mean": 0.7642459869384766, "KL/rejected_KL_mean": -1.8143062591552734, "KL/std": 8.467608451843262, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.157101631164551, "fcm_dpo/q_t": 0.4871388077735901, "grad_norm": 28.240171432495117, "learning_rate": 4.964622763700252e-07, "logits/chosen": -0.692705512046814, "logits/rejected": -0.7039174437522888, "logps/chosen": -233.70248413085938, "logps/ref_chosen": -237.0452880859375, "logps/ref_rejected": -252.7946319580078, "logps/rejected": -254.6089324951172, "loss": 5.3542, "margin_dpo/margin_mean": 5.157101631164551, "margin_dpo/margin_std": 10.475278854370117, "step": 72 }, { "KL/chosen_KL_mean": 4.400068283081055, "KL/mean": 1.9903484582901, "KL/rejected_KL_mean": -0.41938209533691406, "KL/std": 8.966333389282227, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.81945276260376, "fcm_dpo/q_t": 0.4879787862300873, "grad_norm": 27.833248138427734, "learning_rate": 4.961487700566646e-07, "logits/chosen": -0.6518418192863464, "logits/rejected": -0.668941855430603, "logps/chosen": -268.65301513671875, "logps/ref_chosen": -273.0531005859375, "logps/ref_rejected": -246.8330841064453, "logps/rejected": -247.25247192382812, "loss": 5.3711, "margin_dpo/margin_mean": 4.81945276260376, "margin_dpo/margin_std": 12.24702262878418, "step": 73 }, { "KL/chosen_KL_mean": 3.4003162384033203, "KL/mean": 1.1663212776184082, "KL/rejected_KL_mean": -1.0676746368408203, "KL/std": 9.370096206665039, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.4679975509643555, "fcm_dpo/q_t": 0.48887601494789124, "grad_norm": 30.320070266723633, "learning_rate": 4.958220635317885e-07, "logits/chosen": -0.7227351665496826, "logits/rejected": -0.7008697986602783, "logps/chosen": -338.881591796875, "logps/ref_chosen": -342.2818908691406, "logps/ref_rejected": -330.0293884277344, "logps/rejected": -331.0970153808594, "loss": 5.3832, "margin_dpo/margin_mean": 4.4679975509643555, "margin_dpo/margin_std": 11.65971565246582, "step": 74 }, { "KL/chosen_KL_mean": 4.72509765625, "KL/mean": 1.3391616344451904, "KL/rejected_KL_mean": -2.0467777252197266, "KL/std": 9.442550659179688, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.771880149841309, "fcm_dpo/q_t": 0.4831286668777466, "grad_norm": 29.469329833984375, "learning_rate": 4.954821743156767e-07, "logits/chosen": -0.6357704401016235, "logits/rejected": -0.6371195316314697, "logps/chosen": -262.1390380859375, "logps/ref_chosen": -266.8641662597656, "logps/ref_rejected": -276.8699951171875, "logps/rejected": -278.9167785644531, "loss": 5.2922, "margin_dpo/margin_mean": 6.771879196166992, "margin_dpo/margin_std": 10.950206756591797, "step": 75 }, { "KL/chosen_KL_mean": 4.118385314941406, "KL/mean": 0.5083938837051392, "KL/rejected_KL_mean": -3.101591110229492, "KL/std": 9.978694915771484, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.219972610473633, "fcm_dpo/q_t": 0.48206230998039246, "grad_norm": 29.70305061340332, "learning_rate": 4.951291206355559e-07, "logits/chosen": -0.7215156555175781, "logits/rejected": -0.72944575548172, "logps/chosen": -277.05621337890625, "logps/ref_chosen": -281.174560546875, "logps/ref_rejected": -263.6067199707031, "logps/rejected": -266.70831298828125, "loss": 5.2794, "margin_dpo/margin_mean": 7.219972610473633, "margin_dpo/margin_std": 12.839584350585938, "step": 76 }, { "KL/chosen_KL_mean": 3.7498321533203125, "KL/mean": 0.8297483921051025, "KL/rejected_KL_mean": -2.0903377532958984, "KL/std": 11.009910583496094, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.840177059173584, "fcm_dpo/q_t": 0.4854981005191803, "grad_norm": 33.02766418457031, "learning_rate": 4.947629214246236e-07, "logits/chosen": -0.5660206079483032, "logits/rejected": -0.5753567814826965, "logps/chosen": -302.345458984375, "logps/ref_chosen": -306.09527587890625, "logps/ref_rejected": -253.49569702148438, "logps/rejected": -255.58602905273438, "loss": 5.3359, "margin_dpo/margin_mean": 5.840176582336426, "margin_dpo/margin_std": 13.89721393585205, "step": 77 }, { "KL/chosen_KL_mean": 3.9143428802490234, "KL/mean": -0.4267261326313019, "KL/rejected_KL_mean": -4.767789840698242, "KL/std": 10.995488166809082, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.68213176727295, "fcm_dpo/q_t": 0.4784572124481201, "grad_norm": 29.675912857055664, "learning_rate": 4.943835963210323e-07, "logits/chosen": -0.6788281202316284, "logits/rejected": -0.6731718182563782, "logps/chosen": -252.9879913330078, "logps/ref_chosen": -256.90234375, "logps/ref_rejected": -211.57154846191406, "logps/rejected": -216.33935546875, "loss": 5.227, "margin_dpo/margin_mean": 8.68213176727295, "margin_dpo/margin_std": 14.381519317626953, "step": 78 }, { "KL/chosen_KL_mean": 3.0857925415039062, "KL/mean": -1.0710463523864746, "KL/rejected_KL_mean": -5.227884292602539, "KL/std": 11.816179275512695, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.313674926757812, "fcm_dpo/q_t": 0.4793742299079895, "grad_norm": 30.00199317932129, "learning_rate": 4.939911656668361e-07, "logits/chosen": -0.6557655334472656, "logits/rejected": -0.6764158010482788, "logps/chosen": -263.18780517578125, "logps/ref_chosen": -266.2735595703125, "logps/ref_rejected": -251.57257080078125, "logps/rejected": -256.8004455566406, "loss": 5.2437, "margin_dpo/margin_mean": 8.313674926757812, "margin_dpo/margin_std": 15.012711524963379, "step": 79 }, { "KL/chosen_KL_mean": 1.8700141906738281, "KL/mean": -1.4277009963989258, "KL/rejected_KL_mean": -4.725408554077148, "KL/std": 11.388775825500488, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.595414161682129, "fcm_dpo/q_t": 0.48363250494003296, "grad_norm": 28.94468879699707, "learning_rate": 4.935856505068998e-07, "logits/chosen": -0.6733145117759705, "logits/rejected": -0.7031750082969666, "logps/chosen": -285.98095703125, "logps/ref_chosen": -287.8509826660156, "logps/ref_rejected": -256.0766296386719, "logps/rejected": -260.802001953125, "loss": 5.3057, "margin_dpo/margin_mean": 6.595414638519287, "margin_dpo/margin_std": 13.258899688720703, "step": 80 }, { "KL/chosen_KL_mean": 2.3981895446777344, "KL/mean": -1.178763747215271, "KL/rejected_KL_mean": -4.755710601806641, "KL/std": 12.212273597717285, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.153894424438477, "fcm_dpo/q_t": 0.48225945234298706, "grad_norm": 28.060094833374023, "learning_rate": 4.93167072587771e-07, "logits/chosen": -0.6365951299667358, "logits/rejected": -0.6353590488433838, "logps/chosen": -266.1250305175781, "logps/ref_chosen": -268.5232238769531, "logps/ref_rejected": -237.81137084960938, "logps/rejected": -242.56707763671875, "loss": 5.2938, "margin_dpo/margin_mean": 7.153894901275635, "margin_dpo/margin_std": 16.709213256835938, "step": 81 }, { "KL/chosen_KL_mean": 2.450326919555664, "KL/mean": -1.5248596668243408, "KL/rejected_KL_mean": -5.500043869018555, "KL/std": 12.273836135864258, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.950379848480225, "fcm_dpo/q_t": 0.48033344745635986, "grad_norm": 27.83127784729004, "learning_rate": 4.92735454356513e-07, "logits/chosen": -0.719543993473053, "logits/rejected": -0.7259870171546936, "logps/chosen": -276.9136047363281, "logps/ref_chosen": -279.36395263671875, "logps/ref_rejected": -236.51365661621094, "logps/rejected": -242.01370239257812, "loss": 5.2621, "margin_dpo/margin_mean": 7.950380325317383, "margin_dpo/margin_std": 15.83857250213623, "step": 82 }, { "KL/chosen_KL_mean": 0.28804588317871094, "KL/mean": -4.03283166885376, "KL/rejected_KL_mean": -8.35371208190918, "KL/std": 13.607312202453613, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.641751289367676, "fcm_dpo/q_t": 0.47854292392730713, "grad_norm": 30.7332706451416, "learning_rate": 4.922908189595017e-07, "logits/chosen": -0.6931532621383667, "logits/rejected": -0.6767294406890869, "logps/chosen": -273.9311828613281, "logps/ref_chosen": -274.21923828125, "logps/ref_rejected": -276.2212219238281, "logps/rejected": -284.5749206542969, "loss": 5.2434, "margin_dpo/margin_mean": 8.64175033569336, "margin_dpo/margin_std": 18.098514556884766, "step": 83 }, { "KL/chosen_KL_mean": 0.5773792266845703, "KL/mean": -3.0747742652893066, "KL/rejected_KL_mean": -6.726924896240234, "KL/std": 13.29410171508789, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.304309368133545, "fcm_dpo/q_t": 0.4818825423717499, "grad_norm": 29.943376541137695, "learning_rate": 4.918331902411841e-07, "logits/chosen": -0.7306097149848938, "logits/rejected": -0.7447658181190491, "logps/chosen": -293.8201904296875, "logps/ref_chosen": -294.3975524902344, "logps/ref_rejected": -279.81884765625, "logps/rejected": -286.5457458496094, "loss": 5.2894, "margin_dpo/margin_mean": 7.3043107986450195, "margin_dpo/margin_std": 16.863800048828125, "step": 84 }, { "KL/chosen_KL_mean": -1.4663372039794922, "KL/mean": -4.343188762664795, "KL/rejected_KL_mean": -7.220043182373047, "KL/std": 12.284307479858398, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.753708362579346, "fcm_dpo/q_t": 0.4857223331928253, "grad_norm": 29.357282638549805, "learning_rate": 4.913625927427995e-07, "logits/chosen": -0.6696852445602417, "logits/rejected": -0.6779837608337402, "logps/chosen": -245.12852478027344, "logps/ref_chosen": -243.66220092773438, "logps/ref_rejected": -263.9421691894531, "logps/rejected": -271.1622314453125, "loss": 5.345, "margin_dpo/margin_mean": 5.753708839416504, "margin_dpo/margin_std": 15.366795539855957, "step": 85 }, { "KL/chosen_KL_mean": 1.204885482788086, "KL/mean": -3.256032705307007, "KL/rejected_KL_mean": -7.716947555541992, "KL/std": 13.279273986816406, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.92184066772461, "fcm_dpo/q_t": 0.477853924036026, "grad_norm": 36.05186462402344, "learning_rate": 4.908790517010636e-07, "logits/chosen": -0.694229245185852, "logits/rejected": -0.6882424354553223, "logps/chosen": -308.2257995605469, "logps/ref_chosen": -309.4306945800781, "logps/ref_rejected": -290.91278076171875, "logps/rejected": -298.6297607421875, "loss": 5.2279, "margin_dpo/margin_mean": 8.921839714050293, "margin_dpo/margin_std": 17.340333938598633, "step": 86 }, { "KL/chosen_KL_mean": -0.13620948791503906, "KL/mean": -5.250948429107666, "KL/rejected_KL_mean": -10.365686416625977, "KL/std": 15.311408042907715, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.22948169708252, "fcm_dpo/q_t": 0.4747437834739685, "grad_norm": 29.675899505615234, "learning_rate": 4.903825930468148e-07, "logits/chosen": -0.7572658061981201, "logits/rejected": -0.7517373561859131, "logps/chosen": -278.1639404296875, "logps/ref_chosen": -278.0277099609375, "logps/ref_rejected": -245.70123291015625, "logps/rejected": -256.06689453125, "loss": 5.1852, "margin_dpo/margin_mean": 10.22948169708252, "margin_dpo/margin_std": 18.946619033813477, "step": 87 }, { "KL/chosen_KL_mean": -2.036558151245117, "KL/mean": -6.464334011077881, "KL/rejected_KL_mean": -10.892110824584961, "KL/std": 15.0281400680542, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.855554580688477, "fcm_dpo/q_t": 0.4781079888343811, "grad_norm": 28.795900344848633, "learning_rate": 4.898732434036243e-07, "logits/chosen": -0.7639174461364746, "logits/rejected": -0.7798334956169128, "logps/chosen": -268.5513610839844, "logps/ref_chosen": -266.5148010253906, "logps/ref_rejected": -265.90081787109375, "logps/rejected": -276.7929382324219, "loss": 5.2366, "margin_dpo/margin_mean": 8.855554580688477, "margin_dpo/margin_std": 19.017532348632812, "step": 88 }, { "KL/chosen_KL_mean": 0.0593719482421875, "KL/mean": -4.611057281494141, "KL/rejected_KL_mean": -9.281488418579102, "KL/std": 14.272215843200684, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.34085750579834, "fcm_dpo/q_t": 0.4768635332584381, "grad_norm": 30.523344039916992, "learning_rate": 4.893510300863676e-07, "logits/chosen": -0.7424763441085815, "logits/rejected": -0.7329275608062744, "logps/chosen": -265.6299133300781, "logps/ref_chosen": -265.6893005371094, "logps/ref_rejected": -251.49314880371094, "logps/rejected": -260.7746276855469, "loss": 5.2174, "margin_dpo/margin_mean": 9.340858459472656, "margin_dpo/margin_std": 18.225364685058594, "step": 89 }, { "KL/chosen_KL_mean": -1.1600418090820312, "KL/mean": -5.250759124755859, "KL/rejected_KL_mean": -9.341472625732422, "KL/std": 14.324559211730957, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.181434631347656, "fcm_dpo/q_t": 0.47971487045288086, "grad_norm": 29.79361343383789, "learning_rate": 4.8881598109976e-07, "logits/chosen": -0.7358546257019043, "logits/rejected": -0.7441498637199402, "logps/chosen": -308.5850524902344, "logps/ref_chosen": -307.4250183105469, "logps/ref_rejected": -265.7172546386719, "logps/rejected": -275.0587158203125, "loss": 5.2588, "margin_dpo/margin_mean": 8.181434631347656, "margin_dpo/margin_std": 17.716218948364258, "step": 90 }, { "KL/chosen_KL_mean": -2.122282028198242, "KL/mean": -6.840453624725342, "KL/rejected_KL_mean": -11.558622360229492, "KL/std": 14.71864128112793, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.436348915100098, "fcm_dpo/q_t": 0.47672733664512634, "grad_norm": 32.89813232421875, "learning_rate": 4.882681251368548e-07, "logits/chosen": -0.6823728084564209, "logits/rejected": -0.699794352054596, "logps/chosen": -237.86328125, "logps/ref_chosen": -235.74098205566406, "logps/ref_rejected": -226.6428985595703, "logps/rejected": -238.20152282714844, "loss": 5.2185, "margin_dpo/margin_mean": 9.436347961425781, "margin_dpo/margin_std": 19.16326141357422, "step": 91 }, { "KL/chosen_KL_mean": -2.9682865142822266, "KL/mean": -8.081915855407715, "KL/rejected_KL_mean": -13.195533752441406, "KL/std": 17.40971565246582, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.227250099182129, "fcm_dpo/q_t": 0.474727988243103, "grad_norm": 33.458744049072266, "learning_rate": 4.877074915775048e-07, "logits/chosen": -0.7365894317626953, "logits/rejected": -0.7199798822402954, "logps/chosen": -286.41583251953125, "logps/ref_chosen": -283.4475402832031, "logps/ref_rejected": -273.134033203125, "logps/rejected": -286.3295593261719, "loss": 5.194, "margin_dpo/margin_mean": 10.227250099182129, "margin_dpo/margin_std": 21.2034912109375, "step": 92 }, { "KL/chosen_KL_mean": -2.397554397583008, "KL/mean": -7.223630905151367, "KL/rejected_KL_mean": -12.049707412719727, "KL/std": 15.443262100219727, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.652153015136719, "fcm_dpo/q_t": 0.476184606552124, "grad_norm": 28.543886184692383, "learning_rate": 4.871341104867864e-07, "logits/chosen": -0.7294468283653259, "logits/rejected": -0.7528958916664124, "logps/chosen": -235.73471069335938, "logps/ref_chosen": -233.33714294433594, "logps/ref_rejected": -230.54273986816406, "logps/rejected": -242.59243774414062, "loss": 5.2092, "margin_dpo/margin_mean": 9.652153968811035, "margin_dpo/margin_std": 19.43905258178711, "step": 93 }, { "KL/chosen_KL_mean": -2.360097885131836, "KL/mean": -7.027770519256592, "KL/rejected_KL_mean": -11.695440292358398, "KL/std": 16.456880569458008, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.33534049987793, "fcm_dpo/q_t": 0.47690367698669434, "grad_norm": 31.50308609008789, "learning_rate": 4.865480126133871e-07, "logits/chosen": -0.6894798278808594, "logits/rejected": -0.7111689448356628, "logps/chosen": -297.01287841796875, "logps/ref_chosen": -294.6528015136719, "logps/ref_rejected": -283.657958984375, "logps/rejected": -295.3534240722656, "loss": 5.2318, "margin_dpo/margin_mean": 9.335341453552246, "margin_dpo/margin_std": 21.897310256958008, "step": 94 }, { "KL/chosen_KL_mean": -3.2690467834472656, "KL/mean": -8.3305082321167, "KL/rejected_KL_mean": -13.391958236694336, "KL/std": 18.094928741455078, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.122918128967285, "fcm_dpo/q_t": 0.47508278489112854, "grad_norm": 33.46260452270508, "learning_rate": 4.859492293879573e-07, "logits/chosen": -0.7189546227455139, "logits/rejected": -0.7395079731941223, "logps/chosen": -314.9387512207031, "logps/ref_chosen": -311.6697082519531, "logps/ref_rejected": -262.7471923828125, "logps/rejected": -276.13916015625, "loss": 5.2083, "margin_dpo/margin_mean": 10.122917175292969, "margin_dpo/margin_std": 22.639379501342773, "step": 95 }, { "KL/chosen_KL_mean": -4.606351852416992, "KL/mean": -9.789222717285156, "KL/rejected_KL_mean": -14.972097396850586, "KL/std": 19.352949142456055, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.365747451782227, "fcm_dpo/q_t": 0.47450557351112366, "grad_norm": 33.26982879638672, "learning_rate": 4.853377929214243e-07, "logits/chosen": -0.7056884765625, "logits/rejected": -0.7183771133422852, "logps/chosen": -287.162353515625, "logps/ref_chosen": -282.55596923828125, "logps/ref_rejected": -242.71588134765625, "logps/rejected": -257.6879577636719, "loss": 5.1996, "margin_dpo/margin_mean": 10.365745544433594, "margin_dpo/margin_std": 23.32305908203125, "step": 96 }, { "KL/chosen_KL_mean": -4.363153457641602, "KL/mean": -10.424278259277344, "KL/rejected_KL_mean": -16.485395431518555, "KL/std": 17.310468673706055, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.122227668762207, "fcm_dpo/q_t": 0.47013965249061584, "grad_norm": 32.01287841796875, "learning_rate": 4.847137360032699e-07, "logits/chosen": -0.7406127452850342, "logits/rejected": -0.727538526058197, "logps/chosen": -307.9409484863281, "logps/ref_chosen": -303.57781982421875, "logps/ref_rejected": -264.22491455078125, "logps/rejected": -280.7103271484375, "loss": 5.1279, "margin_dpo/margin_mean": 12.122227668762207, "margin_dpo/margin_std": 22.35078239440918, "step": 97 }, { "KL/chosen_KL_mean": -4.8404998779296875, "KL/mean": -10.949996948242188, "KL/rejected_KL_mean": -17.059505462646484, "KL/std": 18.937652587890625, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.010055022314190865, "fcm_dpo/delta": 0.013165837153792381, "fcm_dpo/margin": 12.219008445739746, "fcm_dpo/q_t": 0.47002190351486206, "grad_norm": 34.929656982421875, "learning_rate": 4.84077092099773e-07, "logits/chosen": -0.7656075954437256, "logits/rejected": -0.7773485779762268, "logps/chosen": -291.67083740234375, "logps/ref_chosen": -286.8303527832031, "logps/ref_rejected": -278.08331298828125, "logps/rejected": -295.1427917480469, "loss": 5.1346, "margin_dpo/margin_mean": 12.219010353088379, "margin_dpo/margin_std": 22.74274253845215, "step": 98 }, { "KL/chosen_KL_mean": -5.254631042480469, "KL/mean": -11.531015396118164, "KL/rejected_KL_mean": -17.80739974975586, "KL/std": 19.79965591430664, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.010154977440834045, "fcm_dpo/delta": 0.009678316302597523, "fcm_dpo/margin": 12.552767753601074, "fcm_dpo/q_t": 0.4687933325767517, "grad_norm": 32.95009231567383, "learning_rate": 4.834278953522137e-07, "logits/chosen": -0.7380152344703674, "logits/rejected": -0.7510902285575867, "logps/chosen": -285.17584228515625, "logps/ref_chosen": -279.92120361328125, "logps/ref_rejected": -250.3365478515625, "logps/rejected": -268.1439514160156, "loss": 5.1327, "margin_dpo/margin_mean": 12.552766799926758, "margin_dpo/margin_std": 27.131689071655273, "step": 99 }, { "KL/chosen_KL_mean": -7.756633758544922, "KL/mean": -14.034221649169922, "KL/rejected_KL_mean": -20.311803817749023, "KL/std": 21.95221710205078, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.010277610272169113, "fcm_dpo/delta": 0.0157307181507349, "fcm_dpo/margin": 12.555181503295898, "fcm_dpo/q_t": 0.4685354232788086, "grad_norm": 36.376800537109375, "learning_rate": 4.827661805750437e-07, "logits/chosen": -0.7792176604270935, "logits/rejected": -0.7921389937400818, "logps/chosen": -304.5843505859375, "logps/ref_chosen": -296.8276672363281, "logps/ref_rejected": -275.56146240234375, "logps/rejected": -295.8732604980469, "loss": 5.1137, "margin_dpo/margin_mean": 12.555181503295898, "margin_dpo/margin_std": 24.093582153320312, "step": 100 }, { "KL/chosen_KL_mean": -5.120674133300781, "KL/mean": -12.59157943725586, "KL/rejected_KL_mean": -20.062488555908203, "KL/std": 22.75184440612793, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.010311111807823181, "fcm_dpo/delta": -0.0004459689953364432, "fcm_dpo/margin": 14.941802978515625, "fcm_dpo/q_t": 0.46235236525535583, "grad_norm": 33.297786712646484, "learning_rate": 4.820919832540181e-07, "logits/chosen": -0.7734586000442505, "logits/rejected": -0.7872774004936218, "logps/chosen": -257.8627014160156, "logps/ref_chosen": -252.74203491210938, "logps/ref_rejected": -276.4185485839844, "logps/rejected": -296.4810791015625, "loss": 5.0315, "margin_dpo/margin_mean": 14.941804885864258, "margin_dpo/margin_std": 26.297649383544922, "step": 101 }, { "KL/chosen_KL_mean": -5.081005096435547, "KL/mean": -12.523896217346191, "KL/rejected_KL_mean": -19.96678352355957, "KL/std": 19.68603515625, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.01035170815885067, "fcm_dpo/delta": 0.01568884216248989, "fcm_dpo/margin": 14.88577938079834, "fcm_dpo/q_t": 0.4620976448059082, "grad_norm": 33.22737121582031, "learning_rate": 4.814053395442932e-07, "logits/chosen": -0.7597612738609314, "logits/rejected": -0.7561210989952087, "logps/chosen": -224.63473510742188, "logps/ref_chosen": -219.5537109375, "logps/ref_rejected": -231.90853881835938, "logps/rejected": -251.8753204345703, "loss": 5.0247, "margin_dpo/margin_mean": 14.885781288146973, "margin_dpo/margin_std": 24.928041458129883, "step": 102 }, { "KL/chosen_KL_mean": -4.84321403503418, "KL/mean": -11.587666511535645, "KL/rejected_KL_mean": -18.33211898803711, "KL/std": 21.602670669555664, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.010488416068255901, "fcm_dpo/delta": 0.01155367773026228, "fcm_dpo/margin": 13.488908767700195, "fcm_dpo/q_t": 0.4657100439071655, "grad_norm": 34.28849411010742, "learning_rate": 4.807062862684873e-07, "logits/chosen": -0.7750781774520874, "logits/rejected": -0.7720986008644104, "logps/chosen": -264.51824951171875, "logps/ref_chosen": -259.6750793457031, "logps/ref_rejected": -278.7400817871094, "logps/rejected": -297.07220458984375, "loss": 5.0773, "margin_dpo/margin_mean": 13.488907814025879, "margin_dpo/margin_std": 25.666749954223633, "step": 103 }, { "KL/chosen_KL_mean": -9.041154861450195, "KL/mean": -14.049955368041992, "KL/rejected_KL_mean": -19.05875015258789, "KL/std": 22.522296905517578, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.010536887682974339, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.017589569091797, "fcm_dpo/q_t": 0.47418421506881714, "grad_norm": 34.80829620361328, "learning_rate": 4.799948609147061e-07, "logits/chosen": -0.7744643688201904, "logits/rejected": -0.7815247774124146, "logps/chosen": -277.01531982421875, "logps/ref_chosen": -267.9741516113281, "logps/ref_rejected": -230.5306396484375, "logps/rejected": -249.58938598632812, "loss": 5.2141, "margin_dpo/margin_mean": 10.01759147644043, "margin_dpo/margin_std": 26.329910278320312, "step": 104 }, { "KL/chosen_KL_mean": -5.195383071899414, "KL/mean": -15.491511344909668, "KL/rejected_KL_mean": -25.787647247314453, "KL/std": 25.20000457763672, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.010571734979748726, "fcm_dpo/delta": 0.00043234386248514056, "fcm_dpo/margin": 20.592254638671875, "fcm_dpo/q_t": 0.4467693567276001, "grad_norm": 36.65842819213867, "learning_rate": 4.792711016345321e-07, "logits/chosen": -0.7523764371871948, "logits/rejected": -0.7615402340888977, "logps/chosen": -327.4501953125, "logps/ref_chosen": -322.25482177734375, "logps/ref_rejected": -279.02978515625, "logps/rejected": -304.8174133300781, "loss": 4.8127, "margin_dpo/margin_mean": 20.592256546020508, "margin_dpo/margin_std": 27.057451248168945, "step": 105 }, { "KL/chosen_KL_mean": -12.456212997436523, "KL/mean": -18.74678611755371, "KL/rejected_KL_mean": -25.037351608276367, "KL/std": 23.551477432250977, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.010547379031777382, "fcm_dpo/delta": 0.003242878010496497, "fcm_dpo/margin": 12.581135749816895, "fcm_dpo/q_t": 0.46765029430389404, "grad_norm": 39.138153076171875, "learning_rate": 4.785350472409791e-07, "logits/chosen": -0.7371970415115356, "logits/rejected": -0.7723554968833923, "logps/chosen": -308.614013671875, "logps/ref_chosen": -296.15777587890625, "logps/ref_rejected": -266.2691650390625, "logps/rejected": -291.3065185546875, "loss": 5.1337, "margin_dpo/margin_mean": 12.581133842468262, "margin_dpo/margin_std": 29.417034149169922, "step": 106 }, { "KL/chosen_KL_mean": -4.330564498901367, "KL/mean": -14.176000595092773, "KL/rejected_KL_mean": -24.021432876586914, "KL/std": 22.089252471923828, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.01077681127935648, "fcm_dpo/delta": 0.043501920998096466, "fcm_dpo/margin": 19.69086265563965, "fcm_dpo/q_t": 0.4485030770301819, "grad_norm": 40.70358657836914, "learning_rate": 4.777867372064105e-07, "logits/chosen": -0.7831494212150574, "logits/rejected": -0.7764496207237244, "logps/chosen": -311.3268737792969, "logps/ref_chosen": -306.996337890625, "logps/ref_rejected": -296.79412841796875, "logps/rejected": -320.8155517578125, "loss": 4.8351, "margin_dpo/margin_mean": 19.69086265563965, "margin_dpo/margin_std": 27.672435760498047, "step": 107 }, { "KL/chosen_KL_mean": -5.463014602661133, "KL/mean": -14.59699535369873, "KL/rejected_KL_mean": -23.730979919433594, "KL/std": 25.10581398010254, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.011032754555344582, "fcm_dpo/delta": 0.01777113415300846, "fcm_dpo/margin": 18.267986297607422, "fcm_dpo/q_t": 0.4515891969203949, "grad_norm": 112.73442077636719, "learning_rate": 4.770262116604223e-07, "logits/chosen": -0.7686399817466736, "logits/rejected": -0.7808905839920044, "logps/chosen": -300.61566162109375, "logps/ref_chosen": -295.1526794433594, "logps/ref_rejected": -235.974853515625, "logps/rejected": -259.7058410644531, "loss": 4.9053, "margin_dpo/margin_mean": 18.267986297607422, "margin_dpo/margin_std": 30.327621459960938, "step": 108 }, { "KL/chosen_KL_mean": -8.268291473388672, "KL/mean": -18.426597595214844, "KL/rejected_KL_mean": -28.584897994995117, "KL/std": 25.283273696899414, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.01130305789411068, "fcm_dpo/delta": 0.04050566256046295, "fcm_dpo/margin": 20.31660270690918, "fcm_dpo/q_t": 0.4453127086162567, "grad_norm": 42.34718322753906, "learning_rate": 4.7625351138769166e-07, "logits/chosen": -0.7919974327087402, "logits/rejected": -0.7888400554656982, "logps/chosen": -334.1930847167969, "logps/ref_chosen": -325.9248046875, "logps/ref_rejected": -279.15423583984375, "logps/rejected": -307.7391052246094, "loss": 4.8109, "margin_dpo/margin_mean": 20.31660270690918, "margin_dpo/margin_std": 31.698184967041016, "step": 109 }, { "KL/chosen_KL_mean": -8.925228118896484, "KL/mean": -18.446353912353516, "KL/rejected_KL_mean": -27.96748161315918, "KL/std": 27.263511657714844, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.011561272665858269, "fcm_dpo/delta": 0.012318210676312447, "fcm_dpo/margin": 19.042247772216797, "fcm_dpo/q_t": 0.4473344683647156, "grad_norm": 39.34458923339844, "learning_rate": 4.75468677825789e-07, "logits/chosen": -0.8064714670181274, "logits/rejected": -0.7954632043838501, "logps/chosen": -283.36444091796875, "logps/ref_chosen": -274.439208984375, "logps/ref_rejected": -260.0552062988281, "logps/rejected": -288.022705078125, "loss": 4.8673, "margin_dpo/margin_mean": 19.04224967956543, "margin_dpo/margin_std": 33.730098724365234, "step": 110 }, { "KL/chosen_KL_mean": -9.773136138916016, "KL/mean": -19.87885093688965, "KL/rejected_KL_mean": -29.98457908630371, "KL/std": 27.962778091430664, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.011709115467965603, "fcm_dpo/delta": 0.020104624330997467, "fcm_dpo/margin": 20.211442947387695, "fcm_dpo/q_t": 0.44378963112831116, "grad_norm": 43.68085861206055, "learning_rate": 4.7467175306295647e-07, "logits/chosen": -0.8389703631401062, "logits/rejected": -0.8186056613922119, "logps/chosen": -339.00927734375, "logps/ref_chosen": -329.2361755371094, "logps/ref_rejected": -287.82830810546875, "logps/rejected": -317.8128662109375, "loss": 4.8241, "margin_dpo/margin_mean": 20.211441040039062, "margin_dpo/margin_std": 34.38101577758789, "step": 111 }, { "KL/chosen_KL_mean": -15.453165054321289, "KL/mean": -22.01694107055664, "KL/rejected_KL_mean": -28.580717086791992, "KL/std": 26.992534637451172, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.011764442548155785, "fcm_dpo/delta": 0.009524336084723473, "fcm_dpo/margin": 13.127549171447754, "fcm_dpo/q_t": 0.4634850323200226, "grad_norm": 46.13030242919922, "learning_rate": 4.7386277983585053e-07, "logits/chosen": -0.7355121970176697, "logits/rejected": -0.7651410698890686, "logps/chosen": -272.5125427246094, "logps/ref_chosen": -257.0593566894531, "logps/ref_rejected": -272.9595031738281, "logps/rejected": -301.5401916503906, "loss": 5.1565, "margin_dpo/margin_mean": 13.12755012512207, "margin_dpo/margin_std": 36.32170867919922, "step": 112 }, { "KL/chosen_KL_mean": -12.787643432617188, "KL/mean": -25.045021057128906, "KL/rejected_KL_mean": -37.302391052246094, "KL/std": 30.94366455078125, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.011613179929554462, "fcm_dpo/delta": -0.032145921140909195, "fcm_dpo/margin": 24.514759063720703, "fcm_dpo/q_t": 0.4332182705402374, "grad_norm": 47.04500961303711, "learning_rate": 4.7304180152725024e-07, "logits/chosen": -0.8058240413665771, "logits/rejected": -0.8091129660606384, "logps/chosen": -298.8293151855469, "logps/ref_chosen": -286.0416564941406, "logps/ref_rejected": -270.374267578125, "logps/rejected": -307.6766357421875, "loss": 4.6973, "margin_dpo/margin_mean": 24.514759063720703, "margin_dpo/margin_std": 39.107757568359375, "step": 113 }, { "KL/chosen_KL_mean": -16.7172794342041, "KL/mean": -23.604433059692383, "KL/rejected_KL_mean": -30.491592407226562, "KL/std": 27.874691009521484, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.011623743921518326, "fcm_dpo/delta": 0.012387579306960106, "fcm_dpo/margin": 13.774309158325195, "fcm_dpo/q_t": 0.4614120125770569, "grad_norm": 45.54480743408203, "learning_rate": 4.7220886216373085e-07, "logits/chosen": -0.8526460528373718, "logits/rejected": -0.854054868221283, "logps/chosen": -276.7257385253906, "logps/ref_chosen": -260.0084533691406, "logps/ref_rejected": -246.67190551757812, "logps/rejected": -277.1634826660156, "loss": 5.1092, "margin_dpo/margin_mean": 13.774311065673828, "margin_dpo/margin_std": 35.25768280029297, "step": 114 }, { "KL/chosen_KL_mean": -18.6426944732666, "KL/mean": -25.6184139251709, "KL/rejected_KL_mean": -32.594139099121094, "KL/std": 30.649730682373047, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.01156668271869421, "fcm_dpo/delta": -0.014433400705456734, "fcm_dpo/margin": 13.951436042785645, "fcm_dpo/q_t": 0.462697297334671, "grad_norm": 50.22829055786133, "learning_rate": 4.7136400641330245e-07, "logits/chosen": -0.8506746888160706, "logits/rejected": -0.8111424446105957, "logps/chosen": -318.0656433105469, "logps/ref_chosen": -299.4229736328125, "logps/ref_rejected": -272.1186828613281, "logps/rejected": -304.71282958984375, "loss": 5.1336, "margin_dpo/margin_mean": 13.951436996459961, "margin_dpo/margin_std": 38.57262420654297, "step": 115 }, { "KL/chosen_KL_mean": -14.825180053710938, "KL/mean": -23.868736267089844, "KL/rejected_KL_mean": -32.91230010986328, "KL/std": 28.654550552368164, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.011589784175157547, "fcm_dpo/delta": 0.010382231324911118, "fcm_dpo/margin": 18.087129592895508, "fcm_dpo/q_t": 0.44975095987319946, "grad_norm": 43.76200485229492, "learning_rate": 4.70507279583015e-07, "logits/chosen": -0.8624769449234009, "logits/rejected": -0.8245735764503479, "logps/chosen": -294.089111328125, "logps/ref_chosen": -279.263916015625, "logps/ref_rejected": -253.6192169189453, "logps/rejected": -286.531494140625, "loss": 4.9196, "margin_dpo/margin_mean": 18.08713150024414, "margin_dpo/margin_std": 34.976829528808594, "step": 116 }, { "KL/chosen_KL_mean": -18.559932708740234, "KL/mean": -28.868959426879883, "KL/rejected_KL_mean": -39.177978515625, "KL/std": 27.26178741455078, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.011582687497138977, "fcm_dpo/delta": -0.038109198212623596, "fcm_dpo/margin": 20.61803436279297, "fcm_dpo/q_t": 0.44285690784454346, "grad_norm": 47.86320495605469, "learning_rate": 4.6963872761652834e-07, "logits/chosen": -0.8096694350242615, "logits/rejected": -0.8094600439071655, "logps/chosen": -277.7848205566406, "logps/ref_chosen": -259.2248840332031, "logps/ref_rejected": -229.3042755126953, "logps/rejected": -268.48223876953125, "loss": 4.8158, "margin_dpo/margin_mean": 20.61803436279297, "margin_dpo/margin_std": 31.60936164855957, "step": 117 }, { "KL/chosen_KL_mean": -24.599096298217773, "KL/mean": -36.10367965698242, "KL/rejected_KL_mean": -47.60826873779297, "KL/std": 34.55908203125, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.011433707550168037, "fcm_dpo/delta": 0.008326723240315914, "fcm_dpo/margin": 23.00917625427246, "fcm_dpo/q_t": 0.4388441741466522, "grad_norm": 50.52167892456055, "learning_rate": 4.687583970916486e-07, "logits/chosen": -0.8109974265098572, "logits/rejected": -0.798851490020752, "logps/chosen": -291.6697692871094, "logps/ref_chosen": -267.0707092285156, "logps/ref_rejected": -272.7322082519531, "logps/rejected": -320.3404541015625, "loss": 4.8061, "margin_dpo/margin_mean": 23.009178161621094, "margin_dpo/margin_std": 42.8681640625, "step": 118 }, { "KL/chosen_KL_mean": -29.91927719116211, "KL/mean": -38.381263732910156, "KL/rejected_KL_mean": -46.8432502746582, "KL/std": 33.76998519897461, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.011522539891302586, "fcm_dpo/delta": 0.013245711103081703, "fcm_dpo/margin": 16.92397689819336, "fcm_dpo/q_t": 0.4548139274120331, "grad_norm": 51.030372619628906, "learning_rate": 4.6786633521783005e-07, "logits/chosen": -0.8936063051223755, "logits/rejected": -0.8938306570053101, "logps/chosen": -354.595947265625, "logps/ref_chosen": -324.6766357421875, "logps/ref_rejected": -306.0322265625, "logps/rejected": -352.87548828125, "loss": 5.0481, "margin_dpo/margin_mean": 16.923974990844727, "margin_dpo/margin_std": 41.06504821777344, "step": 119 }, { "KL/chosen_KL_mean": -26.801218032836914, "KL/mean": -36.47585678100586, "KL/rejected_KL_mean": -46.150489807128906, "KL/std": 32.804019927978516, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.011345265433192253, "fcm_dpo/delta": -0.025357699021697044, "fcm_dpo/margin": 19.34927749633789, "fcm_dpo/q_t": 0.4487614631652832, "grad_norm": 46.38136672973633, "learning_rate": 4.669625898336438e-07, "logits/chosen": -0.841278076171875, "logits/rejected": -0.8530943989753723, "logps/chosen": -342.06298828125, "logps/ref_chosen": -315.2617492675781, "logps/ref_rejected": -265.32501220703125, "logps/rejected": -311.4754943847656, "loss": 4.9559, "margin_dpo/margin_mean": 19.349279403686523, "margin_dpo/margin_std": 40.81332778930664, "step": 120 }, { "KL/chosen_KL_mean": -31.404523849487305, "KL/mean": -38.39643859863281, "KL/rejected_KL_mean": -45.38835144042969, "KL/std": 31.411802291870117, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.011385759338736534, "fcm_dpo/delta": 0.026212375611066818, "fcm_dpo/margin": 13.983831405639648, "fcm_dpo/q_t": 0.46326711773872375, "grad_norm": 62.95341491699219, "learning_rate": 4.6604720940421207e-07, "logits/chosen": -0.8449280261993408, "logits/rejected": -0.8593005537986755, "logps/chosen": -254.400634765625, "logps/ref_chosen": -222.99609375, "logps/ref_rejected": -226.92860412597656, "logps/rejected": -272.31695556640625, "loss": 5.144, "margin_dpo/margin_mean": 13.983829498291016, "margin_dpo/margin_std": 39.88007354736328, "step": 121 }, { "KL/chosen_KL_mean": -31.899211883544922, "KL/mean": -41.62091064453125, "KL/rejected_KL_mean": -51.34260177612305, "KL/std": 36.922428131103516, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.011439606547355652, "fcm_dpo/delta": -0.004928136244416237, "fcm_dpo/margin": 19.44339942932129, "fcm_dpo/q_t": 0.44821009039878845, "grad_norm": 58.85353469848633, "learning_rate": 4.651202430186092e-07, "logits/chosen": -0.9108130931854248, "logits/rejected": -0.8724952936172485, "logps/chosen": -307.9255065917969, "logps/ref_chosen": -276.02630615234375, "logps/ref_rejected": -277.97418212890625, "logps/rejected": -329.3167724609375, "loss": 4.9831, "margin_dpo/margin_mean": 19.44339942932129, "margin_dpo/margin_std": 45.87887191772461, "step": 122 }, { "KL/chosen_KL_mean": -24.832054138183594, "KL/mean": -37.42338943481445, "KL/rejected_KL_mean": -50.01473617553711, "KL/std": 36.377105712890625, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.011429657228291035, "fcm_dpo/delta": -0.007541924715042114, "fcm_dpo/margin": 25.18267059326172, "fcm_dpo/q_t": 0.4322325587272644, "grad_norm": 54.087303161621094, "learning_rate": 4.6418174038722924e-07, "logits/chosen": -0.8302006125450134, "logits/rejected": -0.821265697479248, "logps/chosen": -352.9866943359375, "logps/ref_chosen": -328.1546325683594, "logps/ref_rejected": -280.6911315917969, "logps/rejected": -330.70587158203125, "loss": 4.7131, "margin_dpo/margin_mean": 25.18267059326172, "margin_dpo/margin_std": 41.76557159423828, "step": 123 }, { "KL/chosen_KL_mean": -25.639102935791016, "KL/mean": -36.52279281616211, "KL/rejected_KL_mean": -47.40650177001953, "KL/std": 34.944732666015625, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.011149123311042786, "fcm_dpo/delta": -0.033872563391923904, "fcm_dpo/margin": 21.76740837097168, "fcm_dpo/q_t": 0.4419231116771698, "grad_norm": 55.05583953857422, "learning_rate": 4.6323175183912023e-07, "logits/chosen": -0.8557197451591492, "logits/rejected": -0.8204752802848816, "logps/chosen": -301.3352966308594, "logps/ref_chosen": -275.6961975097656, "logps/ref_rejected": -225.361572265625, "logps/rejected": -272.76806640625, "loss": 4.837, "margin_dpo/margin_mean": 21.767410278320312, "margin_dpo/margin_std": 38.42668914794922, "step": 124 }, { "KL/chosen_KL_mean": -27.239627838134766, "KL/mean": -37.06490707397461, "KL/rejected_KL_mean": -46.89017868041992, "KL/std": 36.27045822143555, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.0110806655138731, "fcm_dpo/delta": -0.004720758646726608, "fcm_dpo/margin": 19.650556564331055, "fcm_dpo/q_t": 0.4501379430294037, "grad_norm": 55.957820892333984, "learning_rate": 4.6227032831928483e-07, "logits/chosen": -0.8200584053993225, "logits/rejected": -0.7745668888092041, "logps/chosen": -305.30938720703125, "logps/ref_chosen": -278.06976318359375, "logps/ref_rejected": -265.63873291015625, "logps/rejected": -312.5289001464844, "loss": 5.0283, "margin_dpo/margin_mean": 19.650556564331055, "margin_dpo/margin_std": 49.57725524902344, "step": 125 }, { "KL/chosen_KL_mean": -23.706384658813477, "KL/mean": -34.74204635620117, "KL/rejected_KL_mean": -45.777713775634766, "KL/std": 37.79378128051758, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.010993003845214844, "fcm_dpo/delta": -0.00607214355841279, "fcm_dpo/margin": 22.071332931518555, "fcm_dpo/q_t": 0.44283780455589294, "grad_norm": 53.37355041503906, "learning_rate": 4.612975213859487e-07, "logits/chosen": -0.8317501544952393, "logits/rejected": -0.8419036865234375, "logps/chosen": -345.10247802734375, "logps/ref_chosen": -321.3960876464844, "logps/ref_rejected": -285.37664794921875, "logps/rejected": -331.15435791015625, "loss": 4.8702, "margin_dpo/margin_mean": 22.071331024169922, "margin_dpo/margin_std": 44.08152770996094, "step": 126 }, { "KL/chosen_KL_mean": -22.35991096496582, "KL/mean": -35.4288330078125, "KL/rejected_KL_mean": -48.49776077270508, "KL/std": 39.06257629394531, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.010855698958039284, "fcm_dpo/delta": -0.05256950855255127, "fcm_dpo/margin": 26.137863159179688, "fcm_dpo/q_t": 0.4339562654495239, "grad_norm": 52.98072814941406, "learning_rate": 4.603133832077953e-07, "logits/chosen": -0.9140666127204895, "logits/rejected": -0.8634395003318787, "logps/chosen": -328.9186706542969, "logps/ref_chosen": -306.55877685546875, "logps/ref_rejected": -274.8651428222656, "logps/rejected": -323.3628845214844, "loss": 4.7582, "margin_dpo/margin_mean": 26.13786506652832, "margin_dpo/margin_std": 44.73991394042969, "step": 127 }, { "KL/chosen_KL_mean": -17.275562286376953, "KL/mean": -32.32242202758789, "KL/rejected_KL_mean": -47.3692741394043, "KL/std": 40.74695587158203, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.010318944230675697, "fcm_dpo/delta": -0.034354813396930695, "fcm_dpo/margin": 30.093717575073242, "fcm_dpo/q_t": 0.427314817905426, "grad_norm": 48.40740203857422, "learning_rate": 4.5931796656116837e-07, "logits/chosen": -0.7917481660842896, "logits/rejected": -0.7879784107208252, "logps/chosen": -282.6729431152344, "logps/ref_chosen": -265.3973693847656, "logps/ref_rejected": -250.9737548828125, "logps/rejected": -298.343017578125, "loss": 4.6545, "margin_dpo/margin_mean": 30.093721389770508, "margin_dpo/margin_std": 48.82278060913086, "step": 128 }, { "KL/chosen_KL_mean": -18.557172775268555, "KL/mean": -32.06791305541992, "KL/rejected_KL_mean": -45.57864761352539, "KL/std": 38.27027893066406, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.010120774619281292, "fcm_dpo/delta": -0.0329061821103096, "fcm_dpo/margin": 27.021467208862305, "fcm_dpo/q_t": 0.43612736463546753, "grad_norm": 45.152587890625, "learning_rate": 4.5831132482724193e-07, "logits/chosen": -0.8318417072296143, "logits/rejected": -0.8305982947349548, "logps/chosen": -321.7156066894531, "logps/ref_chosen": -303.158447265625, "logps/ref_rejected": -275.9891052246094, "logps/rejected": -321.5677490234375, "loss": 4.7692, "margin_dpo/margin_mean": 27.021469116210938, "margin_dpo/margin_std": 47.870628356933594, "step": 129 }, { "KL/chosen_KL_mean": -21.258207321166992, "KL/mean": -34.769718170166016, "KL/rejected_KL_mean": -48.28123474121094, "KL/std": 37.819278717041016, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.009886497631669044, "fcm_dpo/delta": -0.0007312544621527195, "fcm_dpo/margin": 27.023021697998047, "fcm_dpo/q_t": 0.4373343586921692, "grad_norm": 55.255393981933594, "learning_rate": 4.5729351198915705e-07, "logits/chosen": -0.8017497062683105, "logits/rejected": -0.8390852808952332, "logps/chosen": -307.6655578613281, "logps/ref_chosen": -286.4073486328125, "logps/ref_rejected": -294.38665771484375, "logps/rejected": -342.66790771484375, "loss": 4.7757, "margin_dpo/margin_mean": 27.02302360534668, "margin_dpo/margin_std": 48.35560989379883, "step": 130 }, { "KL/chosen_KL_mean": -25.688549041748047, "KL/mean": -36.71128463745117, "KL/rejected_KL_mean": -47.734012603759766, "KL/std": 42.36379623413086, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.009975353255867958, "fcm_dpo/delta": 0.0071073779836297035, "fcm_dpo/margin": 22.045459747314453, "fcm_dpo/q_t": 0.44867265224456787, "grad_norm": 58.347599029541016, "learning_rate": 4.5626458262912735e-07, "logits/chosen": -0.8796436190605164, "logits/rejected": -0.8274049162864685, "logps/chosen": -337.2536315917969, "logps/ref_chosen": -311.5650634765625, "logps/ref_rejected": -291.62432861328125, "logps/rejected": -339.3583679199219, "loss": 4.9858, "margin_dpo/margin_mean": 22.045461654663086, "margin_dpo/margin_std": 51.318870544433594, "step": 131 }, { "KL/chosen_KL_mean": -21.675294876098633, "KL/mean": -37.08460998535156, "KL/rejected_KL_mean": -52.493934631347656, "KL/std": 42.055477142333984, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.009898187592625618, "fcm_dpo/delta": -0.05365057289600372, "fcm_dpo/margin": 30.81864356994629, "fcm_dpo/q_t": 0.4293951690196991, "grad_norm": 60.90685272216797, "learning_rate": 4.5522459192551166e-07, "logits/chosen": -0.8308587074279785, "logits/rejected": -0.8142504692077637, "logps/chosen": -291.7571105957031, "logps/ref_chosen": -270.0818176269531, "logps/ref_rejected": -284.3084411621094, "logps/rejected": -336.8023681640625, "loss": 4.7056, "margin_dpo/margin_mean": 30.81864356994629, "margin_dpo/margin_std": 52.920997619628906, "step": 132 }, { "KL/chosen_KL_mean": -25.237079620361328, "KL/mean": -39.2971305847168, "KL/rejected_KL_mean": -53.35718536376953, "KL/std": 39.58035659790039, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.00956808403134346, "fcm_dpo/delta": -0.005704056471586227, "fcm_dpo/margin": 28.12010383605957, "fcm_dpo/q_t": 0.43685850501060486, "grad_norm": 41.762489318847656, "learning_rate": 4.541735956498554e-07, "logits/chosen": -0.861563503742218, "logits/rejected": -0.8600152730941772, "logps/chosen": -310.85845947265625, "logps/ref_chosen": -285.6213684082031, "logps/ref_rejected": -251.19386291503906, "logps/rejected": -304.551025390625, "loss": 4.7711, "margin_dpo/margin_mean": 28.12010383605957, "margin_dpo/margin_std": 49.382789611816406, "step": 133 }, { "KL/chosen_KL_mean": -37.50963592529297, "KL/mean": -48.0689697265625, "KL/rejected_KL_mean": -58.62830352783203, "KL/std": 41.11132049560547, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.009509644471108913, "fcm_dpo/delta": -0.009044441394507885, "fcm_dpo/margin": 21.118667602539062, "fcm_dpo/q_t": 0.45208245515823364, "grad_norm": 48.476036071777344, "learning_rate": 4.5311165016389914e-07, "logits/chosen": -0.8715083599090576, "logits/rejected": -0.8705750703811646, "logps/chosen": -356.43048095703125, "logps/ref_chosen": -318.92083740234375, "logps/ref_rejected": -293.1894836425781, "logps/rejected": -351.8177490234375, "loss": 4.9727, "margin_dpo/margin_mean": 21.118667602539062, "margin_dpo/margin_std": 44.15083694458008, "step": 134 }, { "KL/chosen_KL_mean": -34.88170623779297, "KL/mean": -47.295005798339844, "KL/rejected_KL_mean": -59.70830154418945, "KL/std": 43.2322998046875, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.009455316700041294, "fcm_dpo/delta": -0.0223441980779171, "fcm_dpo/margin": 24.82658576965332, "fcm_dpo/q_t": 0.44377559423446655, "grad_norm": 57.29498291015625, "learning_rate": 4.520388124165564e-07, "logits/chosen": -0.7520244717597961, "logits/rejected": -0.7897548079490662, "logps/chosen": -327.7034606933594, "logps/ref_chosen": -292.8217468261719, "logps/ref_rejected": -269.2896728515625, "logps/rejected": -328.99798583984375, "loss": 4.8953, "margin_dpo/margin_mean": 24.826589584350586, "margin_dpo/margin_std": 48.393245697021484, "step": 135 }, { "KL/chosen_KL_mean": -36.76615905761719, "KL/mean": -50.90818405151367, "KL/rejected_KL_mean": -65.05021667480469, "KL/std": 43.206573486328125, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.009310539811849594, "fcm_dpo/delta": -0.014653807505965233, "fcm_dpo/margin": 28.2840633392334, "fcm_dpo/q_t": 0.4387061595916748, "grad_norm": 64.72450256347656, "learning_rate": 4.5095513994085974e-07, "logits/chosen": -0.8104878664016724, "logits/rejected": -0.8044265508651733, "logps/chosen": -309.6186828613281, "logps/ref_chosen": -272.8525390625, "logps/ref_rejected": -252.68202209472656, "logps/rejected": -317.73223876953125, "loss": 4.8238, "margin_dpo/margin_mean": 28.284059524536133, "margin_dpo/margin_std": 54.00830841064453, "step": 136 }, { "KL/chosen_KL_mean": -41.005741119384766, "KL/mean": -53.573143005371094, "KL/rejected_KL_mean": -66.14056396484375, "KL/std": 45.26162338256836, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.009244083426892757, "fcm_dpo/delta": -0.0003261703532189131, "fcm_dpo/margin": 25.134830474853516, "fcm_dpo/q_t": 0.4460538625717163, "grad_norm": 55.087921142578125, "learning_rate": 4.498606908508753e-07, "logits/chosen": -0.865790843963623, "logits/rejected": -0.8519459366798401, "logps/chosen": -341.7579650878906, "logps/ref_chosen": -300.7522277832031, "logps/ref_rejected": -286.1935119628906, "logps/rejected": -352.33404541015625, "loss": 4.9047, "margin_dpo/margin_mean": 25.13483428955078, "margin_dpo/margin_std": 51.73419952392578, "step": 137 }, { "KL/chosen_KL_mean": -38.480438232421875, "KL/mean": -53.19182205200195, "KL/rejected_KL_mean": -67.90321350097656, "KL/std": 46.54023361206055, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.009153381921350956, "fcm_dpo/delta": -0.008525258861482143, "fcm_dpo/margin": 29.422779083251953, "fcm_dpo/q_t": 0.43822887539863586, "grad_norm": 53.85645294189453, "learning_rate": 4.487555238385862e-07, "logits/chosen": -0.7926889061927795, "logits/rejected": -0.7721691727638245, "logps/chosen": -327.4174499511719, "logps/ref_chosen": -288.9369812011719, "logps/ref_rejected": -263.7076416015625, "logps/rejected": -331.61083984375, "loss": 4.8456, "margin_dpo/margin_mean": 29.422779083251953, "margin_dpo/margin_std": 60.80415725708008, "step": 138 }, { "KL/chosen_KL_mean": -35.84889221191406, "KL/mean": -46.137901306152344, "KL/rejected_KL_mean": -56.42690658569336, "KL/std": 41.7497673034668, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.009096279740333557, "fcm_dpo/delta": -0.008328789845108986, "fcm_dpo/margin": 20.578014373779297, "fcm_dpo/q_t": 0.4562779664993286, "grad_norm": 55.9819450378418, "learning_rate": 4.476396981707453e-07, "logits/chosen": -0.8190897703170776, "logits/rejected": -0.8487043380737305, "logps/chosen": -305.8932189941406, "logps/ref_chosen": -270.0443115234375, "logps/ref_rejected": -267.3226013183594, "logps/rejected": -323.7494812011719, "loss": 5.0658, "margin_dpo/margin_mean": 20.578014373779297, "margin_dpo/margin_std": 51.444358825683594, "step": 139 }, { "KL/chosen_KL_mean": -33.45403289794922, "KL/mean": -49.221893310546875, "KL/rejected_KL_mean": -64.98976135253906, "KL/std": 44.43138885498047, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.008965959772467613, "fcm_dpo/delta": -0.04622640460729599, "fcm_dpo/margin": 31.53574562072754, "fcm_dpo/q_t": 0.4328317940235138, "grad_norm": 48.354827880859375, "learning_rate": 4.4651327368569684e-07, "logits/chosen": -0.8959548473358154, "logits/rejected": -0.861539363861084, "logps/chosen": -316.40960693359375, "logps/ref_chosen": -282.9555969238281, "logps/ref_rejected": -251.17181396484375, "logps/rejected": -316.1615905761719, "loss": 4.6938, "margin_dpo/margin_mean": 31.53574562072754, "margin_dpo/margin_std": 48.106788635253906, "step": 140 }, { "KL/chosen_KL_mean": -32.11634826660156, "KL/mean": -48.44844055175781, "KL/rejected_KL_mean": -64.78053283691406, "KL/std": 45.843807220458984, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.008571082726120949, "fcm_dpo/delta": -0.03963158652186394, "fcm_dpo/margin": 32.664180755615234, "fcm_dpo/q_t": 0.4331643581390381, "grad_norm": 54.81271743774414, "learning_rate": 4.453763107901675e-07, "logits/chosen": -0.8185430765151978, "logits/rejected": -0.812373697757721, "logps/chosen": -328.41650390625, "logps/ref_chosen": -296.3001708984375, "logps/ref_rejected": -279.8486633300781, "logps/rejected": -344.6291809082031, "loss": 4.7447, "margin_dpo/margin_mean": 32.66417694091797, "margin_dpo/margin_std": 56.69047546386719, "step": 141 }, { "KL/chosen_KL_mean": -33.568763732910156, "KL/mean": -46.956512451171875, "KL/rejected_KL_mean": -60.34426498413086, "KL/std": 46.972068786621094, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.008418058045208454, "fcm_dpo/delta": -0.010385151952505112, "fcm_dpo/margin": 26.775487899780273, "fcm_dpo/q_t": 0.44701626896858215, "grad_norm": 51.90497970581055, "learning_rate": 4.4422887045602674e-07, "logits/chosen": -0.836083173751831, "logits/rejected": -0.8355280160903931, "logps/chosen": -334.1346130371094, "logps/ref_chosen": -300.56585693359375, "logps/ref_rejected": -231.43316650390625, "logps/rejected": -291.7774353027344, "loss": 4.9431, "margin_dpo/margin_mean": 26.775489807128906, "margin_dpo/margin_std": 57.83775329589844, "step": 142 }, { "KL/chosen_KL_mean": -33.348106384277344, "KL/mean": -49.812156677246094, "KL/rejected_KL_mean": -66.27620697021484, "KL/std": 44.9078483581543, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.008253632113337517, "fcm_dpo/delta": -0.035776134580373764, "fcm_dpo/margin": 32.9280891418457, "fcm_dpo/q_t": 0.4356859028339386, "grad_norm": 49.289283752441406, "learning_rate": 4.4307101421701755e-07, "logits/chosen": -0.8303281664848328, "logits/rejected": -0.8063556551933289, "logps/chosen": -330.0804443359375, "logps/ref_chosen": -296.73236083984375, "logps/ref_rejected": -266.45257568359375, "logps/rejected": -332.72882080078125, "loss": 4.725, "margin_dpo/margin_mean": 32.9280891418457, "margin_dpo/margin_std": 52.243892669677734, "step": 143 }, { "KL/chosen_KL_mean": -33.928253173828125, "KL/mean": -47.52806854248047, "KL/rejected_KL_mean": -61.12789535522461, "KL/std": 48.23427200317383, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.008003258146345615, "fcm_dpo/delta": -0.02481032907962799, "fcm_dpo/margin": 27.19965171813965, "fcm_dpo/q_t": 0.4491684138774872, "grad_norm": 45.131038665771484, "learning_rate": 4.419028041654559e-07, "logits/chosen": -0.8930144309997559, "logits/rejected": -0.8774906992912292, "logps/chosen": -332.772216796875, "logps/ref_chosen": -298.843994140625, "logps/ref_rejected": -266.120849609375, "logps/rejected": -327.24871826171875, "loss": 4.9486, "margin_dpo/margin_mean": 27.19965362548828, "margin_dpo/margin_std": 58.43843078613281, "step": 144 }, { "KL/chosen_KL_mean": -30.280269622802734, "KL/mean": -47.94648742675781, "KL/rejected_KL_mean": -65.61270141601562, "KL/std": 44.90657424926758, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.007799885701388121, "fcm_dpo/delta": -0.04888714477419853, "fcm_dpo/margin": 35.33243179321289, "fcm_dpo/q_t": 0.4344967007637024, "grad_norm": 50.87854766845703, "learning_rate": 4.4072430294890166e-07, "logits/chosen": -0.8770700097084045, "logits/rejected": -0.8764016628265381, "logps/chosen": -306.03314208984375, "logps/ref_chosen": -275.7528381347656, "logps/ref_rejected": -214.74807739257812, "logps/rejected": -280.36077880859375, "loss": 4.7152, "margin_dpo/margin_mean": 35.33243179321289, "margin_dpo/margin_std": 54.74494171142578, "step": 145 }, { "KL/chosen_KL_mean": -40.7142219543457, "KL/mean": -58.507232666015625, "KL/rejected_KL_mean": -76.30026245117188, "KL/std": 50.367061614990234, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.007565884385257959, "fcm_dpo/delta": -0.025456473231315613, "fcm_dpo/margin": 35.5860481262207, "fcm_dpo/q_t": 0.43644025921821594, "grad_norm": 54.75914764404297, "learning_rate": 4.395355737667985e-07, "logits/chosen": -0.8618282675743103, "logits/rejected": -0.8532693982124329, "logps/chosen": -317.8124084472656, "logps/ref_chosen": -277.09820556640625, "logps/ref_rejected": -265.41046142578125, "logps/rejected": -341.7107238769531, "loss": 4.7365, "margin_dpo/margin_mean": 35.5860481262207, "margin_dpo/margin_std": 56.042728424072266, "step": 146 }, { "KL/chosen_KL_mean": -42.89373779296875, "KL/mean": -56.810821533203125, "KL/rejected_KL_mean": -70.7279052734375, "KL/std": 52.10712432861328, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.007512391544878483, "fcm_dpo/delta": -0.02998882159590721, "fcm_dpo/margin": 27.83416748046875, "fcm_dpo/q_t": 0.4501326084136963, "grad_norm": 44.88176345825195, "learning_rate": 4.3833668036708483e-07, "logits/chosen": -0.8450511693954468, "logits/rejected": -0.8371117115020752, "logps/chosen": -334.31231689453125, "logps/ref_chosen": -291.4185791015625, "logps/ref_rejected": -253.43051147460938, "logps/rejected": -324.15838623046875, "loss": 4.9668, "margin_dpo/margin_mean": 27.834169387817383, "margin_dpo/margin_std": 57.25261688232422, "step": 147 }, { "KL/chosen_KL_mean": -42.17375564575195, "KL/mean": -57.056846618652344, "KL/rejected_KL_mean": -71.93993377685547, "KL/std": 55.181541442871094, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.007428798824548721, "fcm_dpo/delta": 0.017847800627350807, "fcm_dpo/margin": 29.76618003845215, "fcm_dpo/q_t": 0.4492494761943817, "grad_norm": 52.519874572753906, "learning_rate": 4.3712768704277524e-07, "logits/chosen": -0.9135507345199585, "logits/rejected": -0.9141757488250732, "logps/chosen": -278.9222717285156, "logps/ref_chosen": -236.74850463867188, "logps/ref_rejected": -231.4674072265625, "logps/rejected": -303.40728759765625, "loss": 4.9502, "margin_dpo/margin_mean": 29.76618003845215, "margin_dpo/margin_std": 66.35972595214844, "step": 148 }, { "KL/chosen_KL_mean": -49.42608642578125, "KL/mean": -69.45520782470703, "KL/rejected_KL_mean": -89.48432922363281, "KL/std": 50.71747970581055, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.007459109649062157, "fcm_dpo/delta": 0.001031714491546154, "fcm_dpo/margin": 40.05824661254883, "fcm_dpo/q_t": 0.429160475730896, "grad_norm": 54.98072814941406, "learning_rate": 4.3590865862851263e-07, "logits/chosen": -0.8939695358276367, "logits/rejected": -0.8774114847183228, "logps/chosen": -369.35455322265625, "logps/ref_chosen": -319.9284973144531, "logps/ref_rejected": -308.20233154296875, "logps/rejected": -397.6866760253906, "loss": 4.6089, "margin_dpo/margin_mean": 40.05824661254883, "margin_dpo/margin_std": 55.38935470581055, "step": 149 }, { "KL/chosen_KL_mean": -56.57506561279297, "KL/mean": -71.85128021240234, "KL/rejected_KL_mean": -87.12750244140625, "KL/std": 52.93745422363281, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.0074956417083740234, "fcm_dpo/delta": 0.0003224576357752085, "fcm_dpo/margin": 30.552452087402344, "fcm_dpo/q_t": 0.44559797644615173, "grad_norm": 57.156436920166016, "learning_rate": 4.346796604970912e-07, "logits/chosen": -0.860993504524231, "logits/rejected": -0.8432093262672424, "logps/chosen": -332.893310546875, "logps/ref_chosen": -276.3182373046875, "logps/ref_rejected": -273.02215576171875, "logps/rejected": -360.1496276855469, "loss": 4.8725, "margin_dpo/margin_mean": 30.552452087402344, "margin_dpo/margin_std": 57.681915283203125, "step": 150 }, { "KL/chosen_KL_mean": -54.340576171875, "KL/mean": -76.96028137207031, "KL/rejected_KL_mean": -99.57998657226562, "KL/std": 53.98088455200195, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.007399183232337236, "fcm_dpo/delta": -0.03600255399942398, "fcm_dpo/margin": 45.239410400390625, "fcm_dpo/q_t": 0.42125940322875977, "grad_norm": 66.83990478515625, "learning_rate": 4.3344075855595097e-07, "logits/chosen": -0.8806943297386169, "logits/rejected": -0.8683636784553528, "logps/chosen": -351.6534118652344, "logps/ref_chosen": -297.31280517578125, "logps/ref_rejected": -266.1003723144531, "logps/rejected": -365.68035888671875, "loss": 4.541, "margin_dpo/margin_mean": 45.239410400390625, "margin_dpo/margin_std": 62.988826751708984, "step": 151 }, { "KL/chosen_KL_mean": -47.37806701660156, "KL/mean": -67.31233978271484, "KL/rejected_KL_mean": -87.2466049194336, "KL/std": 57.862388610839844, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.007059152238070965, "fcm_dpo/delta": -0.05571059137582779, "fcm_dpo/margin": 39.8685417175293, "fcm_dpo/q_t": 0.4344164729118347, "grad_norm": 50.730186462402344, "learning_rate": 4.3219201924364323e-07, "logits/chosen": -0.8954218029975891, "logits/rejected": -0.8904637098312378, "logps/chosen": -317.6250915527344, "logps/ref_chosen": -270.2470397949219, "logps/ref_rejected": -269.7749328613281, "logps/rejected": -357.02154541015625, "loss": 4.7382, "margin_dpo/margin_mean": 39.86853790283203, "margin_dpo/margin_std": 67.04737854003906, "step": 152 }, { "KL/chosen_KL_mean": -54.68328094482422, "KL/mean": -82.62890625, "KL/rejected_KL_mean": -110.57453918457031, "KL/std": 62.350685119628906, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.006717229261994362, "fcm_dpo/delta": -0.07843609899282455, "fcm_dpo/margin": 55.89125442504883, "fcm_dpo/q_t": 0.41156265139579773, "grad_norm": 52.00371551513672, "learning_rate": 4.309335095262675e-07, "logits/chosen": -0.8805234432220459, "logits/rejected": -0.8666960597038269, "logps/chosen": -328.4623107910156, "logps/ref_chosen": -273.779052734375, "logps/ref_rejected": -280.9530944824219, "logps/rejected": -391.5276184082031, "loss": 4.373, "margin_dpo/margin_mean": 55.89125442504883, "margin_dpo/margin_std": 64.85965728759766, "step": 153 }, { "KL/chosen_KL_mean": -53.89643859863281, "KL/mean": -76.14729309082031, "KL/rejected_KL_mean": -98.39814758300781, "KL/std": 64.30680084228516, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.00642622634768486, "fcm_dpo/delta": -0.034452252089977264, "fcm_dpo/margin": 44.501712799072266, "fcm_dpo/q_t": 0.43304443359375, "grad_norm": 47.76982879638672, "learning_rate": 4.2966529689388064e-07, "logits/chosen": -0.920985221862793, "logits/rejected": -0.9008965492248535, "logps/chosen": -343.7996520996094, "logps/ref_chosen": -289.9031982421875, "logps/ref_rejected": -261.5166320800781, "logps/rejected": -359.9147644042969, "loss": 4.7259, "margin_dpo/margin_mean": 44.501712799072266, "margin_dpo/margin_std": 75.18342590332031, "step": 154 }, { "KL/chosen_KL_mean": -59.23719787597656, "KL/mean": -78.5732650756836, "KL/rejected_KL_mean": -97.90933227539062, "KL/std": 61.958526611328125, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.0063635921105742455, "fcm_dpo/delta": 0.0016274466179311275, "fcm_dpo/margin": 38.672122955322266, "fcm_dpo/q_t": 0.4432119131088257, "grad_norm": 59.706298828125, "learning_rate": 4.2838744935687716e-07, "logits/chosen": -0.8491299152374268, "logits/rejected": -0.8450287580490112, "logps/chosen": -345.098388671875, "logps/ref_chosen": -285.8612060546875, "logps/ref_rejected": -300.1272888183594, "logps/rejected": -398.03662109375, "loss": 4.8296, "margin_dpo/margin_mean": 38.672122955322266, "margin_dpo/margin_std": 70.07493591308594, "step": 155 }, { "KL/chosen_KL_mean": -52.546714782714844, "KL/mean": -79.25215148925781, "KL/rejected_KL_mean": -105.95761108398438, "KL/std": 64.53652954101562, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.006244411692023277, "fcm_dpo/delta": -0.07311277091503143, "fcm_dpo/margin": 53.410911560058594, "fcm_dpo/q_t": 0.42212918400764465, "grad_norm": 51.060523986816406, "learning_rate": 4.271000354423425e-07, "logits/chosen": -0.8737393617630005, "logits/rejected": -0.8699842095375061, "logps/chosen": -331.5821533203125, "logps/ref_chosen": -279.0354919433594, "logps/ref_rejected": -244.2198486328125, "logps/rejected": -350.177490234375, "loss": 4.5922, "margin_dpo/margin_mean": 53.410911560058594, "margin_dpo/margin_std": 81.18497467041016, "step": 156 }, { "KL/chosen_KL_mean": -54.379493713378906, "KL/mean": -75.57967376708984, "KL/rejected_KL_mean": -96.77983856201172, "KL/std": 63.81241226196289, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.005938378162682056, "fcm_dpo/delta": -0.021369297057390213, "fcm_dpo/margin": 42.40034103393555, "fcm_dpo/q_t": 0.4399886727333069, "grad_norm": 52.73342514038086, "learning_rate": 4.258031241903777e-07, "logits/chosen": -0.9237609505653381, "logits/rejected": -0.9217565059661865, "logps/chosen": -325.2098083496094, "logps/ref_chosen": -270.830322265625, "logps/ref_rejected": -259.08319091796875, "logps/rejected": -355.863037109375, "loss": 4.7865, "margin_dpo/margin_mean": 42.40034103393555, "margin_dpo/margin_std": 69.72222900390625, "step": 157 }, { "KL/chosen_KL_mean": -52.62913131713867, "KL/mean": -76.1375732421875, "KL/rejected_KL_mean": -99.64601135253906, "KL/std": 64.6611099243164, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.005866985768079758, "fcm_dpo/delta": -0.02696356363594532, "fcm_dpo/margin": 47.016876220703125, "fcm_dpo/q_t": 0.43413034081459045, "grad_norm": 62.1641731262207, "learning_rate": 4.2449678515039743e-07, "logits/chosen": -0.8632103204727173, "logits/rejected": -0.848564863204956, "logps/chosen": -342.595458984375, "logps/ref_chosen": -289.9663391113281, "logps/ref_rejected": -271.335693359375, "logps/rejected": -370.9817199707031, "loss": 4.7019, "margin_dpo/margin_mean": 47.016876220703125, "margin_dpo/margin_std": 69.72993469238281, "step": 158 }, { "KL/chosen_KL_mean": -62.773353576660156, "KL/mean": -76.6348876953125, "KL/rejected_KL_mean": -90.49641418457031, "KL/std": 60.769474029541016, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.005801479797810316, "fcm_dpo/delta": 0.011277447454631329, "fcm_dpo/margin": 27.723072052001953, "fcm_dpo/q_t": 0.46263986825942993, "grad_norm": 68.88479614257812, "learning_rate": 4.2318108837739986e-07, "logits/chosen": -0.9515292644500732, "logits/rejected": -0.9028602838516235, "logps/chosen": -384.1517333984375, "logps/ref_chosen": -321.37835693359375, "logps/ref_rejected": -250.45652770996094, "logps/rejected": -340.95294189453125, "loss": 5.1459, "margin_dpo/margin_mean": 27.72307014465332, "margin_dpo/margin_std": 76.83480072021484, "step": 159 }, { "KL/chosen_KL_mean": -53.467838287353516, "KL/mean": -79.95763397216797, "KL/rejected_KL_mean": -106.44743347167969, "KL/std": 64.74884796142578, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.005812094546854496, "fcm_dpo/delta": -0.008517347276210785, "fcm_dpo/margin": 52.97959899902344, "fcm_dpo/q_t": 0.4265155494213104, "grad_norm": 48.53911209106445, "learning_rate": 4.218561044282098e-07, "logits/chosen": -0.8837435245513916, "logits/rejected": -0.8850059509277344, "logps/chosen": -329.7513427734375, "logps/ref_chosen": -276.28350830078125, "logps/ref_rejected": -262.7477722167969, "logps/rejected": -369.1951904296875, "loss": 4.5471, "margin_dpo/margin_mean": 52.97959899902344, "margin_dpo/margin_std": 62.66265869140625, "step": 160 }, { "KL/chosen_KL_mean": -65.78504943847656, "KL/mean": -89.10560607910156, "KL/rejected_KL_mean": -112.4261703491211, "KL/std": 68.93439483642578, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.005809702444821596, "fcm_dpo/delta": -0.0074594165198504925, "fcm_dpo/margin": 46.641117095947266, "fcm_dpo/q_t": 0.4373480975627899, "grad_norm": 62.1060905456543, "learning_rate": 4.2052190435769554e-07, "logits/chosen": -0.9010162949562073, "logits/rejected": -0.8825740218162537, "logps/chosen": -376.27783203125, "logps/ref_chosen": -310.4927978515625, "logps/ref_rejected": -250.25347900390625, "logps/rejected": -362.67962646484375, "loss": 4.7666, "margin_dpo/margin_mean": 46.64112091064453, "margin_dpo/margin_std": 81.07203674316406, "step": 161 }, { "KL/chosen_KL_mean": -62.530601501464844, "KL/mean": -84.44666290283203, "KL/rejected_KL_mean": -106.36271667480469, "KL/std": 64.6400375366211, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.005650822538882494, "fcm_dpo/delta": -0.010540883988142014, "fcm_dpo/margin": 43.832122802734375, "fcm_dpo/q_t": 0.44126516580581665, "grad_norm": 40.41975784301758, "learning_rate": 4.1917855971495763e-07, "logits/chosen": -0.8757296204566956, "logits/rejected": -0.8666384816169739, "logps/chosen": -358.6410827636719, "logps/ref_chosen": -296.1105041503906, "logps/ref_rejected": -253.4247589111328, "logps/rejected": -359.7874450683594, "loss": 4.7996, "margin_dpo/margin_mean": 43.832122802734375, "margin_dpo/margin_std": 72.63944244384766, "step": 162 }, { "KL/chosen_KL_mean": -67.80076599121094, "KL/mean": -92.6597900390625, "KL/rejected_KL_mean": -117.51881408691406, "KL/std": 63.900360107421875, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.005691584199666977, "fcm_dpo/delta": -0.013408953323960304, "fcm_dpo/margin": 49.7180290222168, "fcm_dpo/q_t": 0.43232640624046326, "grad_norm": 62.017704010009766, "learning_rate": 4.1782614253949255e-07, "logits/chosen": -0.9154207110404968, "logits/rejected": -0.9098785519599915, "logps/chosen": -361.3006591796875, "logps/ref_chosen": -293.4999084472656, "logps/ref_rejected": -266.7116394042969, "logps/rejected": -384.23046875, "loss": 4.6703, "margin_dpo/margin_mean": 49.7180290222168, "margin_dpo/margin_std": 71.16876983642578, "step": 163 }, { "KL/chosen_KL_mean": -80.27838134765625, "KL/mean": -102.35275268554688, "KL/rejected_KL_mean": -124.4271469116211, "KL/std": 70.86463928222656, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.005669012665748596, "fcm_dpo/delta": -0.007750518620014191, "fcm_dpo/margin": 44.148780822753906, "fcm_dpo/q_t": 0.4416029751300812, "grad_norm": 58.888221740722656, "learning_rate": 4.164647253573289e-07, "logits/chosen": -0.8757019639015198, "logits/rejected": -0.8851001262664795, "logps/chosen": -347.327880859375, "logps/ref_chosen": -267.04949951171875, "logps/ref_rejected": -215.9768829345703, "logps/rejected": -340.4040222167969, "loss": 4.8288, "margin_dpo/margin_mean": 44.148780822753906, "margin_dpo/margin_std": 81.29022216796875, "step": 164 }, { "KL/chosen_KL_mean": -69.51595306396484, "KL/mean": -85.5526351928711, "KL/rejected_KL_mean": -101.5893325805664, "KL/std": 59.26508712768555, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.00558866374194622, "fcm_dpo/delta": -0.004711986985057592, "fcm_dpo/margin": 32.07337188720703, "fcm_dpo/q_t": 0.4569927155971527, "grad_norm": 62.76590347290039, "learning_rate": 4.1509438117713863e-07, "logits/chosen": -0.8961373567581177, "logits/rejected": -0.8679142594337463, "logps/chosen": -347.57745361328125, "logps/ref_chosen": -278.06146240234375, "logps/ref_rejected": -260.4288635253906, "logps/rejected": -362.0182189941406, "loss": 5.0112, "margin_dpo/margin_mean": 32.07337188720703, "margin_dpo/margin_std": 66.55110168457031, "step": 165 }, { "KL/chosen_KL_mean": -64.54472351074219, "KL/mean": -85.60026550292969, "KL/rejected_KL_mean": -106.65580749511719, "KL/std": 63.61589431762695, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.0055479020811617374, "fcm_dpo/delta": -0.0022045508958399296, "fcm_dpo/margin": 42.111087799072266, "fcm_dpo/q_t": 0.44493845105171204, "grad_norm": 52.08544921875, "learning_rate": 4.137151834863213e-07, "logits/chosen": -0.8519665598869324, "logits/rejected": -0.8151019215583801, "logps/chosen": -340.4938049316406, "logps/ref_chosen": -275.9490661621094, "logps/ref_rejected": -232.13473510742188, "logps/rejected": -338.79058837890625, "loss": 4.8543, "margin_dpo/margin_mean": 42.111087799072266, "margin_dpo/margin_std": 77.61107635498047, "step": 166 }, { "KL/chosen_KL_mean": -74.52401733398438, "KL/mean": -98.62837219238281, "KL/rejected_KL_mean": -122.73274230957031, "KL/std": 69.09346008300781, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.005625554360449314, "fcm_dpo/delta": 0.007754523307085037, "fcm_dpo/margin": 48.208717346191406, "fcm_dpo/q_t": 0.4354207515716553, "grad_norm": 49.939430236816406, "learning_rate": 4.123272062470633e-07, "logits/chosen": -0.8557583689689636, "logits/rejected": -0.8407419323921204, "logps/chosen": -355.075439453125, "logps/ref_chosen": -280.5514221191406, "logps/ref_rejected": -255.2896728515625, "logps/rejected": -378.02239990234375, "loss": 4.7377, "margin_dpo/margin_mean": 48.20872116088867, "margin_dpo/margin_std": 78.89886474609375, "step": 167 }, { "KL/chosen_KL_mean": -76.38279724121094, "KL/mean": -102.09847259521484, "KL/rejected_KL_mean": -127.81414794921875, "KL/std": 71.54838562011719, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.005538075231015682, "fcm_dpo/delta": -0.018495822325348854, "fcm_dpo/margin": 51.43134689331055, "fcm_dpo/q_t": 0.4321715533733368, "grad_norm": 106.25069427490234, "learning_rate": 4.1093052389237174e-07, "logits/chosen": -0.8275035619735718, "logits/rejected": -0.7970058917999268, "logps/chosen": -392.1810302734375, "logps/ref_chosen": -315.7982177734375, "logps/ref_rejected": -291.48406982421875, "logps/rejected": -419.2982482910156, "loss": 4.6839, "margin_dpo/margin_mean": 51.43134689331055, "margin_dpo/margin_std": 78.42271423339844, "step": 168 }, { "KL/chosen_KL_mean": -71.67886352539062, "KL/mean": -102.86964416503906, "KL/rejected_KL_mean": -134.06040954589844, "KL/std": 73.42317962646484, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.00543187977746129, "fcm_dpo/delta": -0.05743662640452385, "fcm_dpo/margin": 62.381561279296875, "fcm_dpo/q_t": 0.41999682784080505, "grad_norm": 57.55976104736328, "learning_rate": 4.0952521132208267e-07, "logits/chosen": -0.823381245136261, "logits/rejected": -0.8265929818153381, "logps/chosen": -332.74310302734375, "logps/ref_chosen": -261.06427001953125, "logps/ref_rejected": -235.40663146972656, "logps/rejected": -369.467041015625, "loss": 4.5049, "margin_dpo/margin_mean": 62.381561279296875, "margin_dpo/margin_std": 80.02377319335938, "step": 169 }, { "KL/chosen_KL_mean": -71.20687103271484, "KL/mean": -96.61824798583984, "KL/rejected_KL_mean": -122.02960205078125, "KL/std": 77.275146484375, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.005277743563055992, "fcm_dpo/delta": 0.012714497745037079, "fcm_dpo/margin": 50.82273864746094, "fcm_dpo/q_t": 0.4381609857082367, "grad_norm": 54.236793518066406, "learning_rate": 4.081113438988443e-07, "logits/chosen": -0.7995795607566833, "logits/rejected": -0.7993738055229187, "logps/chosen": -380.17413330078125, "logps/ref_chosen": -308.96722412109375, "logps/ref_rejected": -263.8466796875, "logps/rejected": -385.87628173828125, "loss": 4.8003, "margin_dpo/margin_mean": 50.82273864746094, "margin_dpo/margin_std": 97.3071517944336, "step": 170 }, { "KL/chosen_KL_mean": -58.641273498535156, "KL/mean": -90.96955871582031, "KL/rejected_KL_mean": -123.29782104492188, "KL/std": 73.8992919921875, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.005161117296665907, "fcm_dpo/delta": -0.0646912157535553, "fcm_dpo/margin": 64.65653228759766, "fcm_dpo/q_t": 0.4205283522605896, "grad_norm": 64.7024154663086, "learning_rate": 4.0668899744407567e-07, "logits/chosen": -0.8300844430923462, "logits/rejected": -0.8436256647109985, "logps/chosen": -317.5302734375, "logps/ref_chosen": -258.8890380859375, "logps/ref_rejected": -262.19140625, "logps/rejected": -385.4892272949219, "loss": 4.4977, "margin_dpo/margin_mean": 64.65653228759766, "margin_dpo/margin_std": 79.60781860351562, "step": 171 }, { "KL/chosen_KL_mean": -66.7177505493164, "KL/mean": -84.35318756103516, "KL/rejected_KL_mean": -101.9886474609375, "KL/std": 65.91461181640625, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.005088986828923225, "fcm_dpo/delta": 0.008377095684409142, "fcm_dpo/margin": 35.27089309692383, "fcm_dpo/q_t": 0.45685499906539917, "grad_norm": 41.981468200683594, "learning_rate": 4.0525824823390043e-07, "logits/chosen": -0.8545408844947815, "logits/rejected": -0.8687371015548706, "logps/chosen": -405.74005126953125, "logps/ref_chosen": -339.0223388671875, "logps/ref_rejected": -295.78759765625, "logps/rejected": -397.7762451171875, "loss": 5.0282, "margin_dpo/margin_mean": 35.27088928222656, "margin_dpo/margin_std": 80.5746841430664, "step": 172 }, { "KL/chosen_KL_mean": -65.19223022460938, "KL/mean": -88.45635223388672, "KL/rejected_KL_mean": -111.72047424316406, "KL/std": 68.98754119873047, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.005017148330807686, "fcm_dpo/delta": -0.033796440809965134, "fcm_dpo/margin": 46.52825164794922, "fcm_dpo/q_t": 0.4449205994606018, "grad_norm": 60.493263244628906, "learning_rate": 4.0381917299505686e-07, "logits/chosen": -0.8733835220336914, "logits/rejected": -0.8692581057548523, "logps/chosen": -365.3037109375, "logps/ref_chosen": -300.1114501953125, "logps/ref_rejected": -273.78460693359375, "logps/rejected": -385.505126953125, "loss": 4.8467, "margin_dpo/margin_mean": 46.52825164794922, "margin_dpo/margin_std": 81.14070129394531, "step": 173 }, { "KL/chosen_KL_mean": -62.053688049316406, "KL/mean": -90.67073822021484, "KL/rejected_KL_mean": -119.28778076171875, "KL/std": 70.07064819335938, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.004972523078322411, "fcm_dpo/delta": 0.003636527806520462, "fcm_dpo/margin": 57.23409652709961, "fcm_dpo/q_t": 0.431392639875412, "grad_norm": 71.27803039550781, "learning_rate": 4.0237184890078243e-07, "logits/chosen": -0.8437986373901367, "logits/rejected": -0.8229135870933533, "logps/chosen": -397.1075744628906, "logps/ref_chosen": -335.0538635253906, "logps/ref_rejected": -257.4646911621094, "logps/rejected": -376.7524719238281, "loss": 4.6288, "margin_dpo/margin_mean": 57.234092712402344, "margin_dpo/margin_std": 74.53538513183594, "step": 174 }, { "KL/chosen_KL_mean": -80.63742065429688, "KL/mean": -106.42805480957031, "KL/rejected_KL_mean": -132.21868896484375, "KL/std": 80.53546905517578, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.0049472167156636715, "fcm_dpo/delta": -0.037538737058639526, "fcm_dpo/margin": 51.581260681152344, "fcm_dpo/q_t": 0.44112372398376465, "grad_norm": 57.902069091796875, "learning_rate": 4.00916353566676e-07, "logits/chosen": -0.8539243936538696, "logits/rejected": -0.8540430665016174, "logps/chosen": -365.0329895019531, "logps/ref_chosen": -284.39556884765625, "logps/ref_rejected": -283.3876647949219, "logps/rejected": -415.6062927246094, "loss": 4.8182, "margin_dpo/margin_mean": 51.58126449584961, "margin_dpo/margin_std": 90.54949951171875, "step": 175 }, { "KL/chosen_KL_mean": -78.06394958496094, "KL/mean": -98.68791198730469, "KL/rejected_KL_mean": -119.31187438964844, "KL/std": 74.86479187011719, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.004791987594217062, "fcm_dpo/delta": -0.025556650012731552, "fcm_dpo/margin": 41.2479248046875, "fcm_dpo/q_t": 0.45347633957862854, "grad_norm": 61.08745193481445, "learning_rate": 3.994527650465352e-07, "logits/chosen": -0.8255434036254883, "logits/rejected": -0.8293647170066833, "logps/chosen": -329.8767395019531, "logps/ref_chosen": -251.81280517578125, "logps/ref_rejected": -242.05328369140625, "logps/rejected": -361.36517333984375, "loss": 5.0155, "margin_dpo/margin_mean": 41.247928619384766, "margin_dpo/margin_std": 94.69705200195312, "step": 176 }, { "KL/chosen_KL_mean": -86.40694427490234, "KL/mean": -107.02310180664062, "KL/rejected_KL_mean": -127.63926696777344, "KL/std": 72.94503784179688, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.004743529949337244, "fcm_dpo/delta": -0.0020989251788705587, "fcm_dpo/margin": 41.23231887817383, "fcm_dpo/q_t": 0.4538949728012085, "grad_norm": 60.08957290649414, "learning_rate": 3.979811618281705e-07, "logits/chosen": -0.9129557013511658, "logits/rejected": -0.8802610635757446, "logps/chosen": -385.0533447265625, "logps/ref_chosen": -298.6463928222656, "logps/ref_rejected": -295.66534423828125, "logps/rejected": -423.3045959472656, "loss": 5.0155, "margin_dpo/margin_mean": 41.23231887817383, "margin_dpo/margin_std": 89.64039611816406, "step": 177 }, { "KL/chosen_KL_mean": -74.41709899902344, "KL/mean": -104.72712707519531, "KL/rejected_KL_mean": -135.0371551513672, "KL/std": 80.7784652709961, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.004676537588238716, "fcm_dpo/delta": -0.01576678641140461, "fcm_dpo/margin": 60.62005615234375, "fcm_dpo/q_t": 0.4334092140197754, "grad_norm": 44.437416076660156, "learning_rate": 3.9650162282919654e-07, "logits/chosen": -0.842369019985199, "logits/rejected": -0.8331227898597717, "logps/chosen": -360.67474365234375, "logps/ref_chosen": -286.2576599121094, "logps/ref_rejected": -243.97491455078125, "logps/rejected": -379.0120544433594, "loss": 4.6885, "margin_dpo/margin_mean": 60.620059967041016, "margin_dpo/margin_std": 93.60894775390625, "step": 178 }, { "KL/chosen_KL_mean": -85.76069641113281, "KL/mean": -113.523681640625, "KL/rejected_KL_mean": -141.2866973876953, "KL/std": 82.93985748291016, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.004612220451235771, "fcm_dpo/delta": -0.01727444678544998, "fcm_dpo/margin": 55.525997161865234, "fcm_dpo/q_t": 0.4389217495918274, "grad_norm": 58.93293380737305, "learning_rate": 3.9501422739279953e-07, "logits/chosen": -0.8293529748916626, "logits/rejected": -0.8276381492614746, "logps/chosen": -345.49774169921875, "logps/ref_chosen": -259.737060546875, "logps/ref_rejected": -277.8813171386719, "logps/rejected": -419.16802978515625, "loss": 4.7825, "margin_dpo/margin_mean": 55.525997161865234, "margin_dpo/margin_std": 91.86540985107422, "step": 179 }, { "KL/chosen_KL_mean": -93.309326171875, "KL/mean": -121.91683959960938, "KL/rejected_KL_mean": -150.52435302734375, "KL/std": 79.11772918701172, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.00460518104955554, "fcm_dpo/delta": 0.0006139341276139021, "fcm_dpo/margin": 57.21503448486328, "fcm_dpo/q_t": 0.43886619806289673, "grad_norm": 78.7430648803711, "learning_rate": 3.935190552834828e-07, "logits/chosen": -0.8608454465866089, "logits/rejected": -0.8818348050117493, "logps/chosen": -360.61822509765625, "logps/ref_chosen": -267.30889892578125, "logps/ref_rejected": -230.4376983642578, "logps/rejected": -380.9620666503906, "loss": 4.7406, "margin_dpo/margin_mean": 57.21503448486328, "margin_dpo/margin_std": 90.0433578491211, "step": 180 }, { "KL/chosen_KL_mean": -109.72969055175781, "KL/mean": -134.2457275390625, "KL/rejected_KL_mean": -158.7617645263672, "KL/std": 81.4984130859375, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.0046082790940999985, "fcm_dpo/delta": 0.011829550378024578, "fcm_dpo/margin": 49.0320930480957, "fcm_dpo/q_t": 0.4466300904750824, "grad_norm": 73.56534576416016, "learning_rate": 3.920161866827889e-07, "logits/chosen": -0.8686573505401611, "logits/rejected": -0.8654269576072693, "logps/chosen": -410.2210998535156, "logps/ref_chosen": -300.49139404296875, "logps/ref_rejected": -278.98284912109375, "logps/rejected": -437.74462890625, "loss": 4.8905, "margin_dpo/margin_mean": 49.0320930480957, "margin_dpo/margin_std": 95.10774993896484, "step": 181 }, { "KL/chosen_KL_mean": -108.689453125, "KL/mean": -143.95114135742188, "KL/rejected_KL_mean": -179.2128143310547, "KL/std": 81.61646270751953, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.0045288666151463985, "fcm_dpo/delta": -0.06309524923563004, "fcm_dpo/margin": 70.52334594726562, "fcm_dpo/q_t": 0.42456480860710144, "grad_norm": 56.7416877746582, "learning_rate": 3.90505702185e-07, "logits/chosen": -0.8651958107948303, "logits/rejected": -0.8845742344856262, "logps/chosen": -388.1876220703125, "logps/ref_chosen": -279.4981689453125, "logps/ref_rejected": -263.6926574707031, "logps/rejected": -442.90545654296875, "loss": 4.5715, "margin_dpo/margin_mean": 70.52334594726562, "margin_dpo/margin_std": 96.27135467529297, "step": 182 }, { "KL/chosen_KL_mean": -111.27757263183594, "KL/mean": -141.33160400390625, "KL/rejected_KL_mean": -171.38565063476562, "KL/std": 83.98210144042969, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.004440652672201395, "fcm_dpo/delta": 0.01649186573922634, "fcm_dpo/margin": 60.108070373535156, "fcm_dpo/q_t": 0.437813401222229, "grad_norm": 64.96882629394531, "learning_rate": 3.889876827928156e-07, "logits/chosen": -0.9078041911125183, "logits/rejected": -0.9069268703460693, "logps/chosen": -382.4832458496094, "logps/ref_chosen": -271.2057189941406, "logps/ref_rejected": -243.91549682617188, "logps/rejected": -415.3011474609375, "loss": 4.7459, "margin_dpo/margin_mean": 60.10807418823242, "margin_dpo/margin_std": 99.40080261230469, "step": 183 }, { "KL/chosen_KL_mean": -110.44056701660156, "KL/mean": -153.08717346191406, "KL/rejected_KL_mean": -195.73379516601562, "KL/std": 93.96009063720703, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.004284189082682133, "fcm_dpo/delta": -0.069338858127594, "fcm_dpo/margin": 85.2932357788086, "fcm_dpo/q_t": 0.4131174087524414, "grad_norm": 68.06148529052734, "learning_rate": 3.874622099130087e-07, "logits/chosen": -0.9283655881881714, "logits/rejected": -0.8976894021034241, "logps/chosen": -428.8863220214844, "logps/ref_chosen": -318.4457702636719, "logps/ref_rejected": -266.640869140625, "logps/rejected": -462.3746643066406, "loss": 4.4215, "margin_dpo/margin_mean": 85.2932357788086, "margin_dpo/margin_std": 105.1895980834961, "step": 184 }, { "KL/chosen_KL_mean": -122.83193969726562, "KL/mean": -152.273193359375, "KL/rejected_KL_mean": -181.71441650390625, "KL/std": 99.07866668701172, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.004218521062284708, "fcm_dpo/delta": 0.006759931333363056, "fcm_dpo/margin": 58.88247299194336, "fcm_dpo/q_t": 0.4418259263038635, "grad_norm": 83.13562774658203, "learning_rate": 3.859293653520604e-07, "logits/chosen": -0.9097890853881836, "logits/rejected": -0.9024062752723694, "logps/chosen": -397.1407775878906, "logps/ref_chosen": -274.308837890625, "logps/ref_rejected": -260.7274169921875, "logps/rejected": -442.44183349609375, "loss": 4.8157, "margin_dpo/margin_mean": 58.88247299194336, "margin_dpo/margin_std": 104.70620727539062, "step": 185 }, { "KL/chosen_KL_mean": -120.67276000976562, "KL/mean": -149.5914764404297, "KL/rejected_KL_mean": -178.51016235351562, "KL/std": 88.66747283935547, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.004291029646992683, "fcm_dpo/delta": 0.005338459741324186, "fcm_dpo/margin": 57.8374137878418, "fcm_dpo/q_t": 0.44115057587623596, "grad_norm": 60.644100189208984, "learning_rate": 3.8438923131177237e-07, "logits/chosen": -0.921852171421051, "logits/rejected": -0.9182274341583252, "logps/chosen": -419.6781005859375, "logps/ref_chosen": -299.00537109375, "logps/ref_rejected": -274.4014587402344, "logps/rejected": -452.9115905761719, "loss": 4.7951, "margin_dpo/margin_mean": 57.8374137878418, "margin_dpo/margin_std": 96.82072448730469, "step": 186 }, { "KL/chosen_KL_mean": -130.63047790527344, "KL/mean": -154.88661193847656, "KL/rejected_KL_mean": -179.14276123046875, "KL/std": 94.44642639160156, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.0042493510991334915, "fcm_dpo/delta": -0.010592436417937279, "fcm_dpo/margin": 48.51228713989258, "fcm_dpo/q_t": 0.44973719120025635, "grad_norm": 70.30011749267578, "learning_rate": 3.828418903848593e-07, "logits/chosen": -0.8689478039741516, "logits/rejected": -0.851067066192627, "logps/chosen": -460.45574951171875, "logps/ref_chosen": -329.8253173828125, "logps/ref_rejected": -263.73175048828125, "logps/rejected": -442.8745422363281, "loss": 5.0081, "margin_dpo/margin_mean": 48.51228713989258, "margin_dpo/margin_std": 113.23208618164062, "step": 187 }, { "KL/chosen_KL_mean": -113.98969268798828, "KL/mean": -141.99594116210938, "KL/rejected_KL_mean": -170.00218200683594, "KL/std": 95.48326873779297, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.004231072496622801, "fcm_dpo/delta": -0.0032284194603562355, "fcm_dpo/margin": 56.012489318847656, "fcm_dpo/q_t": 0.4451254904270172, "grad_norm": 73.19304656982422, "learning_rate": 3.812874255505191e-07, "logits/chosen": -0.8525975346565247, "logits/rejected": -0.8392002582550049, "logps/chosen": -376.9953308105469, "logps/ref_chosen": -263.005615234375, "logps/ref_rejected": -247.08668518066406, "logps/rejected": -417.08880615234375, "loss": 4.9198, "margin_dpo/margin_mean": 56.012489318847656, "margin_dpo/margin_std": 120.3454360961914, "step": 188 }, { "KL/chosen_KL_mean": -92.47537231445312, "KL/mean": -129.1150665283203, "KL/rejected_KL_mean": -165.75473022460938, "KL/std": 93.57108306884766, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.004115546587854624, "fcm_dpo/delta": -0.04511391744017601, "fcm_dpo/margin": 73.27936553955078, "fcm_dpo/q_t": 0.4287566840648651, "grad_norm": 63.68251037597656, "learning_rate": 3.797259201699833e-07, "logits/chosen": -0.8904998898506165, "logits/rejected": -0.8967751860618591, "logps/chosen": -365.4357604980469, "logps/ref_chosen": -272.96038818359375, "logps/ref_rejected": -275.13238525390625, "logps/rejected": -440.88714599609375, "loss": 4.6142, "margin_dpo/margin_mean": 73.27935791015625, "margin_dpo/margin_std": 102.02257537841797, "step": 189 }, { "KL/chosen_KL_mean": -90.63619995117188, "KL/mean": -123.51648712158203, "KL/rejected_KL_mean": -156.39678955078125, "KL/std": 85.31573486328125, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.00409154687076807, "fcm_dpo/delta": 0.01720324158668518, "fcm_dpo/margin": 65.7605972290039, "fcm_dpo/q_t": 0.43600544333457947, "grad_norm": 63.41102981567383, "learning_rate": 3.781574579820464e-07, "logits/chosen": -0.8776698708534241, "logits/rejected": -0.834459662437439, "logps/chosen": -348.4337463378906, "logps/ref_chosen": -257.79754638671875, "logps/ref_rejected": -225.2164306640625, "logps/rejected": -381.6131896972656, "loss": 4.6957, "margin_dpo/margin_mean": 65.76058959960938, "margin_dpo/margin_std": 94.38175964355469, "step": 190 }, { "KL/chosen_KL_mean": -84.44734954833984, "KL/mean": -114.27680206298828, "KL/rejected_KL_mean": -144.1062469482422, "KL/std": 86.19569396972656, "epoch": 0.4, "fcm_dpo/beta": 0.004162714816629887, "fcm_dpo/delta": 0.016047921031713486, "fcm_dpo/margin": 59.65888977050781, "fcm_dpo/q_t": 0.44123804569244385, "grad_norm": 62.25904083251953, "learning_rate": 3.765821230985757e-07, "logits/chosen": -0.9132973551750183, "logits/rejected": -0.914715588092804, "logps/chosen": -328.305908203125, "logps/ref_chosen": -243.8585205078125, "logps/ref_rejected": -245.12136840820312, "logps/rejected": -389.22760009765625, "loss": 4.8031, "margin_dpo/margin_mean": 59.65888977050781, "margin_dpo/margin_std": 103.41494750976562, "step": 191 }, { "KL/chosen_KL_mean": -92.90057373046875, "KL/mean": -118.03880310058594, "KL/rejected_KL_mean": -143.17701721191406, "KL/std": 79.73211669921875, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.004252881743013859, "fcm_dpo/delta": 0.03286261856555939, "fcm_dpo/margin": 50.27644348144531, "fcm_dpo/q_t": 0.44909244775772095, "grad_norm": 93.99234008789062, "learning_rate": 3.75e-07, "logits/chosen": -0.8387249708175659, "logits/rejected": -0.8206347227096558, "logps/chosen": -359.8804931640625, "logps/ref_chosen": -266.9799499511719, "logps/ref_rejected": -260.1697082519531, "logps/rejected": -403.3467102050781, "loss": 4.8962, "margin_dpo/margin_mean": 50.27644729614258, "margin_dpo/margin_std": 90.78395080566406, "step": 192 }, { "KL/chosen_KL_mean": -108.05162811279297, "KL/mean": -137.0741729736328, "KL/rejected_KL_mean": -166.09671020507812, "KL/std": 91.33769989013672, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.0042908648028969765, "fcm_dpo/delta": -0.0064502134919166565, "fcm_dpo/margin": 58.045074462890625, "fcm_dpo/q_t": 0.4416932761669159, "grad_norm": 55.59531021118164, "learning_rate": 3.734111735307796e-07, "logits/chosen": -0.8882525563240051, "logits/rejected": -0.8587220311164856, "logps/chosen": -388.3048400878906, "logps/ref_chosen": -280.25323486328125, "logps/ref_rejected": -291.0348815917969, "logps/rejected": -457.1316223144531, "loss": 4.8238, "margin_dpo/margin_mean": 58.04507827758789, "margin_dpo/margin_std": 103.87382507324219, "step": 193 }, { "KL/chosen_KL_mean": -106.48265075683594, "KL/mean": -130.47157287597656, "KL/rejected_KL_mean": -154.46047973632812, "KL/std": 87.91346740722656, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.004275742452591658, "fcm_dpo/delta": 0.00011287536472082138, "fcm_dpo/margin": 47.977813720703125, "fcm_dpo/q_t": 0.45183444023132324, "grad_norm": 50.88446807861328, "learning_rate": 3.7181572889485623e-07, "logits/chosen": -0.8614975214004517, "logits/rejected": -0.8511518239974976, "logps/chosen": -394.62213134765625, "logps/ref_chosen": -288.13946533203125, "logps/ref_rejected": -251.31529235839844, "logps/rejected": -405.7757568359375, "loss": 4.943, "margin_dpo/margin_mean": 47.977813720703125, "margin_dpo/margin_std": 95.73290252685547, "step": 194 }, { "KL/chosen_KL_mean": -108.96260833740234, "KL/mean": -132.38340759277344, "KL/rejected_KL_mean": -155.80416870117188, "KL/std": 83.44720458984375, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.004298967309296131, "fcm_dpo/delta": 0.013229867443442345, "fcm_dpo/margin": 46.84156036376953, "fcm_dpo/q_t": 0.453215628862381, "grad_norm": 47.520111083984375, "learning_rate": 3.7021375165108377e-07, "logits/chosen": -0.8580233454704285, "logits/rejected": -0.8627651333808899, "logps/chosen": -382.9632568359375, "logps/ref_chosen": -274.0006408691406, "logps/ref_rejected": -280.22723388671875, "logps/rejected": -436.0314025878906, "loss": 4.9577, "margin_dpo/margin_mean": 46.84156036376953, "margin_dpo/margin_std": 93.48262023925781, "step": 195 }, { "KL/chosen_KL_mean": -93.70986938476562, "KL/mean": -121.1141357421875, "KL/rejected_KL_mean": -148.51840209960938, "KL/std": 85.61984252929688, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.004368783440440893, "fcm_dpo/delta": 0.010245390236377716, "fcm_dpo/margin": 54.80853271484375, "fcm_dpo/q_t": 0.44323647022247314, "grad_norm": 61.44898223876953, "learning_rate": 3.6860532770864005e-07, "logits/chosen": -0.8427191376686096, "logits/rejected": -0.8498020172119141, "logps/chosen": -368.6105651855469, "logps/ref_chosen": -274.90069580078125, "logps/ref_rejected": -248.7281951904297, "logps/rejected": -397.2466125488281, "loss": 4.8345, "margin_dpo/margin_mean": 54.80853271484375, "margin_dpo/margin_std": 98.43161010742188, "step": 196 }, { "KL/chosen_KL_mean": -102.14750671386719, "KL/mean": -134.74530029296875, "KL/rejected_KL_mean": -167.3431396484375, "KL/std": 87.57936096191406, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.0043055955320596695, "fcm_dpo/delta": -0.03260016813874245, "fcm_dpo/margin": 65.19561004638672, "fcm_dpo/q_t": 0.4334312677383423, "grad_norm": 73.9769287109375, "learning_rate": 3.6699054332241985e-07, "logits/chosen": -0.8774630427360535, "logits/rejected": -0.8628032207489014, "logps/chosen": -411.68231201171875, "logps/ref_chosen": -309.5348205566406, "logps/ref_rejected": -264.3179931640625, "logps/rejected": -431.6611022949219, "loss": 4.6842, "margin_dpo/margin_mean": 65.19561004638672, "margin_dpo/margin_std": 94.93258666992188, "step": 197 }, { "KL/chosen_KL_mean": -99.85154724121094, "KL/mean": -134.7466583251953, "KL/rejected_KL_mean": -169.6417694091797, "KL/std": 91.86687469482422, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.0042022340930998325, "fcm_dpo/delta": -0.010694863274693489, "fcm_dpo/margin": 69.79022979736328, "fcm_dpo/q_t": 0.4308726191520691, "grad_norm": 61.50657653808594, "learning_rate": 3.653694850884091e-07, "logits/chosen": -0.8791506290435791, "logits/rejected": -0.8535560369491577, "logps/chosen": -400.8650207519531, "logps/ref_chosen": -301.0134582519531, "logps/ref_rejected": -292.84185791015625, "logps/rejected": -462.4836120605469, "loss": 4.6516, "margin_dpo/margin_mean": 69.79023742675781, "margin_dpo/margin_std": 103.8303451538086, "step": 198 }, { "KL/chosen_KL_mean": -110.65092468261719, "KL/mean": -140.47079467773438, "KL/rejected_KL_mean": -170.29063415527344, "KL/std": 86.93942260742188, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.004240399692207575, "fcm_dpo/delta": 0.0082355672493577, "fcm_dpo/margin": 59.63971710205078, "fcm_dpo/q_t": 0.4405139088630676, "grad_norm": 103.90625762939453, "learning_rate": 3.6374223993904124e-07, "logits/chosen": -0.848025381565094, "logits/rejected": -0.8029439449310303, "logps/chosen": -375.2567443847656, "logps/ref_chosen": -264.6058654785156, "logps/ref_rejected": -214.9014892578125, "logps/rejected": -385.1921081542969, "loss": 4.8064, "margin_dpo/margin_mean": 59.63971710205078, "margin_dpo/margin_std": 107.35498046875, "step": 199 }, { "KL/chosen_KL_mean": -123.91107940673828, "KL/mean": -150.3993377685547, "KL/rejected_KL_mean": -176.88763427734375, "KL/std": 90.9786376953125, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.004204145632684231, "fcm_dpo/delta": -0.014344342984259129, "fcm_dpo/margin": 52.97654342651367, "fcm_dpo/q_t": 0.44768112897872925, "grad_norm": 70.7446060180664, "learning_rate": 3.621088951385353e-07, "logits/chosen": -0.9124985933303833, "logits/rejected": -0.8872630000114441, "logps/chosen": -448.0699462890625, "logps/ref_chosen": -324.1588134765625, "logps/ref_rejected": -277.80218505859375, "logps/rejected": -454.6898193359375, "loss": 4.9611, "margin_dpo/margin_mean": 52.97654724121094, "margin_dpo/margin_std": 117.82571411132812, "step": 200 }, { "epoch": 0.418848167539267, "eval_KL/chosen_KL_mean": -132.41326904296875, "eval_KL/mean": -163.7840576171875, "eval_KL/rejected_KL_mean": -195.15484619140625, "eval_KL/std": 96.79059600830078, "eval_fcm_dpo/beta": 0.004193730652332306, "eval_logits/chosen": -0.8912529349327087, "eval_logits/rejected": -0.8724632859230042, "eval_logps/chosen": -420.2400207519531, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -462.0848693847656, "eval_loss": 0.6005851030349731, "eval_margin_dpo/margin_mean": 62.7415885925293, "eval_margin_dpo/margin_std": 107.95939636230469, "eval_runtime": 78.042, "eval_samples_per_second": 25.627, "eval_steps_per_second": 3.203, "step": 200 }, { "KL/chosen_KL_mean": -131.6876220703125, "KL/mean": -163.39187622070312, "KL/rejected_KL_mean": -195.0961151123047, "KL/std": 98.21855163574219, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.004204457625746727, "fcm_dpo/delta": -0.014064101502299309, "fcm_dpo/margin": 63.40850067138672, "fcm_dpo/q_t": 0.4373506009578705, "grad_norm": 70.647216796875, "learning_rate": 3.604695382782159e-07, "logits/chosen": -0.8758408427238464, "logits/rejected": -0.8588773012161255, "logps/chosen": -403.1833190917969, "logps/ref_chosen": -271.49566650390625, "logps/ref_rejected": -245.71414184570312, "logps/rejected": -440.8102722167969, "loss": 4.7786, "margin_dpo/margin_mean": 63.40850830078125, "margin_dpo/margin_std": 111.03482818603516, "step": 201 }, { "KL/chosen_KL_mean": -132.4500732421875, "KL/mean": -164.19525146484375, "KL/rejected_KL_mean": -195.94044494628906, "KL/std": 97.9029769897461, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.004103401210159063, "fcm_dpo/delta": -0.00629676878452301, "fcm_dpo/margin": 63.49036407470703, "fcm_dpo/q_t": 0.4392518997192383, "grad_norm": 82.53160858154297, "learning_rate": 3.588242572718162e-07, "logits/chosen": -0.8929991722106934, "logits/rejected": -0.8781752586364746, "logps/chosen": -404.5479736328125, "logps/ref_chosen": -272.0979309082031, "logps/ref_rejected": -235.94805908203125, "logps/rejected": -431.88848876953125, "loss": 4.8147, "margin_dpo/margin_mean": 63.49037170410156, "margin_dpo/margin_std": 119.6777114868164, "step": 202 }, { "KL/chosen_KL_mean": -143.78916931152344, "KL/mean": -167.3878173828125, "KL/rejected_KL_mean": -190.98648071289062, "KL/std": 96.77014923095703, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.00414351187646389, "fcm_dpo/delta": 0.007255503907799721, "fcm_dpo/margin": 47.19731521606445, "fcm_dpo/q_t": 0.4533665180206299, "grad_norm": 82.75169372558594, "learning_rate": 3.571731403507635e-07, "logits/chosen": -0.8697388172149658, "logits/rejected": -0.8752584457397461, "logps/chosen": -424.01129150390625, "logps/ref_chosen": -280.2221374511719, "logps/ref_rejected": -251.79798889160156, "logps/rejected": -442.78448486328125, "loss": 4.9882, "margin_dpo/margin_mean": 47.19731521606445, "margin_dpo/margin_std": 101.90586853027344, "step": 203 }, { "KL/chosen_KL_mean": -135.544921875, "KL/mean": -169.3643341064453, "KL/rejected_KL_mean": -203.18374633789062, "KL/std": 96.953369140625, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.0041145868599414825, "fcm_dpo/delta": -0.010046249255537987, "fcm_dpo/margin": 67.6387939453125, "fcm_dpo/q_t": 0.4341426193714142, "grad_norm": 76.59571838378906, "learning_rate": 3.5551627605944746e-07, "logits/chosen": -0.914879560470581, "logits/rejected": -0.8851953744888306, "logps/chosen": -454.3409729003906, "logps/ref_chosen": -318.7960510253906, "logps/ref_rejected": -269.69921875, "logps/rejected": -472.8829650878906, "loss": 4.7134, "margin_dpo/margin_mean": 67.6387939453125, "margin_dpo/margin_std": 109.43064880371094, "step": 204 }, { "KL/chosen_KL_mean": -131.20462036132812, "KL/mean": -172.88485717773438, "KL/rejected_KL_mean": -214.56509399414062, "KL/std": 105.96263122558594, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.004047570750117302, "fcm_dpo/delta": -0.03919146582484245, "fcm_dpo/margin": 83.3604736328125, "fcm_dpo/q_t": 0.4227820038795471, "grad_norm": 58.605960845947266, "learning_rate": 3.5385375325047163e-07, "logits/chosen": -0.8598192930221558, "logits/rejected": -0.8276709914207458, "logps/chosen": -414.9666748046875, "logps/ref_chosen": -283.7620544433594, "logps/ref_rejected": -297.69439697265625, "logps/rejected": -512.259521484375, "loss": 4.5619, "margin_dpo/margin_mean": 83.3604736328125, "margin_dpo/margin_std": 123.52423095703125, "step": 205 }, { "KL/chosen_KL_mean": -140.71893310546875, "KL/mean": -172.0816192626953, "KL/rejected_KL_mean": -203.44432067871094, "KL/std": 101.86876678466797, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.003931830637156963, "fcm_dpo/delta": -0.03239187225699425, "fcm_dpo/margin": 62.725379943847656, "fcm_dpo/q_t": 0.44201433658599854, "grad_norm": 59.901485443115234, "learning_rate": 3.5218566107988867e-07, "logits/chosen": -0.8909974694252014, "logits/rejected": -0.9125638008117676, "logps/chosen": -434.38275146484375, "logps/ref_chosen": -293.66387939453125, "logps/ref_rejected": -291.3056640625, "logps/rejected": -494.75, "loss": 4.8594, "margin_dpo/margin_mean": 62.725379943847656, "margin_dpo/margin_std": 120.58600616455078, "step": 206 }, { "KL/chosen_KL_mean": -133.72694396972656, "KL/mean": -165.00857543945312, "KL/rejected_KL_mean": -196.29022216796875, "KL/std": 113.79143524169922, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.0038526002317667007, "fcm_dpo/delta": -0.010718029923737049, "fcm_dpo/margin": 62.56328201293945, "fcm_dpo/q_t": 0.44273653626441956, "grad_norm": 85.31092834472656, "learning_rate": 3.505120890024195e-07, "logits/chosen": -0.8365156650543213, "logits/rejected": -0.8453404307365417, "logps/chosen": -404.26202392578125, "logps/ref_chosen": -270.5350646972656, "logps/ref_rejected": -278.7747497558594, "logps/rejected": -475.0649719238281, "loss": 4.9322, "margin_dpo/margin_mean": 62.56327438354492, "margin_dpo/margin_std": 138.38185119628906, "step": 207 }, { "KL/chosen_KL_mean": -134.51309204101562, "KL/mean": -175.3253173828125, "KL/rejected_KL_mean": -216.13755798339844, "KL/std": 110.93104553222656, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.0038300526794046164, "fcm_dpo/delta": -0.026927048340439796, "fcm_dpo/margin": 81.62447357177734, "fcm_dpo/q_t": 0.4281120300292969, "grad_norm": 68.95054626464844, "learning_rate": 3.4883312676665534e-07, "logits/chosen": -0.8949654698371887, "logits/rejected": -0.846905529499054, "logps/chosen": -414.0958557128906, "logps/ref_chosen": -279.582763671875, "logps/ref_rejected": -290.041015625, "logps/rejected": -506.1785888671875, "loss": 4.6102, "margin_dpo/margin_mean": 81.62447357177734, "margin_dpo/margin_std": 121.49321746826172, "step": 208 }, { "KL/chosen_KL_mean": -149.1902313232422, "KL/mean": -173.98388671875, "KL/rejected_KL_mean": -198.7775115966797, "KL/std": 104.27743530273438, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.0037748278118669987, "fcm_dpo/delta": 0.0028195905033499002, "fcm_dpo/margin": 49.58729553222656, "fcm_dpo/q_t": 0.4552307724952698, "grad_norm": 62.14313888549805, "learning_rate": 3.4714886441024573e-07, "logits/chosen": -0.8140920996665955, "logits/rejected": -0.8075211644172668, "logps/chosen": -468.0627136230469, "logps/ref_chosen": -318.8725280761719, "logps/ref_rejected": -270.64324951171875, "logps/rejected": -469.4207763671875, "loss": 5.0178, "margin_dpo/margin_mean": 49.58729553222656, "margin_dpo/margin_std": 111.15589904785156, "step": 209 }, { "KL/chosen_KL_mean": -133.66421508789062, "KL/mean": -168.675048828125, "KL/rejected_KL_mean": -203.68588256835938, "KL/std": 101.76795959472656, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.0037062368355691433, "fcm_dpo/delta": -0.02896423637866974, "fcm_dpo/margin": 70.02168273925781, "fcm_dpo/q_t": 0.43855100870132446, "grad_norm": 68.48916625976562, "learning_rate": 3.454593922550693e-07, "logits/chosen": -0.840144693851471, "logits/rejected": -0.8310486078262329, "logps/chosen": -416.80450439453125, "logps/ref_chosen": -283.14031982421875, "logps/ref_rejected": -287.2986755371094, "logps/rejected": -490.98455810546875, "loss": 4.7707, "margin_dpo/margin_mean": 70.02169036865234, "margin_dpo/margin_std": 117.46251678466797, "step": 210 }, { "KL/chosen_KL_mean": -119.03889465332031, "KL/mean": -163.0735626220703, "KL/rejected_KL_mean": -207.1082000732422, "KL/std": 99.52161407470703, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.0035809765104204416, "fcm_dpo/delta": -0.06169634312391281, "fcm_dpo/margin": 88.06932067871094, "fcm_dpo/q_t": 0.4240798354148865, "grad_norm": 62.72855758666992, "learning_rate": 3.4376480090239047e-07, "logits/chosen": -0.865166187286377, "logits/rejected": -0.8419003486633301, "logps/chosen": -395.4617614746094, "logps/ref_chosen": -276.4228515625, "logps/ref_rejected": -252.40603637695312, "logps/rejected": -459.5142517089844, "loss": 4.5323, "margin_dpo/margin_mean": 88.0693130493164, "margin_dpo/margin_std": 102.9296646118164, "step": 211 }, { "KL/chosen_KL_mean": -138.11749267578125, "KL/mean": -164.1044158935547, "KL/rejected_KL_mean": -190.09130859375, "KL/std": 96.616943359375, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.003526175394654274, "fcm_dpo/delta": 0.015544392168521881, "fcm_dpo/margin": 51.97382354736328, "fcm_dpo/q_t": 0.45574235916137695, "grad_norm": 79.6462173461914, "learning_rate": 3.4206518122800055e-07, "logits/chosen": -0.8534685373306274, "logits/rejected": -0.8499578237533569, "logps/chosen": -409.82305908203125, "logps/ref_chosen": -271.7055358886719, "logps/ref_rejected": -241.18511962890625, "logps/rejected": -431.2764587402344, "loss": 5.0072, "margin_dpo/margin_mean": 51.97381591796875, "margin_dpo/margin_std": 109.19019317626953, "step": 212 }, { "KL/chosen_KL_mean": -145.60855102539062, "KL/mean": -175.45535278320312, "KL/rejected_KL_mean": -205.3021697998047, "KL/std": 116.7935562133789, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.003567378269508481, "fcm_dpo/delta": 0.0018861833959817886, "fcm_dpo/margin": 59.69361877441406, "fcm_dpo/q_t": 0.45073333382606506, "grad_norm": 71.9309310913086, "learning_rate": 3.403606243773448e-07, "logits/chosen": -0.8502581119537354, "logits/rejected": -0.8634451031684875, "logps/chosen": -447.90618896484375, "logps/ref_chosen": -302.2976379394531, "logps/ref_rejected": -303.6202087402344, "logps/rejected": -508.9223327636719, "loss": 4.9608, "margin_dpo/margin_mean": 59.69361877441406, "margin_dpo/margin_std": 130.421142578125, "step": 213 }, { "KL/chosen_KL_mean": -159.0684814453125, "KL/mean": -196.1217041015625, "KL/rejected_KL_mean": -233.17489624023438, "KL/std": 104.86323547363281, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.0035179981496185064, "fcm_dpo/delta": -0.0281633660197258, "fcm_dpo/margin": 74.10640716552734, "fcm_dpo/q_t": 0.4384738802909851, "grad_norm": 71.70868682861328, "learning_rate": 3.3865122176063385e-07, "logits/chosen": -0.8574197292327881, "logits/rejected": -0.8577027916908264, "logps/chosen": -431.20111083984375, "logps/ref_chosen": -272.13262939453125, "logps/ref_rejected": -294.82354736328125, "logps/rejected": -527.9984130859375, "loss": 4.7398, "margin_dpo/margin_mean": 74.10640716552734, "margin_dpo/margin_std": 111.84681701660156, "step": 214 }, { "KL/chosen_KL_mean": -164.32371520996094, "KL/mean": -187.74632263183594, "KL/rejected_KL_mean": -211.16893005371094, "KL/std": 117.98490142822266, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.0034630722366273403, "fcm_dpo/delta": -0.017829960212111473, "fcm_dpo/margin": 46.845191955566406, "fcm_dpo/q_t": 0.46209806203842163, "grad_norm": 78.56700897216797, "learning_rate": 3.3693706504794243e-07, "logits/chosen": -0.8971854448318481, "logits/rejected": -0.8750616908073425, "logps/chosen": -455.70208740234375, "logps/ref_chosen": -291.3782958984375, "logps/ref_rejected": -261.05792236328125, "logps/rejected": -472.22686767578125, "loss": 5.1106, "margin_dpo/margin_mean": 46.84518814086914, "margin_dpo/margin_std": 120.64424133300781, "step": 215 }, { "KL/chosen_KL_mean": -151.2208251953125, "KL/mean": -186.5756378173828, "KL/rejected_KL_mean": -221.93045043945312, "KL/std": 109.45191955566406, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.00344485929235816, "fcm_dpo/delta": -0.002506938762962818, "fcm_dpo/margin": 70.70963287353516, "fcm_dpo/q_t": 0.44286611676216125, "grad_norm": 60.88117980957031, "learning_rate": 3.3521824616429284e-07, "logits/chosen": -0.9232491254806519, "logits/rejected": -0.9159709215164185, "logps/chosen": -489.72625732421875, "logps/ref_chosen": -338.50543212890625, "logps/ref_rejected": -305.76104736328125, "logps/rejected": -527.6914672851562, "loss": 4.8403, "margin_dpo/margin_mean": 70.70964050292969, "margin_dpo/margin_std": 130.25973510742188, "step": 216 }, { "KL/chosen_KL_mean": -157.36825561523438, "KL/mean": -200.84681701660156, "KL/rejected_KL_mean": -244.32540893554688, "KL/std": 115.38238525390625, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.0033965944312512875, "fcm_dpo/delta": -0.01266030129045248, "fcm_dpo/margin": 86.9571533203125, "fcm_dpo/q_t": 0.43058374524116516, "grad_norm": 45.89756393432617, "learning_rate": 3.334948572847253e-07, "logits/chosen": -0.8301356434822083, "logits/rejected": -0.7868068814277649, "logps/chosen": -450.91802978515625, "logps/ref_chosen": -293.5498046875, "logps/ref_rejected": -256.7830810546875, "logps/rejected": -501.10845947265625, "loss": 4.6518, "margin_dpo/margin_mean": 86.95714569091797, "margin_dpo/margin_std": 131.319091796875, "step": 217 }, { "KL/chosen_KL_mean": -157.33384704589844, "KL/mean": -196.21994018554688, "KL/rejected_KL_mean": -235.10601806640625, "KL/std": 113.00043487548828, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.0033359171357005835, "fcm_dpo/delta": -0.02211921662092209, "fcm_dpo/margin": 77.77216339111328, "fcm_dpo/q_t": 0.43903183937072754, "grad_norm": 53.51487350463867, "learning_rate": 3.317669908293554e-07, "logits/chosen": -0.8625221252441406, "logits/rejected": -0.8787875175476074, "logps/chosen": -477.9132080078125, "logps/ref_chosen": -320.579345703125, "logps/ref_rejected": -294.0381164550781, "logps/rejected": -529.1441650390625, "loss": 4.7594, "margin_dpo/margin_mean": 77.77216339111328, "margin_dpo/margin_std": 127.42372131347656, "step": 218 }, { "KL/chosen_KL_mean": -159.63987731933594, "KL/mean": -200.8522491455078, "KL/rejected_KL_mean": -242.0646209716797, "KL/std": 109.93661499023438, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.003269998822361231, "fcm_dpo/delta": -0.035739410668611526, "fcm_dpo/margin": 82.42474365234375, "fcm_dpo/q_t": 0.43642672896385193, "grad_norm": 61.093894958496094, "learning_rate": 3.300347394584172e-07, "logits/chosen": -0.8373451828956604, "logits/rejected": -0.8628327250480652, "logps/chosen": -428.0585021972656, "logps/ref_chosen": -268.4186096191406, "logps/ref_rejected": -265.7808837890625, "logps/rejected": -507.8454895019531, "loss": 4.7514, "margin_dpo/margin_mean": 82.42474365234375, "margin_dpo/margin_std": 134.4219512939453, "step": 219 }, { "KL/chosen_KL_mean": -158.48060607910156, "KL/mean": -197.44924926757812, "KL/rejected_KL_mean": -236.41790771484375, "KL/std": 106.5591049194336, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.003238196484744549, "fcm_dpo/delta": 0.010954329743981361, "fcm_dpo/margin": 77.93732452392578, "fcm_dpo/q_t": 0.439789742231369, "grad_norm": 55.85917663574219, "learning_rate": 3.2829819606729477e-07, "logits/chosen": -0.9119465947151184, "logits/rejected": -0.8783890008926392, "logps/chosen": -471.3670654296875, "logps/ref_chosen": -312.8864440917969, "logps/ref_rejected": -259.5191955566406, "logps/rejected": -495.9371337890625, "loss": 4.7505, "margin_dpo/margin_mean": 77.93733215332031, "margin_dpo/margin_std": 118.18209838867188, "step": 220 }, { "KL/chosen_KL_mean": -172.79757690429688, "KL/mean": -207.20826721191406, "KL/rejected_KL_mean": -241.6189422607422, "KL/std": 113.18028259277344, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.0032698106952011585, "fcm_dpo/delta": -0.015935488045215607, "fcm_dpo/margin": 68.82136535644531, "fcm_dpo/q_t": 0.447135865688324, "grad_norm": 58.454261779785156, "learning_rate": 3.265574537815398e-07, "logits/chosen": -0.8103448152542114, "logits/rejected": -0.8210662603378296, "logps/chosen": -473.1233825683594, "logps/ref_chosen": -300.32586669921875, "logps/ref_rejected": -286.312255859375, "logps/rejected": -527.9312133789062, "loss": 4.9031, "margin_dpo/margin_mean": 68.82136535644531, "margin_dpo/margin_std": 134.83969116210938, "step": 221 }, { "KL/chosen_KL_mean": -169.22169494628906, "KL/mean": -202.10203552246094, "KL/rejected_KL_mean": -234.98236083984375, "KL/std": 103.63680267333984, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.0032462161034345627, "fcm_dpo/delta": 0.005263763479888439, "fcm_dpo/margin": 65.76068115234375, "fcm_dpo/q_t": 0.448941707611084, "grad_norm": 52.64696502685547, "learning_rate": 3.248126059518784e-07, "logits/chosen": -0.9082949757575989, "logits/rejected": -0.8724523186683655, "logps/chosen": -466.3330383300781, "logps/ref_chosen": -297.1113586425781, "logps/ref_rejected": -235.53146362304688, "logps/rejected": -470.5138244628906, "loss": 4.894, "margin_dpo/margin_mean": 65.76068115234375, "margin_dpo/margin_std": 120.36772155761719, "step": 222 }, { "KL/chosen_KL_mean": -174.15005493164062, "KL/mean": -210.77581787109375, "KL/rejected_KL_mean": -247.4015655517578, "KL/std": 112.02201843261719, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.0032290732488036156, "fcm_dpo/delta": -0.0026750918477773666, "fcm_dpo/margin": 73.25151062011719, "fcm_dpo/q_t": 0.4433066248893738, "grad_norm": 51.26232147216797, "learning_rate": 3.230637461492043e-07, "logits/chosen": -0.8510481715202332, "logits/rejected": -0.8167670369148254, "logps/chosen": -460.56512451171875, "logps/ref_chosen": -286.41510009765625, "logps/ref_rejected": -241.1181640625, "logps/rejected": -488.5197448730469, "loss": 4.8299, "margin_dpo/margin_mean": 73.25151062011719, "margin_dpo/margin_std": 127.65164184570312, "step": 223 }, { "KL/chosen_KL_mean": -169.56051635742188, "KL/mean": -205.94590759277344, "KL/rejected_KL_mean": -242.33132934570312, "KL/std": 109.94374084472656, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.0032642222940921783, "fcm_dpo/delta": 0.0056016454473137856, "fcm_dpo/margin": 72.77080535888672, "fcm_dpo/q_t": 0.44394826889038086, "grad_norm": 64.43927764892578, "learning_rate": 3.213109681595612e-07, "logits/chosen": -0.8066267371177673, "logits/rejected": -0.8236450552940369, "logps/chosen": -419.0528564453125, "logps/ref_chosen": -249.49234008789062, "logps/ref_rejected": -233.10752868652344, "logps/rejected": -475.4388427734375, "loss": 4.832, "margin_dpo/margin_mean": 72.77080535888672, "margin_dpo/margin_std": 126.9173812866211, "step": 224 }, { "KL/chosen_KL_mean": -191.29263305664062, "KL/mean": -226.8955535888672, "KL/rejected_KL_mean": -262.4984130859375, "KL/std": 117.79441833496094, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.0032615442760288715, "fcm_dpo/delta": 0.009283446706831455, "fcm_dpo/margin": 71.2057876586914, "fcm_dpo/q_t": 0.4457131624221802, "grad_norm": 65.1972885131836, "learning_rate": 3.1955436597911315e-07, "logits/chosen": -0.8588619828224182, "logits/rejected": -0.8319200873374939, "logps/chosen": -503.1509704589844, "logps/ref_chosen": -311.8583679199219, "logps/ref_rejected": -336.8523864746094, "logps/rejected": -599.3507690429688, "loss": 4.8845, "margin_dpo/margin_mean": 71.2057876586914, "margin_dpo/margin_std": 140.93931579589844, "step": 225 }, { "KL/chosen_KL_mean": -175.48651123046875, "KL/mean": -205.9941864013672, "KL/rejected_KL_mean": -236.5018768310547, "KL/std": 112.99928283691406, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.00331744528375566, "fcm_dpo/delta": 0.015448614954948425, "fcm_dpo/margin": 61.015350341796875, "fcm_dpo/q_t": 0.45238372683525085, "grad_norm": 58.19024658203125, "learning_rate": 3.1779403380910425e-07, "logits/chosen": -0.874785304069519, "logits/rejected": -0.8616780638694763, "logps/chosen": -427.687744140625, "logps/ref_chosen": -252.20123291015625, "logps/ref_rejected": -254.41162109375, "logps/rejected": -490.9134826660156, "loss": 4.9792, "margin_dpo/margin_mean": 61.015350341796875, "margin_dpo/margin_std": 135.62869262695312, "step": 226 }, { "KL/chosen_KL_mean": -170.14938354492188, "KL/mean": -212.30055236816406, "KL/rejected_KL_mean": -254.45169067382812, "KL/std": 110.70320892333984, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.0032987315207719803, "fcm_dpo/delta": -0.015660453587770462, "fcm_dpo/margin": 84.30230712890625, "fcm_dpo/q_t": 0.4337405860424042, "grad_norm": 55.077152252197266, "learning_rate": 3.160300660508064e-07, "logits/chosen": -0.8456053733825684, "logits/rejected": -0.8431981801986694, "logps/chosen": -455.4088134765625, "logps/ref_chosen": -285.25946044921875, "logps/ref_rejected": -261.3220520019531, "logps/rejected": -515.773681640625, "loss": 4.7053, "margin_dpo/margin_mean": 84.30230712890625, "margin_dpo/margin_std": 131.59988403320312, "step": 227 }, { "KL/chosen_KL_mean": -158.64784240722656, "KL/mean": -197.88645935058594, "KL/rejected_KL_mean": -237.12509155273438, "KL/std": 105.5765380859375, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.003275742521509528, "fcm_dpo/delta": -0.015454575419425964, "fcm_dpo/margin": 78.47726440429688, "fcm_dpo/q_t": 0.438965380191803, "grad_norm": 57.37860870361328, "learning_rate": 3.1426255730045695e-07, "logits/chosen": -0.8636391162872314, "logits/rejected": -0.8306207656860352, "logps/chosen": -472.4666442871094, "logps/ref_chosen": -313.81878662109375, "logps/ref_rejected": -258.07061767578125, "logps/rejected": -495.1957092285156, "loss": 4.7617, "margin_dpo/margin_mean": 78.47726440429688, "margin_dpo/margin_std": 124.56838989257812, "step": 228 }, { "KL/chosen_KL_mean": -173.101806640625, "KL/mean": -220.78363037109375, "KL/rejected_KL_mean": -268.4654235839844, "KL/std": 115.71029663085938, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.0031459908932447433, "fcm_dpo/delta": -0.05162987485527992, "fcm_dpo/margin": 95.36361694335938, "fcm_dpo/q_t": 0.4281517565250397, "grad_norm": 42.334407806396484, "learning_rate": 3.1249160234418644e-07, "logits/chosen": -0.8365947008132935, "logits/rejected": -0.8395388126373291, "logps/chosen": -465.07257080078125, "logps/ref_chosen": -291.9707946777344, "logps/ref_rejected": -263.42059326171875, "logps/rejected": -531.885986328125, "loss": 4.6133, "margin_dpo/margin_mean": 95.36361694335938, "margin_dpo/margin_std": 127.28816223144531, "step": 229 }, { "KL/chosen_KL_mean": -157.16651916503906, "KL/mean": -198.59124755859375, "KL/rejected_KL_mean": -240.01600646972656, "KL/std": 123.6738052368164, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.0030816374346613884, "fcm_dpo/delta": -0.005881241988390684, "fcm_dpo/margin": 82.84947204589844, "fcm_dpo/q_t": 0.44025325775146484, "grad_norm": 49.02077102661133, "learning_rate": 3.1071729615293424e-07, "logits/chosen": -0.8741714358329773, "logits/rejected": -0.8733953833580017, "logps/chosen": -390.4266357421875, "logps/ref_chosen": -233.2601318359375, "logps/ref_rejected": -238.922119140625, "logps/rejected": -478.9381103515625, "loss": 4.7806, "margin_dpo/margin_mean": 82.8494644165039, "margin_dpo/margin_std": 139.558349609375, "step": 230 }, { "KL/chosen_KL_mean": -168.19107055664062, "KL/mean": -202.2855682373047, "KL/rejected_KL_mean": -236.38006591796875, "KL/std": 106.24060821533203, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.003070762613788247, "fcm_dpo/delta": -0.0014391806907951832, "fcm_dpo/margin": 68.18899536132812, "fcm_dpo/q_t": 0.4494929909706116, "grad_norm": 62.14220428466797, "learning_rate": 3.0893973387735683e-07, "logits/chosen": -0.8697140216827393, "logits/rejected": -0.8564260005950928, "logps/chosen": -490.3462829589844, "logps/ref_chosen": -322.1551818847656, "logps/ref_rejected": -280.97613525390625, "logps/rejected": -517.356201171875, "loss": 4.8992, "margin_dpo/margin_mean": 68.18899536132812, "margin_dpo/margin_std": 120.236328125, "step": 231 }, { "KL/chosen_KL_mean": -168.53921508789062, "KL/mean": -201.57940673828125, "KL/rejected_KL_mean": -234.6195831298828, "KL/std": 117.83100128173828, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.003069432219490409, "fcm_dpo/delta": -0.005725067108869553, "fcm_dpo/margin": 66.08037567138672, "fcm_dpo/q_t": 0.45206257700920105, "grad_norm": 58.8627815246582, "learning_rate": 3.071590108427243e-07, "logits/chosen": -0.8262802362442017, "logits/rejected": -0.8069007396697998, "logps/chosen": -440.28302001953125, "logps/ref_chosen": -271.7437744140625, "logps/ref_rejected": -249.94981384277344, "logps/rejected": -484.56939697265625, "loss": 4.9796, "margin_dpo/margin_mean": 66.08036804199219, "margin_dpo/margin_std": 138.81605529785156, "step": 232 }, { "KL/chosen_KL_mean": -159.59194946289062, "KL/mean": -203.31967163085938, "KL/rejected_KL_mean": -247.0474090576172, "KL/std": 114.54499053955078, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.0030200586188584566, "fcm_dpo/delta": -0.03824126720428467, "fcm_dpo/margin": 87.45545196533203, "fcm_dpo/q_t": 0.4372973144054413, "grad_norm": 45.34437561035156, "learning_rate": 3.05375222543809e-07, "logits/chosen": -0.8714592456817627, "logits/rejected": -0.8599870204925537, "logps/chosen": -444.93426513671875, "logps/ref_chosen": -285.3423156738281, "logps/ref_rejected": -266.34320068359375, "logps/rejected": -513.3905639648438, "loss": 4.7357, "margin_dpo/margin_mean": 87.45545196533203, "margin_dpo/margin_std": 130.9783935546875, "step": 233 }, { "KL/chosen_KL_mean": -161.26751708984375, "KL/mean": -199.64366149902344, "KL/rejected_KL_mean": -238.01988220214844, "KL/std": 113.85478973388672, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.003014399204403162, "fcm_dpo/delta": 0.028298456221818924, "fcm_dpo/margin": 76.75235748291016, "fcm_dpo/q_t": 0.44490310549736023, "grad_norm": 42.643096923828125, "learning_rate": 3.035884646397637e-07, "logits/chosen": -0.8460695743560791, "logits/rejected": -0.8349598050117493, "logps/chosen": -456.17327880859375, "logps/ref_chosen": -294.9057312011719, "logps/ref_rejected": -299.37054443359375, "logps/rejected": -537.3904418945312, "loss": 4.8618, "margin_dpo/margin_mean": 76.75235748291016, "margin_dpo/margin_std": 141.786865234375, "step": 234 }, { "KL/chosen_KL_mean": -162.90518188476562, "KL/mean": -201.39187622070312, "KL/rejected_KL_mean": -239.87855529785156, "KL/std": 123.73214721679688, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.0030970873776823282, "fcm_dpo/delta": 0.02609279938042164, "fcm_dpo/margin": 76.973388671875, "fcm_dpo/q_t": 0.4432239234447479, "grad_norm": 47.34357452392578, "learning_rate": 3.017988329489923e-07, "logits/chosen": -0.8745311498641968, "logits/rejected": -0.862747073173523, "logps/chosen": -452.40277099609375, "logps/ref_chosen": -289.49755859375, "logps/ref_rejected": -247.55076599121094, "logps/rejected": -487.4293212890625, "loss": 4.8187, "margin_dpo/margin_mean": 76.973388671875, "margin_dpo/margin_std": 132.9673309326172, "step": 235 }, { "KL/chosen_KL_mean": -153.37669372558594, "KL/mean": -191.69894409179688, "KL/rejected_KL_mean": -230.02120971679688, "KL/std": 117.47241973876953, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.0031566726975142956, "fcm_dpo/delta": 0.016904598101973534, "fcm_dpo/margin": 76.64453125, "fcm_dpo/q_t": 0.4432297945022583, "grad_norm": 57.95139694213867, "learning_rate": 3.000064234440111e-07, "logits/chosen": -0.8885783553123474, "logits/rejected": -0.8885743618011475, "logps/chosen": -442.2613525390625, "logps/ref_chosen": -288.8846435546875, "logps/ref_rejected": -242.0452880859375, "logps/rejected": -472.06646728515625, "loss": 4.8325, "margin_dpo/margin_mean": 76.64453125, "margin_dpo/margin_std": 139.19834899902344, "step": 236 }, { "KL/chosen_KL_mean": -160.40975952148438, "KL/mean": -200.35916137695312, "KL/rejected_KL_mean": -240.30856323242188, "KL/std": 103.72799682617188, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.003175287041813135, "fcm_dpo/delta": 0.009177761152386665, "fcm_dpo/margin": 79.89883422851562, "fcm_dpo/q_t": 0.4399101138114929, "grad_norm": 42.846649169921875, "learning_rate": 2.9821133224630223e-07, "logits/chosen": -0.8538817167282104, "logits/rejected": -0.8301167488098145, "logps/chosen": -425.888427734375, "logps/ref_chosen": -265.47869873046875, "logps/ref_rejected": -267.9891357421875, "logps/rejected": -508.2976989746094, "loss": 4.7534, "margin_dpo/margin_mean": 79.8988265991211, "margin_dpo/margin_std": 124.49240112304688, "step": 237 }, { "KL/chosen_KL_mean": -169.00497436523438, "KL/mean": -208.2441864013672, "KL/rejected_KL_mean": -247.48336791992188, "KL/std": 114.40408325195312, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.0031809653155505657, "fcm_dpo/delta": 0.0017670519882813096, "fcm_dpo/margin": 78.4783706665039, "fcm_dpo/q_t": 0.44169679284095764, "grad_norm": 57.76200485229492, "learning_rate": 2.964136556211588e-07, "logits/chosen": -0.862713634967804, "logits/rejected": -0.8289677500724792, "logps/chosen": -481.00762939453125, "logps/ref_chosen": -312.0026550292969, "logps/ref_rejected": -270.0257263183594, "logps/rejected": -517.5090942382812, "loss": 4.7887, "margin_dpo/margin_mean": 78.4783706665039, "margin_dpo/margin_std": 131.0048065185547, "step": 238 }, { "KL/chosen_KL_mean": -155.6094970703125, "KL/mean": -191.5946807861328, "KL/rejected_KL_mean": -227.57989501953125, "KL/std": 121.70386505126953, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.003189996350556612, "fcm_dpo/delta": 0.004023873247206211, "fcm_dpo/margin": 71.97037506103516, "fcm_dpo/q_t": 0.44634878635406494, "grad_norm": 68.1906967163086, "learning_rate": 2.946134899725226e-07, "logits/chosen": -0.8460709452629089, "logits/rejected": -0.8845388889312744, "logps/chosen": -422.7767639160156, "logps/ref_chosen": -267.167236328125, "logps/ref_rejected": -275.99468994140625, "logps/rejected": -503.5745544433594, "loss": 4.9432, "margin_dpo/margin_mean": 71.97037506103516, "margin_dpo/margin_std": 159.57943725585938, "step": 239 }, { "KL/chosen_KL_mean": -157.0327606201172, "KL/mean": -200.23236083984375, "KL/rejected_KL_mean": -243.43197631835938, "KL/std": 119.24127960205078, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.0031683642882853746, "fcm_dpo/delta": -0.00633890088647604, "fcm_dpo/margin": 86.39923095703125, "fcm_dpo/q_t": 0.4365653991699219, "grad_norm": 67.59850311279297, "learning_rate": 2.9281093183781403e-07, "logits/chosen": -0.9054303765296936, "logits/rejected": -0.8970333337783813, "logps/chosen": -443.01239013671875, "logps/ref_chosen": -285.9796142578125, "logps/ref_rejected": -256.8258056640625, "logps/rejected": -500.2577819824219, "loss": 4.7286, "margin_dpo/margin_mean": 86.39923095703125, "margin_dpo/margin_std": 141.60586547851562, "step": 240 }, { "KL/chosen_KL_mean": -176.8218231201172, "KL/mean": -210.4013214111328, "KL/rejected_KL_mean": -243.9808349609375, "KL/std": 128.3367919921875, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.0031660483218729496, "fcm_dpo/delta": 0.00029843486845493317, "fcm_dpo/margin": 67.15899658203125, "fcm_dpo/q_t": 0.45050010085105896, "grad_norm": 50.04656982421875, "learning_rate": 2.910060778827554e-07, "logits/chosen": -0.8268923759460449, "logits/rejected": -0.8014039397239685, "logps/chosen": -438.33868408203125, "logps/ref_chosen": -261.516845703125, "logps/ref_rejected": -250.2250518798828, "logps/rejected": -494.2059020996094, "loss": 4.9681, "margin_dpo/margin_mean": 67.15899658203125, "margin_dpo/margin_std": 144.54986572265625, "step": 241 }, { "KL/chosen_KL_mean": -161.33384704589844, "KL/mean": -197.4034881591797, "KL/rejected_KL_mean": -233.47311401367188, "KL/std": 123.67906188964844, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.0031525217927992344, "fcm_dpo/delta": 0.003016672795638442, "fcm_dpo/margin": 72.1392822265625, "fcm_dpo/q_t": 0.44707444310188293, "grad_norm": 65.69779205322266, "learning_rate": 2.891990248961871e-07, "logits/chosen": -0.8855007886886597, "logits/rejected": -0.8583632707595825, "logps/chosen": -431.8477783203125, "logps/ref_chosen": -270.51397705078125, "logps/ref_rejected": -244.8560791015625, "logps/rejected": -478.3291931152344, "loss": 4.8718, "margin_dpo/margin_mean": 72.1392822265625, "margin_dpo/margin_std": 136.6427001953125, "step": 242 }, { "KL/chosen_KL_mean": -154.03050231933594, "KL/mean": -196.08612060546875, "KL/rejected_KL_mean": -238.14178466796875, "KL/std": 124.8993148803711, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.0031572661828249693, "fcm_dpo/delta": -0.03359325975179672, "fcm_dpo/margin": 84.11128234863281, "fcm_dpo/q_t": 0.4410480856895447, "grad_norm": 62.51230239868164, "learning_rate": 2.873898697848762e-07, "logits/chosen": -0.8862600326538086, "logits/rejected": -0.8692023754119873, "logps/chosen": -478.7125549316406, "logps/ref_chosen": -324.68206787109375, "logps/ref_rejected": -307.1111755371094, "logps/rejected": -545.2529296875, "loss": 4.7993, "margin_dpo/margin_mean": 84.11128234863281, "margin_dpo/margin_std": 155.82298278808594, "step": 243 }, { "KL/chosen_KL_mean": -163.61180114746094, "KL/mean": -206.80072021484375, "KL/rejected_KL_mean": -249.98959350585938, "KL/std": 115.1811752319336, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.0030796793289482594, "fcm_dpo/delta": -0.0004585727583616972, "fcm_dpo/margin": 86.37779235839844, "fcm_dpo/q_t": 0.43625351786613464, "grad_norm": 52.980777740478516, "learning_rate": 2.8557870956832133e-07, "logits/chosen": -0.867234468460083, "logits/rejected": -0.8243776559829712, "logps/chosen": -482.591064453125, "logps/ref_chosen": -318.979248046875, "logps/ref_rejected": -269.67572021484375, "logps/rejected": -519.6653442382812, "loss": 4.7323, "margin_dpo/margin_mean": 86.37779235839844, "margin_dpo/margin_std": 129.40206909179688, "step": 244 }, { "KL/chosen_KL_mean": -154.42869567871094, "KL/mean": -198.804931640625, "KL/rejected_KL_mean": -243.18118286132812, "KL/std": 110.18680572509766, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.003070926060900092, "fcm_dpo/delta": -0.018872009590268135, "fcm_dpo/margin": 88.75247192382812, "fcm_dpo/q_t": 0.43488627672195435, "grad_norm": 62.36658477783203, "learning_rate": 2.837656413735479e-07, "logits/chosen": -0.8802121877670288, "logits/rejected": -0.8869256377220154, "logps/chosen": -449.32672119140625, "logps/ref_chosen": -294.8980712890625, "logps/ref_rejected": -239.8111114501953, "logps/rejected": -482.9922790527344, "loss": 4.6985, "margin_dpo/margin_mean": 88.75247192382812, "margin_dpo/margin_std": 128.5753631591797, "step": 245 }, { "KL/chosen_KL_mean": -186.30972290039062, "KL/mean": -216.44203186035156, "KL/rejected_KL_mean": -246.57437133789062, "KL/std": 116.18084716796875, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.003061179770156741, "fcm_dpo/delta": 0.012664351612329483, "fcm_dpo/margin": 60.264644622802734, "fcm_dpo/q_t": 0.457215815782547, "grad_norm": 64.10836791992188, "learning_rate": 2.8195076242990116e-07, "logits/chosen": -0.8674138784408569, "logits/rejected": -0.8710952401161194, "logps/chosen": -466.9951171875, "logps/ref_chosen": -280.6854248046875, "logps/ref_rejected": -253.65382385253906, "logps/rejected": -500.2281799316406, "loss": 5.0715, "margin_dpo/margin_mean": 60.264644622802734, "margin_dpo/margin_std": 155.37069702148438, "step": 246 }, { "KL/chosen_KL_mean": -174.0077667236328, "KL/mean": -212.17257690429688, "KL/rejected_KL_mean": -250.33738708496094, "KL/std": 119.93651580810547, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.0030726096592843533, "fcm_dpo/delta": -0.012722790241241455, "fcm_dpo/margin": 76.32959747314453, "fcm_dpo/q_t": 0.44508832693099976, "grad_norm": 56.767269134521484, "learning_rate": 2.801341700638307e-07, "logits/chosen": -0.8562105298042297, "logits/rejected": -0.8530141115188599, "logps/chosen": -455.1168518066406, "logps/ref_chosen": -281.1091003417969, "logps/ref_rejected": -260.3700866699219, "logps/rejected": -510.7074890136719, "loss": 4.8493, "margin_dpo/margin_mean": 76.32959747314453, "margin_dpo/margin_std": 135.0543212890625, "step": 247 }, { "KL/chosen_KL_mean": -178.17813110351562, "KL/mean": -217.06997680664062, "KL/rejected_KL_mean": -255.96185302734375, "KL/std": 118.73770904541016, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.0030005162116140127, "fcm_dpo/delta": -0.03079632669687271, "fcm_dpo/margin": 77.78372192382812, "fcm_dpo/q_t": 0.4442315101623535, "grad_norm": 67.58417510986328, "learning_rate": 2.7831596169367227e-07, "logits/chosen": -0.817348301410675, "logits/rejected": -0.8214455842971802, "logps/chosen": -448.49652099609375, "logps/ref_chosen": -270.318359375, "logps/ref_rejected": -233.46778869628906, "logps/rejected": -489.42962646484375, "loss": 4.8472, "margin_dpo/margin_mean": 77.78372192382812, "margin_dpo/margin_std": 135.59178161621094, "step": 248 }, { "KL/chosen_KL_mean": -205.08566284179688, "KL/mean": -236.98524475097656, "KL/rejected_KL_mean": -268.88482666015625, "KL/std": 122.88720703125, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.002997219329699874, "fcm_dpo/delta": 0.01746796816587448, "fcm_dpo/margin": 63.79915237426758, "fcm_dpo/q_t": 0.4539327025413513, "grad_norm": 51.39647674560547, "learning_rate": 2.7649623482442274e-07, "logits/chosen": -0.854729175567627, "logits/rejected": -0.8268671631813049, "logps/chosen": -480.8945007324219, "logps/ref_chosen": -275.8088684082031, "logps/ref_rejected": -243.45138549804688, "logps/rejected": -512.336181640625, "loss": 5.0311, "margin_dpo/margin_mean": 63.79915237426758, "margin_dpo/margin_std": 153.8164825439453, "step": 249 }, { "KL/chosen_KL_mean": -214.7335662841797, "KL/mean": -255.48167419433594, "KL/rejected_KL_mean": -296.22979736328125, "KL/std": 139.7268524169922, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.0029838732443749905, "fcm_dpo/delta": -0.01467887032777071, "fcm_dpo/margin": 81.49626159667969, "fcm_dpo/q_t": 0.443011999130249, "grad_norm": 68.4245834350586, "learning_rate": 2.7467508704251135e-07, "logits/chosen": -0.8430991768836975, "logits/rejected": -0.8344937562942505, "logps/chosen": -507.2281188964844, "logps/ref_chosen": -292.4945373535156, "logps/ref_rejected": -284.2869567871094, "logps/rejected": -580.5167236328125, "loss": 4.8745, "margin_dpo/margin_mean": 81.49626159667969, "margin_dpo/margin_std": 161.67291259765625, "step": 250 }, { "KL/chosen_KL_mean": -203.28543090820312, "KL/mean": -243.52606201171875, "KL/rejected_KL_mean": -283.7666931152344, "KL/std": 141.32611083984375, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.002941292244940996, "fcm_dpo/delta": -0.046136289834976196, "fcm_dpo/margin": 80.48124694824219, "fcm_dpo/q_t": 0.4475208818912506, "grad_norm": 59.001190185546875, "learning_rate": 2.7285261601056697e-07, "logits/chosen": -0.8501484394073486, "logits/rejected": -0.8327538967132568, "logps/chosen": -485.0220031738281, "logps/ref_chosen": -281.736572265625, "logps/ref_rejected": -255.9419708251953, "logps/rejected": -539.7086791992188, "loss": 4.907, "margin_dpo/margin_mean": 80.48124694824219, "margin_dpo/margin_std": 165.2854766845703, "step": 251 }, { "KL/chosen_KL_mean": -218.41604614257812, "KL/mean": -259.9915771484375, "KL/rejected_KL_mean": -301.5671081542969, "KL/std": 131.10438537597656, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.002828023862093687, "fcm_dpo/delta": -0.023578261956572533, "fcm_dpo/margin": 83.15106201171875, "fcm_dpo/q_t": 0.44388335943222046, "grad_norm": 65.67149353027344, "learning_rate": 2.7102891946217994e-07, "logits/chosen": -0.9110385775566101, "logits/rejected": -0.884746789932251, "logps/chosen": -514.3834228515625, "logps/ref_chosen": -295.9674072265625, "logps/ref_rejected": -280.111572265625, "logps/rejected": -581.6787109375, "loss": 4.8493, "margin_dpo/margin_mean": 83.15106201171875, "margin_dpo/margin_std": 149.3827362060547, "step": 252 }, { "KL/chosen_KL_mean": -233.10960388183594, "KL/mean": -268.5941467285156, "KL/rejected_KL_mean": -304.07867431640625, "KL/std": 137.26515197753906, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.002824939088895917, "fcm_dpo/delta": 0.0016382570611312985, "fcm_dpo/margin": 70.96907043457031, "fcm_dpo/q_t": 0.45377257466316223, "grad_norm": 51.97392272949219, "learning_rate": 2.692040951966617e-07, "logits/chosen": -0.8730481863021851, "logits/rejected": -0.8639348745346069, "logps/chosen": -510.181884765625, "logps/ref_chosen": -277.072265625, "logps/ref_rejected": -247.31643676757812, "logps/rejected": -551.3951416015625, "loss": 5.025, "margin_dpo/margin_mean": 70.96907043457031, "margin_dpo/margin_std": 168.4381103515625, "step": 253 }, { "KL/chosen_KL_mean": -216.6527099609375, "KL/mean": -260.6445007324219, "KL/rejected_KL_mean": -304.6363220214844, "KL/std": 141.9154052734375, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.0028039044700562954, "fcm_dpo/delta": -0.033436983823776245, "fcm_dpo/margin": 87.98359680175781, "fcm_dpo/q_t": 0.4428294897079468, "grad_norm": 53.65641403198242, "learning_rate": 2.6737824107379947e-07, "logits/chosen": -0.8167764544487, "logits/rejected": -0.795870304107666, "logps/chosen": -486.6005554199219, "logps/ref_chosen": -269.9478454589844, "logps/ref_rejected": -249.45005798339844, "logps/rejected": -554.0863647460938, "loss": 4.8213, "margin_dpo/margin_mean": 87.98358917236328, "margin_dpo/margin_std": 156.28126525878906, "step": 254 }, { "KL/chosen_KL_mean": -208.58926391601562, "KL/mean": -257.9679260253906, "KL/rejected_KL_mean": -307.34661865234375, "KL/std": 138.3050994873047, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.0027517471462488174, "fcm_dpo/delta": 0.006953006610274315, "fcm_dpo/margin": 98.75733947753906, "fcm_dpo/q_t": 0.4354252517223358, "grad_norm": 51.104637145996094, "learning_rate": 2.655514550086086e-07, "logits/chosen": -0.8255376219749451, "logits/rejected": -0.7873697280883789, "logps/chosen": -515.2445678710938, "logps/ref_chosen": -306.6552734375, "logps/ref_rejected": -254.47528076171875, "logps/rejected": -561.8218383789062, "loss": 4.7701, "margin_dpo/margin_mean": 98.75733947753906, "margin_dpo/margin_std": 176.37081909179688, "step": 255 }, { "KL/chosen_KL_mean": -212.90078735351562, "KL/mean": -253.50555419921875, "KL/rejected_KL_mean": -294.1103210449219, "KL/std": 134.76048278808594, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.002711032284423709, "fcm_dpo/delta": -0.04468809440732002, "fcm_dpo/margin": 81.20955657958984, "fcm_dpo/q_t": 0.44711193442344666, "grad_norm": 60.70124816894531, "learning_rate": 2.6372383496608186e-07, "logits/chosen": -0.8649431467056274, "logits/rejected": -0.8482561707496643, "logps/chosen": -536.6189575195312, "logps/ref_chosen": -323.7181701660156, "logps/ref_rejected": -254.1871337890625, "logps/rejected": -548.2974853515625, "loss": 4.9389, "margin_dpo/margin_mean": 81.20955657958984, "margin_dpo/margin_std": 163.05526733398438, "step": 256 }, { "KL/chosen_KL_mean": -213.99270629882812, "KL/mean": -261.0357360839844, "KL/rejected_KL_mean": -308.07879638671875, "KL/std": 134.3519287109375, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.0026609383057802916, "fcm_dpo/delta": -0.0028730593621730804, "fcm_dpo/margin": 94.08607482910156, "fcm_dpo/q_t": 0.44072961807250977, "grad_norm": 55.27256774902344, "learning_rate": 2.618954789559356e-07, "logits/chosen": -0.8478982448577881, "logits/rejected": -0.8403754234313965, "logps/chosen": -481.2048034667969, "logps/ref_chosen": -267.21209716796875, "logps/ref_rejected": -249.12579345703125, "logps/rejected": -557.20458984375, "loss": 4.7854, "margin_dpo/margin_mean": 94.08607482910156, "margin_dpo/margin_std": 158.97830200195312, "step": 257 }, { "KL/chosen_KL_mean": -220.2513427734375, "KL/mean": -272.7309875488281, "KL/rejected_KL_mean": -325.2106628417969, "KL/std": 131.34759521484375, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.0025932216085493565, "fcm_dpo/delta": -0.044448841363191605, "fcm_dpo/margin": 104.95930480957031, "fcm_dpo/q_t": 0.43503230810165405, "grad_norm": 51.729408264160156, "learning_rate": 2.600664850273538e-07, "logits/chosen": -0.8749293088912964, "logits/rejected": -0.8454850316047668, "logps/chosen": -497.93408203125, "logps/ref_chosen": -277.6827392578125, "logps/ref_rejected": -250.73385620117188, "logps/rejected": -575.9445190429688, "loss": 4.698, "margin_dpo/margin_mean": 104.95930480957031, "margin_dpo/margin_std": 149.8773193359375, "step": 258 }, { "KL/chosen_KL_mean": -205.57168579101562, "KL/mean": -254.19036865234375, "KL/rejected_KL_mean": -302.80902099609375, "KL/std": 146.01449584960938, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.0025686267763376236, "fcm_dpo/delta": -0.010285234078764915, "fcm_dpo/margin": 97.23735809326172, "fcm_dpo/q_t": 0.4419627785682678, "grad_norm": 61.2537956237793, "learning_rate": 2.582369512637302e-07, "logits/chosen": -0.8877295255661011, "logits/rejected": -0.8807848691940308, "logps/chosen": -500.1816711425781, "logps/ref_chosen": -294.6099853515625, "logps/ref_rejected": -272.2725830078125, "logps/rejected": -575.0816040039062, "loss": 4.8003, "margin_dpo/margin_mean": 97.23736572265625, "margin_dpo/margin_std": 166.01547241210938, "step": 259 }, { "KL/chosen_KL_mean": -215.47994995117188, "KL/mean": -236.25408935546875, "KL/rejected_KL_mean": -257.0281982421875, "KL/std": 120.56246948242188, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.0025487085804343224, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.548255920410156, "fcm_dpo/q_t": 0.4746982753276825, "grad_norm": 50.7750129699707, "learning_rate": 2.5640697577740815e-07, "logits/chosen": -0.8832182884216309, "logits/rejected": -0.8753829002380371, "logps/chosen": -506.3370666503906, "logps/ref_chosen": -290.85711669921875, "logps/ref_rejected": -277.5970153808594, "logps/rejected": -534.625244140625, "loss": 5.265, "margin_dpo/margin_mean": 41.548255920410156, "margin_dpo/margin_std": 138.51763916015625, "step": 260 }, { "KL/chosen_KL_mean": -227.56674194335938, "KL/mean": -264.64959716796875, "KL/rejected_KL_mean": -301.732421875, "KL/std": 133.1595458984375, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.0025418729055672884, "fcm_dpo/delta": -0.007170156575739384, "fcm_dpo/margin": 74.16572570800781, "fcm_dpo/q_t": 0.4553813934326172, "grad_norm": 65.28130340576172, "learning_rate": 2.5457665670441937e-07, "logits/chosen": -0.7683281302452087, "logits/rejected": -0.7774943113327026, "logps/chosen": -478.6989440917969, "logps/ref_chosen": -251.13223266601562, "logps/ref_rejected": -244.76016235351562, "logps/rejected": -546.4925537109375, "loss": 5.0165, "margin_dpo/margin_mean": 74.16571807861328, "margin_dpo/margin_std": 169.27870178222656, "step": 261 }, { "KL/chosen_KL_mean": -209.81301879882812, "KL/mean": -261.0389099121094, "KL/rejected_KL_mean": -312.2647705078125, "KL/std": 132.10711669921875, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.002556585008278489, "fcm_dpo/delta": -0.00010207295417785645, "fcm_dpo/margin": 102.45171356201172, "fcm_dpo/q_t": 0.4371283948421478, "grad_norm": 63.33259582519531, "learning_rate": 2.527460921992209e-07, "logits/chosen": -0.8112813830375671, "logits/rejected": -0.7984949350357056, "logps/chosen": -509.5347900390625, "logps/ref_chosen": -299.7217712402344, "logps/ref_rejected": -277.0969543457031, "logps/rejected": -589.3616943359375, "loss": 4.7045, "margin_dpo/margin_mean": 102.45170593261719, "margin_dpo/margin_std": 142.5891876220703, "step": 262 }, { "KL/chosen_KL_mean": -216.5228729248047, "KL/mean": -258.5859680175781, "KL/rejected_KL_mean": -300.64898681640625, "KL/std": 135.47119140625, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.0025327729526907206, "fcm_dpo/delta": -0.0021185576915740967, "fcm_dpo/margin": 84.12616729736328, "fcm_dpo/q_t": 0.4490615129470825, "grad_norm": 44.847206115722656, "learning_rate": 2.509153804294318e-07, "logits/chosen": -0.8098288178443909, "logits/rejected": -0.7890791296958923, "logps/chosen": -496.4754333496094, "logps/ref_chosen": -279.95257568359375, "logps/ref_rejected": -256.5327453613281, "logps/rejected": -557.1817626953125, "loss": 4.9059, "margin_dpo/margin_mean": 84.12616729736328, "margin_dpo/margin_std": 155.34945678710938, "step": 263 }, { "KL/chosen_KL_mean": -206.41465759277344, "KL/mean": -260.0697937011719, "KL/rejected_KL_mean": -313.7249450683594, "KL/std": 139.62014770507812, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.0024950942024588585, "fcm_dpo/delta": -0.015230976976454258, "fcm_dpo/margin": 107.31029510498047, "fcm_dpo/q_t": 0.43691185116767883, "grad_norm": 69.77923583984375, "learning_rate": 2.4908461957056825e-07, "logits/chosen": -0.8283612728118896, "logits/rejected": -0.8172589540481567, "logps/chosen": -466.94976806640625, "logps/ref_chosen": -260.53509521484375, "logps/ref_rejected": -255.53799438476562, "logps/rejected": -569.262939453125, "loss": 4.7187, "margin_dpo/margin_mean": 107.31029510498047, "margin_dpo/margin_std": 161.04441833496094, "step": 264 }, { "KL/chosen_KL_mean": -207.57537841796875, "KL/mean": -265.12884521484375, "KL/rejected_KL_mean": -322.6822814941406, "KL/std": 153.11061096191406, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.00249352864921093, "fcm_dpo/delta": -0.007051775231957436, "fcm_dpo/margin": 115.10690307617188, "fcm_dpo/q_t": 0.43265461921691895, "grad_norm": 40.28909683227539, "learning_rate": 2.4725390780077905e-07, "logits/chosen": -0.8889414668083191, "logits/rejected": -0.8872557878494263, "logps/chosen": -491.2884216308594, "logps/ref_chosen": -283.7130432128906, "logps/ref_rejected": -270.3209533691406, "logps/rejected": -593.0032348632812, "loss": 4.6979, "margin_dpo/margin_mean": 115.10690307617188, "margin_dpo/margin_std": 186.2714385986328, "step": 265 }, { "KL/chosen_KL_mean": -189.83436584472656, "KL/mean": -245.71548461914062, "KL/rejected_KL_mean": -301.5965881347656, "KL/std": 136.78062438964844, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.0025241486728191376, "fcm_dpo/delta": 0.017888490110635757, "fcm_dpo/margin": 111.76222229003906, "fcm_dpo/q_t": 0.4332635700702667, "grad_norm": 66.26383972167969, "learning_rate": 2.454233432955807e-07, "logits/chosen": -0.9080507755279541, "logits/rejected": -0.8778403401374817, "logps/chosen": -467.93365478515625, "logps/ref_chosen": -278.09930419921875, "logps/ref_rejected": -260.6734619140625, "logps/rejected": -562.2700805664062, "loss": 4.6501, "margin_dpo/margin_mean": 111.76222229003906, "margin_dpo/margin_std": 153.3962860107422, "step": 266 }, { "KL/chosen_KL_mean": -225.291015625, "KL/mean": -266.1501159667969, "KL/rejected_KL_mean": -307.0091857910156, "KL/std": 149.58706665039062, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.002512805862352252, "fcm_dpo/delta": -0.0011747367680072784, "fcm_dpo/margin": 81.71817016601562, "fcm_dpo/q_t": 0.45182037353515625, "grad_norm": 65.87675476074219, "learning_rate": 2.435930242225919e-07, "logits/chosen": -0.8443954586982727, "logits/rejected": -0.8486557602882385, "logps/chosen": -505.6242370605469, "logps/ref_chosen": -280.33319091796875, "logps/ref_rejected": -247.78099060058594, "logps/rejected": -554.7901611328125, "loss": 4.9719, "margin_dpo/margin_mean": 81.71817016601562, "margin_dpo/margin_std": 172.19381713867188, "step": 267 }, { "KL/chosen_KL_mean": -215.37521362304688, "KL/mean": -266.24896240234375, "KL/rejected_KL_mean": -317.1227111816406, "KL/std": 149.34877014160156, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.002550693228840828, "fcm_dpo/delta": 0.024130037054419518, "fcm_dpo/margin": 101.74750518798828, "fcm_dpo/q_t": 0.43864506483078003, "grad_norm": 51.989707946777344, "learning_rate": 2.4176304873626984e-07, "logits/chosen": -0.8222531080245972, "logits/rejected": -0.8021284341812134, "logps/chosen": -519.5538940429688, "logps/ref_chosen": -304.1787109375, "logps/ref_rejected": -272.80316162109375, "logps/rejected": -589.9259033203125, "loss": 4.7542, "margin_dpo/margin_mean": 101.74750518798828, "margin_dpo/margin_std": 164.10125732421875, "step": 268 }, { "KL/chosen_KL_mean": -209.81631469726562, "KL/mean": -247.71542358398438, "KL/rejected_KL_mean": -285.6145324707031, "KL/std": 126.85572052001953, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.002604001434519887, "fcm_dpo/delta": 0.029024910181760788, "fcm_dpo/margin": 75.79827880859375, "fcm_dpo/q_t": 0.45281317830085754, "grad_norm": 49.523555755615234, "learning_rate": 2.399335149726463e-07, "logits/chosen": -0.8387216329574585, "logits/rejected": -0.8304933309555054, "logps/chosen": -459.6614074707031, "logps/ref_chosen": -249.84512329101562, "logps/ref_rejected": -223.37356567382812, "logps/rejected": -508.9881591796875, "loss": 4.9555, "margin_dpo/margin_mean": 75.79827880859375, "margin_dpo/margin_std": 152.57525634765625, "step": 269 }, { "KL/chosen_KL_mean": -222.31195068359375, "KL/mean": -272.5738525390625, "KL/rejected_KL_mean": -322.8357238769531, "KL/std": 154.9764404296875, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.0026149782352149487, "fcm_dpo/delta": 0.005393762607127428, "fcm_dpo/margin": 100.5237808227539, "fcm_dpo/q_t": 0.4390154480934143, "grad_norm": 45.56135940551758, "learning_rate": 2.381045210440644e-07, "logits/chosen": -0.9296514391899109, "logits/rejected": -0.9286025762557983, "logps/chosen": -540.8743286132812, "logps/ref_chosen": -318.5623779296875, "logps/ref_rejected": -281.1880798339844, "logps/rejected": -604.0238037109375, "loss": 4.81, "margin_dpo/margin_mean": 100.5237808227539, "margin_dpo/margin_std": 193.27618408203125, "step": 270 }, { "KL/chosen_KL_mean": -219.34278869628906, "KL/mean": -264.65020751953125, "KL/rejected_KL_mean": -309.9576721191406, "KL/std": 151.9405517578125, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.002673304872587323, "fcm_dpo/delta": 0.020023522898554802, "fcm_dpo/margin": 90.61488342285156, "fcm_dpo/q_t": 0.4431973099708557, "grad_norm": 49.953712463378906, "learning_rate": 2.3627616503391812e-07, "logits/chosen": -0.7674182057380676, "logits/rejected": -0.7575105428695679, "logps/chosen": -503.44744873046875, "logps/ref_chosen": -284.104736328125, "logps/ref_rejected": -253.9580535888672, "logps/rejected": -563.915771484375, "loss": 4.8207, "margin_dpo/margin_mean": 90.61488342285156, "margin_dpo/margin_std": 158.77781677246094, "step": 271 }, { "KL/chosen_KL_mean": -213.052734375, "KL/mean": -261.3451232910156, "KL/rejected_KL_mean": -309.63751220703125, "KL/std": 154.0564422607422, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.0026661050505936146, "fcm_dpo/delta": -0.0036860520485788584, "fcm_dpo/margin": 96.58477783203125, "fcm_dpo/q_t": 0.44022059440612793, "grad_norm": 59.533260345458984, "learning_rate": 2.344485449913914e-07, "logits/chosen": -0.86882084608078, "logits/rejected": -0.8478980660438538, "logps/chosen": -510.4117431640625, "logps/ref_chosen": -297.3590087890625, "logps/ref_rejected": -279.20196533203125, "logps/rejected": -588.8394775390625, "loss": 4.8307, "margin_dpo/margin_mean": 96.58477783203125, "margin_dpo/margin_std": 188.1719970703125, "step": 272 }, { "KL/chosen_KL_mean": -212.71578979492188, "KL/mean": -263.0025634765625, "KL/rejected_KL_mean": -313.28936767578125, "KL/std": 143.3462371826172, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.002671892289072275, "fcm_dpo/delta": 0.0007058036280795932, "fcm_dpo/margin": 100.5736083984375, "fcm_dpo/q_t": 0.43806958198547363, "grad_norm": 48.56292724609375, "learning_rate": 2.3262175892620062e-07, "logits/chosen": -0.8706230521202087, "logits/rejected": -0.8805627226829529, "logps/chosen": -505.9215393066406, "logps/ref_chosen": -293.20574951171875, "logps/ref_rejected": -274.7646789550781, "logps/rejected": -588.0540771484375, "loss": 4.7907, "margin_dpo/margin_mean": 100.5736083984375, "margin_dpo/margin_std": 187.0547637939453, "step": 273 }, { "KL/chosen_KL_mean": -211.43441772460938, "KL/mean": -272.50616455078125, "KL/rejected_KL_mean": -333.57781982421875, "KL/std": 158.19119262695312, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.00263871718198061, "fcm_dpo/delta": -0.023334428668022156, "fcm_dpo/margin": 122.1434326171875, "fcm_dpo/q_t": 0.4251728653907776, "grad_norm": 49.472415924072266, "learning_rate": 2.3079590480333827e-07, "logits/chosen": -0.8166660070419312, "logits/rejected": -0.7837158441543579, "logps/chosen": -481.9930725097656, "logps/ref_chosen": -270.55865478515625, "logps/ref_rejected": -239.47048950195312, "logps/rejected": -573.0482788085938, "loss": 4.5988, "margin_dpo/margin_mean": 122.1434326171875, "margin_dpo/margin_std": 185.6885528564453, "step": 274 }, { "KL/chosen_KL_mean": -190.10397338867188, "KL/mean": -249.68603515625, "KL/rejected_KL_mean": -309.2680969238281, "KL/std": 151.7843780517578, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.002591077471151948, "fcm_dpo/delta": -0.01983138732612133, "fcm_dpo/margin": 119.16410064697266, "fcm_dpo/q_t": 0.42760196328163147, "grad_norm": 40.00221252441406, "learning_rate": 2.2897108053782e-07, "logits/chosen": -0.8611962795257568, "logits/rejected": -0.8414742946624756, "logps/chosen": -440.4231872558594, "logps/ref_chosen": -250.31922912597656, "logps/ref_rejected": -249.3187255859375, "logps/rejected": -558.5867919921875, "loss": 4.5833, "margin_dpo/margin_mean": 119.16410064697266, "margin_dpo/margin_std": 162.83229064941406, "step": 275 }, { "KL/chosen_KL_mean": -208.79034423828125, "KL/mean": -267.42169189453125, "KL/rejected_KL_mean": -326.05303955078125, "KL/std": 164.59161376953125, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.0025207321159541607, "fcm_dpo/delta": -0.0197414793074131, "fcm_dpo/margin": 117.26268005371094, "fcm_dpo/q_t": 0.43078911304473877, "grad_norm": 56.43882369995117, "learning_rate": 2.2714738398943308e-07, "logits/chosen": -0.9205873012542725, "logits/rejected": -0.8991212844848633, "logps/chosen": -506.4214172363281, "logps/ref_chosen": -297.6310729980469, "logps/ref_rejected": -295.225830078125, "logps/rejected": -621.2788696289062, "loss": 4.6872, "margin_dpo/margin_mean": 117.26268005371094, "margin_dpo/margin_std": 186.11912536621094, "step": 276 }, { "KL/chosen_KL_mean": -236.33641052246094, "KL/mean": -275.8341369628906, "KL/rejected_KL_mean": -315.3318786621094, "KL/std": 163.67776489257812, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.0025246557779610157, "fcm_dpo/delta": -0.01643836311995983, "fcm_dpo/margin": 78.99545288085938, "fcm_dpo/q_t": 0.45376577973365784, "grad_norm": 67.82545471191406, "learning_rate": 2.2532491295748865e-07, "logits/chosen": -0.8489004969596863, "logits/rejected": -0.848082423210144, "logps/chosen": -502.6968994140625, "logps/ref_chosen": -266.3604736328125, "logps/ref_rejected": -253.36767578125, "logps/rejected": -568.6995239257812, "loss": 5.0265, "margin_dpo/margin_mean": 78.99545288085938, "margin_dpo/margin_std": 191.4637451171875, "step": 277 }, { "KL/chosen_KL_mean": -250.80859375, "KL/mean": -280.59112548828125, "KL/rejected_KL_mean": -310.3736572265625, "KL/std": 171.3376922607422, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.002524363109841943, "fcm_dpo/delta": 0.014590254053473473, "fcm_dpo/margin": 59.56508255004883, "fcm_dpo/q_t": 0.46817678213119507, "grad_norm": 93.50398254394531, "learning_rate": 2.2350376517557726e-07, "logits/chosen": -0.8730792999267578, "logits/rejected": -0.8339457511901855, "logps/chosen": -518.2158813476562, "logps/ref_chosen": -267.40728759765625, "logps/ref_rejected": -229.5758514404297, "logps/rejected": -539.949462890625, "loss": 5.2911, "margin_dpo/margin_mean": 59.56508255004883, "margin_dpo/margin_std": 218.88031005859375, "step": 278 }, { "KL/chosen_KL_mean": -235.7555389404297, "KL/mean": -301.6609191894531, "KL/rejected_KL_mean": -367.5663146972656, "KL/std": 183.39990234375, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.002474685199558735, "fcm_dpo/delta": -0.06022359058260918, "fcm_dpo/margin": 131.81077575683594, "fcm_dpo/q_t": 0.42557230591773987, "grad_norm": 52.174964904785156, "learning_rate": 2.2168403830632769e-07, "logits/chosen": -0.8034964799880981, "logits/rejected": -0.7859145998954773, "logps/chosen": -549.123291015625, "logps/ref_chosen": -313.3677978515625, "logps/ref_rejected": -299.1744384765625, "logps/rejected": -666.7407836914062, "loss": 4.6083, "margin_dpo/margin_mean": 131.81077575683594, "margin_dpo/margin_std": 206.10121154785156, "step": 279 }, { "KL/chosen_KL_mean": -223.7259063720703, "KL/mean": -272.3006286621094, "KL/rejected_KL_mean": -320.87530517578125, "KL/std": 148.4961700439453, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.002427027327939868, "fcm_dpo/delta": 0.006918002385646105, "fcm_dpo/margin": 97.14940643310547, "fcm_dpo/q_t": 0.4443333148956299, "grad_norm": 53.57758331298828, "learning_rate": 2.1986582993616925e-07, "logits/chosen": -0.8703023791313171, "logits/rejected": -0.8816097974777222, "logps/chosen": -489.28173828125, "logps/ref_chosen": -265.5558166503906, "logps/ref_rejected": -247.1573944091797, "logps/rejected": -568.03271484375, "loss": 4.8549, "margin_dpo/margin_mean": 97.14940643310547, "margin_dpo/margin_std": 184.31455993652344, "step": 280 }, { "KL/chosen_KL_mean": -253.8318634033203, "KL/mean": -313.9239501953125, "KL/rejected_KL_mean": -374.0159912109375, "KL/std": 170.88157653808594, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.002389302011579275, "fcm_dpo/delta": -0.018726233392953873, "fcm_dpo/margin": 120.18415832519531, "fcm_dpo/q_t": 0.4337007999420166, "grad_norm": 62.008872985839844, "learning_rate": 2.1804923757009882e-07, "logits/chosen": -0.8482406735420227, "logits/rejected": -0.855226993560791, "logps/chosen": -549.1314086914062, "logps/ref_chosen": -295.2995910644531, "logps/ref_rejected": -293.80877685546875, "logps/rejected": -667.82470703125, "loss": 4.6938, "margin_dpo/margin_mean": 120.18414306640625, "margin_dpo/margin_std": 185.703369140625, "step": 281 }, { "KL/chosen_KL_mean": -258.6143493652344, "KL/mean": -303.7931213378906, "KL/rejected_KL_mean": -348.971923828125, "KL/std": 165.28585815429688, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.002437637187540531, "fcm_dpo/delta": 0.035667020827531815, "fcm_dpo/margin": 90.3575668334961, "fcm_dpo/q_t": 0.4478360116481781, "grad_norm": 55.04109573364258, "learning_rate": 2.1623435862645205e-07, "logits/chosen": -0.8456370234489441, "logits/rejected": -0.8471148610115051, "logps/chosen": -577.2515258789062, "logps/ref_chosen": -318.63714599609375, "logps/ref_rejected": -273.5943603515625, "logps/rejected": -622.5662841796875, "loss": 4.9269, "margin_dpo/margin_mean": 90.35755920410156, "margin_dpo/margin_std": 188.9864959716797, "step": 282 }, { "KL/chosen_KL_mean": -248.04953002929688, "KL/mean": -300.7643127441406, "KL/rejected_KL_mean": -353.47906494140625, "KL/std": 167.42236328125, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.0024897807743400335, "fcm_dpo/delta": 0.0018690899014472961, "fcm_dpo/margin": 105.42955780029297, "fcm_dpo/q_t": 0.4403661787509918, "grad_norm": 55.374061584472656, "learning_rate": 2.1442129043167873e-07, "logits/chosen": -0.8618201017379761, "logits/rejected": -0.846630871295929, "logps/chosen": -502.7100830078125, "logps/ref_chosen": -254.66053771972656, "logps/ref_rejected": -236.8627166748047, "logps/rejected": -590.3417358398438, "loss": 4.8116, "margin_dpo/margin_mean": 105.42955780029297, "margin_dpo/margin_std": 198.71615600585938, "step": 283 }, { "KL/chosen_KL_mean": -259.0715026855469, "KL/mean": -313.150390625, "KL/rejected_KL_mean": -367.229248046875, "KL/std": 178.3697509765625, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.002422032179310918, "fcm_dpo/delta": -0.03486540541052818, "fcm_dpo/margin": 108.15774536132812, "fcm_dpo/q_t": 0.4399450421333313, "grad_norm": 78.6396255493164, "learning_rate": 2.1261013021512378e-07, "logits/chosen": -0.8256248235702515, "logits/rejected": -0.7858996987342834, "logps/chosen": -532.4268188476562, "logps/ref_chosen": -273.355224609375, "logps/ref_rejected": -259.84759521484375, "logps/rejected": -627.0768432617188, "loss": 4.8311, "margin_dpo/margin_mean": 108.15774536132812, "margin_dpo/margin_std": 207.28114318847656, "step": 284 }, { "KL/chosen_KL_mean": -270.5787353515625, "KL/mean": -302.9770202636719, "KL/rejected_KL_mean": -335.37530517578125, "KL/std": 163.78836059570312, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.00241269962862134, "fcm_dpo/delta": -0.005318695679306984, "fcm_dpo/margin": 64.7966079711914, "fcm_dpo/q_t": 0.46380117535591125, "grad_norm": 94.97909545898438, "learning_rate": 2.1080097510381294e-07, "logits/chosen": -0.8431457877159119, "logits/rejected": -0.8374038338661194, "logps/chosen": -580.3809204101562, "logps/ref_chosen": -309.8022155761719, "logps/ref_rejected": -279.11846923828125, "logps/rejected": -614.4937744140625, "loss": 5.2171, "margin_dpo/margin_mean": 64.7966079711914, "margin_dpo/margin_std": 207.95433044433594, "step": 285 }, { "KL/chosen_KL_mean": -250.92564392089844, "KL/mean": -298.47711181640625, "KL/rejected_KL_mean": -346.0285339355469, "KL/std": 179.1441192626953, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.002375759882852435, "fcm_dpo/delta": -0.007511192001402378, "fcm_dpo/margin": 95.10287475585938, "fcm_dpo/q_t": 0.44973820447921753, "grad_norm": 66.971435546875, "learning_rate": 2.089939221172446e-07, "logits/chosen": -0.8156773447990417, "logits/rejected": -0.8105944991111755, "logps/chosen": -522.3911743164062, "logps/ref_chosen": -271.4655456542969, "logps/ref_rejected": -279.531494140625, "logps/rejected": -625.5599975585938, "loss": 4.9477, "margin_dpo/margin_mean": 95.10287475585938, "margin_dpo/margin_std": 213.24725341796875, "step": 286 }, { "KL/chosen_KL_mean": -257.23956298828125, "KL/mean": -302.582275390625, "KL/rejected_KL_mean": -347.92498779296875, "KL/std": 168.57205200195312, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.0023798570036888123, "fcm_dpo/delta": 0.008962479420006275, "fcm_dpo/margin": 90.68539428710938, "fcm_dpo/q_t": 0.4496259093284607, "grad_norm": 69.09150695800781, "learning_rate": 2.0718906816218595e-07, "logits/chosen": -0.8373988270759583, "logits/rejected": -0.821157693862915, "logps/chosen": -534.3328247070312, "logps/ref_chosen": -277.0932312011719, "logps/ref_rejected": -233.55599975585938, "logps/rejected": -581.4810180664062, "loss": 4.9939, "margin_dpo/margin_mean": 90.68539428710938, "margin_dpo/margin_std": 213.72222900390625, "step": 287 }, { "KL/chosen_KL_mean": -254.89105224609375, "KL/mean": -298.76812744140625, "KL/rejected_KL_mean": -342.64520263671875, "KL/std": 174.66859436035156, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.00239731278270483, "fcm_dpo/delta": -0.010840969160199165, "fcm_dpo/margin": 87.75419616699219, "fcm_dpo/q_t": 0.45140206813812256, "grad_norm": 61.62644577026367, "learning_rate": 2.053865100274774e-07, "logits/chosen": -0.862617015838623, "logits/rejected": -0.8665178418159485, "logps/chosen": -548.0592041015625, "logps/ref_chosen": -293.1681823730469, "logps/ref_rejected": -263.4059143066406, "logps/rejected": -606.0511474609375, "loss": 4.9783, "margin_dpo/margin_mean": 87.75419616699219, "margin_dpo/margin_std": 197.9239959716797, "step": 288 }, { "KL/chosen_KL_mean": -280.8144226074219, "KL/mean": -305.62109375, "KL/rejected_KL_mean": -330.42779541015625, "KL/std": 159.7655029296875, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.002391617279499769, "fcm_dpo/delta": 0.014636196196079254, "fcm_dpo/margin": 49.613338470458984, "fcm_dpo/q_t": 0.47206318378448486, "grad_norm": 76.14956665039062, "learning_rate": 2.035863443788411e-07, "logits/chosen": -0.8445371985435486, "logits/rejected": -0.8282222747802734, "logps/chosen": -610.7718505859375, "logps/ref_chosen": -329.9574279785156, "logps/ref_rejected": -276.7565002441406, "logps/rejected": -607.1842651367188, "loss": 5.2943, "margin_dpo/margin_mean": 49.61334228515625, "margin_dpo/margin_std": 183.8115234375, "step": 289 }, { "KL/chosen_KL_mean": -241.115966796875, "KL/mean": -295.4197998046875, "KL/rejected_KL_mean": -349.7236328125, "KL/std": 173.45321655273438, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.0023513035848736763, "fcm_dpo/delta": -0.02749420329928398, "fcm_dpo/margin": 108.60765838623047, "fcm_dpo/q_t": 0.4406154751777649, "grad_norm": 70.177490234375, "learning_rate": 2.0178866775369774e-07, "logits/chosen": -0.8551790714263916, "logits/rejected": -0.7867915034294128, "logps/chosen": -565.7849731445312, "logps/ref_chosen": -324.6690673828125, "logps/ref_rejected": -311.8439636230469, "logps/rejected": -661.5675659179688, "loss": 4.8276, "margin_dpo/margin_mean": 108.60765838623047, "margin_dpo/margin_std": 204.76686096191406, "step": 290 }, { "KL/chosen_KL_mean": -229.23654174804688, "KL/mean": -290.4278869628906, "KL/rejected_KL_mean": -351.6192321777344, "KL/std": 162.2596435546875, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.002302813110873103, "fcm_dpo/delta": -0.028997551649808884, "fcm_dpo/margin": 122.38272857666016, "fcm_dpo/q_t": 0.43339967727661133, "grad_norm": 71.22866821289062, "learning_rate": 1.9999357655598891e-07, "logits/chosen": -0.832295298576355, "logits/rejected": -0.8221176862716675, "logps/chosen": -503.380615234375, "logps/ref_chosen": -274.1440734863281, "logps/ref_rejected": -278.07208251953125, "logps/rejected": -629.6913452148438, "loss": 4.6779, "margin_dpo/margin_mean": 122.38272094726562, "margin_dpo/margin_std": 177.83932495117188, "step": 291 }, { "KL/chosen_KL_mean": -220.65379333496094, "KL/mean": -276.26177978515625, "KL/rejected_KL_mean": -331.8697204589844, "KL/std": 173.44187927246094, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.0022653606720268726, "fcm_dpo/delta": -0.005662755109369755, "fcm_dpo/margin": 111.21595764160156, "fcm_dpo/q_t": 0.4417262077331543, "grad_norm": 73.43364715576172, "learning_rate": 1.9820116705100775e-07, "logits/chosen": -0.8171704411506653, "logits/rejected": -0.8130745887756348, "logps/chosen": -480.01751708984375, "logps/ref_chosen": -259.3636779785156, "logps/ref_rejected": -279.30218505859375, "logps/rejected": -611.1719970703125, "loss": 4.8279, "margin_dpo/margin_mean": 111.21595764160156, "margin_dpo/margin_std": 205.4509735107422, "step": 292 }, { "KL/chosen_KL_mean": -248.36492919921875, "KL/mean": -304.3353576660156, "KL/rejected_KL_mean": -360.3057861328125, "KL/std": 182.17123413085938, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.0022749018389731646, "fcm_dpo/delta": -0.007784893270581961, "fcm_dpo/margin": 111.94085693359375, "fcm_dpo/q_t": 0.4398689568042755, "grad_norm": 80.19559478759766, "learning_rate": 1.9641153536023642e-07, "logits/chosen": -0.9110856652259827, "logits/rejected": -0.871292233467102, "logps/chosen": -552.1357421875, "logps/ref_chosen": -303.77081298828125, "logps/ref_rejected": -270.07513427734375, "logps/rejected": -630.3809204101562, "loss": 4.782, "margin_dpo/margin_mean": 111.94085693359375, "margin_dpo/margin_std": 186.9821014404297, "step": 293 }, { "KL/chosen_KL_mean": -241.12615966796875, "KL/mean": -287.281494140625, "KL/rejected_KL_mean": -333.4368591308594, "KL/std": 172.89413452148438, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.002252732403576374, "fcm_dpo/delta": -0.025135207921266556, "fcm_dpo/margin": 92.31072235107422, "fcm_dpo/q_t": 0.4520157277584076, "grad_norm": 46.71921157836914, "learning_rate": 1.9462477745619106e-07, "logits/chosen": -0.8031244277954102, "logits/rejected": -0.8073158264160156, "logps/chosen": -481.364501953125, "logps/ref_chosen": -240.23831176757812, "logps/ref_rejected": -229.187744140625, "logps/rejected": -562.6246337890625, "loss": 5.0051, "margin_dpo/margin_mean": 92.31072235107422, "margin_dpo/margin_std": 210.6240234375, "step": 294 }, { "KL/chosen_KL_mean": -238.98159790039062, "KL/mean": -284.9803771972656, "KL/rejected_KL_mean": -330.9790954589844, "KL/std": 149.19898986816406, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.002246445743367076, "fcm_dpo/delta": 0.018294546753168106, "fcm_dpo/margin": 91.99751281738281, "fcm_dpo/q_t": 0.4525008201599121, "grad_norm": 48.98105239868164, "learning_rate": 1.928409891572757e-07, "logits/chosen": -0.8140034675598145, "logits/rejected": -0.8149075508117676, "logps/chosen": -489.99127197265625, "logps/ref_chosen": -251.00970458984375, "logps/ref_rejected": -244.15142822265625, "logps/rejected": -575.1305541992188, "loss": 4.9526, "margin_dpo/margin_mean": 91.99751281738281, "margin_dpo/margin_std": 192.78147888183594, "step": 295 }, { "KL/chosen_KL_mean": -246.5187530517578, "KL/mean": -312.3707580566406, "KL/rejected_KL_mean": -378.2227478027344, "KL/std": 179.55752563476562, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.002214499283581972, "fcm_dpo/delta": -0.0431789830327034, "fcm_dpo/margin": 131.70401000976562, "fcm_dpo/q_t": 0.43224072456359863, "grad_norm": 54.046234130859375, "learning_rate": 1.9106026612264315e-07, "logits/chosen": -0.8091863989830017, "logits/rejected": -0.774419903755188, "logps/chosen": -540.3992919921875, "logps/ref_chosen": -293.880615234375, "logps/ref_rejected": -283.4175720214844, "logps/rejected": -661.640380859375, "loss": 4.7058, "margin_dpo/margin_mean": 131.70401000976562, "margin_dpo/margin_std": 217.57437133789062, "step": 296 }, { "KL/chosen_KL_mean": -244.05921936035156, "KL/mean": -290.41455078125, "KL/rejected_KL_mean": -336.7698669433594, "KL/std": 159.8158721923828, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.0021560671739280224, "fcm_dpo/delta": -0.01855265535414219, "fcm_dpo/margin": 92.71065521240234, "fcm_dpo/q_t": 0.45290887355804443, "grad_norm": 45.659339904785156, "learning_rate": 1.8928270384706582e-07, "logits/chosen": -0.8852605819702148, "logits/rejected": -0.8780304193496704, "logps/chosen": -533.519287109375, "logps/ref_chosen": -289.4600830078125, "logps/ref_rejected": -283.69110107421875, "logps/rejected": -620.4609375, "loss": 4.9569, "margin_dpo/margin_mean": 92.71065521240234, "margin_dpo/margin_std": 185.48695373535156, "step": 297 }, { "KL/chosen_KL_mean": -233.01348876953125, "KL/mean": -288.38299560546875, "KL/rejected_KL_mean": -343.7524719238281, "KL/std": 172.53945922851562, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.002129694912582636, "fcm_dpo/delta": -0.02039862424135208, "fcm_dpo/margin": 110.73900604248047, "fcm_dpo/q_t": 0.4452170133590698, "grad_norm": 39.55707931518555, "learning_rate": 1.875083976558136e-07, "logits/chosen": -0.8103281259536743, "logits/rejected": -0.7954323887825012, "logps/chosen": -539.5285034179688, "logps/ref_chosen": -306.5150146484375, "logps/ref_rejected": -280.6969909667969, "logps/rejected": -624.449462890625, "loss": 4.884, "margin_dpo/margin_mean": 110.73900604248047, "margin_dpo/margin_std": 216.20150756835938, "step": 298 }, { "KL/chosen_KL_mean": -272.2557373046875, "KL/mean": -314.5762634277344, "KL/rejected_KL_mean": -356.8967590332031, "KL/std": 171.6256103515625, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.0021101348102092743, "fcm_dpo/delta": -0.007379084825515747, "fcm_dpo/margin": 84.64105224609375, "fcm_dpo/q_t": 0.45750677585601807, "grad_norm": 48.66347122192383, "learning_rate": 1.8573744269954297e-07, "logits/chosen": -0.8078640699386597, "logits/rejected": -0.7908865809440613, "logps/chosen": -553.6195678710938, "logps/ref_chosen": -281.36376953125, "logps/ref_rejected": -270.39508056640625, "logps/rejected": -627.2918701171875, "loss": 5.0207, "margin_dpo/margin_mean": 84.64105224609375, "margin_dpo/margin_std": 174.2266082763672, "step": 299 }, { "KL/chosen_KL_mean": -270.9239501953125, "KL/mean": -316.153564453125, "KL/rejected_KL_mean": -361.38323974609375, "KL/std": 182.1024627685547, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.002133725443854928, "fcm_dpo/delta": 0.029630564153194427, "fcm_dpo/margin": 90.45928192138672, "fcm_dpo/q_t": 0.4544250965118408, "grad_norm": 60.00415802001953, "learning_rate": 1.839699339491937e-07, "logits/chosen": -0.8176003694534302, "logits/rejected": -0.7979527115821838, "logps/chosen": -585.7597045898438, "logps/ref_chosen": -314.83575439453125, "logps/ref_rejected": -269.1154479980469, "logps/rejected": -630.4987182617188, "loss": 5.0057, "margin_dpo/margin_mean": 90.45928192138672, "margin_dpo/margin_std": 204.20758056640625, "step": 300 }, { "KL/chosen_KL_mean": -255.38441467285156, "KL/mean": -300.5905456542969, "KL/rejected_KL_mean": -345.79669189453125, "KL/std": 168.21604919433594, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.0021687678527086973, "fcm_dpo/delta": 0.005485248751938343, "fcm_dpo/margin": 90.41228485107422, "fcm_dpo/q_t": 0.45402035117149353, "grad_norm": 65.89996337890625, "learning_rate": 1.8220596619089573e-07, "logits/chosen": -0.8691757321357727, "logits/rejected": -0.8592531681060791, "logps/chosen": -535.2789306640625, "logps/ref_chosen": -279.89453125, "logps/ref_rejected": -271.6694641113281, "logps/rejected": -617.4661865234375, "loss": 4.9779, "margin_dpo/margin_mean": 90.41228485107422, "margin_dpo/margin_std": 187.416748046875, "step": 301 }, { "KL/chosen_KL_mean": -254.57908630371094, "KL/mean": -314.37725830078125, "KL/rejected_KL_mean": -374.1754150390625, "KL/std": 175.4994659423828, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.002159257186576724, "fcm_dpo/delta": -0.00899820402264595, "fcm_dpo/margin": 119.59630584716797, "fcm_dpo/q_t": 0.439103364944458, "grad_norm": 77.14640808105469, "learning_rate": 1.8044563402088682e-07, "logits/chosen": -0.8205274343490601, "logits/rejected": -0.8089447617530823, "logps/chosen": -525.910888671875, "logps/ref_chosen": -271.3318176269531, "logps/ref_rejected": -256.5587158203125, "logps/rejected": -630.734130859375, "loss": 4.7509, "margin_dpo/margin_mean": 119.59629821777344, "margin_dpo/margin_std": 186.8571319580078, "step": 302 }, { "KL/chosen_KL_mean": -256.7622375488281, "KL/mean": -310.6941833496094, "KL/rejected_KL_mean": -364.6260986328125, "KL/std": 153.02818298339844, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.002148208674043417, "fcm_dpo/delta": -0.004830796271562576, "fcm_dpo/margin": 107.8638916015625, "fcm_dpo/q_t": 0.44455963373184204, "grad_norm": 52.09537124633789, "learning_rate": 1.7868903184043885e-07, "logits/chosen": -0.8106588125228882, "logits/rejected": -0.7836918830871582, "logps/chosen": -561.6432495117188, "logps/ref_chosen": -304.88104248046875, "logps/ref_rejected": -269.063720703125, "logps/rejected": -633.6898803710938, "loss": 4.8106, "margin_dpo/margin_mean": 107.8638916015625, "margin_dpo/margin_std": 165.25503540039062, "step": 303 }, { "KL/chosen_KL_mean": -276.5693359375, "KL/mean": -339.6380920410156, "KL/rejected_KL_mean": -402.7069091796875, "KL/std": 195.83033752441406, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.0021316506899893284, "fcm_dpo/delta": 0.0033886905293911695, "fcm_dpo/margin": 126.1375732421875, "fcm_dpo/q_t": 0.438024640083313, "grad_norm": 60.87114334106445, "learning_rate": 1.7693625385079574e-07, "logits/chosen": -0.8092816472053528, "logits/rejected": -0.813895046710968, "logps/chosen": -567.2802734375, "logps/ref_chosen": -290.7109680175781, "logps/ref_rejected": -237.6885986328125, "logps/rejected": -640.3955688476562, "loss": 4.757, "margin_dpo/margin_mean": 126.13758850097656, "margin_dpo/margin_std": 216.04942321777344, "step": 304 }, { "KL/chosen_KL_mean": -237.2429656982422, "KL/mean": -328.9360046386719, "KL/rejected_KL_mean": -420.62908935546875, "KL/std": 190.5575714111328, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.0020590811036527157, "fcm_dpo/delta": -0.10773821175098419, "fcm_dpo/margin": 183.38613891601562, "fcm_dpo/q_t": 0.41207101941108704, "grad_norm": 81.0551528930664, "learning_rate": 1.7518739404812155e-07, "logits/chosen": -0.8565566539764404, "logits/rejected": -0.8202612996101379, "logps/chosen": -493.7269592285156, "logps/ref_chosen": -256.4839782714844, "logps/ref_rejected": -266.4063415527344, "logps/rejected": -687.035400390625, "loss": 4.3971, "margin_dpo/margin_mean": 183.38613891601562, "margin_dpo/margin_std": 223.84494018554688, "step": 305 }, { "KL/chosen_KL_mean": -277.597900390625, "KL/mean": -328.07940673828125, "KL/rejected_KL_mean": -378.5609130859375, "KL/std": 166.4034423828125, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.001981166424229741, "fcm_dpo/delta": 0.015996161848306656, "fcm_dpo/margin": 100.9630355834961, "fcm_dpo/q_t": 0.45244964957237244, "grad_norm": 73.6423568725586, "learning_rate": 1.7344254621846017e-07, "logits/chosen": -0.8491054177284241, "logits/rejected": -0.8253453969955444, "logps/chosen": -598.2471313476562, "logps/ref_chosen": -320.6492004394531, "logps/ref_rejected": -273.36773681640625, "logps/rejected": -651.9286499023438, "loss": 4.9412, "margin_dpo/margin_mean": 100.9630355834961, "margin_dpo/margin_std": 185.91299438476562, "step": 306 }, { "KL/chosen_KL_mean": -311.2239074707031, "KL/mean": -363.1423645019531, "KL/rejected_KL_mean": -415.060791015625, "KL/std": 186.05921936035156, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.00200508045963943, "fcm_dpo/delta": 0.010745943523943424, "fcm_dpo/margin": 103.8368911743164, "fcm_dpo/q_t": 0.44979315996170044, "grad_norm": 51.963165283203125, "learning_rate": 1.717018039327053e-07, "logits/chosen": -0.7687966227531433, "logits/rejected": -0.781086266040802, "logps/chosen": -590.6781005859375, "logps/ref_chosen": -279.4541931152344, "logps/ref_rejected": -240.3796844482422, "logps/rejected": -655.4404907226562, "loss": 4.8875, "margin_dpo/margin_mean": 103.83688354492188, "margin_dpo/margin_std": 176.36830139160156, "step": 307 }, { "KL/chosen_KL_mean": -348.136474609375, "KL/mean": -388.1585693359375, "KL/rejected_KL_mean": -428.1806335449219, "KL/std": 201.38768005371094, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.0020293970592319965, "fcm_dpo/delta": 0.011533215641975403, "fcm_dpo/margin": 80.04412841796875, "fcm_dpo/q_t": 0.46260982751846313, "grad_norm": 68.01890563964844, "learning_rate": 1.699652605415828e-07, "logits/chosen": -0.8337502479553223, "logits/rejected": -0.84173983335495, "logps/chosen": -644.73486328125, "logps/ref_chosen": -296.598388671875, "logps/ref_rejected": -258.6953430175781, "logps/rejected": -686.8759765625, "loss": 5.1313, "margin_dpo/margin_mean": 80.04411315917969, "margin_dpo/margin_std": 218.29266357421875, "step": 308 }, { "KL/chosen_KL_mean": -329.8775939941406, "KL/mean": -393.9901123046875, "KL/rejected_KL_mean": -458.1026916503906, "KL/std": 194.3191375732422, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.0020433913450688124, "fcm_dpo/delta": 0.0097114909440279, "fcm_dpo/margin": 128.22503662109375, "fcm_dpo/q_t": 0.43997424840927124, "grad_norm": 47.546016693115234, "learning_rate": 1.6823300917064458e-07, "logits/chosen": -0.823945164680481, "logits/rejected": -0.8316885232925415, "logps/chosen": -611.2657470703125, "logps/ref_chosen": -281.3881530761719, "logps/ref_rejected": -262.458740234375, "logps/rejected": -720.5614013671875, "loss": 4.7889, "margin_dpo/margin_mean": 128.2250518798828, "margin_dpo/margin_std": 234.49032592773438, "step": 309 }, { "KL/chosen_KL_mean": -325.3269348144531, "KL/mean": -389.1058044433594, "KL/rejected_KL_mean": -452.8846130371094, "KL/std": 203.00721740722656, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.002034237841144204, "fcm_dpo/delta": -0.013699397444725037, "fcm_dpo/margin": 127.55763244628906, "fcm_dpo/q_t": 0.44047310948371887, "grad_norm": 61.21415710449219, "learning_rate": 1.6650514271527465e-07, "logits/chosen": -0.8205724954605103, "logits/rejected": -0.7908133864402771, "logps/chosen": -604.5142211914062, "logps/ref_chosen": -279.1872863769531, "logps/ref_rejected": -261.8279724121094, "logps/rejected": -714.7125244140625, "loss": 4.7957, "margin_dpo/margin_mean": 127.5576171875, "margin_dpo/margin_std": 224.91964721679688, "step": 310 }, { "KL/chosen_KL_mean": -325.9942321777344, "KL/mean": -393.7972412109375, "KL/rejected_KL_mean": -461.60028076171875, "KL/std": 239.32907104492188, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.0019492747960612178, "fcm_dpo/delta": -0.049720633774995804, "fcm_dpo/margin": 135.60604858398438, "fcm_dpo/q_t": 0.44296321272850037, "grad_norm": 53.1006965637207, "learning_rate": 1.647817538357072e-07, "logits/chosen": -0.806901216506958, "logits/rejected": -0.7805233001708984, "logps/chosen": -597.3923950195312, "logps/ref_chosen": -271.39813232421875, "logps/ref_rejected": -266.12701416015625, "logps/rejected": -727.727294921875, "loss": 4.8429, "margin_dpo/margin_mean": 135.6060791015625, "margin_dpo/margin_std": 269.9830627441406, "step": 311 }, { "KL/chosen_KL_mean": -323.4100341796875, "KL/mean": -384.630126953125, "KL/rejected_KL_mean": -445.8502197265625, "KL/std": 218.39796447753906, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.0019210537429898977, "fcm_dpo/delta": -0.01848769001662731, "fcm_dpo/margin": 122.44014739990234, "fcm_dpo/q_t": 0.44824573397636414, "grad_norm": 80.48198699951172, "learning_rate": 1.6306293495205755e-07, "logits/chosen": -0.8279159665107727, "logits/rejected": -0.7945848703384399, "logps/chosen": -605.7950439453125, "logps/ref_chosen": -282.3850402832031, "logps/ref_rejected": -246.35389709472656, "logps/rejected": -692.2041015625, "loss": 4.9828, "margin_dpo/margin_mean": 122.44013977050781, "margin_dpo/margin_std": 290.7316589355469, "step": 312 }, { "KL/chosen_KL_mean": -329.6253662109375, "KL/mean": -385.30535888671875, "KL/rejected_KL_mean": -440.98541259765625, "KL/std": 222.88763427734375, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.0019223717972636223, "fcm_dpo/delta": -0.002858624327927828, "fcm_dpo/margin": 111.36000061035156, "fcm_dpo/q_t": 0.4549371302127838, "grad_norm": 56.645931243896484, "learning_rate": 1.6134877823936607e-07, "logits/chosen": -0.8647723197937012, "logits/rejected": -0.8453179001808167, "logps/chosen": -633.2562255859375, "logps/ref_chosen": -303.630859375, "logps/ref_rejected": -273.1156921386719, "logps/rejected": -714.10107421875, "loss": 5.0238, "margin_dpo/margin_mean": 111.36000061035156, "margin_dpo/margin_std": 269.7938537597656, "step": 313 }, { "KL/chosen_KL_mean": -324.699462890625, "KL/mean": -377.6182556152344, "KL/rejected_KL_mean": -430.53704833984375, "KL/std": 193.24305725097656, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.0019222039263695478, "fcm_dpo/delta": 0.01650712452828884, "fcm_dpo/margin": 105.83755493164062, "fcm_dpo/q_t": 0.45183467864990234, "grad_norm": 50.479618072509766, "learning_rate": 1.5963937562265522e-07, "logits/chosen": -0.876418948173523, "logits/rejected": -0.8731269836425781, "logps/chosen": -627.0037231445312, "logps/ref_chosen": -302.3042907714844, "logps/ref_rejected": -273.6416015625, "logps/rejected": -704.1786499023438, "loss": 4.9631, "margin_dpo/margin_mean": 105.83756256103516, "margin_dpo/margin_std": 222.26353454589844, "step": 314 }, { "KL/chosen_KL_mean": -307.293212890625, "KL/mean": -383.67724609375, "KL/rejected_KL_mean": -460.061279296875, "KL/std": 201.8465118408203, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.0019190021557733417, "fcm_dpo/delta": -0.030157245695590973, "fcm_dpo/margin": 152.7681121826172, "fcm_dpo/q_t": 0.4318777322769165, "grad_norm": 58.37327194213867, "learning_rate": 1.5793481877199943e-07, "logits/chosen": -0.8553957939147949, "logits/rejected": -0.8330000042915344, "logps/chosen": -610.0224609375, "logps/ref_chosen": -302.729248046875, "logps/ref_rejected": -270.26910400390625, "logps/rejected": -730.3304443359375, "loss": 4.6619, "margin_dpo/margin_mean": 152.76809692382812, "margin_dpo/margin_std": 228.41522216796875, "step": 315 }, { "KL/chosen_KL_mean": -319.1265869140625, "KL/mean": -379.0169372558594, "KL/rejected_KL_mean": -438.9073486328125, "KL/std": 191.73924255371094, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.0019111887086182833, "fcm_dpo/delta": -0.0019163042306900024, "fcm_dpo/margin": 119.78076171875, "fcm_dpo/q_t": 0.44601792097091675, "grad_norm": 51.57410430908203, "learning_rate": 1.562351990976095e-07, "logits/chosen": -0.882705807685852, "logits/rejected": -0.8593080043792725, "logps/chosen": -629.6972045898438, "logps/ref_chosen": -310.5706481933594, "logps/ref_rejected": -272.9354553222656, "logps/rejected": -711.8428344726562, "loss": 4.8727, "margin_dpo/margin_mean": 119.78076934814453, "margin_dpo/margin_std": 223.42430114746094, "step": 316 }, { "KL/chosen_KL_mean": -270.0894775390625, "KL/mean": -324.0589599609375, "KL/rejected_KL_mean": -378.0285339355469, "KL/std": 167.6583251953125, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.0019239266403019428, "fcm_dpo/delta": 0.03239323943853378, "fcm_dpo/margin": 107.93901824951172, "fcm_dpo/q_t": 0.45031195878982544, "grad_norm": 51.05884552001953, "learning_rate": 1.5454060774493065e-07, "logits/chosen": -0.8534022569656372, "logits/rejected": -0.8216595649719238, "logps/chosen": -523.9898681640625, "logps/ref_chosen": -253.90036010742188, "logps/ref_rejected": -218.74078369140625, "logps/rejected": -596.769287109375, "loss": 4.8902, "margin_dpo/margin_mean": 107.93902587890625, "margin_dpo/margin_std": 182.3351287841797, "step": 317 }, { "KL/chosen_KL_mean": -276.93927001953125, "KL/mean": -353.7762451171875, "KL/rejected_KL_mean": -430.61322021484375, "KL/std": 195.12513732910156, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.0018967122305184603, "fcm_dpo/delta": -0.02675527334213257, "fcm_dpo/margin": 153.67393493652344, "fcm_dpo/q_t": 0.43146854639053345, "grad_norm": 60.334529876708984, "learning_rate": 1.5285113558975427e-07, "logits/chosen": -0.8724962472915649, "logits/rejected": -0.8433178663253784, "logps/chosen": -547.7621459960938, "logps/ref_chosen": -270.8228759765625, "logps/ref_rejected": -255.30972290039062, "logps/rejected": -685.9229125976562, "loss": 4.6496, "margin_dpo/margin_mean": 153.67393493652344, "margin_dpo/margin_std": 221.29598999023438, "step": 318 }, { "KL/chosen_KL_mean": -292.36883544921875, "KL/mean": -358.0066833496094, "KL/rejected_KL_mean": -423.64459228515625, "KL/std": 185.0775909423828, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.0018959176959469914, "fcm_dpo/delta": 0.0009724181145429611, "fcm_dpo/margin": 131.27574157714844, "fcm_dpo/q_t": 0.4395703971385956, "grad_norm": 47.7817497253418, "learning_rate": 1.5116687323334464e-07, "logits/chosen": -0.8555124998092651, "logits/rejected": -0.8220670819282532, "logps/chosen": -593.3716430664062, "logps/ref_chosen": -301.0028076171875, "logps/ref_rejected": -242.39002990722656, "logps/rejected": -666.0346069335938, "loss": 4.7536, "margin_dpo/margin_mean": 131.27574157714844, "margin_dpo/margin_std": 191.61190795898438, "step": 319 }, { "KL/chosen_KL_mean": -291.9874267578125, "KL/mean": -353.7680358886719, "KL/rejected_KL_mean": -415.5486755371094, "KL/std": 201.40379333496094, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.0018904343014582992, "fcm_dpo/delta": -0.003565825056284666, "fcm_dpo/margin": 123.561279296875, "fcm_dpo/q_t": 0.44610100984573364, "grad_norm": 71.67448425292969, "learning_rate": 1.4948791099758052e-07, "logits/chosen": -0.8239633440971375, "logits/rejected": -0.8111226558685303, "logps/chosen": -595.6099853515625, "logps/ref_chosen": -303.6225891113281, "logps/ref_rejected": -280.85174560546875, "logps/rejected": -696.400390625, "loss": 4.8644, "margin_dpo/margin_mean": 123.561279296875, "margin_dpo/margin_std": 236.85836791992188, "step": 320 }, { "KL/chosen_KL_mean": -319.300537109375, "KL/mean": -363.65484619140625, "KL/rejected_KL_mean": -408.0091857910156, "KL/std": 196.95912170410156, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.0019133113091811538, "fcm_dpo/delta": 0.015708304941654205, "fcm_dpo/margin": 88.7086181640625, "fcm_dpo/q_t": 0.4608987867832184, "grad_norm": 51.643287658691406, "learning_rate": 1.478143389201113e-07, "logits/chosen": -0.8439365029335022, "logits/rejected": -0.811326801776886, "logps/chosen": -608.286376953125, "logps/ref_chosen": -288.98583984375, "logps/ref_rejected": -241.1822052001953, "logps/rejected": -649.1913452148438, "loss": 5.0942, "margin_dpo/margin_mean": 88.7086181640625, "margin_dpo/margin_std": 222.06124877929688, "step": 321 }, { "KL/chosen_KL_mean": -333.3482666015625, "KL/mean": -380.2379455566406, "KL/rejected_KL_mean": -427.12762451171875, "KL/std": 206.414306640625, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.001928423997014761, "fcm_dpo/delta": 0.017177335917949677, "fcm_dpo/margin": 93.77931213378906, "fcm_dpo/q_t": 0.4577179551124573, "grad_norm": 72.54539489746094, "learning_rate": 1.461462467495284e-07, "logits/chosen": -0.8990727066993713, "logits/rejected": -0.865135908126831, "logps/chosen": -641.8917846679688, "logps/ref_chosen": -308.54345703125, "logps/ref_rejected": -269.7995910644531, "logps/rejected": -696.9271850585938, "loss": 5.0935, "margin_dpo/margin_mean": 93.77930450439453, "margin_dpo/margin_std": 251.90206909179688, "step": 322 }, { "KL/chosen_KL_mean": -305.1156311035156, "KL/mean": -348.0940856933594, "KL/rejected_KL_mean": -391.07257080078125, "KL/std": 183.42276000976562, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.0019287847680971026, "fcm_dpo/delta": -0.01517847552895546, "fcm_dpo/margin": 85.95698547363281, "fcm_dpo/q_t": 0.46190139651298523, "grad_norm": 76.46353149414062, "learning_rate": 1.4448372394055246e-07, "logits/chosen": -0.8569475412368774, "logits/rejected": -0.8476258516311646, "logps/chosen": -587.6092529296875, "logps/ref_chosen": -282.49365234375, "logps/ref_rejected": -227.7105255126953, "logps/rejected": -618.7831420898438, "loss": 5.1093, "margin_dpo/margin_mean": 85.95698547363281, "margin_dpo/margin_std": 220.92446899414062, "step": 323 }, { "KL/chosen_KL_mean": -270.2607421875, "KL/mean": -339.736328125, "KL/rejected_KL_mean": -409.2118835449219, "KL/std": 195.49070739746094, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.0018783288542181253, "fcm_dpo/delta": -0.012117970734834671, "fcm_dpo/margin": 138.95114135742188, "fcm_dpo/q_t": 0.4381677508354187, "grad_norm": 49.30532455444336, "learning_rate": 1.428268596492364e-07, "logits/chosen": -0.7851729989051819, "logits/rejected": -0.7778810262680054, "logps/chosen": -509.59912109375, "logps/ref_chosen": -239.33836364746094, "logps/ref_rejected": -230.53775024414062, "logps/rejected": -639.7496337890625, "loss": 4.7328, "margin_dpo/margin_mean": 138.95114135742188, "margin_dpo/margin_std": 205.01913452148438, "step": 324 }, { "KL/chosen_KL_mean": -310.2890625, "KL/mean": -369.69451904296875, "KL/rejected_KL_mean": -429.0999755859375, "KL/std": 213.21319580078125, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.0018853824585676193, "fcm_dpo/delta": -0.007832746021449566, "fcm_dpo/margin": 118.81090545654297, "fcm_dpo/q_t": 0.4491155743598938, "grad_norm": 67.0599594116211, "learning_rate": 1.4117574272818386e-07, "logits/chosen": -0.7914143204689026, "logits/rejected": -0.7728296518325806, "logps/chosen": -590.9180297851562, "logps/ref_chosen": -280.62896728515625, "logps/ref_rejected": -270.5085754394531, "logps/rejected": -699.6085205078125, "loss": 4.9665, "margin_dpo/margin_mean": 118.8109130859375, "margin_dpo/margin_std": 269.921142578125, "step": 325 }, { "KL/chosen_KL_mean": -282.23529052734375, "KL/mean": -345.9592590332031, "KL/rejected_KL_mean": -409.68316650390625, "KL/std": 194.88714599609375, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.0019169156439602375, "fcm_dpo/delta": 0.019461024552583694, "fcm_dpo/margin": 127.44788360595703, "fcm_dpo/q_t": 0.44288212060928345, "grad_norm": 94.46167755126953, "learning_rate": 1.3953046172178413e-07, "logits/chosen": -0.9129518866539001, "logits/rejected": -0.9026789665222168, "logps/chosen": -523.222412109375, "logps/ref_chosen": -240.9871368408203, "logps/ref_rejected": -261.0238342285156, "logps/rejected": -670.7069702148438, "loss": 4.8168, "margin_dpo/margin_mean": 127.44788360595703, "margin_dpo/margin_std": 225.53358459472656, "step": 326 }, { "KL/chosen_KL_mean": -272.2273254394531, "KL/mean": -343.879150390625, "KL/rejected_KL_mean": -415.531005859375, "KL/std": 187.35333251953125, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.0019026086665689945, "fcm_dpo/delta": -0.026806041598320007, "fcm_dpo/margin": 143.3036651611328, "fcm_dpo/q_t": 0.43724074959754944, "grad_norm": 67.36668395996094, "learning_rate": 1.3789110486146468e-07, "logits/chosen": -0.8426380753517151, "logits/rejected": -0.8181723952293396, "logps/chosen": -551.747314453125, "logps/ref_chosen": -279.52001953125, "logps/ref_rejected": -269.51824951171875, "logps/rejected": -685.0491943359375, "loss": 4.7248, "margin_dpo/margin_mean": 143.3036651611328, "margin_dpo/margin_std": 221.62916564941406, "step": 327 }, { "KL/chosen_KL_mean": -292.79803466796875, "KL/mean": -357.90460205078125, "KL/rejected_KL_mean": -423.01116943359375, "KL/std": 195.24517822265625, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.0018517444841563702, "fcm_dpo/delta": -0.01919415406882763, "fcm_dpo/margin": 130.21310424804688, "fcm_dpo/q_t": 0.4430963397026062, "grad_norm": 58.3312873840332, "learning_rate": 1.362577600609588e-07, "logits/chosen": -0.8256867527961731, "logits/rejected": -0.8203503489494324, "logps/chosen": -593.8314819335938, "logps/ref_chosen": -301.033447265625, "logps/ref_rejected": -284.2101135253906, "logps/rejected": -707.2212524414062, "loss": 4.822, "margin_dpo/margin_mean": 130.21310424804688, "margin_dpo/margin_std": 222.81100463867188, "step": 328 }, { "KL/chosen_KL_mean": -317.37786865234375, "KL/mean": -394.31024169921875, "KL/rejected_KL_mean": -471.24267578125, "KL/std": 254.76437377929688, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.0018192044226452708, "fcm_dpo/delta": -0.03148316219449043, "fcm_dpo/margin": 153.8647918701172, "fcm_dpo/q_t": 0.43870800733566284, "grad_norm": 66.76805114746094, "learning_rate": 1.3463051491159093e-07, "logits/chosen": -0.8388152122497559, "logits/rejected": -0.7993863821029663, "logps/chosen": -637.3667602539062, "logps/ref_chosen": -319.9888610839844, "logps/ref_rejected": -307.5588684082031, "logps/rejected": -778.8015747070312, "loss": 4.7845, "margin_dpo/margin_mean": 153.8647918701172, "margin_dpo/margin_std": 286.5224304199219, "step": 329 }, { "KL/chosen_KL_mean": -318.2249755859375, "KL/mean": -386.0837707519531, "KL/rejected_KL_mean": -453.9425048828125, "KL/std": 222.40377807617188, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.0017876912606880069, "fcm_dpo/delta": -0.022983934730291367, "fcm_dpo/margin": 135.717529296875, "fcm_dpo/q_t": 0.444413959980011, "grad_norm": 64.91725158691406, "learning_rate": 1.3300945667758012e-07, "logits/chosen": -0.8319533467292786, "logits/rejected": -0.8325479030609131, "logps/chosen": -619.3397216796875, "logps/ref_chosen": -301.11474609375, "logps/ref_rejected": -299.673095703125, "logps/rejected": -753.6156005859375, "loss": 4.8354, "margin_dpo/margin_mean": 135.71754455566406, "margin_dpo/margin_std": 240.82728576660156, "step": 330 }, { "KL/chosen_KL_mean": -303.1584777832031, "KL/mean": -362.89227294921875, "KL/rejected_KL_mean": -422.6261291503906, "KL/std": 216.93350219726562, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.001791123067960143, "fcm_dpo/delta": 0.03097469173371792, "fcm_dpo/margin": 119.4676513671875, "fcm_dpo/q_t": 0.45106902718544006, "grad_norm": 50.95379638671875, "learning_rate": 1.3139467229135998e-07, "logits/chosen": -0.8472961187362671, "logits/rejected": -0.8238837718963623, "logps/chosen": -580.7499389648438, "logps/ref_chosen": -277.59149169921875, "logps/ref_rejected": -256.025634765625, "logps/rejected": -678.6517944335938, "loss": 4.963, "margin_dpo/margin_mean": 119.4676513671875, "margin_dpo/margin_std": 267.2186584472656, "step": 331 }, { "KL/chosen_KL_mean": -322.1370849609375, "KL/mean": -387.9303894042969, "KL/rejected_KL_mean": -453.72369384765625, "KL/std": 211.57984924316406, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.001803787425160408, "fcm_dpo/delta": -0.002744631841778755, "fcm_dpo/margin": 131.58660888671875, "fcm_dpo/q_t": 0.44477543234825134, "grad_norm": 52.000823974609375, "learning_rate": 1.2978624834891626e-07, "logits/chosen": -0.834603488445282, "logits/rejected": -0.8034043908119202, "logps/chosen": -592.11083984375, "logps/ref_chosen": -269.97369384765625, "logps/ref_rejected": -235.03164672851562, "logps/rejected": -688.75537109375, "loss": 4.911, "margin_dpo/margin_mean": 131.58660888671875, "margin_dpo/margin_std": 275.0579833984375, "step": 332 }, { "KL/chosen_KL_mean": -320.29888916015625, "KL/mean": -371.5259094238281, "KL/rejected_KL_mean": -422.75299072265625, "KL/std": 204.25921630859375, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.0018104577902704477, "fcm_dpo/delta": 0.011496592313051224, "fcm_dpo/margin": 102.45411682128906, "fcm_dpo/q_t": 0.4570245146751404, "grad_norm": 61.41099548339844, "learning_rate": 1.281842711051438e-07, "logits/chosen": -0.9206676483154297, "logits/rejected": -0.8802468776702881, "logps/chosen": -617.0618896484375, "logps/ref_chosen": -296.76300048828125, "logps/ref_rejected": -265.97991943359375, "logps/rejected": -688.7329711914062, "loss": 5.0439, "margin_dpo/margin_mean": 102.45410919189453, "margin_dpo/margin_std": 249.8954620361328, "step": 333 }, { "KL/chosen_KL_mean": -309.74859619140625, "KL/mean": -369.7897033691406, "KL/rejected_KL_mean": -429.8308410644531, "KL/std": 202.07411193847656, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.0018201847560703754, "fcm_dpo/delta": -0.018839359283447266, "fcm_dpo/margin": 120.08220672607422, "fcm_dpo/q_t": 0.44838762283325195, "grad_norm": 74.61259460449219, "learning_rate": 1.2658882646922033e-07, "logits/chosen": -0.8214524984359741, "logits/rejected": -0.7901473045349121, "logps/chosen": -610.785400390625, "logps/ref_chosen": -301.0367431640625, "logps/ref_rejected": -268.87652587890625, "logps/rejected": -698.7073364257812, "loss": 4.9181, "margin_dpo/margin_mean": 120.08220672607422, "margin_dpo/margin_std": 236.2801055908203, "step": 334 }, { "KL/chosen_KL_mean": -325.9380187988281, "KL/mean": -387.39630126953125, "KL/rejected_KL_mean": -448.8546447753906, "KL/std": 182.33425903320312, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.0017861748347058892, "fcm_dpo/delta": -0.0105055533349514, "fcm_dpo/margin": 122.9166259765625, "fcm_dpo/q_t": 0.44840696454048157, "grad_norm": 78.46752166748047, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8165170550346375, "logits/rejected": -0.8090022802352905, "logps/chosen": -602.0708618164062, "logps/ref_chosen": -276.13275146484375, "logps/ref_rejected": -243.44203186035156, "logps/rejected": -692.296630859375, "loss": 4.8941, "margin_dpo/margin_mean": 122.9166259765625, "margin_dpo/margin_std": 230.23648071289062, "step": 335 }, { "KL/chosen_KL_mean": -330.6165771484375, "KL/mean": -392.3404846191406, "KL/rejected_KL_mean": -454.0643310546875, "KL/std": 247.4468994140625, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.0017896379576995969, "fcm_dpo/delta": 0.007889148779213428, "fcm_dpo/margin": 123.44775390625, "fcm_dpo/q_t": 0.4509285092353821, "grad_norm": 63.69434356689453, "learning_rate": 1.2341787690142435e-07, "logits/chosen": -0.8447047472000122, "logits/rejected": -0.7878402471542358, "logps/chosen": -576.8792724609375, "logps/ref_chosen": -246.2626495361328, "logps/ref_rejected": -261.0617980957031, "logps/rejected": -715.126220703125, "loss": 4.9596, "margin_dpo/margin_mean": 123.44773864746094, "margin_dpo/margin_std": 272.46478271484375, "step": 336 }, { "KL/chosen_KL_mean": -309.1364440917969, "KL/mean": -379.6464538574219, "KL/rejected_KL_mean": -450.1564636230469, "KL/std": 201.87640380859375, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.0017988713225349784, "fcm_dpo/delta": -0.01950749382376671, "fcm_dpo/margin": 141.02003479003906, "fcm_dpo/q_t": 0.4397309422492981, "grad_norm": 67.87557220458984, "learning_rate": 1.2184254201795363e-07, "logits/chosen": -0.8472563028335571, "logits/rejected": -0.8143435120582581, "logps/chosen": -576.1302490234375, "logps/ref_chosen": -266.9937744140625, "logps/ref_rejected": -253.015625, "logps/rejected": -703.172119140625, "loss": 4.7793, "margin_dpo/margin_mean": 141.02003479003906, "margin_dpo/margin_std": 226.84255981445312, "step": 337 }, { "KL/chosen_KL_mean": -302.8250427246094, "KL/mean": -373.4712219238281, "KL/rejected_KL_mean": -444.117431640625, "KL/std": 216.23487854003906, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.0017496687360107899, "fcm_dpo/delta": -0.004362808540463448, "fcm_dpo/margin": 141.2923583984375, "fcm_dpo/q_t": 0.4422909617424011, "grad_norm": 79.64447021484375, "learning_rate": 1.202740798300168e-07, "logits/chosen": -0.8437353372573853, "logits/rejected": -0.8279985785484314, "logps/chosen": -579.4176025390625, "logps/ref_chosen": -276.5925598144531, "logps/ref_rejected": -233.979248046875, "logps/rejected": -678.0966186523438, "loss": 4.8058, "margin_dpo/margin_mean": 141.2923583984375, "margin_dpo/margin_std": 242.50144958496094, "step": 338 }, { "KL/chosen_KL_mean": -316.61181640625, "KL/mean": -395.79437255859375, "KL/rejected_KL_mean": -474.9768981933594, "KL/std": 224.5152587890625, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.0017471557948738337, "fcm_dpo/delta": -0.03965400159358978, "fcm_dpo/margin": 158.36505126953125, "fcm_dpo/q_t": 0.4362345039844513, "grad_norm": 69.49354553222656, "learning_rate": 1.1871257444948096e-07, "logits/chosen": -0.8592104911804199, "logits/rejected": -0.8398917317390442, "logps/chosen": -620.1395874023438, "logps/ref_chosen": -303.5277404785156, "logps/ref_rejected": -283.11676025390625, "logps/rejected": -758.09375, "loss": 4.745, "margin_dpo/margin_mean": 158.36505126953125, "margin_dpo/margin_std": 268.96624755859375, "step": 339 }, { "KL/chosen_KL_mean": -326.1214599609375, "KL/mean": -385.92706298828125, "KL/rejected_KL_mean": -445.7326965332031, "KL/std": 232.37582397460938, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.0017085066065192223, "fcm_dpo/delta": 0.008730532601475716, "fcm_dpo/margin": 119.61127471923828, "fcm_dpo/q_t": 0.4503565728664398, "grad_norm": 50.96295928955078, "learning_rate": 1.1715810961514072e-07, "logits/chosen": -0.8013584613800049, "logits/rejected": -0.7939292192459106, "logps/chosen": -587.647216796875, "logps/ref_chosen": -261.5257568359375, "logps/ref_rejected": -259.39862060546875, "logps/rejected": -705.1312866210938, "loss": 5.0456, "margin_dpo/margin_mean": 119.61126708984375, "margin_dpo/margin_std": 292.8406066894531, "step": 340 }, { "KL/chosen_KL_mean": -386.3853454589844, "KL/mean": -446.016357421875, "KL/rejected_KL_mean": -505.6473693847656, "KL/std": 270.52874755859375, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.0017137773102149367, "fcm_dpo/delta": -0.013240095227956772, "fcm_dpo/margin": 119.26201629638672, "fcm_dpo/q_t": 0.4538622498512268, "grad_norm": 76.47930908203125, "learning_rate": 1.1561076868822755e-07, "logits/chosen": -0.823747992515564, "logits/rejected": -0.7963781356811523, "logps/chosen": -702.2888793945312, "logps/ref_chosen": -315.903564453125, "logps/ref_rejected": -308.02392578125, "logps/rejected": -813.6712646484375, "loss": 5.0467, "margin_dpo/margin_mean": 119.26200866699219, "margin_dpo/margin_std": 304.12493896484375, "step": 341 }, { "KL/chosen_KL_mean": -317.0606689453125, "KL/mean": -373.2706298828125, "KL/rejected_KL_mean": -429.48052978515625, "KL/std": 199.8746337890625, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.0017002663807943463, "fcm_dpo/delta": -0.0062491451390087605, "fcm_dpo/margin": 112.41980743408203, "fcm_dpo/q_t": 0.4558378756046295, "grad_norm": 62.66633987426758, "learning_rate": 1.1407063464793965e-07, "logits/chosen": -0.818219780921936, "logits/rejected": -0.8124086260795593, "logps/chosen": -586.2393798828125, "logps/ref_chosen": -269.17864990234375, "logps/ref_rejected": -260.8977355957031, "logps/rejected": -690.3782958984375, "loss": 4.9878, "margin_dpo/margin_mean": 112.41981506347656, "margin_dpo/margin_std": 235.89794921875, "step": 342 }, { "KL/chosen_KL_mean": -344.4051818847656, "KL/mean": -385.9061279296875, "KL/rejected_KL_mean": -427.4070129394531, "KL/std": 199.83212280273438, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.0016928425757214427, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.00184631347656, "fcm_dpo/q_t": 0.46619269251823425, "grad_norm": 54.531524658203125, "learning_rate": 1.125377900869913e-07, "logits/chosen": -0.8236934542655945, "logits/rejected": -0.798531711101532, "logps/chosen": -655.1251831054688, "logps/ref_chosen": -310.719970703125, "logps/ref_rejected": -263.5224914550781, "logps/rejected": -690.9295654296875, "loss": 5.1421, "margin_dpo/margin_mean": 83.00184631347656, "margin_dpo/margin_std": 217.48834228515625, "step": 343 }, { "KL/chosen_KL_mean": -331.8240966796875, "KL/mean": -385.7796630859375, "KL/rejected_KL_mean": -439.7351989746094, "KL/std": 196.72119140625, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.0016955842729657888, "fcm_dpo/delta": 0.00766171608120203, "fcm_dpo/margin": 107.91111755371094, "fcm_dpo/q_t": 0.4565431475639343, "grad_norm": 53.93059539794922, "learning_rate": 1.110123172071844e-07, "logits/chosen": -0.8151747584342957, "logits/rejected": -0.8005221486091614, "logps/chosen": -633.6240234375, "logps/ref_chosen": -301.7999267578125, "logps/ref_rejected": -257.9061584472656, "logps/rejected": -697.641357421875, "loss": 5.0036, "margin_dpo/margin_mean": 107.91112518310547, "margin_dpo/margin_std": 228.1441650390625, "step": 344 }, { "KL/chosen_KL_mean": -310.6817626953125, "KL/mean": -362.8054504394531, "KL/rejected_KL_mean": -414.9290771484375, "KL/std": 194.0869903564453, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.0016927202232182026, "fcm_dpo/delta": -0.012885820120573044, "fcm_dpo/margin": 104.247314453125, "fcm_dpo/q_t": 0.45815154910087585, "grad_norm": 64.63471984863281, "learning_rate": 1.09494297815e-07, "logits/chosen": -0.8192985653877258, "logits/rejected": -0.8202333450317383, "logps/chosen": -593.7001953125, "logps/ref_chosen": -283.0184326171875, "logps/ref_rejected": -266.8457336425781, "logps/rejected": -681.7747802734375, "loss": 5.0197, "margin_dpo/margin_mean": 104.247314453125, "margin_dpo/margin_std": 206.515625, "step": 345 }, { "KL/chosen_KL_mean": -328.94573974609375, "KL/mean": -400.6337890625, "KL/rejected_KL_mean": -472.32177734375, "KL/std": 245.28079223632812, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.0016636288492009044, "fcm_dpo/delta": -0.02877185493707657, "fcm_dpo/margin": 143.3760528564453, "fcm_dpo/q_t": 0.44495314359664917, "grad_norm": 38.55996322631836, "learning_rate": 1.0798381331721107e-07, "logits/chosen": -0.8979187607765198, "logits/rejected": -0.846237063407898, "logps/chosen": -597.386962890625, "logps/ref_chosen": -268.44122314453125, "logps/ref_rejected": -227.8225860595703, "logps/rejected": -700.1443481445312, "loss": 4.8526, "margin_dpo/margin_mean": 143.3760528564453, "margin_dpo/margin_std": 257.0928955078125, "step": 346 }, { "KL/chosen_KL_mean": -304.1871337890625, "KL/mean": -360.26837158203125, "KL/rejected_KL_mean": -416.3497009277344, "KL/std": 195.80422973632812, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.0016611977480351925, "fcm_dpo/delta": 0.01643402874469757, "fcm_dpo/margin": 112.16256713867188, "fcm_dpo/q_t": 0.4552745223045349, "grad_norm": 60.81416702270508, "learning_rate": 1.0648094471651722e-07, "logits/chosen": -0.745126485824585, "logits/rejected": -0.7690558433532715, "logps/chosen": -577.890625, "logps/ref_chosen": -273.70355224609375, "logps/ref_rejected": -243.65521240234375, "logps/rejected": -660.0048217773438, "loss": 4.9932, "margin_dpo/margin_mean": 112.16256713867188, "margin_dpo/margin_std": 225.79522705078125, "step": 347 }, { "KL/chosen_KL_mean": -297.6252136230469, "KL/mean": -350.5988464355469, "KL/rejected_KL_mean": -403.5724792480469, "KL/std": 215.3185577392578, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.0016852599801495671, "fcm_dpo/delta": 0.017294129356741905, "fcm_dpo/margin": 105.94729614257812, "fcm_dpo/q_t": 0.45961958169937134, "grad_norm": 58.32246017456055, "learning_rate": 1.0498577260720048e-07, "logits/chosen": -0.8524982929229736, "logits/rejected": -0.8348946571350098, "logps/chosen": -583.2666015625, "logps/ref_chosen": -285.64141845703125, "logps/ref_rejected": -265.6270446777344, "logps/rejected": -669.1995849609375, "loss": 5.0505, "margin_dpo/margin_mean": 105.9472885131836, "margin_dpo/margin_std": 251.26690673828125, "step": 348 }, { "KL/chosen_KL_mean": -297.42645263671875, "KL/mean": -378.73858642578125, "KL/rejected_KL_mean": -460.05078125, "KL/std": 242.02818298339844, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.0016933238366618752, "fcm_dpo/delta": -0.01814393885433674, "fcm_dpo/margin": 162.62429809570312, "fcm_dpo/q_t": 0.43410930037498474, "grad_norm": 54.12470626831055, "learning_rate": 1.0349837717080347e-07, "logits/chosen": -0.8172075152397156, "logits/rejected": -0.8003545999526978, "logps/chosen": -625.7439575195312, "logps/ref_chosen": -328.3175048828125, "logps/ref_rejected": -292.37872314453125, "logps/rejected": -752.429443359375, "loss": 4.7733, "margin_dpo/margin_mean": 162.62429809570312, "margin_dpo/margin_std": 289.4478454589844, "step": 349 }, { "KL/chosen_KL_mean": -318.63232421875, "KL/mean": -383.8518981933594, "KL/rejected_KL_mean": -449.071533203125, "KL/std": 229.48977661132812, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.0016819657757878304, "fcm_dpo/delta": 0.02026461437344551, "fcm_dpo/margin": 130.43919372558594, "fcm_dpo/q_t": 0.44916805624961853, "grad_norm": 65.67074584960938, "learning_rate": 1.0201883817182949e-07, "logits/chosen": -0.7871538996696472, "logits/rejected": -0.7973438501358032, "logps/chosen": -611.4369506835938, "logps/ref_chosen": -292.8046569824219, "logps/ref_rejected": -250.35504150390625, "logps/rejected": -699.426513671875, "loss": 4.9129, "margin_dpo/margin_mean": 130.439208984375, "margin_dpo/margin_std": 262.13153076171875, "step": 350 }, { "KL/chosen_KL_mean": -307.9429931640625, "KL/mean": -345.2463073730469, "KL/rejected_KL_mean": -382.5496520996094, "KL/std": 196.58447265625, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.0016986449481919408, "fcm_dpo/delta": 0.0035051475279033184, "fcm_dpo/margin": 74.606689453125, "fcm_dpo/q_t": 0.4699043929576874, "grad_norm": 44.76028060913086, "learning_rate": 1.0054723495346482e-07, "logits/chosen": -0.8658809065818787, "logits/rejected": -0.8433598279953003, "logps/chosen": -619.83203125, "logps/ref_chosen": -311.8890380859375, "logps/ref_rejected": -263.59033203125, "logps/rejected": -646.1400146484375, "loss": 5.2382, "margin_dpo/margin_mean": 74.606689453125, "margin_dpo/margin_std": 245.6429443359375, "step": 351 }, { "KL/chosen_KL_mean": -285.75799560546875, "KL/mean": -362.57232666015625, "KL/rejected_KL_mean": -439.3866271972656, "KL/std": 213.55348205566406, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.001679836306720972, "fcm_dpo/delta": -0.0261016096919775, "fcm_dpo/margin": 153.6286163330078, "fcm_dpo/q_t": 0.4385840892791748, "grad_norm": 82.70178985595703, "learning_rate": 9.908364643332398e-08, "logits/chosen": -0.7824245095252991, "logits/rejected": -0.7497365474700928, "logps/chosen": -540.6658935546875, "logps/ref_chosen": -254.9078826904297, "logps/ref_rejected": -257.1688232421875, "logps/rejected": -696.555419921875, "loss": 4.759, "margin_dpo/margin_mean": 153.6286163330078, "margin_dpo/margin_std": 241.4222412109375, "step": 352 }, { "KL/chosen_KL_mean": -301.4290771484375, "KL/mean": -370.8741149902344, "KL/rejected_KL_mean": -440.3191223144531, "KL/std": 203.89584350585938, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.0016587604768574238, "fcm_dpo/delta": -0.01662537083029747, "fcm_dpo/margin": 138.89007568359375, "fcm_dpo/q_t": 0.4453839659690857, "grad_norm": 61.3695182800293, "learning_rate": 9.76281510992176e-08, "logits/chosen": -0.7979697585105896, "logits/rejected": -0.7893396615982056, "logps/chosen": -571.8051147460938, "logps/ref_chosen": -270.3760681152344, "logps/ref_rejected": -264.65234375, "logps/rejected": -704.9714965820312, "loss": 4.8318, "margin_dpo/margin_mean": 138.89007568359375, "margin_dpo/margin_std": 225.83079528808594, "step": 353 }, { "KL/chosen_KL_mean": -303.3616943359375, "KL/mean": -355.75030517578125, "KL/rejected_KL_mean": -408.1388854980469, "KL/std": 229.685302734375, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.0016466137021780014, "fcm_dpo/delta": 0.0029272071551531553, "fcm_dpo/margin": 104.77714538574219, "fcm_dpo/q_t": 0.46023455262184143, "grad_norm": 51.536128997802734, "learning_rate": 9.618082700494318e-08, "logits/chosen": -0.7839207053184509, "logits/rejected": -0.8151556253433228, "logps/chosen": -561.01025390625, "logps/ref_chosen": -257.6485595703125, "logps/ref_rejected": -246.94203186035156, "logps/rejected": -655.0808715820312, "loss": 5.0629, "margin_dpo/margin_mean": 104.77714538574219, "margin_dpo/margin_std": 247.20835876464844, "step": 354 }, { "KL/chosen_KL_mean": -307.9608154296875, "KL/mean": -374.2004089355469, "KL/rejected_KL_mean": -440.4400329589844, "KL/std": 232.50241088867188, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.001645258511416614, "fcm_dpo/delta": 0.0026130429469048977, "fcm_dpo/margin": 132.4791717529297, "fcm_dpo/q_t": 0.44836515188217163, "grad_norm": 49.24905776977539, "learning_rate": 9.474175176609956e-08, "logits/chosen": -0.8556162714958191, "logits/rejected": -0.8463337421417236, "logps/chosen": -601.3141479492188, "logps/ref_chosen": -293.35333251953125, "logps/ref_rejected": -275.6051940917969, "logps/rejected": -716.0452270507812, "loss": 4.9783, "margin_dpo/margin_mean": 132.4791717529297, "margin_dpo/margin_std": 294.04217529296875, "step": 355 }, { "KL/chosen_KL_mean": -286.2415771484375, "KL/mean": -336.8193359375, "KL/rejected_KL_mean": -387.39703369140625, "KL/std": 191.365234375, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.0016517550684511662, "fcm_dpo/delta": -0.0037825205363333225, "fcm_dpo/margin": 101.15542602539062, "fcm_dpo/q_t": 0.46026772260665894, "grad_norm": 59.42401123046875, "learning_rate": 9.331100255592436e-08, "logits/chosen": -0.7560597658157349, "logits/rejected": -0.7795335650444031, "logps/chosen": -490.4970703125, "logps/ref_chosen": -204.25550842285156, "logps/ref_rejected": -213.467529296875, "logps/rejected": -600.864501953125, "loss": 5.0282, "margin_dpo/margin_mean": 101.15543365478516, "margin_dpo/margin_std": 203.6053466796875, "step": 356 }, { "KL/chosen_KL_mean": -323.9365539550781, "KL/mean": -389.21868896484375, "KL/rejected_KL_mean": -454.5008239746094, "KL/std": 225.989990234375, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.0016592942411080003, "fcm_dpo/delta": 0.024898748844861984, "fcm_dpo/margin": 130.5642547607422, "fcm_dpo/q_t": 0.4498782753944397, "grad_norm": 54.635040283203125, "learning_rate": 9.18886561011557e-08, "logits/chosen": -0.7446423172950745, "logits/rejected": -0.7396554946899414, "logps/chosen": -590.30712890625, "logps/ref_chosen": -266.3705749511719, "logps/ref_rejected": -239.04490661621094, "logps/rejected": -693.5457153320312, "loss": 4.9304, "margin_dpo/margin_mean": 130.56427001953125, "margin_dpo/margin_std": 261.66534423828125, "step": 357 }, { "KL/chosen_KL_mean": -291.0216369628906, "KL/mean": -364.53607177734375, "KL/rejected_KL_mean": -438.05047607421875, "KL/std": 207.47088623046875, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.0016887328820303082, "fcm_dpo/delta": 0.008424624800682068, "fcm_dpo/margin": 147.02883911132812, "fcm_dpo/q_t": 0.4417012631893158, "grad_norm": 45.379417419433594, "learning_rate": 9.047478867791731e-08, "logits/chosen": -0.840856671333313, "logits/rejected": -0.8128774166107178, "logps/chosen": -590.1691284179688, "logps/ref_chosen": -299.1474609375, "logps/ref_rejected": -257.2531433105469, "logps/rejected": -695.3037109375, "loss": 4.8174, "margin_dpo/margin_mean": 147.02883911132812, "margin_dpo/margin_std": 259.8493957519531, "step": 358 }, { "KL/chosen_KL_mean": -311.9315490722656, "KL/mean": -373.98565673828125, "KL/rejected_KL_mean": -436.0397033691406, "KL/std": 224.18618774414062, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.0017018289072439075, "fcm_dpo/delta": 0.012597101740539074, "fcm_dpo/margin": 124.10816955566406, "fcm_dpo/q_t": 0.45063310861587524, "grad_norm": 70.6742935180664, "learning_rate": 8.906947610762825e-08, "logits/chosen": -0.8120812177658081, "logits/rejected": -0.8148164749145508, "logps/chosen": -614.9293212890625, "logps/ref_chosen": -302.99786376953125, "logps/ref_rejected": -260.4137268066406, "logps/rejected": -696.4534301757812, "loss": 4.9726, "margin_dpo/margin_mean": 124.10816192626953, "margin_dpo/margin_std": 277.532470703125, "step": 359 }, { "KL/chosen_KL_mean": -323.221923828125, "KL/mean": -384.31170654296875, "KL/rejected_KL_mean": -445.4015808105469, "KL/std": 263.3553466796875, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.0016766900662332773, "fcm_dpo/delta": -0.02853110060095787, "fcm_dpo/margin": 122.17969512939453, "fcm_dpo/q_t": 0.45524489879608154, "grad_norm": 63.280357360839844, "learning_rate": 8.76727937529367e-08, "logits/chosen": -0.8388208746910095, "logits/rejected": -0.8119864463806152, "logps/chosen": -632.8333129882812, "logps/ref_chosen": -309.6114501953125, "logps/ref_rejected": -256.64031982421875, "logps/rejected": -702.0418701171875, "loss": 5.1195, "margin_dpo/margin_mean": 122.17970275878906, "margin_dpo/margin_std": 326.59295654296875, "step": 360 }, { "KL/chosen_KL_mean": -273.9930419921875, "KL/mean": -359.9635314941406, "KL/rejected_KL_mean": -445.93402099609375, "KL/std": 247.8087158203125, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.0016680698608979583, "fcm_dpo/delta": -0.004171290434896946, "fcm_dpo/margin": 171.94100952148438, "fcm_dpo/q_t": 0.43602752685546875, "grad_norm": 57.07918930053711, "learning_rate": 8.628481651367875e-08, "logits/chosen": -0.7725020051002502, "logits/rejected": -0.7491423487663269, "logps/chosen": -537.3727416992188, "logps/ref_chosen": -263.3797607421875, "logps/ref_rejected": -271.18157958984375, "logps/rejected": -717.1156005859375, "loss": 4.742, "margin_dpo/margin_mean": 171.94100952148438, "margin_dpo/margin_std": 304.07904052734375, "step": 361 }, { "KL/chosen_KL_mean": -307.6124267578125, "KL/mean": -366.4735412597656, "KL/rejected_KL_mean": -425.3346862792969, "KL/std": 190.72222900390625, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.0016689738258719444, "fcm_dpo/delta": 0.010700585320591927, "fcm_dpo/margin": 117.72223663330078, "fcm_dpo/q_t": 0.45483019948005676, "grad_norm": 67.26338195800781, "learning_rate": 8.490561882286135e-08, "logits/chosen": -0.8269234895706177, "logits/rejected": -0.8027861714363098, "logps/chosen": -610.870849609375, "logps/ref_chosen": -303.2583923339844, "logps/ref_rejected": -243.22891235351562, "logps/rejected": -668.5635375976562, "loss": 4.949, "margin_dpo/margin_mean": 117.72223663330078, "margin_dpo/margin_std": 217.4143829345703, "step": 362 }, { "KL/chosen_KL_mean": -318.3824462890625, "KL/mean": -391.4795837402344, "KL/rejected_KL_mean": -464.5767822265625, "KL/std": 235.78480529785156, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.0016669887118041515, "fcm_dpo/delta": -0.028279222548007965, "fcm_dpo/margin": 146.19439697265625, "fcm_dpo/q_t": 0.444366991519928, "grad_norm": 56.83020782470703, "learning_rate": 8.353527464267104e-08, "logits/chosen": -0.8206988573074341, "logits/rejected": -0.7746375799179077, "logps/chosen": -621.7296752929688, "logps/ref_chosen": -303.34722900390625, "logps/ref_rejected": -262.05419921875, "logps/rejected": -726.6309814453125, "loss": 4.8521, "margin_dpo/margin_mean": 146.19439697265625, "margin_dpo/margin_std": 274.79852294921875, "step": 363 }, { "KL/chosen_KL_mean": -361.5899658203125, "KL/mean": -448.52484130859375, "KL/rejected_KL_mean": -535.459716796875, "KL/std": 327.3614807128906, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.0016274906229227781, "fcm_dpo/delta": -0.05278400331735611, "fcm_dpo/margin": 173.8697509765625, "fcm_dpo/q_t": 0.4439254403114319, "grad_norm": 57.509544372558594, "learning_rate": 8.217385746050742e-08, "logits/chosen": -0.7986475825309753, "logits/rejected": -0.7825050354003906, "logps/chosen": -647.1337280273438, "logps/ref_chosen": -285.54376220703125, "logps/ref_rejected": -284.84619140625, "logps/rejected": -820.3059692382812, "loss": 4.8973, "margin_dpo/margin_mean": 173.86978149414062, "margin_dpo/margin_std": 399.4956970214844, "step": 364 }, { "KL/chosen_KL_mean": -339.6693115234375, "KL/mean": -399.89129638671875, "KL/rejected_KL_mean": -460.11328125, "KL/std": 253.50515747070312, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.0015868466580286622, "fcm_dpo/delta": 0.014260734431445599, "fcm_dpo/margin": 120.44401550292969, "fcm_dpo/q_t": 0.45612239837646484, "grad_norm": 60.942710876464844, "learning_rate": 8.082144028504231e-08, "logits/chosen": -0.8087151050567627, "logits/rejected": -0.804487943649292, "logps/chosen": -614.4570922851562, "logps/ref_chosen": -274.7878112792969, "logps/ref_rejected": -256.5738220214844, "logps/rejected": -716.6871337890625, "loss": 5.0254, "margin_dpo/margin_mean": 120.44401550292969, "margin_dpo/margin_std": 279.97271728515625, "step": 365 }, { "KL/chosen_KL_mean": -317.1668701171875, "KL/mean": -382.4986572265625, "KL/rejected_KL_mean": -447.8304138183594, "KL/std": 217.1991424560547, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.0016188733279705048, "fcm_dpo/delta": 0.03934203460812569, "fcm_dpo/margin": 130.66355895996094, "fcm_dpo/q_t": 0.45035284757614136, "grad_norm": 73.36628723144531, "learning_rate": 7.947809564230445e-08, "logits/chosen": -0.7664698362350464, "logits/rejected": -0.7650561928749084, "logps/chosen": -603.8164672851562, "logps/ref_chosen": -286.6496276855469, "logps/ref_rejected": -251.97140502929688, "logps/rejected": -699.8018188476562, "loss": 4.9317, "margin_dpo/margin_mean": 130.66355895996094, "margin_dpo/margin_std": 264.78948974609375, "step": 366 }, { "KL/chosen_KL_mean": -322.634521484375, "KL/mean": -398.5595703125, "KL/rejected_KL_mean": -474.48455810546875, "KL/std": 227.88592529296875, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.001617192174308002, "fcm_dpo/delta": -0.01441990491002798, "fcm_dpo/margin": 151.8500518798828, "fcm_dpo/q_t": 0.44398194551467896, "grad_norm": 52.11629867553711, "learning_rate": 7.814389557179016e-08, "logits/chosen": -0.809653639793396, "logits/rejected": -0.7744249105453491, "logps/chosen": -624.5795288085938, "logps/ref_chosen": -301.9449768066406, "logps/ref_rejected": -265.5677185058594, "logps/rejected": -740.0523071289062, "loss": 4.8384, "margin_dpo/margin_mean": 151.85003662109375, "margin_dpo/margin_std": 275.8656311035156, "step": 367 }, { "KL/chosen_KL_mean": -310.06488037109375, "KL/mean": -400.94287109375, "KL/rejected_KL_mean": -491.8209228515625, "KL/std": 228.97950744628906, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.0015950199449434876, "fcm_dpo/delta": -0.037169210612773895, "fcm_dpo/margin": 181.75608825683594, "fcm_dpo/q_t": 0.43195655941963196, "grad_norm": 53.12458801269531, "learning_rate": 7.681891162260015e-08, "logits/chosen": -0.7793463468551636, "logits/rejected": -0.77353835105896, "logps/chosen": -604.6913452148438, "logps/ref_chosen": -294.62652587890625, "logps/ref_rejected": -258.7628479003906, "logps/rejected": -750.583740234375, "loss": 4.6447, "margin_dpo/margin_mean": 181.75608825683594, "margin_dpo/margin_std": 250.93226623535156, "step": 368 }, { "KL/chosen_KL_mean": -319.5187683105469, "KL/mean": -384.4464111328125, "KL/rejected_KL_mean": -449.3740234375, "KL/std": 208.54287719726562, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.0015673936577513814, "fcm_dpo/delta": -0.004795985296368599, "fcm_dpo/margin": 129.85523986816406, "fcm_dpo/q_t": 0.45139309763908386, "grad_norm": 75.6568374633789, "learning_rate": 7.550321484960251e-08, "logits/chosen": -0.8449119329452515, "logits/rejected": -0.8261924982070923, "logps/chosen": -602.0244750976562, "logps/ref_chosen": -282.5057373046875, "logps/ref_rejected": -266.41607666015625, "logps/rejected": -715.7900390625, "loss": 4.8941, "margin_dpo/margin_mean": 129.85523986816406, "margin_dpo/margin_std": 217.25881958007812, "step": 369 }, { "KL/chosen_KL_mean": -300.08746337890625, "KL/mean": -385.3553771972656, "KL/rejected_KL_mean": -470.623291015625, "KL/std": 238.8286590576172, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.0015783013077452779, "fcm_dpo/delta": 0.0026395046152174473, "fcm_dpo/margin": 170.53579711914062, "fcm_dpo/q_t": 0.43791812658309937, "grad_norm": 65.24024200439453, "learning_rate": 7.419687580962222e-08, "logits/chosen": -0.8119034767150879, "logits/rejected": -0.8299652338027954, "logps/chosen": -551.0939331054688, "logps/ref_chosen": -251.00640869140625, "logps/ref_rejected": -238.12542724609375, "logps/rejected": -708.7487182617188, "loss": 4.7508, "margin_dpo/margin_mean": 170.53579711914062, "margin_dpo/margin_std": 288.934326171875, "step": 370 }, { "KL/chosen_KL_mean": -358.78350830078125, "KL/mean": -403.6153564453125, "KL/rejected_KL_mean": -448.4471740722656, "KL/std": 235.52256774902344, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.001578000606968999, "fcm_dpo/delta": 0.0009506037458777428, "fcm_dpo/margin": 89.66363525390625, "fcm_dpo/q_t": 0.4663865566253662, "grad_norm": 57.798397064208984, "learning_rate": 7.289996455765748e-08, "logits/chosen": -0.7915385961532593, "logits/rejected": -0.762095034122467, "logps/chosen": -655.442626953125, "logps/ref_chosen": -296.6591491699219, "logps/ref_rejected": -251.14675903320312, "logps/rejected": -699.5939331054688, "loss": 5.1806, "margin_dpo/margin_mean": 89.66363525390625, "margin_dpo/margin_std": 257.93023681640625, "step": 371 }, { "KL/chosen_KL_mean": -348.8611145019531, "KL/mean": -430.9217834472656, "KL/rejected_KL_mean": -512.982421875, "KL/std": 264.5611267089844, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.0015493937535211444, "fcm_dpo/delta": -0.015690520405769348, "fcm_dpo/margin": 164.12132263183594, "fcm_dpo/q_t": 0.44282326102256775, "grad_norm": 55.466941833496094, "learning_rate": 7.161255064312283e-08, "logits/chosen": -0.7481983304023743, "logits/rejected": -0.7245327234268188, "logps/chosen": -680.2325439453125, "logps/ref_chosen": -331.3714599609375, "logps/ref_rejected": -285.56805419921875, "logps/rejected": -798.5504760742188, "loss": 4.8437, "margin_dpo/margin_mean": 164.12132263183594, "margin_dpo/margin_std": 297.5496826171875, "step": 372 }, { "KL/chosen_KL_mean": -335.027099609375, "KL/mean": -411.2965087890625, "KL/rejected_KL_mean": -487.56591796875, "KL/std": 254.25425720214844, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.001557708252221346, "fcm_dpo/delta": -0.0038446406833827496, "fcm_dpo/margin": 152.53880310058594, "fcm_dpo/q_t": 0.44847139716148376, "grad_norm": 49.32960891723633, "learning_rate": 7.033470310611945e-08, "logits/chosen": -0.8375375866889954, "logits/rejected": -0.7990292310714722, "logps/chosen": -656.9700927734375, "logps/ref_chosen": -321.9429931640625, "logps/ref_rejected": -271.2288513183594, "logps/rejected": -758.7947998046875, "loss": 4.9111, "margin_dpo/margin_mean": 152.53878784179688, "margin_dpo/margin_std": 324.3055419921875, "step": 373 }, { "KL/chosen_KL_mean": -341.29376220703125, "KL/mean": -412.0097961425781, "KL/rejected_KL_mean": -482.7259826660156, "KL/std": 252.2704315185547, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.0015299688093364239, "fcm_dpo/delta": -0.02197812870144844, "fcm_dpo/margin": 141.4322052001953, "fcm_dpo/q_t": 0.44911688566207886, "grad_norm": 37.8529167175293, "learning_rate": 6.906649047373245e-08, "logits/chosen": -0.8442825675010681, "logits/rejected": -0.8310015797615051, "logps/chosen": -660.4622802734375, "logps/ref_chosen": -319.1685485839844, "logps/ref_rejected": -284.6263732910156, "logps/rejected": -767.352294921875, "loss": 4.8947, "margin_dpo/margin_mean": 141.43222045898438, "margin_dpo/margin_std": 258.3140563964844, "step": 374 }, { "KL/chosen_KL_mean": -364.37353515625, "KL/mean": -419.4544677734375, "KL/rejected_KL_mean": -474.5354309082031, "KL/std": 210.785400390625, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.0015292322495952249, "fcm_dpo/delta": 0.005789346527308226, "fcm_dpo/margin": 110.16191864013672, "fcm_dpo/q_t": 0.4604509472846985, "grad_norm": 45.025943756103516, "learning_rate": 6.780798075635675e-08, "logits/chosen": -0.8373956084251404, "logits/rejected": -0.814143717288971, "logps/chosen": -679.2493286132812, "logps/ref_chosen": -314.87579345703125, "logps/ref_rejected": -259.1965026855469, "logps/rejected": -733.73193359375, "loss": 5.0522, "margin_dpo/margin_mean": 110.16191864013672, "margin_dpo/margin_std": 243.85162353515625, "step": 375 }, { "KL/chosen_KL_mean": -353.1634216308594, "KL/mean": -430.255859375, "KL/rejected_KL_mean": -507.3483581542969, "KL/std": 252.78001403808594, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.001551387831568718, "fcm_dpo/delta": 0.02010853961110115, "fcm_dpo/margin": 154.18492126464844, "fcm_dpo/q_t": 0.44513237476348877, "grad_norm": 48.4976806640625, "learning_rate": 6.655924144404906e-08, "logits/chosen": -0.8034533262252808, "logits/rejected": -0.7987213134765625, "logps/chosen": -640.836669921875, "logps/ref_chosen": -287.6732482910156, "logps/ref_rejected": -256.6697082519531, "logps/rejected": -764.01806640625, "loss": 4.8674, "margin_dpo/margin_mean": 154.18492126464844, "margin_dpo/margin_std": 304.42388916015625, "step": 376 }, { "KL/chosen_KL_mean": -386.02972412109375, "KL/mean": -438.955322265625, "KL/rejected_KL_mean": -491.88092041015625, "KL/std": 254.73690795898438, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.0015574595890939236, "fcm_dpo/delta": 0.011809874325990677, "fcm_dpo/margin": 105.85123443603516, "fcm_dpo/q_t": 0.46167030930519104, "grad_norm": 64.08843231201172, "learning_rate": 6.532033950290885e-08, "logits/chosen": -0.7984158396720886, "logits/rejected": -0.7836294174194336, "logps/chosen": -691.2911987304688, "logps/ref_chosen": -305.261474609375, "logps/ref_rejected": -271.8887023925781, "logps/rejected": -763.7696533203125, "loss": 5.145, "margin_dpo/margin_mean": 105.85124206542969, "margin_dpo/margin_std": 296.171630859375, "step": 377 }, { "KL/chosen_KL_mean": -372.6801452636719, "KL/mean": -446.7356262207031, "KL/rejected_KL_mean": -520.7911376953125, "KL/std": 272.634521484375, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.0015734572662040591, "fcm_dpo/delta": 0.004630648531019688, "fcm_dpo/margin": 148.11105346679688, "fcm_dpo/q_t": 0.4467093348503113, "grad_norm": 57.25436782836914, "learning_rate": 6.409134137148736e-08, "logits/chosen": -0.778599739074707, "logits/rejected": -0.7654741406440735, "logps/chosen": -654.209716796875, "logps/ref_chosen": -281.5295715332031, "logps/ref_rejected": -296.980224609375, "logps/rejected": -817.7713623046875, "loss": 4.9086, "margin_dpo/margin_mean": 148.11105346679688, "margin_dpo/margin_std": 312.68133544921875, "step": 378 }, { "KL/chosen_KL_mean": -342.051513671875, "KL/mean": -402.2879333496094, "KL/rejected_KL_mean": -462.52435302734375, "KL/std": 208.310302734375, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.0015739906812086701, "fcm_dpo/delta": -0.005612190812826157, "fcm_dpo/margin": 120.47281646728516, "fcm_dpo/q_t": 0.45524847507476807, "grad_norm": 55.82075500488281, "learning_rate": 6.28723129572247e-08, "logits/chosen": -0.8357870578765869, "logits/rejected": -0.8165754675865173, "logps/chosen": -607.13232421875, "logps/ref_chosen": -265.0807800292969, "logps/ref_rejected": -230.58932495117188, "logps/rejected": -693.1136474609375, "loss": 5.0035, "margin_dpo/margin_mean": 120.47280883789062, "margin_dpo/margin_std": 265.7042236328125, "step": 379 }, { "KL/chosen_KL_mean": -374.452880859375, "KL/mean": -457.7342834472656, "KL/rejected_KL_mean": -541.015625, "KL/std": 298.7972106933594, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.0015483014285564423, "fcm_dpo/delta": -0.030254196375608444, "fcm_dpo/margin": 166.56275939941406, "fcm_dpo/q_t": 0.4449843466281891, "grad_norm": 62.38126754760742, "learning_rate": 6.166331963291519e-08, "logits/chosen": -0.8070506453514099, "logits/rejected": -0.7720668911933899, "logps/chosen": -680.3612670898438, "logps/ref_chosen": -305.90838623046875, "logps/ref_rejected": -286.5906677246094, "logps/rejected": -827.6063842773438, "loss": 4.9223, "margin_dpo/margin_mean": 166.56275939941406, "margin_dpo/margin_std": 373.67486572265625, "step": 380 }, { "KL/chosen_KL_mean": -369.968505859375, "KL/mean": -463.00933837890625, "KL/rejected_KL_mean": -556.0502319335938, "KL/std": 326.89111328125, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.0015199973713606596, "fcm_dpo/delta": -0.028817355632781982, "fcm_dpo/margin": 186.08172607421875, "fcm_dpo/q_t": 0.4386086165904999, "grad_norm": 55.27873992919922, "learning_rate": 6.046442623320145e-08, "logits/chosen": -0.7852144837379456, "logits/rejected": -0.7426830530166626, "logps/chosen": -622.8392333984375, "logps/ref_chosen": -252.87066650390625, "logps/ref_rejected": -261.1927490234375, "logps/rejected": -817.2429809570312, "loss": 4.7924, "margin_dpo/margin_mean": 186.08172607421875, "margin_dpo/margin_std": 358.6522216796875, "step": 381 }, { "KL/chosen_KL_mean": -367.1063537597656, "KL/mean": -461.32720947265625, "KL/rejected_KL_mean": -555.5479736328125, "KL/std": 248.60533142089844, "epoch": 0.8, "fcm_dpo/beta": 0.0014805117389187217, "fcm_dpo/delta": -0.010750288143754005, "fcm_dpo/margin": 188.44158935546875, "fcm_dpo/q_t": 0.43490761518478394, "grad_norm": 65.76939392089844, "learning_rate": 5.9275697051098275e-08, "logits/chosen": -0.8315447568893433, "logits/rejected": -0.8279691934585571, "logps/chosen": -656.31787109375, "logps/ref_chosen": -289.2114562988281, "logps/ref_rejected": -278.45751953125, "logps/rejected": -834.0054321289062, "loss": 4.7007, "margin_dpo/margin_mean": 188.44158935546875, "margin_dpo/margin_std": 298.6485595703125, "step": 382 }, { "KL/chosen_KL_mean": -354.55718994140625, "KL/mean": -444.1139221191406, "KL/rejected_KL_mean": -533.6705932617188, "KL/std": 305.8647766113281, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.0015076538547873497, "fcm_dpo/delta": -0.013801611959934235, "fcm_dpo/margin": 179.1134796142578, "fcm_dpo/q_t": 0.44126763939857483, "grad_norm": 61.43276596069336, "learning_rate": 5.809719583454414e-08, "logits/chosen": -0.7901857495307922, "logits/rejected": -0.7668880820274353, "logps/chosen": -628.1880493164062, "logps/ref_chosen": -273.630859375, "logps/ref_rejected": -261.44024658203125, "logps/rejected": -795.1109008789062, "loss": 4.8441, "margin_dpo/margin_mean": 179.1134796142578, "margin_dpo/margin_std": 363.99066162109375, "step": 383 }, { "KL/chosen_KL_mean": -368.296630859375, "KL/mean": -438.9754333496094, "KL/rejected_KL_mean": -509.65423583984375, "KL/std": 221.85972595214844, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.001479036407545209, "fcm_dpo/delta": 0.02152104675769806, "fcm_dpo/margin": 141.35757446289062, "fcm_dpo/q_t": 0.450982928276062, "grad_norm": 50.3799934387207, "learning_rate": 5.6928985782982524e-08, "logits/chosen": -0.8385475277900696, "logits/rejected": -0.8302735686302185, "logps/chosen": -642.86669921875, "logps/ref_chosen": -274.5699462890625, "logps/ref_rejected": -285.8253479003906, "logps/rejected": -795.4794921875, "loss": 4.9229, "margin_dpo/margin_mean": 141.35757446289062, "margin_dpo/margin_std": 276.80145263671875, "step": 384 }, { "KL/chosen_KL_mean": -368.0294494628906, "KL/mean": -428.7261962890625, "KL/rejected_KL_mean": -489.4228515625, "KL/std": 226.50360107421875, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.0015037511475384235, "fcm_dpo/delta": 0.007470243610441685, "fcm_dpo/margin": 121.39341735839844, "fcm_dpo/q_t": 0.4575641453266144, "grad_norm": 60.63737869262695, "learning_rate": 5.57711295439732e-08, "logits/chosen": -0.7681106328964233, "logits/rejected": -0.7596469521522522, "logps/chosen": -652.1801147460938, "logps/ref_chosen": -284.150634765625, "logps/ref_rejected": -244.87921142578125, "logps/rejected": -734.3021240234375, "loss": 5.0108, "margin_dpo/margin_mean": 121.39341735839844, "margin_dpo/margin_std": 255.0092010498047, "step": 385 }, { "KL/chosen_KL_mean": -365.3989562988281, "KL/mean": -464.4845275878906, "KL/rejected_KL_mean": -563.5701293945312, "KL/std": 260.08233642578125, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.001453772303648293, "fcm_dpo/delta": -0.05015706643462181, "fcm_dpo/margin": 198.17123413085938, "fcm_dpo/q_t": 0.4336777925491333, "grad_norm": 91.76905822753906, "learning_rate": 5.4623689209832484e-08, "logits/chosen": -0.7455501556396484, "logits/rejected": -0.7397761940956116, "logps/chosen": -685.5752563476562, "logps/ref_chosen": -320.1762390136719, "logps/ref_rejected": -302.05023193359375, "logps/rejected": -865.620361328125, "loss": 4.6863, "margin_dpo/margin_mean": 198.17123413085938, "margin_dpo/margin_std": 294.59130859375, "step": 386 }, { "KL/chosen_KL_mean": -407.8135986328125, "KL/mean": -475.7935485839844, "KL/rejected_KL_mean": -543.7735595703125, "KL/std": 272.47930908203125, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.001420615240931511, "fcm_dpo/delta": -0.033063966780900955, "fcm_dpo/margin": 135.95997619628906, "fcm_dpo/q_t": 0.4548041522502899, "grad_norm": 57.505863189697266, "learning_rate": 5.3486726314303175e-08, "logits/chosen": -0.7548569440841675, "logits/rejected": -0.7590454816818237, "logps/chosen": -680.09375, "logps/ref_chosen": -272.2801513671875, "logps/ref_rejected": -265.1615905761719, "logps/rejected": -808.9351806640625, "loss": 5.076, "margin_dpo/margin_mean": 135.95997619628906, "margin_dpo/margin_std": 337.6916809082031, "step": 387 }, { "KL/chosen_KL_mean": -430.6405334472656, "KL/mean": -481.8836669921875, "KL/rejected_KL_mean": -533.1268310546875, "KL/std": 267.2966003417969, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.0014033719198778272, "fcm_dpo/delta": 0.007294456474483013, "fcm_dpo/margin": 102.4863052368164, "fcm_dpo/q_t": 0.46697717905044556, "grad_norm": 49.18757629394531, "learning_rate": 5.2360301829254745e-08, "logits/chosen": -0.7436924576759338, "logits/rejected": -0.7248135209083557, "logps/chosen": -703.171875, "logps/ref_chosen": -272.5313415527344, "logps/ref_rejected": -239.55735778808594, "logps/rejected": -772.6842041015625, "loss": 5.2023, "margin_dpo/margin_mean": 102.4863052368164, "margin_dpo/margin_std": 319.1353759765625, "step": 388 }, { "KL/chosen_KL_mean": -439.1097717285156, "KL/mean": -521.8563232421875, "KL/rejected_KL_mean": -604.6029052734375, "KL/std": 302.55511474609375, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.0013935244642198086, "fcm_dpo/delta": -0.028961144387722015, "fcm_dpo/margin": 165.49313354492188, "fcm_dpo/q_t": 0.44787466526031494, "grad_norm": 57.25077438354492, "learning_rate": 5.1244476161413806e-08, "logits/chosen": -0.7774551510810852, "logits/rejected": -0.7715262174606323, "logps/chosen": -720.198974609375, "logps/ref_chosen": -281.0892639160156, "logps/ref_rejected": -246.50045776367188, "logps/rejected": -851.1033935546875, "loss": 4.9262, "margin_dpo/margin_mean": 165.49314880371094, "margin_dpo/margin_std": 342.1744384765625, "step": 389 }, { "KL/chosen_KL_mean": -406.08447265625, "KL/mean": -487.11151123046875, "KL/rejected_KL_mean": -568.1385498046875, "KL/std": 261.64813232421875, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.0013833637349307537, "fcm_dpo/delta": 0.0005270633846521378, "fcm_dpo/margin": 162.0540771484375, "fcm_dpo/q_t": 0.4468287527561188, "grad_norm": 63.353206634521484, "learning_rate": 5.013930914912476e-08, "logits/chosen": -0.7989583015441895, "logits/rejected": -0.8104404807090759, "logps/chosen": -690.072021484375, "logps/ref_chosen": -283.98748779296875, "logps/ref_rejected": -283.465087890625, "logps/rejected": -851.6036376953125, "loss": 4.8718, "margin_dpo/margin_mean": 162.05409240722656, "margin_dpo/margin_std": 298.63543701171875, "step": 390 }, { "KL/chosen_KL_mean": -428.979736328125, "KL/mean": -515.8250122070312, "KL/rejected_KL_mean": -602.6703491210938, "KL/std": 311.1672668457031, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.001366346376016736, "fcm_dpo/delta": -0.023520534858107567, "fcm_dpo/margin": 173.6905517578125, "fcm_dpo/q_t": 0.4458865821361542, "grad_norm": 53.48771667480469, "learning_rate": 4.904486005914027e-08, "logits/chosen": -0.7384014129638672, "logits/rejected": -0.7049020528793335, "logps/chosen": -712.8411254882812, "logps/ref_chosen": -283.86138916015625, "logps/ref_rejected": -263.5093688964844, "logps/rejected": -866.1796875, "loss": 4.887, "margin_dpo/margin_mean": 173.69053649902344, "margin_dpo/margin_std": 344.82696533203125, "step": 391 }, { "KL/chosen_KL_mean": -400.332763671875, "KL/mean": -495.0168762207031, "KL/rejected_KL_mean": -589.7010498046875, "KL/std": 263.923583984375, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.0013336706906557083, "fcm_dpo/delta": -0.014936976134777069, "fcm_dpo/margin": 189.36825561523438, "fcm_dpo/q_t": 0.4395950138568878, "grad_norm": 65.17903900146484, "learning_rate": 4.796118758344353e-08, "logits/chosen": -0.7459010481834412, "logits/rejected": -0.7519693374633789, "logps/chosen": -710.40283203125, "logps/ref_chosen": -310.070068359375, "logps/ref_rejected": -252.89817810058594, "logps/rejected": -842.5991821289062, "loss": 4.779, "margin_dpo/margin_mean": 189.36825561523438, "margin_dpo/margin_std": 308.4510803222656, "step": 392 }, { "KL/chosen_KL_mean": -422.2491455078125, "KL/mean": -484.3158874511719, "KL/rejected_KL_mean": -546.3826293945312, "KL/std": 261.5521545410156, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.0013305129250511527, "fcm_dpo/delta": -0.00916867796331644, "fcm_dpo/margin": 124.13351440429688, "fcm_dpo/q_t": 0.4618360996246338, "grad_norm": 50.20282745361328, "learning_rate": 4.688834983610082e-08, "logits/chosen": -0.7697139978408813, "logits/rejected": -0.7453039288520813, "logps/chosen": -708.9647216796875, "logps/ref_chosen": -286.7156677246094, "logps/ref_rejected": -230.00357055664062, "logps/rejected": -776.38623046875, "loss": 5.1041, "margin_dpo/margin_mean": 124.13351440429688, "margin_dpo/margin_std": 312.4925537109375, "step": 393 }, { "KL/chosen_KL_mean": -431.40948486328125, "KL/mean": -507.337890625, "KL/rejected_KL_mean": -583.2664184570312, "KL/std": 355.6380920410156, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.001326891127973795, "fcm_dpo/delta": -0.0004829922690987587, "fcm_dpo/margin": 151.85691833496094, "fcm_dpo/q_t": 0.45768600702285767, "grad_norm": 45.90632629394531, "learning_rate": 4.582640435014459e-08, "logits/chosen": -0.8130242228507996, "logits/rejected": -0.7946957349777222, "logps/chosen": -757.4029541015625, "logps/ref_chosen": -325.9934387207031, "logps/ref_rejected": -317.42706298828125, "logps/rejected": -900.6934814453125, "loss": 5.0776, "margin_dpo/margin_mean": 151.85691833496094, "margin_dpo/margin_std": 398.54766845703125, "step": 394 }, { "KL/chosen_KL_mean": -396.9059753417969, "KL/mean": -497.17181396484375, "KL/rejected_KL_mean": -597.4376220703125, "KL/std": 304.17828369140625, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.0013342795427888632, "fcm_dpo/delta": -0.008068141527473927, "fcm_dpo/margin": 200.5316619873047, "fcm_dpo/q_t": 0.44144290685653687, "grad_norm": 54.122161865234375, "learning_rate": 4.477540807448832e-08, "logits/chosen": -0.7593112587928772, "logits/rejected": -0.7634783983230591, "logps/chosen": -665.8067626953125, "logps/ref_chosen": -268.90081787109375, "logps/ref_rejected": -272.85809326171875, "logps/rejected": -870.2957763671875, "loss": 4.8032, "margin_dpo/margin_mean": 200.5316619873047, "margin_dpo/margin_std": 380.6814880371094, "step": 395 }, { "KL/chosen_KL_mean": -439.490478515625, "KL/mean": -507.13873291015625, "KL/rejected_KL_mean": -574.7869873046875, "KL/std": 280.7339782714844, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.0013154197949916124, "fcm_dpo/delta": -0.006766099948436022, "fcm_dpo/margin": 135.29653930664062, "fcm_dpo/q_t": 0.4586159884929657, "grad_norm": 77.64254760742188, "learning_rate": 4.373541737087263e-08, "logits/chosen": -0.7809940576553345, "logits/rejected": -0.7539770603179932, "logps/chosen": -730.688720703125, "logps/ref_chosen": -291.19830322265625, "logps/ref_rejected": -253.2803955078125, "logps/rejected": -828.0673828125, "loss": 5.0548, "margin_dpo/margin_mean": 135.29653930664062, "margin_dpo/margin_std": 323.10113525390625, "step": 396 }, { "KL/chosen_KL_mean": -422.6109619140625, "KL/mean": -488.63763427734375, "KL/rejected_KL_mean": -554.6641845703125, "KL/std": 269.7872619628906, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.0012935028644278646, "fcm_dpo/delta": -0.021948356181383133, "fcm_dpo/margin": 132.0531768798828, "fcm_dpo/q_t": 0.46082544326782227, "grad_norm": 43.36281204223633, "learning_rate": 4.270648801084295e-08, "logits/chosen": -0.784430742263794, "logits/rejected": -0.7664919495582581, "logps/chosen": -732.4334106445312, "logps/ref_chosen": -309.8224182128906, "logps/ref_rejected": -291.9057922363281, "logps/rejected": -846.5699462890625, "loss": 5.0907, "margin_dpo/margin_mean": 132.05319213867188, "margin_dpo/margin_std": 318.5513916015625, "step": 397 }, { "KL/chosen_KL_mean": -475.15447998046875, "KL/mean": -527.6256713867188, "KL/rejected_KL_mean": -580.0968017578125, "KL/std": 301.6562805175781, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.0012898289132863283, "fcm_dpo/delta": 0.005256129428744316, "fcm_dpo/margin": 104.9423599243164, "fcm_dpo/q_t": 0.4673859477043152, "grad_norm": 83.37042999267578, "learning_rate": 4.168867517275806e-08, "logits/chosen": -0.6738908886909485, "logits/rejected": -0.711742103099823, "logps/chosen": -772.968017578125, "logps/ref_chosen": -297.8135070800781, "logps/ref_rejected": -270.5025634765625, "logps/rejected": -850.5994873046875, "loss": 5.3166, "margin_dpo/margin_mean": 104.94235229492188, "margin_dpo/margin_std": 399.0448913574219, "step": 398 }, { "KL/chosen_KL_mean": -434.46160888671875, "KL/mean": -512.9297485351562, "KL/rejected_KL_mean": -591.3978271484375, "KL/std": 286.52301025390625, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.0013032241258770227, "fcm_dpo/delta": 0.01705552078783512, "fcm_dpo/margin": 156.936279296875, "fcm_dpo/q_t": 0.4539187550544739, "grad_norm": 78.72877502441406, "learning_rate": 4.0682033438831584e-08, "logits/chosen": -0.7885871529579163, "logits/rejected": -0.7415661811828613, "logps/chosen": -727.3084106445312, "logps/ref_chosen": -292.8467712402344, "logps/ref_rejected": -268.3638916015625, "logps/rejected": -859.76171875, "loss": 4.9874, "margin_dpo/margin_mean": 156.93626403808594, "margin_dpo/margin_std": 351.0825500488281, "step": 399 }, { "KL/chosen_KL_mean": -427.16387939453125, "KL/mean": -506.3305969238281, "KL/rejected_KL_mean": -585.497314453125, "KL/std": 281.44110107421875, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.0013135882327333093, "fcm_dpo/delta": 0.005816956982016563, "fcm_dpo/margin": 158.33343505859375, "fcm_dpo/q_t": 0.4515479803085327, "grad_norm": 66.58218383789062, "learning_rate": 3.968661679220467e-08, "logits/chosen": -0.8234080076217651, "logits/rejected": -0.8150714039802551, "logps/chosen": -690.84033203125, "logps/ref_chosen": -263.6763916015625, "logps/ref_rejected": -258.67266845703125, "logps/rejected": -844.1699829101562, "loss": 4.9672, "margin_dpo/margin_mean": 158.3334197998047, "margin_dpo/margin_std": 340.52227783203125, "step": 400 }, { "epoch": 0.837696335078534, "eval_KL/chosen_KL_mean": -431.93609619140625, "eval_KL/mean": -511.6028137207031, "eval_KL/rejected_KL_mean": -591.2696533203125, "eval_KL/std": 291.858154296875, "eval_fcm_dpo/beta": 0.0013183961855247617, "eval_logits/chosen": -0.7613639235496521, "eval_logits/rejected": -0.7369002103805542, "eval_logps/chosen": -719.7628173828125, "eval_logps/ref_chosen": -287.8268127441406, "eval_logps/ref_rejected": -266.9300231933594, "eval_logps/rejected": -858.19970703125, "eval_loss": 0.6247319579124451, "eval_margin_dpo/margin_mean": 159.33351135253906, "eval_margin_dpo/margin_std": 334.47113037109375, "eval_runtime": 78.1079, "eval_samples_per_second": 25.606, "eval_steps_per_second": 3.201, "step": 400 }, { "KL/chosen_KL_mean": -426.7398376464844, "KL/mean": -502.2171630859375, "KL/rejected_KL_mean": -577.6944580078125, "KL/std": 288.7701416015625, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.001324369921348989, "fcm_dpo/delta": 0.019296318292617798, "fcm_dpo/margin": 150.95465087890625, "fcm_dpo/q_t": 0.4532890021800995, "grad_norm": 71.27273559570312, "learning_rate": 3.8702478614051345e-08, "logits/chosen": -0.7519879937171936, "logits/rejected": -0.7463111877441406, "logps/chosen": -745.025146484375, "logps/ref_chosen": -318.2853088378906, "logps/ref_rejected": -293.75225830078125, "logps/rejected": -871.4466552734375, "loss": 4.9883, "margin_dpo/margin_mean": 150.95465087890625, "margin_dpo/margin_std": 336.1131896972656, "step": 401 }, { "KL/chosen_KL_mean": -430.9375305175781, "KL/mean": -494.4723205566406, "KL/rejected_KL_mean": -558.0069580078125, "KL/std": 295.1917724609375, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.0013450951082631946, "fcm_dpo/delta": 0.007680067326873541, "fcm_dpo/margin": 127.0694580078125, "fcm_dpo/q_t": 0.46227654814720154, "grad_norm": 67.7490463256836, "learning_rate": 3.772967168071517e-08, "logits/chosen": -0.8251144886016846, "logits/rejected": -0.7937527298927307, "logps/chosen": -740.3653564453125, "logps/ref_chosen": -309.4278564453125, "logps/ref_rejected": -282.0279846191406, "logps/rejected": -840.0350341796875, "loss": 5.1424, "margin_dpo/margin_mean": 127.0694580078125, "margin_dpo/margin_std": 378.865966796875, "step": 402 }, { "KL/chosen_KL_mean": -396.99566650390625, "KL/mean": -511.38201904296875, "KL/rejected_KL_mean": -625.768310546875, "KL/std": 321.3643798828125, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.0013335788389667869, "fcm_dpo/delta": -0.03909500688314438, "fcm_dpo/margin": 228.77272033691406, "fcm_dpo/q_t": 0.4304249584674835, "grad_norm": 51.24015426635742, "learning_rate": 3.676824816087978e-08, "logits/chosen": -0.8037683963775635, "logits/rejected": -0.7648081183433533, "logps/chosen": -706.0241088867188, "logps/ref_chosen": -309.0284729003906, "logps/ref_rejected": -272.9622497558594, "logps/rejected": -898.7305908203125, "loss": 4.6302, "margin_dpo/margin_mean": 228.772705078125, "margin_dpo/margin_std": 343.154541015625, "step": 403 }, { "KL/chosen_KL_mean": -442.41656494140625, "KL/mean": -525.9095458984375, "KL/rejected_KL_mean": -609.4025268554688, "KL/std": 310.6871032714844, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.001324630226008594, "fcm_dpo/delta": 0.01602954789996147, "fcm_dpo/margin": 166.98602294921875, "fcm_dpo/q_t": 0.449731707572937, "grad_norm": 60.760292053222656, "learning_rate": 3.581825961277074e-08, "logits/chosen": -0.8088729381561279, "logits/rejected": -0.7796494364738464, "logps/chosen": -739.7001953125, "logps/ref_chosen": -297.2837219238281, "logps/ref_rejected": -256.99041748046875, "logps/rejected": -866.3929443359375, "loss": 4.9848, "margin_dpo/margin_mean": 166.98602294921875, "margin_dpo/margin_std": 390.0989990234375, "step": 404 }, { "KL/chosen_KL_mean": -426.5012512207031, "KL/mean": -526.6109008789062, "KL/rejected_KL_mean": -626.7206420898438, "KL/std": 317.7232971191406, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.0013220272958278656, "fcm_dpo/delta": -6.324471905827522e-05, "fcm_dpo/margin": 200.21939086914062, "fcm_dpo/q_t": 0.43894970417022705, "grad_norm": 57.350894927978516, "learning_rate": 3.487975698139084e-08, "logits/chosen": -0.7303048372268677, "logits/rejected": -0.7393031120300293, "logps/chosen": -684.4666137695312, "logps/ref_chosen": -257.96533203125, "logps/ref_rejected": -255.811279296875, "logps/rejected": -882.531982421875, "loss": 4.784, "margin_dpo/margin_mean": 200.21939086914062, "margin_dpo/margin_std": 362.07623291015625, "step": 405 }, { "KL/chosen_KL_mean": -448.4387512207031, "KL/mean": -505.62249755859375, "KL/rejected_KL_mean": -562.8062133789062, "KL/std": 272.3173828125, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.0013311943039298058, "fcm_dpo/delta": 0.01564777083694935, "fcm_dpo/margin": 114.367431640625, "fcm_dpo/q_t": 0.4627969264984131, "grad_norm": 58.29486846923828, "learning_rate": 3.3952790595787986e-08, "logits/chosen": -0.7839182615280151, "logits/rejected": -0.7578977346420288, "logps/chosen": -733.619873046875, "logps/ref_chosen": -285.1810607910156, "logps/ref_rejected": -264.41351318359375, "logps/rejected": -827.2197265625, "loss": 5.1225, "margin_dpo/margin_mean": 114.367431640625, "margin_dpo/margin_std": 295.78497314453125, "step": 406 }, { "KL/chosen_KL_mean": -450.4339599609375, "KL/mean": -536.018798828125, "KL/rejected_KL_mean": -621.6035766601562, "KL/std": 315.64569091796875, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.0013364061014726758, "fcm_dpo/delta": -0.014766542240977287, "fcm_dpo/margin": 171.16966247558594, "fcm_dpo/q_t": 0.4484463036060333, "grad_norm": 60.3465461730957, "learning_rate": 3.303741016635614e-08, "logits/chosen": -0.7542386054992676, "logits/rejected": -0.7638047933578491, "logps/chosen": -715.6720581054688, "logps/ref_chosen": -265.23809814453125, "logps/ref_rejected": -219.0631561279297, "logps/rejected": -840.6668090820312, "loss": 4.9662, "margin_dpo/margin_mean": 171.16966247558594, "margin_dpo/margin_std": 389.5382995605469, "step": 407 }, { "KL/chosen_KL_mean": -413.54681396484375, "KL/mean": -505.22528076171875, "KL/rejected_KL_mean": -596.9036865234375, "KL/std": 283.1024169921875, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.0013287973124533892, "fcm_dpo/delta": -0.018995679914951324, "fcm_dpo/margin": 183.35684204101562, "fcm_dpo/q_t": 0.4423255920410156, "grad_norm": 65.34120178222656, "learning_rate": 3.2133664782169944e-08, "logits/chosen": -0.7940729856491089, "logits/rejected": -0.7762957215309143, "logps/chosen": -710.5194091796875, "logps/ref_chosen": -296.9726257324219, "logps/ref_rejected": -295.4786376953125, "logps/rejected": -892.38232421875, "loss": 4.8044, "margin_dpo/margin_mean": 183.35687255859375, "margin_dpo/margin_std": 300.9643859863281, "step": 408 }, { "KL/chosen_KL_mean": -426.305419921875, "KL/mean": -522.0606689453125, "KL/rejected_KL_mean": -617.8158569335938, "KL/std": 355.56585693359375, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.001289794105105102, "fcm_dpo/delta": -0.027987752109766006, "fcm_dpo/margin": 191.5104522705078, "fcm_dpo/q_t": 0.4499598741531372, "grad_norm": 46.15365982055664, "learning_rate": 3.12416029083514e-08, "logits/chosen": -0.7818073034286499, "logits/rejected": -0.761074960231781, "logps/chosen": -713.684814453125, "logps/ref_chosen": -287.37933349609375, "logps/ref_rejected": -275.80291748046875, "logps/rejected": -893.6187744140625, "loss": 4.9385, "margin_dpo/margin_mean": 191.51046752929688, "margin_dpo/margin_std": 431.0859069824219, "step": 409 }, { "KL/chosen_KL_mean": -437.843505859375, "KL/mean": -509.63983154296875, "KL/rejected_KL_mean": -581.43603515625, "KL/std": 298.229736328125, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.001277453382499516, "fcm_dpo/delta": 0.0034497997257858515, "fcm_dpo/margin": 143.592529296875, "fcm_dpo/q_t": 0.45774397253990173, "grad_norm": 70.2773208618164, "learning_rate": 3.036127238347164e-08, "logits/chosen": -0.7765858769416809, "logits/rejected": -0.7771980166435242, "logps/chosen": -719.6236572265625, "logps/ref_chosen": -281.7801818847656, "logps/ref_rejected": -266.7550354003906, "logps/rejected": -848.1910400390625, "loss": 5.0507, "margin_dpo/margin_mean": 143.592529296875, "margin_dpo/margin_std": 343.8995056152344, "step": 410 }, { "KL/chosen_KL_mean": -411.8667297363281, "KL/mean": -501.0542907714844, "KL/rejected_KL_mean": -590.2418823242188, "KL/std": 304.47467041015625, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.0012669181451201439, "fcm_dpo/delta": -0.0407031811773777, "fcm_dpo/margin": 178.37509155273438, "fcm_dpo/q_t": 0.4484909772872925, "grad_norm": 55.282936096191406, "learning_rate": 2.9492720416985e-08, "logits/chosen": -0.7966690063476562, "logits/rejected": -0.7400557398796082, "logps/chosen": -693.4540405273438, "logps/ref_chosen": -281.5872497558594, "logps/ref_rejected": -254.78916931152344, "logps/rejected": -845.031005859375, "loss": 4.8709, "margin_dpo/margin_mean": 178.37509155273438, "margin_dpo/margin_std": 299.1616516113281, "step": 411 }, { "KL/chosen_KL_mean": -471.7283630371094, "KL/mean": -547.4363403320312, "KL/rejected_KL_mean": -623.1444702148438, "KL/std": 309.1171569824219, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.001206240151077509, "fcm_dpo/delta": -0.03707820922136307, "fcm_dpo/margin": 151.41604614257812, "fcm_dpo/q_t": 0.4574580192565918, "grad_norm": 91.96940612792969, "learning_rate": 2.863599358669755e-08, "logits/chosen": -0.7530825734138489, "logits/rejected": -0.7635226249694824, "logps/chosen": -748.2625122070312, "logps/ref_chosen": -276.5341796875, "logps/ref_rejected": -273.8751220703125, "logps/rejected": -897.01953125, "loss": 5.0485, "margin_dpo/margin_mean": 151.4160614013672, "margin_dpo/margin_std": 343.7683410644531, "step": 412 }, { "KL/chosen_KL_mean": -426.9269104003906, "KL/mean": -498.89007568359375, "KL/rejected_KL_mean": -570.8531494140625, "KL/std": 293.0097351074219, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.0012071667006239295, "fcm_dpo/delta": 0.00854087807238102, "fcm_dpo/margin": 143.92626953125, "fcm_dpo/q_t": 0.46002674102783203, "grad_norm": 51.58241653442383, "learning_rate": 2.7791137836269158e-08, "logits/chosen": -0.778816819190979, "logits/rejected": -0.7682943344116211, "logps/chosen": -698.2015380859375, "logps/ref_chosen": -271.2745666503906, "logps/ref_rejected": -270.16912841796875, "logps/rejected": -841.0222778320312, "loss": 5.0608, "margin_dpo/margin_mean": 143.92626953125, "margin_dpo/margin_std": 354.32513427734375, "step": 413 }, { "KL/chosen_KL_mean": -444.8292236328125, "KL/mean": -537.6610107421875, "KL/rejected_KL_mean": -630.49267578125, "KL/std": 300.1920166015625, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.0012186833191663027, "fcm_dpo/delta": 0.010454244911670685, "fcm_dpo/margin": 185.66348266601562, "fcm_dpo/q_t": 0.4482007324695587, "grad_norm": 49.763938903808594, "learning_rate": 2.6958198472749717e-08, "logits/chosen": -0.8072595000267029, "logits/rejected": -0.7890864610671997, "logps/chosen": -741.9442749023438, "logps/ref_chosen": -297.11505126953125, "logps/ref_rejected": -271.7034606933594, "logps/rejected": -902.1961059570312, "loss": 4.882, "margin_dpo/margin_mean": 185.66348266601562, "margin_dpo/margin_std": 349.4181213378906, "step": 414 }, { "KL/chosen_KL_mean": -425.5120849609375, "KL/mean": -520.7711791992188, "KL/rejected_KL_mean": -616.0302734375, "KL/std": 286.868896484375, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.0012264670804142952, "fcm_dpo/delta": 0.006376807112246752, "fcm_dpo/margin": 190.51815795898438, "fcm_dpo/q_t": 0.4448871314525604, "grad_norm": 91.31519317626953, "learning_rate": 2.613722016414943e-08, "logits/chosen": -0.7885741591453552, "logits/rejected": -0.7686821818351746, "logps/chosen": -723.204833984375, "logps/ref_chosen": -297.6926574707031, "logps/ref_rejected": -279.0503234863281, "logps/rejected": -895.08056640625, "loss": 4.8312, "margin_dpo/margin_mean": 190.51815795898438, "margin_dpo/margin_std": 332.1375732421875, "step": 415 }, { "KL/chosen_KL_mean": -427.74713134765625, "KL/mean": -514.8232421875, "KL/rejected_KL_mean": -601.8994750976562, "KL/std": 273.8400573730469, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.0012284396216273308, "fcm_dpo/delta": 0.01868375763297081, "fcm_dpo/margin": 174.15232849121094, "fcm_dpo/q_t": 0.449332594871521, "grad_norm": 81.10550689697266, "learning_rate": 2.5328246937043525e-08, "logits/chosen": -0.7960155606269836, "logits/rejected": -0.7860805988311768, "logps/chosen": -739.5726318359375, "logps/ref_chosen": -311.8255615234375, "logps/ref_rejected": -268.6170654296875, "logps/rejected": -870.5166015625, "loss": 4.8818, "margin_dpo/margin_mean": 174.15231323242188, "margin_dpo/margin_std": 305.1326904296875, "step": 416 }, { "KL/chosen_KL_mean": -471.95123291015625, "KL/mean": -555.9248657226562, "KL/rejected_KL_mean": -639.8984985351562, "KL/std": 336.646240234375, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.0012344487477093935, "fcm_dpo/delta": -0.029500747099518776, "fcm_dpo/margin": 167.94720458984375, "fcm_dpo/q_t": 0.45440053939819336, "grad_norm": 57.704097747802734, "learning_rate": 2.4531322174210973e-08, "logits/chosen": -0.7565368413925171, "logits/rejected": -0.734302282333374, "logps/chosen": -782.3880615234375, "logps/ref_chosen": -310.43682861328125, "logps/ref_rejected": -277.15283203125, "logps/rejected": -917.05126953125, "loss": 5.0212, "margin_dpo/margin_mean": 167.9472198486328, "margin_dpo/margin_std": 395.55621337890625, "step": 417 }, { "KL/chosen_KL_mean": -478.63140869140625, "KL/mean": -567.9556274414062, "KL/rejected_KL_mean": -657.2798461914062, "KL/std": 373.185302734375, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.001195128308609128, "fcm_dpo/delta": -0.022249775007367134, "fcm_dpo/margin": 178.6484832763672, "fcm_dpo/q_t": 0.4534074366092682, "grad_norm": 60.83293151855469, "learning_rate": 2.3746488612308295e-08, "logits/chosen": -0.7093433141708374, "logits/rejected": -0.6751872897148132, "logps/chosen": -757.1273193359375, "logps/ref_chosen": -278.49591064453125, "logps/ref_rejected": -276.56671142578125, "logps/rejected": -933.8466796875, "loss": 5.0329, "margin_dpo/margin_mean": 178.6484832763672, "margin_dpo/margin_std": 459.3668518066406, "step": 418 }, { "KL/chosen_KL_mean": -466.22515869140625, "KL/mean": -540.742919921875, "KL/rejected_KL_mean": -615.2605590820312, "KL/std": 327.9731750488281, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.001201170147396624, "fcm_dpo/delta": 0.011327693238854408, "fcm_dpo/margin": 149.03546142578125, "fcm_dpo/q_t": 0.4571627676486969, "grad_norm": 66.67911529541016, "learning_rate": 2.297378833957761e-08, "logits/chosen": -0.7795159220695496, "logits/rejected": -0.7457402944564819, "logps/chosen": -765.1253662109375, "logps/ref_chosen": -298.9002380371094, "logps/ref_rejected": -246.1540985107422, "logps/rejected": -861.4146728515625, "loss": 5.1194, "margin_dpo/margin_mean": 149.0354461669922, "margin_dpo/margin_std": 419.2374572753906, "step": 419 }, { "KL/chosen_KL_mean": -451.4075927734375, "KL/mean": -554.6902465820312, "KL/rejected_KL_mean": -657.972900390625, "KL/std": 334.60174560546875, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.0011963802389800549, "fcm_dpo/delta": -0.025561338290572166, "fcm_dpo/margin": 206.5653076171875, "fcm_dpo/q_t": 0.44405436515808105, "grad_norm": 48.52541732788086, "learning_rate": 2.2213262793589482e-08, "logits/chosen": -0.7045433521270752, "logits/rejected": -0.6653756499290466, "logps/chosen": -715.968505859375, "logps/ref_chosen": -264.5608825683594, "logps/ref_rejected": -245.67031860351562, "logps/rejected": -903.6431884765625, "loss": 4.8964, "margin_dpo/margin_mean": 206.5653076171875, "margin_dpo/margin_std": 432.84625244140625, "step": 420 }, { "KL/chosen_KL_mean": -413.5687561035156, "KL/mean": -502.61492919921875, "KL/rejected_KL_mean": -591.6611938476562, "KL/std": 279.97216796875, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.001170460251159966, "fcm_dpo/delta": -0.023245420306921005, "fcm_dpo/margin": 178.09242248535156, "fcm_dpo/q_t": 0.44988635182380676, "grad_norm": 61.88900375366211, "learning_rate": 2.1464952759020856e-08, "logits/chosen": -0.7998079657554626, "logits/rejected": -0.7664088010787964, "logps/chosen": -711.2737426757812, "logps/ref_chosen": -297.70501708984375, "logps/ref_rejected": -243.74771118164062, "logps/rejected": -835.4088745117188, "loss": 4.9225, "margin_dpo/margin_mean": 178.09242248535156, "margin_dpo/margin_std": 329.73651123046875, "step": 421 }, { "KL/chosen_KL_mean": -449.1634826660156, "KL/mean": -527.736083984375, "KL/rejected_KL_mean": -606.3087158203125, "KL/std": 306.22967529296875, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.001158454455435276, "fcm_dpo/delta": 0.0001966398995136842, "fcm_dpo/margin": 157.14520263671875, "fcm_dpo/q_t": 0.4581235349178314, "grad_norm": 46.273345947265625, "learning_rate": 2.07288983654679e-08, "logits/chosen": -0.6590052247047424, "logits/rejected": -0.6952117085456848, "logps/chosen": -737.522216796875, "logps/ref_chosen": -288.3587646484375, "logps/ref_rejected": -256.4377746582031, "logps/rejected": -862.7464599609375, "loss": 5.0234, "margin_dpo/margin_mean": 157.14520263671875, "margin_dpo/margin_std": 353.7700500488281, "step": 422 }, { "KL/chosen_KL_mean": -467.9991455078125, "KL/mean": -561.8355712890625, "KL/rejected_KL_mean": -655.6720581054688, "KL/std": 315.5198974609375, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.0011576899560168386, "fcm_dpo/delta": -0.0015649726847186685, "fcm_dpo/margin": 187.67288208007812, "fcm_dpo/q_t": 0.4489739239215851, "grad_norm": 48.21940231323242, "learning_rate": 2.0005139085293942e-08, "logits/chosen": -0.7844016551971436, "logits/rejected": -0.7567226886749268, "logps/chosen": -764.0061645507812, "logps/ref_chosen": -296.00701904296875, "logps/ref_rejected": -261.3480529785156, "logps/rejected": -917.0200805664062, "loss": 4.9226, "margin_dpo/margin_mean": 187.6728973388672, "margin_dpo/margin_std": 380.403564453125, "step": 423 }, { "KL/chosen_KL_mean": -434.7855224609375, "KL/mean": -512.031494140625, "KL/rejected_KL_mean": -589.2774658203125, "KL/std": 287.8274230957031, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.0011533235665410757, "fcm_dpo/delta": -0.00939516257494688, "fcm_dpo/margin": 154.49185180664062, "fcm_dpo/q_t": 0.45800527930259705, "grad_norm": 76.39921569824219, "learning_rate": 1.9293713731512673e-08, "logits/chosen": -0.7860153317451477, "logits/rejected": -0.76336669921875, "logps/chosen": -744.2073974609375, "logps/ref_chosen": -309.421875, "logps/ref_rejected": -249.14886474609375, "logps/rejected": -838.4263305664062, "loss": 5.0235, "margin_dpo/margin_mean": 154.49185180664062, "margin_dpo/margin_std": 335.57342529296875, "step": 424 }, { "KL/chosen_KL_mean": -412.7100524902344, "KL/mean": -507.1483154296875, "KL/rejected_KL_mean": -601.5865478515625, "KL/std": 304.760986328125, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.0011293534189462662, "fcm_dpo/delta": -0.029587510973215103, "fcm_dpo/margin": 188.87649536132812, "fcm_dpo/q_t": 0.4508245587348938, "grad_norm": 64.6044692993164, "learning_rate": 1.8594660455706763e-08, "logits/chosen": -0.7572231888771057, "logits/rejected": -0.7515621781349182, "logps/chosen": -693.2191772460938, "logps/ref_chosen": -280.50909423828125, "logps/ref_rejected": -276.8252258300781, "logps/rejected": -878.4116821289062, "loss": 4.9217, "margin_dpo/margin_mean": 188.87649536132812, "margin_dpo/margin_std": 359.30194091796875, "step": 425 }, { "KL/chosen_KL_mean": -442.26385498046875, "KL/mean": -525.7176513671875, "KL/rejected_KL_mean": -609.1713256835938, "KL/std": 298.6646728515625, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.0011355035239830613, "fcm_dpo/delta": 0.027290547266602516, "fcm_dpo/margin": 166.90748596191406, "fcm_dpo/q_t": 0.4546186625957489, "grad_norm": 55.23300552368164, "learning_rate": 1.7908016745981856e-08, "logits/chosen": -0.7826669812202454, "logits/rejected": -0.7525278925895691, "logps/chosen": -735.049072265625, "logps/ref_chosen": -292.78521728515625, "logps/ref_rejected": -255.62698364257812, "logps/rejected": -864.79833984375, "loss": 4.9927, "margin_dpo/margin_mean": 166.90748596191406, "margin_dpo/margin_std": 347.90826416015625, "step": 426 }, { "KL/chosen_KL_mean": -442.0292053222656, "KL/mean": -572.7102661132812, "KL/rejected_KL_mean": -703.3912353515625, "KL/std": 374.0984191894531, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.001141170971095562, "fcm_dpo/delta": -0.0355389341711998, "fcm_dpo/margin": 261.36199951171875, "fcm_dpo/q_t": 0.43112075328826904, "grad_norm": 64.85372924804688, "learning_rate": 1.7233819424956247e-08, "logits/chosen": -0.759992778301239, "logits/rejected": -0.7070995569229126, "logps/chosen": -730.7979125976562, "logps/ref_chosen": -288.7687072753906, "logps/ref_rejected": -268.4986572265625, "logps/rejected": -971.889892578125, "loss": 4.7296, "margin_dpo/margin_mean": 261.36199951171875, "margin_dpo/margin_std": 463.7349853515625, "step": 427 }, { "KL/chosen_KL_mean": -482.4189453125, "KL/mean": -601.1357421875, "KL/rejected_KL_mean": -719.8524169921875, "KL/std": 355.86456298828125, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.0010970557341352105, "fcm_dpo/delta": -0.02731183171272278, "fcm_dpo/margin": 237.43350219726562, "fcm_dpo/q_t": 0.4391084909439087, "grad_norm": 63.699703216552734, "learning_rate": 1.6572104647786245e-08, "logits/chosen": -0.7069486975669861, "logits/rejected": -0.7331293225288391, "logps/chosen": -777.93994140625, "logps/ref_chosen": -295.5209655761719, "logps/ref_rejected": -275.71026611328125, "logps/rejected": -995.5628051757812, "loss": 4.7554, "margin_dpo/margin_mean": 237.43350219726562, "margin_dpo/margin_std": 377.74169921875, "step": 428 }, { "KL/chosen_KL_mean": -476.097412109375, "KL/mean": -550.6402587890625, "KL/rejected_KL_mean": -625.1830444335938, "KL/std": 336.6489562988281, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.0011044524144381285, "fcm_dpo/delta": 0.02802928164601326, "fcm_dpo/margin": 149.08566284179688, "fcm_dpo/q_t": 0.46070483326911926, "grad_norm": 84.84398651123047, "learning_rate": 1.5922907900227017e-08, "logits/chosen": -0.7112962603569031, "logits/rejected": -0.7193475365638733, "logps/chosen": -750.48974609375, "logps/ref_chosen": -274.392333984375, "logps/ref_rejected": -258.574462890625, "logps/rejected": -883.7575073242188, "loss": 5.1361, "margin_dpo/margin_mean": 149.08566284179688, "margin_dpo/margin_std": 409.7275695800781, "step": 429 }, { "KL/chosen_KL_mean": -446.72955322265625, "KL/mean": -537.5370483398438, "KL/rejected_KL_mean": -628.344482421875, "KL/std": 328.6794738769531, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.0011219914304092526, "fcm_dpo/delta": 0.002371245063841343, "fcm_dpo/margin": 181.61495971679688, "fcm_dpo/q_t": 0.4525456428527832, "grad_norm": 79.65233612060547, "learning_rate": 1.5286263996730026e-08, "logits/chosen": -0.7959480285644531, "logits/rejected": -0.7541887760162354, "logps/chosen": -735.4686889648438, "logps/ref_chosen": -288.7391357421875, "logps/ref_rejected": -268.6106262207031, "logps/rejected": -896.955078125, "loss": 4.9536, "margin_dpo/margin_mean": 181.6149444580078, "margin_dpo/margin_std": 366.8076477050781, "step": 430 }, { "KL/chosen_KL_mean": -441.72039794921875, "KL/mean": -512.4733276367188, "KL/rejected_KL_mean": -583.2261962890625, "KL/std": 299.0691833496094, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.001115912338718772, "fcm_dpo/delta": -0.009300373494625092, "fcm_dpo/margin": 141.50582885742188, "fcm_dpo/q_t": 0.4632718563079834, "grad_norm": 49.450313568115234, "learning_rate": 1.4662207078575684e-08, "logits/chosen": -0.7692259550094604, "logits/rejected": -0.7354046702384949, "logps/chosen": -717.4451904296875, "logps/ref_chosen": -275.7247314453125, "logps/ref_rejected": -268.91729736328125, "logps/rejected": -852.1436157226562, "loss": 5.1099, "margin_dpo/margin_mean": 141.50582885742188, "margin_dpo/margin_std": 358.96746826171875, "step": 431 }, { "KL/chosen_KL_mean": -396.63092041015625, "KL/mean": -489.38275146484375, "KL/rejected_KL_mean": -582.1345825195312, "KL/std": 305.49884033203125, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.0011289321118965745, "fcm_dpo/delta": 0.0328177772462368, "fcm_dpo/margin": 185.50364685058594, "fcm_dpo/q_t": 0.45187050104141235, "grad_norm": 48.00371170043945, "learning_rate": 1.40507706120426e-08, "logits/chosen": -0.817834734916687, "logits/rejected": -0.788792073726654, "logps/chosen": -688.051025390625, "logps/ref_chosen": -291.42010498046875, "logps/ref_rejected": -255.48202514648438, "logps/rejected": -837.6165771484375, "loss": 4.9446, "margin_dpo/margin_mean": 185.50363159179688, "margin_dpo/margin_std": 378.95550537109375, "step": 432 }, { "KL/chosen_KL_mean": -459.69891357421875, "KL/mean": -535.0228271484375, "KL/rejected_KL_mean": -610.3468017578125, "KL/std": 313.0767822265625, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.001144950743764639, "fcm_dpo/delta": 0.013033966533839703, "fcm_dpo/margin": 150.6478729248047, "fcm_dpo/q_t": 0.4605104327201843, "grad_norm": 76.42973327636719, "learning_rate": 1.345198738661285e-08, "logits/chosen": -0.7439653277397156, "logits/rejected": -0.7333388328552246, "logps/chosen": -705.92578125, "logps/ref_chosen": -246.2268829345703, "logps/ref_rejected": -253.65924072265625, "logps/rejected": -864.0060424804688, "loss": 5.0636, "margin_dpo/margin_mean": 150.64788818359375, "margin_dpo/margin_std": 367.3023376464844, "step": 433 }, { "KL/chosen_KL_mean": -501.3629150390625, "KL/mean": -568.4837036132812, "KL/rejected_KL_mean": -635.6045532226562, "KL/std": 315.385986328125, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.001152915065176785, "fcm_dpo/delta": 0.004903012420982122, "fcm_dpo/margin": 134.24163818359375, "fcm_dpo/q_t": 0.4633429944515228, "grad_norm": 64.78780364990234, "learning_rate": 1.2865889513213628e-08, "logits/chosen": -0.7485510110855103, "logits/rejected": -0.7544351816177368, "logps/chosen": -796.82470703125, "logps/ref_chosen": -295.4618225097656, "logps/ref_rejected": -256.2254333496094, "logps/rejected": -891.8299560546875, "loss": 5.1368, "margin_dpo/margin_mean": 134.2416229248047, "margin_dpo/margin_std": 361.8308410644531, "step": 434 }, { "KL/chosen_KL_mean": -415.9727783203125, "KL/mean": -528.1788940429688, "KL/rejected_KL_mean": -640.385009765625, "KL/std": 337.5413513183594, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.001140053616836667, "fcm_dpo/delta": -0.03828362748026848, "fcm_dpo/margin": 224.41220092773438, "fcm_dpo/q_t": 0.44446510076522827, "grad_norm": 68.40642547607422, "learning_rate": 1.2292508422495157e-08, "logits/chosen": -0.7327988147735596, "logits/rejected": -0.7096722722053528, "logps/chosen": -676.711181640625, "logps/ref_chosen": -260.7384033203125, "logps/ref_rejected": -248.5688018798828, "logps/rejected": -888.953857421875, "loss": 4.8225, "margin_dpo/margin_mean": 224.41220092773438, "margin_dpo/margin_std": 394.1939697265625, "step": 435 }, { "KL/chosen_KL_mean": -489.31561279296875, "KL/mean": -566.24462890625, "KL/rejected_KL_mean": -643.1736450195312, "KL/std": 384.1330261230469, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.0011170388897880912, "fcm_dpo/delta": -0.0015771682374179363, "fcm_dpo/margin": 153.85800170898438, "fcm_dpo/q_t": 0.4651756286621094, "grad_norm": 48.129756927490234, "learning_rate": 1.1731874863145142e-08, "logits/chosen": -0.7540550231933594, "logits/rejected": -0.7378331422805786, "logps/chosen": -808.63818359375, "logps/ref_chosen": -319.3224792480469, "logps/ref_rejected": -299.30322265625, "logps/rejected": -942.4769287109375, "loss": 5.141, "margin_dpo/margin_mean": 153.8579864501953, "margin_dpo/margin_std": 445.167724609375, "step": 436 }, { "KL/chosen_KL_mean": -440.3687438964844, "KL/mean": -549.68701171875, "KL/rejected_KL_mean": -659.005126953125, "KL/std": 355.17633056640625, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.0011156515683978796, "fcm_dpo/delta": -0.011046146973967552, "fcm_dpo/margin": 218.63638305664062, "fcm_dpo/q_t": 0.44490429759025574, "grad_norm": 69.38978576660156, "learning_rate": 1.118401890024001e-08, "logits/chosen": -0.7746818661689758, "logits/rejected": -0.742755651473999, "logps/chosen": -719.197509765625, "logps/ref_chosen": -278.82879638671875, "logps/ref_rejected": -272.55303955078125, "logps/rejected": -931.5582275390625, "loss": 4.8757, "margin_dpo/margin_mean": 218.6363983154297, "margin_dpo/margin_std": 423.87152099609375, "step": 437 }, { "KL/chosen_KL_mean": -474.75103759765625, "KL/mean": -517.1573486328125, "KL/rejected_KL_mean": -559.5636596679688, "KL/std": 331.350341796875, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.001117004663683474, "fcm_dpo/delta": 0.006466980557888746, "fcm_dpo/margin": 84.8126449584961, "fcm_dpo/q_t": 0.47826677560806274, "grad_norm": 88.04359436035156, "learning_rate": 1.06489699136324e-08, "logits/chosen": -0.7461004257202148, "logits/rejected": -0.7531372308731079, "logps/chosen": -734.070068359375, "logps/ref_chosen": -259.31903076171875, "logps/ref_rejected": -240.99581909179688, "logps/rejected": -800.5595092773438, "loss": 5.3795, "margin_dpo/margin_mean": 84.81265258789062, "margin_dpo/margin_std": 385.8534851074219, "step": 438 }, { "KL/chosen_KL_mean": -444.52960205078125, "KL/mean": -533.1126708984375, "KL/rejected_KL_mean": -621.6957397460938, "KL/std": 293.2552795410156, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.001108786673285067, "fcm_dpo/delta": -0.006224591284990311, "fcm_dpo/margin": 177.16610717773438, "fcm_dpo/q_t": 0.45308101177215576, "grad_norm": 51.11709213256836, "learning_rate": 1.0126756596375685e-08, "logits/chosen": -0.7281197905540466, "logits/rejected": -0.7132576107978821, "logps/chosen": -701.6539916992188, "logps/ref_chosen": -257.1243896484375, "logps/ref_rejected": -243.20416259765625, "logps/rejected": -864.89990234375, "loss": 4.9507, "margin_dpo/margin_mean": 177.1660919189453, "margin_dpo/margin_std": 342.137451171875, "step": 439 }, { "KL/chosen_KL_mean": -474.2215576171875, "KL/mean": -542.44384765625, "KL/rejected_KL_mean": -610.666259765625, "KL/std": 283.33172607421875, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.0011141917202621698, "fcm_dpo/delta": 0.01364380493760109, "fcm_dpo/margin": 136.44468688964844, "fcm_dpo/q_t": 0.46402812004089355, "grad_norm": 48.65362548828125, "learning_rate": 9.617406953185136e-09, "logits/chosen": -0.8012443780899048, "logits/rejected": -0.7894343733787537, "logps/chosen": -781.7530517578125, "logps/ref_chosen": -307.5315246582031, "logps/ref_rejected": -264.3540954589844, "logps/rejected": -875.0203857421875, "loss": 5.1135, "margin_dpo/margin_mean": 136.44468688964844, "margin_dpo/margin_std": 327.4635009765625, "step": 440 }, { "KL/chosen_KL_mean": -447.04949951171875, "KL/mean": -549.7024536132812, "KL/rejected_KL_mean": -652.3553466796875, "KL/std": 306.68316650390625, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.001123150810599327, "fcm_dpo/delta": -0.0012265940895304084, "fcm_dpo/margin": 205.3058624267578, "fcm_dpo/q_t": 0.44529205560684204, "grad_norm": 68.43964385986328, "learning_rate": 9.12094829893642e-09, "logits/chosen": -0.7399640679359436, "logits/rejected": -0.7328054308891296, "logps/chosen": -757.0314331054688, "logps/ref_chosen": -309.9819641113281, "logps/ref_rejected": -297.4968566894531, "logps/rejected": -949.852294921875, "loss": 4.8237, "margin_dpo/margin_mean": 205.30584716796875, "margin_dpo/margin_std": 332.4583435058594, "step": 441 }, { "KL/chosen_KL_mean": -504.6666564941406, "KL/mean": -573.6967163085938, "KL/rejected_KL_mean": -642.726806640625, "KL/std": 340.78564453125, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.0011333015281707048, "fcm_dpo/delta": 0.019338196143507957, "fcm_dpo/margin": 138.06015014648438, "fcm_dpo/q_t": 0.46324992179870605, "grad_norm": 47.62255096435547, "learning_rate": 8.637407257200496e-09, "logits/chosen": -0.8079502582550049, "logits/rejected": -0.7644075751304626, "logps/chosen": -783.645751953125, "logps/ref_chosen": -278.9791564941406, "logps/ref_rejected": -242.87310791015625, "logps/rejected": -885.5999755859375, "loss": 5.1402, "margin_dpo/margin_mean": 138.06015014648438, "margin_dpo/margin_std": 388.3710632324219, "step": 442 }, { "KL/chosen_KL_mean": -456.517822265625, "KL/mean": -535.2323608398438, "KL/rejected_KL_mean": -613.947021484375, "KL/std": 287.5276794433594, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.0011378020280972123, "fcm_dpo/delta": -0.008695745840668678, "fcm_dpo/margin": 157.42919921875, "fcm_dpo/q_t": 0.45707207918167114, "grad_norm": 54.09843826293945, "learning_rate": 8.166809758815895e-09, "logits/chosen": -0.6914748549461365, "logits/rejected": -0.7063671946525574, "logps/chosen": -730.0767822265625, "logps/ref_chosen": -273.5590515136719, "logps/ref_rejected": -264.0199279785156, "logps/rejected": -877.9669799804688, "loss": 5.0316, "margin_dpo/margin_mean": 157.42919921875, "margin_dpo/margin_std": 345.0313415527344, "step": 443 }, { "KL/chosen_KL_mean": -447.2994079589844, "KL/mean": -540.459716796875, "KL/rejected_KL_mean": -633.6199951171875, "KL/std": 347.92816162109375, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.0011359085328876972, "fcm_dpo/delta": 0.008714323863387108, "fcm_dpo/margin": 186.3206329345703, "fcm_dpo/q_t": 0.45250868797302246, "grad_norm": 54.263832092285156, "learning_rate": 7.709181040498253e-09, "logits/chosen": -0.7050941586494446, "logits/rejected": -0.6862058639526367, "logps/chosen": -745.443603515625, "logps/ref_chosen": -298.1441955566406, "logps/ref_rejected": -268.0572814941406, "logps/rejected": -901.67724609375, "loss": 4.977, "margin_dpo/margin_mean": 186.3206329345703, "margin_dpo/margin_std": 420.0409851074219, "step": 444 }, { "KL/chosen_KL_mean": -447.3858642578125, "KL/mean": -535.5361938476562, "KL/rejected_KL_mean": -623.6864013671875, "KL/std": 328.20489501953125, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.0011325681116431952, "fcm_dpo/delta": -0.020159322768449783, "fcm_dpo/margin": 176.30056762695312, "fcm_dpo/q_t": 0.4535037875175476, "grad_norm": 53.423500061035156, "learning_rate": 7.2645456434869965e-09, "logits/chosen": -0.7933753728866577, "logits/rejected": -0.7875576019287109, "logps/chosen": -701.9265747070312, "logps/ref_chosen": -254.54067993164062, "logps/ref_rejected": -264.2445983886719, "logps/rejected": -887.9310913085938, "loss": 5.007, "margin_dpo/margin_mean": 176.30055236816406, "margin_dpo/margin_std": 394.967041015625, "step": 445 }, { "KL/chosen_KL_mean": -436.194091796875, "KL/mean": -528.354248046875, "KL/rejected_KL_mean": -620.514404296875, "KL/std": 328.3664855957031, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.001101514557376504, "fcm_dpo/delta": -0.017040304839611053, "fcm_dpo/margin": 184.32025146484375, "fcm_dpo/q_t": 0.4546178877353668, "grad_norm": 41.92047882080078, "learning_rate": 6.832927412229017e-09, "logits/chosen": -0.7097505331039429, "logits/rejected": -0.7096059322357178, "logps/chosen": -742.9165649414062, "logps/ref_chosen": -306.72247314453125, "logps/ref_rejected": -266.3735656738281, "logps/rejected": -886.8878784179688, "loss": 4.9714, "margin_dpo/margin_mean": 184.32025146484375, "margin_dpo/margin_std": 382.3779602050781, "step": 446 }, { "KL/chosen_KL_mean": -450.354736328125, "KL/mean": -550.6195678710938, "KL/rejected_KL_mean": -650.8844604492188, "KL/std": 328.66229248046875, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.0010986090637743473, "fcm_dpo/delta": -0.006220587063580751, "fcm_dpo/margin": 200.52969360351562, "fcm_dpo/q_t": 0.4488736689090729, "grad_norm": 46.463623046875, "learning_rate": 6.414349493100129e-09, "logits/chosen": -0.7155401706695557, "logits/rejected": -0.7002200484275818, "logps/chosen": -710.8720703125, "logps/ref_chosen": -260.51727294921875, "logps/ref_rejected": -236.47061157226562, "logps/rejected": -887.3550415039062, "loss": 4.9114, "margin_dpo/margin_mean": 200.52969360351562, "margin_dpo/margin_std": 392.89251708984375, "step": 447 }, { "KL/chosen_KL_mean": -461.6020202636719, "KL/mean": -561.8016967773438, "KL/rejected_KL_mean": -662.00146484375, "KL/std": 313.98712158203125, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.001077001099474728, "fcm_dpo/delta": -0.028215978294610977, "fcm_dpo/margin": 200.39947509765625, "fcm_dpo/q_t": 0.4488033652305603, "grad_norm": 54.6915168762207, "learning_rate": 6.0088343331638756e-09, "logits/chosen": -0.7094584107398987, "logits/rejected": -0.6863497495651245, "logps/chosen": -730.3890991210938, "logps/ref_chosen": -268.78704833984375, "logps/ref_rejected": -262.1703796386719, "logps/rejected": -924.171875, "loss": 4.8867, "margin_dpo/margin_mean": 200.39947509765625, "margin_dpo/margin_std": 350.0815124511719, "step": 448 }, { "KL/chosen_KL_mean": -462.7024230957031, "KL/mean": -540.6029663085938, "KL/rejected_KL_mean": -618.5035400390625, "KL/std": 322.6525573730469, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.0010794418631121516, "fcm_dpo/delta": 0.01929195038974285, "fcm_dpo/margin": 155.8011016845703, "fcm_dpo/q_t": 0.45973268151283264, "grad_norm": 43.563201904296875, "learning_rate": 5.616403678967624e-09, "logits/chosen": -0.835517168045044, "logits/rejected": -0.8153800368309021, "logps/chosen": -793.65380859375, "logps/ref_chosen": -330.9514465332031, "logps/ref_rejected": -239.76974487304688, "logps/rejected": -858.2732543945312, "loss": 5.074, "margin_dpo/margin_mean": 155.80111694335938, "margin_dpo/margin_std": 373.4229431152344, "step": 449 }, { "KL/chosen_KL_mean": -485.2821044921875, "KL/mean": -559.709228515625, "KL/rejected_KL_mean": -634.1363525390625, "KL/std": 333.56817626953125, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.0010939650237560272, "fcm_dpo/delta": 0.0062581561505794525, "fcm_dpo/margin": 148.8542938232422, "fcm_dpo/q_t": 0.46235862374305725, "grad_norm": 54.444618225097656, "learning_rate": 5.2370785753763356e-09, "logits/chosen": -0.7135239243507385, "logits/rejected": -0.6962549090385437, "logps/chosen": -769.5475463867188, "logps/ref_chosen": -284.26544189453125, "logps/ref_rejected": -250.5401611328125, "logps/rejected": -884.6766357421875, "loss": 5.0828, "margin_dpo/margin_mean": 148.8542938232422, "margin_dpo/margin_std": 354.3857421875, "step": 450 }, { "KL/chosen_KL_mean": -498.91839599609375, "KL/mean": -550.2883911132812, "KL/rejected_KL_mean": -601.6583251953125, "KL/std": 294.4558410644531, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.0010946807451546192, "fcm_dpo/delta": -0.0018840640550479293, "fcm_dpo/margin": 102.73995971679688, "fcm_dpo/q_t": 0.4727487862110138, "grad_norm": 80.72921752929688, "learning_rate": 4.8708793644441086e-09, "logits/chosen": -0.7374417185783386, "logits/rejected": -0.702163815498352, "logps/chosen": -801.2393798828125, "logps/ref_chosen": -302.3209228515625, "logps/ref_rejected": -254.09747314453125, "logps/rejected": -855.7559814453125, "loss": 5.2356, "margin_dpo/margin_mean": 102.73995971679688, "margin_dpo/margin_std": 311.53643798828125, "step": 451 }, { "KL/chosen_KL_mean": -456.5367126464844, "KL/mean": -576.4017333984375, "KL/rejected_KL_mean": -696.2666015625, "KL/std": 430.700927734375, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.0010616311337798834, "fcm_dpo/delta": -0.04442259296774864, "fcm_dpo/margin": 239.7299346923828, "fcm_dpo/q_t": 0.4475480318069458, "grad_norm": 47.51118850708008, "learning_rate": 4.517825684323323e-09, "logits/chosen": -0.7811554670333862, "logits/rejected": -0.7392996549606323, "logps/chosen": -755.9288330078125, "logps/ref_chosen": -299.39215087890625, "logps/ref_rejected": -284.3475036621094, "logps/rejected": -980.6141967773438, "loss": 4.9112, "margin_dpo/margin_mean": 239.72991943359375, "margin_dpo/margin_std": 534.2335205078125, "step": 452 }, { "KL/chosen_KL_mean": -476.1728820800781, "KL/mean": -562.9217529296875, "KL/rejected_KL_mean": -649.6705322265625, "KL/std": 316.76861572265625, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.0010666890302672982, "fcm_dpo/delta": 0.020303022116422653, "fcm_dpo/margin": 173.4977264404297, "fcm_dpo/q_t": 0.45698270201683044, "grad_norm": 51.801551818847656, "learning_rate": 4.1779364682113794e-09, "logits/chosen": -0.7304836511611938, "logits/rejected": -0.7136521339416504, "logps/chosen": -800.8245239257812, "logps/ref_chosen": -324.6517028808594, "logps/ref_rejected": -304.1527099609375, "logps/rejected": -953.8233032226562, "loss": 5.0035, "margin_dpo/margin_mean": 173.4976806640625, "margin_dpo/margin_std": 376.6050109863281, "step": 453 }, { "KL/chosen_KL_mean": -467.48114013671875, "KL/mean": -588.2965087890625, "KL/rejected_KL_mean": -709.1119995117188, "KL/std": 378.76043701171875, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.001057203160598874, "fcm_dpo/delta": -0.029810752719640732, "fcm_dpo/margin": 241.630859375, "fcm_dpo/q_t": 0.44075632095336914, "grad_norm": 77.30159759521484, "learning_rate": 3.851229943335393e-09, "logits/chosen": -0.7590242624282837, "logits/rejected": -0.750261664390564, "logps/chosen": -767.0928955078125, "logps/ref_chosen": -299.6117248535156, "logps/ref_rejected": -303.74224853515625, "logps/rejected": -1012.854248046875, "loss": 4.8069, "margin_dpo/margin_mean": 241.630859375, "margin_dpo/margin_std": 433.0992431640625, "step": 454 }, { "KL/chosen_KL_mean": -456.4439697265625, "KL/mean": -515.0791015625, "KL/rejected_KL_mean": -573.7142944335938, "KL/std": 320.5881652832031, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.0010561775416135788, "fcm_dpo/delta": 0.01066396851092577, "fcm_dpo/margin": 117.27033233642578, "fcm_dpo/q_t": 0.4721914529800415, "grad_norm": 46.713623046875, "learning_rate": 3.5377236299748147e-09, "logits/chosen": -0.7215044498443604, "logits/rejected": -0.7258639335632324, "logps/chosen": -730.0556640625, "logps/ref_chosen": -273.6116943359375, "logps/ref_rejected": -274.4293518066406, "logps/rejected": -848.1436157226562, "loss": 5.2412, "margin_dpo/margin_mean": 117.27033233642578, "margin_dpo/margin_std": 370.8341369628906, "step": 455 }, { "KL/chosen_KL_mean": -489.3581848144531, "KL/mean": -581.25244140625, "KL/rejected_KL_mean": -673.1466674804688, "KL/std": 343.29412841796875, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.001065379474312067, "fcm_dpo/delta": 0.016431938856840134, "fcm_dpo/margin": 183.78857421875, "fcm_dpo/q_t": 0.455960214138031, "grad_norm": 42.45829391479492, "learning_rate": 3.2374343405217884e-09, "logits/chosen": -0.6549510955810547, "logits/rejected": -0.6670107841491699, "logps/chosen": -811.5300903320312, "logps/ref_chosen": -322.17193603515625, "logps/ref_rejected": -294.54461669921875, "logps/rejected": -967.6912841796875, "loss": 5.0033, "margin_dpo/margin_mean": 183.78857421875, "margin_dpo/margin_std": 419.4525146484375, "step": 456 }, { "KL/chosen_KL_mean": -504.164306640625, "KL/mean": -611.7514038085938, "KL/rejected_KL_mean": -719.3384399414062, "KL/std": 379.7641906738281, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.0010634178761392832, "fcm_dpo/delta": -0.008057055063545704, "fcm_dpo/margin": 215.17417907714844, "fcm_dpo/q_t": 0.44393303990364075, "grad_norm": 74.09278106689453, "learning_rate": 2.9503781785795713e-09, "logits/chosen": -0.7260345816612244, "logits/rejected": -0.722815752029419, "logps/chosen": -811.9605712890625, "logps/ref_chosen": -307.7962341308594, "logps/ref_rejected": -274.5501403808594, "logps/rejected": -993.888671875, "loss": 4.9483, "margin_dpo/margin_mean": 215.17417907714844, "margin_dpo/margin_std": 461.9056396484375, "step": 457 }, { "KL/chosen_KL_mean": -467.6044616699219, "KL/mean": -549.5364379882812, "KL/rejected_KL_mean": -631.4684448242188, "KL/std": 351.8533630371094, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.0010722451843321323, "fcm_dpo/delta": 0.013735459186136723, "fcm_dpo/margin": 163.86398315429688, "fcm_dpo/q_t": 0.45987468957901, "grad_norm": 42.21095275878906, "learning_rate": 2.6765705380989432e-09, "logits/chosen": -0.7559491395950317, "logits/rejected": -0.7171483635902405, "logps/chosen": -764.6361694335938, "logps/ref_chosen": -297.0316467285156, "logps/ref_rejected": -276.1112365722656, "logps/rejected": -907.5797119140625, "loss": 5.0922, "margin_dpo/margin_mean": 163.86398315429688, "margin_dpo/margin_std": 412.696044921875, "step": 458 }, { "KL/chosen_KL_mean": -458.0815124511719, "KL/mean": -525.6103515625, "KL/rejected_KL_mean": -593.1392822265625, "KL/std": 330.22552490234375, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.0010813046246767044, "fcm_dpo/delta": 0.006299625150859356, "fcm_dpo/margin": 135.0577392578125, "fcm_dpo/q_t": 0.46749192476272583, "grad_norm": 58.347652435302734, "learning_rate": 2.416026102552732e-09, "logits/chosen": -0.8096132874488831, "logits/rejected": -0.8010511994361877, "logps/chosen": -751.6068115234375, "logps/ref_chosen": -293.5252990722656, "logps/ref_rejected": -289.30126953125, "logps/rejected": -882.4405517578125, "loss": 5.1984, "margin_dpo/margin_mean": 135.05772399902344, "margin_dpo/margin_std": 416.4771728515625, "step": 459 }, { "KL/chosen_KL_mean": -477.0860290527344, "KL/mean": -545.5380859375, "KL/rejected_KL_mean": -613.9900512695312, "KL/std": 299.5758361816406, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.0010851901024580002, "fcm_dpo/delta": 0.008401179686188698, "fcm_dpo/margin": 136.9040069580078, "fcm_dpo/q_t": 0.46470630168914795, "grad_norm": 72.48368072509766, "learning_rate": 2.168758844148272e-09, "logits/chosen": -0.7803556323051453, "logits/rejected": -0.7745279669761658, "logps/chosen": -795.866455078125, "logps/ref_chosen": -318.7803649902344, "logps/ref_rejected": -258.7906799316406, "logps/rejected": -872.7807006835938, "loss": 5.1416, "margin_dpo/margin_mean": 136.9040069580078, "margin_dpo/margin_std": 376.03265380859375, "step": 460 }, { "KL/chosen_KL_mean": -461.55438232421875, "KL/mean": -552.3533325195312, "KL/rejected_KL_mean": -643.15234375, "KL/std": 352.3241882324219, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.0010989790316671133, "fcm_dpo/delta": 0.023482128977775574, "fcm_dpo/margin": 181.59786987304688, "fcm_dpo/q_t": 0.4545283019542694, "grad_norm": 63.46205520629883, "learning_rate": 1.9347820230782295e-09, "logits/chosen": -0.7056773900985718, "logits/rejected": -0.7202193140983582, "logps/chosen": -705.4644165039062, "logps/ref_chosen": -243.9099884033203, "logps/ref_rejected": -232.6382293701172, "logps/rejected": -875.7904663085938, "loss": 5.0086, "margin_dpo/margin_mean": 181.59786987304688, "margin_dpo/margin_std": 421.58538818359375, "step": 461 }, { "KL/chosen_KL_mean": -449.98541259765625, "KL/mean": -550.90478515625, "KL/rejected_KL_mean": -651.82421875, "KL/std": 304.707763671875, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.0011157675180584192, "fcm_dpo/delta": 0.003436810802668333, "fcm_dpo/margin": 201.8387908935547, "fcm_dpo/q_t": 0.44782936573028564, "grad_norm": 72.2857437133789, "learning_rate": 1.7141081868094209e-09, "logits/chosen": -0.7916049957275391, "logits/rejected": -0.7290614247322083, "logps/chosen": -794.076416015625, "logps/ref_chosen": -344.09100341796875, "logps/ref_rejected": -252.45037841796875, "logps/rejected": -904.274658203125, "loss": 4.8414, "margin_dpo/margin_mean": 201.8387908935547, "margin_dpo/margin_std": 333.3164978027344, "step": 462 }, { "KL/chosen_KL_mean": -470.310302734375, "KL/mean": -540.879638671875, "KL/rejected_KL_mean": -611.4489135742188, "KL/std": 348.91265869140625, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.0011070480104535818, "fcm_dpo/delta": -0.016562294214963913, "fcm_dpo/margin": 141.1386260986328, "fcm_dpo/q_t": 0.4637291133403778, "grad_norm": 66.43771362304688, "learning_rate": 1.5067491694100153e-09, "logits/chosen": -0.7600305080413818, "logits/rejected": -0.7077437043190002, "logps/chosen": -767.4527587890625, "logps/ref_chosen": -297.1424560546875, "logps/ref_rejected": -234.0208282470703, "logps/rejected": -845.4697875976562, "loss": 5.174, "margin_dpo/margin_mean": 141.1386260986328, "margin_dpo/margin_std": 422.1974182128906, "step": 463 }, { "KL/chosen_KL_mean": -452.6793518066406, "KL/mean": -539.170166015625, "KL/rejected_KL_mean": -625.6610107421875, "KL/std": 322.0829162597656, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.0011113593354821205, "fcm_dpo/delta": 0.016370417550206184, "fcm_dpo/margin": 172.981689453125, "fcm_dpo/q_t": 0.4547066390514374, "grad_norm": 65.45187377929688, "learning_rate": 1.3127160909147672e-09, "logits/chosen": -0.7473437190055847, "logits/rejected": -0.7489007115364075, "logps/chosen": -718.3900756835938, "logps/ref_chosen": -265.71075439453125, "logps/ref_rejected": -256.4108581542969, "logps/rejected": -882.0718994140625, "loss": 4.9785, "margin_dpo/margin_mean": 172.98167419433594, "margin_dpo/margin_std": 365.59429931640625, "step": 464 }, { "KL/chosen_KL_mean": -431.3951416015625, "KL/mean": -539.25048828125, "KL/rejected_KL_mean": -647.1058349609375, "KL/std": 306.6711730957031, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.0010858203750103712, "fcm_dpo/delta": -0.023821821436285973, "fcm_dpo/margin": 215.71067810058594, "fcm_dpo/q_t": 0.4461546540260315, "grad_norm": 58.74069595336914, "learning_rate": 1.1320193567288527e-09, "logits/chosen": -0.8197015523910522, "logits/rejected": -0.7692691683769226, "logps/chosen": -724.5478515625, "logps/ref_chosen": -293.1527404785156, "logps/ref_rejected": -293.70947265625, "logps/rejected": -940.8153686523438, "loss": 4.8416, "margin_dpo/margin_mean": 215.710693359375, "margin_dpo/margin_std": 361.2246398925781, "step": 465 }, { "KL/chosen_KL_mean": -410.092041015625, "KL/mean": -514.951416015625, "KL/rejected_KL_mean": -619.8109130859375, "KL/std": 301.6080322265625, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.0010852470295503736, "fcm_dpo/delta": -0.006281266920268536, "fcm_dpo/margin": 209.71878051757812, "fcm_dpo/q_t": 0.44774800539016724, "grad_norm": 50.34075927734375, "learning_rate": 9.64668657069706e-10, "logits/chosen": -0.7202876210212708, "logits/rejected": -0.6470254063606262, "logps/chosen": -671.5696411132812, "logps/ref_chosen": -261.4775695800781, "logps/ref_rejected": -248.36282348632812, "logps/rejected": -868.1737060546875, "loss": 4.8706, "margin_dpo/margin_mean": 209.71878051757812, "margin_dpo/margin_std": 377.7794494628906, "step": 466 }, { "KL/chosen_KL_mean": -453.5052795410156, "KL/mean": -544.7555541992188, "KL/rejected_KL_mean": -636.0057373046875, "KL/std": 345.9569396972656, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.0010806998470798135, "fcm_dpo/delta": -0.003298811614513397, "fcm_dpo/margin": 182.50048828125, "fcm_dpo/q_t": 0.4551469683647156, "grad_norm": 58.7347412109375, "learning_rate": 8.106729664475176e-10, "logits/chosen": -0.7169447541236877, "logits/rejected": -0.6979930400848389, "logps/chosen": -719.8595581054688, "logps/ref_chosen": -266.354248046875, "logps/ref_rejected": -277.76324462890625, "logps/rejected": -913.7689819335938, "loss": 5.0016, "margin_dpo/margin_mean": 182.50045776367188, "margin_dpo/margin_std": 418.3126220703125, "step": 467 }, { "KL/chosen_KL_mean": -473.4375, "KL/mean": -515.6995849609375, "KL/rejected_KL_mean": -557.9615478515625, "KL/std": 294.5906677246094, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.0010830379324033856, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.52410888671875, "fcm_dpo/q_t": 0.47659796476364136, "grad_norm": 90.00639343261719, "learning_rate": 6.700405431837585e-10, "logits/chosen": -0.7935588955879211, "logits/rejected": -0.7631844282150269, "logps/chosen": -791.4006958007812, "logps/ref_chosen": -317.9631652832031, "logps/ref_rejected": -261.8744201660156, "logps/rejected": -819.8359985351562, "loss": 5.3815, "margin_dpo/margin_mean": 84.52410125732422, "margin_dpo/margin_std": 381.2652587890625, "step": 468 }, { "KL/chosen_KL_mean": -442.52105712890625, "KL/mean": -534.7864379882812, "KL/rejected_KL_mean": -627.0517578125, "KL/std": 324.3078918457031, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.001092868042178452, "fcm_dpo/delta": 0.0077682118862867355, "fcm_dpo/margin": 184.53070068359375, "fcm_dpo/q_t": 0.45302706956863403, "grad_norm": 45.122615814208984, "learning_rate": 5.427789289685347e-10, "logits/chosen": -0.718509316444397, "logits/rejected": -0.6958325505256653, "logps/chosen": -767.4078369140625, "logps/ref_chosen": -324.8868103027344, "logps/ref_rejected": -264.0421447753906, "logps/rejected": -891.0939331054688, "loss": 4.9723, "margin_dpo/margin_mean": 184.53070068359375, "margin_dpo/margin_std": 391.1455383300781, "step": 469 }, { "KL/chosen_KL_mean": -439.5545654296875, "KL/mean": -550.50146484375, "KL/rejected_KL_mean": -661.4483642578125, "KL/std": 344.94183349609375, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.0010674743680283427, "fcm_dpo/delta": -0.032233696430921555, "fcm_dpo/margin": 221.89381408691406, "fcm_dpo/q_t": 0.44584980607032776, "grad_norm": 44.609622955322266, "learning_rate": 4.288949484559934e-10, "logits/chosen": -0.7209665775299072, "logits/rejected": -0.710899293422699, "logps/chosen": -754.2588500976562, "logps/ref_chosen": -314.7042541503906, "logps/ref_rejected": -259.2276611328125, "logps/rejected": -920.6760864257812, "loss": 4.8792, "margin_dpo/margin_mean": 221.893798828125, "margin_dpo/margin_std": 426.10504150390625, "step": 470 }, { "KL/chosen_KL_mean": -462.39056396484375, "KL/mean": -570.186279296875, "KL/rejected_KL_mean": -677.98193359375, "KL/std": 388.74066162109375, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.0010593057377263904, "fcm_dpo/delta": -0.02124343067407608, "fcm_dpo/margin": 215.5913848876953, "fcm_dpo/q_t": 0.45041680335998535, "grad_norm": 76.77764892578125, "learning_rate": 3.2839470889836627e-10, "logits/chosen": -0.7802831530570984, "logits/rejected": -0.7523050308227539, "logps/chosen": -754.9653930664062, "logps/ref_chosen": -292.5748291015625, "logps/ref_rejected": -298.7525329589844, "logps/rejected": -976.7344970703125, "loss": 4.9355, "margin_dpo/margin_mean": 215.5913848876953, "margin_dpo/margin_std": 460.085693359375, "step": 471 }, { "KL/chosen_KL_mean": -429.9429931640625, "KL/mean": -550.7857666015625, "KL/rejected_KL_mean": -671.6284790039062, "KL/std": 356.4366760253906, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.0010364714544266462, "fcm_dpo/delta": -0.011188654229044914, "fcm_dpo/margin": 241.68548583984375, "fcm_dpo/q_t": 0.44299882650375366, "grad_norm": 62.40871810913086, "learning_rate": 2.412835998185092e-10, "logits/chosen": -0.7540468573570251, "logits/rejected": -0.7562943696975708, "logps/chosen": -673.3168334960938, "logps/ref_chosen": -243.37380981445312, "logps/ref_rejected": -251.12109375, "logps/rejected": -922.7495727539062, "loss": 4.8288, "margin_dpo/margin_mean": 241.68548583984375, "margin_dpo/margin_std": 443.82684326171875, "step": 472 }, { "KL/chosen_KL_mean": -455.9682922363281, "KL/mean": -553.0786743164062, "KL/rejected_KL_mean": -650.1890869140625, "KL/std": 289.64495849609375, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.0010364020708948374, "fcm_dpo/delta": 0.0068814936093986034, "fcm_dpo/margin": 194.22076416015625, "fcm_dpo/q_t": 0.4516716003417969, "grad_norm": 58.57929992675781, "learning_rate": 1.6756629272085544e-10, "logits/chosen": -0.7046954035758972, "logits/rejected": -0.694038987159729, "logps/chosen": -742.2969360351562, "logps/ref_chosen": -286.3286437988281, "logps/ref_rejected": -258.6535339355469, "logps/rejected": -908.8426513671875, "loss": 4.9053, "margin_dpo/margin_mean": 194.22076416015625, "margin_dpo/margin_std": 324.9159240722656, "step": 473 }, { "KL/chosen_KL_mean": -456.2850341796875, "KL/mean": -544.32763671875, "KL/rejected_KL_mean": -632.3701782226562, "KL/std": 346.0625305175781, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.0010376791469752789, "fcm_dpo/delta": -0.004413094371557236, "fcm_dpo/margin": 176.0851287841797, "fcm_dpo/q_t": 0.45776984095573425, "grad_norm": 48.36650085449219, "learning_rate": 1.072467408408384e-10, "logits/chosen": -0.7669036984443665, "logits/rejected": -0.7505558729171753, "logps/chosen": -744.3746337890625, "logps/ref_chosen": -288.08966064453125, "logps/ref_rejected": -266.69696044921875, "logps/rejected": -899.067138671875, "loss": 5.0173, "margin_dpo/margin_mean": 176.0851287841797, "margin_dpo/margin_std": 386.46929931640625, "step": 474 }, { "KL/chosen_KL_mean": -405.2410583496094, "KL/mean": -492.22198486328125, "KL/rejected_KL_mean": -579.202880859375, "KL/std": 290.22174072265625, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.0010348742362111807, "fcm_dpo/delta": -0.004362727515399456, "fcm_dpo/margin": 173.9619140625, "fcm_dpo/q_t": 0.4581862986087799, "grad_norm": 47.80220413208008, "learning_rate": 6.032817893297793e-11, "logits/chosen": -0.7423918843269348, "logits/rejected": -0.7410845756530762, "logps/chosen": -661.244140625, "logps/ref_chosen": -256.0030517578125, "logps/ref_rejected": -244.50660705566406, "logps/rejected": -823.7095336914062, "loss": 4.9956, "margin_dpo/margin_mean": 173.96189880371094, "margin_dpo/margin_std": 340.6141052246094, "step": 475 }, { "KL/chosen_KL_mean": -434.2881774902344, "KL/mean": -515.4061279296875, "KL/rejected_KL_mean": -596.5241088867188, "KL/std": 296.0513610839844, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.0010204007849097252, "fcm_dpo/delta": -0.0251779668033123, "fcm_dpo/margin": 162.23599243164062, "fcm_dpo/q_t": 0.46206921339035034, "grad_norm": 38.84898376464844, "learning_rate": 2.6813123097352287e-11, "logits/chosen": -0.8090430498123169, "logits/rejected": -0.7528334856033325, "logps/chosen": -755.7556762695312, "logps/ref_chosen": -321.467529296875, "logps/ref_rejected": -295.0592956542969, "logps/rejected": -891.5834350585938, "loss": 5.0824, "margin_dpo/margin_mean": 162.23599243164062, "margin_dpo/margin_std": 391.11846923828125, "step": 476 }, { "KL/chosen_KL_mean": -469.63360595703125, "KL/mean": -557.083740234375, "KL/rejected_KL_mean": -644.5338134765625, "KL/std": 356.2486572265625, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.0010169773595407605, "fcm_dpo/delta": 0.00865718349814415, "fcm_dpo/margin": 174.90023803710938, "fcm_dpo/q_t": 0.4589536190032959, "grad_norm": 72.80176544189453, "learning_rate": 6.7033706447061635e-12, "logits/chosen": -0.6653886437416077, "logits/rejected": -0.6609338521957397, "logps/chosen": -746.4275512695312, "logps/ref_chosen": -276.7939758300781, "logps/ref_rejected": -244.82919311523438, "logps/rejected": -889.3629760742188, "loss": 5.0882, "margin_dpo/margin_mean": 174.90023803710938, "margin_dpo/margin_std": 460.7952575683594, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 5.002585172153369, "train_runtime": 6534.5428, "train_samples_per_second": 9.356, "train_steps_per_second": 0.073 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }