{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9989528795811519, "eval_steps": 200, "global_step": 477, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": -0.04536247253417969, "KL/mean": -0.05389120429754257, "KL/rejected_KL_mean": -0.06241607666015625, "KL/std": 0.5513200759887695, "epoch": 0.0020942408376963353, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01704825460910797, "fcm_dpo/q_t": 0.499957412481308, "grad_norm": 14.618375778198242, "learning_rate": 0.0, "logits/chosen": 1.9854813814163208, "logits/rejected": 1.7737188339233398, "logps/chosen": -267.0936584472656, "logps/ref_chosen": -267.04827880859375, "logps/ref_rejected": -219.76657104492188, "logps/rejected": -219.82899475097656, "loss": 5.5446, "margin_dpo/margin_mean": 0.01704716682434082, "margin_dpo/margin_std": 0.7467707991600037, "step": 1 }, { "KL/chosen_KL_mean": 0.07550811767578125, "KL/mean": 0.03060000389814377, "KL/rejected_KL_mean": -0.014301300048828125, "KL/std": 0.6403332352638245, "epoch": 0.004188481675392671, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08979703485965729, "fcm_dpo/q_t": 0.4997755289077759, "grad_norm": 15.139668464660645, "learning_rate": 1.0416666666666666e-08, "logits/chosen": 2.1993117332458496, "logits/rejected": 2.1368322372436523, "logps/chosen": -256.5101623535156, "logps/ref_chosen": -256.585693359375, "logps/ref_rejected": -239.92742919921875, "logps/rejected": -239.94171142578125, "loss": 5.5417, "margin_dpo/margin_mean": 0.08979731798171997, "margin_dpo/margin_std": 0.9745746850967407, "step": 2 }, { "KL/chosen_KL_mean": -0.04197883605957031, "KL/mean": -0.04229529947042465, "KL/rejected_KL_mean": -0.04261016845703125, "KL/std": 0.6228253245353699, "epoch": 0.0062827225130890054, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0006283968687057495, "fcm_dpo/q_t": 0.49999845027923584, "grad_norm": 14.598505973815918, "learning_rate": 2.083333333333333e-08, "logits/chosen": 2.2566211223602295, "logits/rejected": 2.2960007190704346, "logps/chosen": -265.71087646484375, "logps/ref_chosen": -265.66888427734375, "logps/ref_rejected": -284.3778991699219, "logps/rejected": -284.4205322265625, "loss": 5.5452, "margin_dpo/margin_mean": 0.0006298869848251343, "margin_dpo/margin_std": 0.8659858107566833, "step": 3 }, { "KL/chosen_KL_mean": 0.003345489501953125, "KL/mean": -0.003935039043426514, "KL/rejected_KL_mean": -0.011201858520507812, "KL/std": 0.6224133968353271, "epoch": 0.008376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01454666256904602, "fcm_dpo/q_t": 0.4999636113643646, "grad_norm": 15.804097175598145, "learning_rate": 3.125e-08, "logits/chosen": 1.9357259273529053, "logits/rejected": 1.8748366832733154, "logps/chosen": -319.5041198730469, "logps/ref_chosen": -319.50750732421875, "logps/ref_rejected": -296.22314453125, "logps/rejected": -296.2343444824219, "loss": 5.5447, "margin_dpo/margin_mean": 0.014546826481819153, "margin_dpo/margin_std": 0.8421736359596252, "step": 4 }, { "KL/chosen_KL_mean": 0.02983856201171875, "KL/mean": 0.04574793577194214, "KL/rejected_KL_mean": 0.0616607666015625, "KL/std": 0.601071298122406, "epoch": 0.010471204188481676, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0318109393119812, "fcm_dpo/q_t": 0.5000795125961304, "grad_norm": 15.678240776062012, "learning_rate": 4.166666666666666e-08, "logits/chosen": 1.8491719961166382, "logits/rejected": 1.851250410079956, "logps/chosen": -292.8431396484375, "logps/ref_chosen": -292.87298583984375, "logps/ref_rejected": -270.3802795410156, "logps/rejected": -270.3186340332031, "loss": 5.5465, "margin_dpo/margin_mean": -0.03181034326553345, "margin_dpo/margin_std": 0.9028931260108948, "step": 5 }, { "KL/chosen_KL_mean": 0.018728256225585938, "KL/mean": 0.017683424055576324, "KL/rejected_KL_mean": 0.0166473388671875, "KL/std": 0.6028397083282471, "epoch": 0.012565445026178011, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.002083107829093933, "fcm_dpo/q_t": 0.49999481439590454, "grad_norm": 15.508500099182129, "learning_rate": 5.208333333333333e-08, "logits/chosen": 2.140822649002075, "logits/rejected": 2.076831340789795, "logps/chosen": -280.2667236328125, "logps/ref_chosen": -280.28546142578125, "logps/ref_rejected": -242.7611541748047, "logps/rejected": -242.74449157714844, "loss": 5.5452, "margin_dpo/margin_mean": 0.0020822137594223022, "margin_dpo/margin_std": 0.8873875141143799, "step": 6 }, { "KL/chosen_KL_mean": 0.002033233642578125, "KL/mean": 0.03911462053656578, "KL/rejected_KL_mean": 0.0761871337890625, "KL/std": 0.6071482300758362, "epoch": 0.014659685863874346, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07415292412042618, "fcm_dpo/q_t": 0.5001853704452515, "grad_norm": 15.60063648223877, "learning_rate": 6.25e-08, "logits/chosen": 2.155738115310669, "logits/rejected": 2.0826399326324463, "logps/chosen": -255.1036834716797, "logps/ref_chosen": -255.10572814941406, "logps/ref_rejected": -251.15402221679688, "logps/rejected": -251.07785034179688, "loss": 5.5482, "margin_dpo/margin_mean": -0.07415376603603363, "margin_dpo/margin_std": 0.8465871214866638, "step": 7 }, { "KL/chosen_KL_mean": -0.026432037353515625, "KL/mean": 0.003932002931833267, "KL/rejected_KL_mean": 0.03430938720703125, "KL/std": 0.7194060683250427, "epoch": 0.016753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06074821203947067, "fcm_dpo/q_t": 0.5001518726348877, "grad_norm": 15.915802001953125, "learning_rate": 7.291666666666667e-08, "logits/chosen": 2.315553903579712, "logits/rejected": 2.1242852210998535, "logps/chosen": -345.2080078125, "logps/ref_chosen": -345.18157958984375, "logps/ref_rejected": -277.3522644042969, "logps/rejected": -277.3179931640625, "loss": 5.5477, "margin_dpo/margin_mean": -0.06074722111225128, "margin_dpo/margin_std": 1.0226110219955444, "step": 8 }, { "KL/chosen_KL_mean": -0.046703338623046875, "KL/mean": -0.05963290482759476, "KL/rejected_KL_mean": -0.07256317138671875, "KL/std": 0.6330029964447021, "epoch": 0.018848167539267015, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025863781571388245, "fcm_dpo/q_t": 0.4999352991580963, "grad_norm": 14.079815864562988, "learning_rate": 8.333333333333333e-08, "logits/chosen": 2.0162174701690674, "logits/rejected": 1.9880930185317993, "logps/chosen": -276.77703857421875, "logps/ref_chosen": -276.7303161621094, "logps/ref_rejected": -266.19952392578125, "logps/rejected": -266.2720642089844, "loss": 5.5442, "margin_dpo/margin_mean": 0.02586349844932556, "margin_dpo/margin_std": 0.8693335056304932, "step": 9 }, { "KL/chosen_KL_mean": -0.02152252197265625, "KL/mean": 0.0014857053756713867, "KL/rejected_KL_mean": 0.024494171142578125, "KL/std": 0.5658552050590515, "epoch": 0.020942408376963352, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04602077603340149, "fcm_dpo/q_t": 0.5001150369644165, "grad_norm": 13.844391822814941, "learning_rate": 9.375e-08, "logits/chosen": 1.8030073642730713, "logits/rejected": 1.8423787355422974, "logps/chosen": -246.5837860107422, "logps/ref_chosen": -246.562255859375, "logps/ref_rejected": -224.70401000976562, "logps/rejected": -224.67950439453125, "loss": 5.5471, "margin_dpo/margin_mean": -0.0460197776556015, "margin_dpo/margin_std": 0.7490549087524414, "step": 10 }, { "KL/chosen_KL_mean": -0.0169677734375, "KL/mean": -0.038240786641836166, "KL/rejected_KL_mean": -0.05950736999511719, "KL/std": 0.6162736415863037, "epoch": 0.023036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.042532242834568024, "fcm_dpo/q_t": 0.4998936653137207, "grad_norm": 15.514847755432129, "learning_rate": 1.0416666666666667e-07, "logits/chosen": 2.0664830207824707, "logits/rejected": 2.0137524604797363, "logps/chosen": -290.51806640625, "logps/ref_chosen": -290.5010986328125, "logps/ref_rejected": -253.9559326171875, "logps/rejected": -254.01541137695312, "loss": 5.5436, "margin_dpo/margin_mean": 0.042532920837402344, "margin_dpo/margin_std": 0.9051563143730164, "step": 11 }, { "KL/chosen_KL_mean": 0.0077972412109375, "KL/mean": -0.016895174980163574, "KL/rejected_KL_mean": -0.04158592224121094, "KL/std": 0.6474024653434753, "epoch": 0.025130890052356022, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04937353730201721, "fcm_dpo/q_t": 0.49987655878067017, "grad_norm": 14.951948165893555, "learning_rate": 1.1458333333333332e-07, "logits/chosen": 1.7271878719329834, "logits/rejected": 1.7156167030334473, "logps/chosen": -278.5107421875, "logps/ref_chosen": -278.5185241699219, "logps/ref_rejected": -273.56207275390625, "logps/rejected": -273.6036682128906, "loss": 5.5433, "margin_dpo/margin_mean": 0.049374401569366455, "margin_dpo/margin_std": 0.8798114061355591, "step": 12 }, { "KL/chosen_KL_mean": -0.017948150634765625, "KL/mean": -0.02015083283185959, "KL/rejected_KL_mean": -0.0223541259765625, "KL/std": 0.5520142912864685, "epoch": 0.027225130890052355, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00440196692943573, "fcm_dpo/q_t": 0.49998903274536133, "grad_norm": 14.794624328613281, "learning_rate": 1.25e-07, "logits/chosen": 1.9182813167572021, "logits/rejected": 2.0527210235595703, "logps/chosen": -264.6750793457031, "logps/ref_chosen": -264.6571350097656, "logps/ref_rejected": -274.3492736816406, "logps/rejected": -274.3716735839844, "loss": 5.5451, "margin_dpo/margin_mean": 0.004402056336402893, "margin_dpo/margin_std": 0.862347424030304, "step": 13 }, { "KL/chosen_KL_mean": 0.053466796875, "KL/mean": 0.00025077909231185913, "KL/rejected_KL_mean": -0.052967071533203125, "KL/std": 0.5912115573883057, "epoch": 0.02931937172774869, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10642853379249573, "fcm_dpo/q_t": 0.49973392486572266, "grad_norm": 15.18773078918457, "learning_rate": 1.3541666666666666e-07, "logits/chosen": 2.0496034622192383, "logits/rejected": 2.0889523029327393, "logps/chosen": -274.06109619140625, "logps/ref_chosen": -274.1145935058594, "logps/ref_rejected": -261.0951843261719, "logps/rejected": -261.1481628417969, "loss": 5.541, "margin_dpo/margin_mean": 0.10642816126346588, "margin_dpo/margin_std": 0.8385832905769348, "step": 14 }, { "KL/chosen_KL_mean": -0.01445770263671875, "KL/mean": -0.0046601369976997375, "KL/rejected_KL_mean": 0.00513458251953125, "KL/std": 0.5840538740158081, "epoch": 0.031413612565445025, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.019589528441429138, "fcm_dpo/q_t": 0.5000489950180054, "grad_norm": 16.10891342163086, "learning_rate": 1.4583333333333335e-07, "logits/chosen": 2.1259677410125732, "logits/rejected": 2.005582094192505, "logps/chosen": -288.5991516113281, "logps/ref_chosen": -288.584716796875, "logps/ref_rejected": -224.8094482421875, "logps/rejected": -224.8043212890625, "loss": 5.546, "margin_dpo/margin_mean": -0.01959007978439331, "margin_dpo/margin_std": 0.8089416027069092, "step": 15 }, { "KL/chosen_KL_mean": 0.035327911376953125, "KL/mean": 0.012278169393539429, "KL/rejected_KL_mean": -0.010770797729492188, "KL/std": 0.600157618522644, "epoch": 0.033507853403141365, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04610002040863037, "fcm_dpo/q_t": 0.4998847544193268, "grad_norm": 16.78972816467285, "learning_rate": 1.5624999999999999e-07, "logits/chosen": 2.119436502456665, "logits/rejected": 1.9895243644714355, "logps/chosen": -283.80780029296875, "logps/ref_chosen": -283.8431091308594, "logps/ref_rejected": -248.63645935058594, "logps/rejected": -248.64723205566406, "loss": 5.5434, "margin_dpo/margin_mean": 0.04610002040863037, "margin_dpo/margin_std": 0.8715221881866455, "step": 16 }, { "KL/chosen_KL_mean": 0.05317878723144531, "KL/mean": 0.01209481805562973, "KL/rejected_KL_mean": -0.02899169921875, "KL/std": 0.5728281736373901, "epoch": 0.0356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08215896785259247, "fcm_dpo/q_t": 0.499794602394104, "grad_norm": 15.140898704528809, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 1.8333988189697266, "logits/rejected": 1.786064624786377, "logps/chosen": -254.5803985595703, "logps/ref_chosen": -254.6335906982422, "logps/ref_rejected": -245.3981170654297, "logps/rejected": -245.4271240234375, "loss": 5.542, "margin_dpo/margin_mean": 0.08215945959091187, "margin_dpo/margin_std": 0.855034351348877, "step": 17 }, { "KL/chosen_KL_mean": -0.07054710388183594, "KL/mean": -0.010584428906440735, "KL/rejected_KL_mean": 0.04937553405761719, "KL/std": 0.6170819997787476, "epoch": 0.03769633507853403, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.11993196606636047, "fcm_dpo/q_t": 0.5002998113632202, "grad_norm": 15.51552963256836, "learning_rate": 1.7708333333333334e-07, "logits/chosen": 1.833971619606018, "logits/rejected": 1.7458878755569458, "logps/chosen": -287.820068359375, "logps/ref_chosen": -287.7495422363281, "logps/ref_rejected": -273.03363037109375, "logps/rejected": -272.9842529296875, "loss": 5.5501, "margin_dpo/margin_mean": -0.11993266642093658, "margin_dpo/margin_std": 0.842696487903595, "step": 18 }, { "KL/chosen_KL_mean": -0.050800323486328125, "KL/mean": -0.01509222760796547, "KL/rejected_KL_mean": 0.020612716674804688, "KL/std": 0.6209220886230469, "epoch": 0.039790575916230364, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07141081243753433, "fcm_dpo/q_t": 0.5001784563064575, "grad_norm": 14.881866455078125, "learning_rate": 1.875e-07, "logits/chosen": 2.0522818565368652, "logits/rejected": 2.0182793140411377, "logps/chosen": -273.4547119140625, "logps/ref_chosen": -273.4039001464844, "logps/ref_rejected": -260.09722900390625, "logps/rejected": -260.0766296386719, "loss": 5.5481, "margin_dpo/margin_mean": -0.07141060382127762, "margin_dpo/margin_std": 0.7947637438774109, "step": 19 }, { "KL/chosen_KL_mean": -0.011041641235351562, "KL/mean": -0.049523673951625824, "KL/rejected_KL_mean": -0.08799934387207031, "KL/std": 0.5975103974342346, "epoch": 0.041884816753926704, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07696382701396942, "fcm_dpo/q_t": 0.49980759620666504, "grad_norm": 15.470925331115723, "learning_rate": 1.9791666666666664e-07, "logits/chosen": 1.8817099332809448, "logits/rejected": 1.8345247507095337, "logps/chosen": -292.97314453125, "logps/ref_chosen": -292.96209716796875, "logps/ref_rejected": -256.602783203125, "logps/rejected": -256.6907958984375, "loss": 5.5422, "margin_dpo/margin_mean": 0.07696413993835449, "margin_dpo/margin_std": 0.8018198013305664, "step": 20 }, { "KL/chosen_KL_mean": -0.0027065277099609375, "KL/mean": 0.011075474321842194, "KL/rejected_KL_mean": 0.02484893798828125, "KL/std": 0.6228327751159668, "epoch": 0.04397905759162304, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02755419909954071, "fcm_dpo/q_t": 0.5000689029693604, "grad_norm": 15.547636032104492, "learning_rate": 2.0833333333333333e-07, "logits/chosen": 2.1370320320129395, "logits/rejected": 2.0824661254882812, "logps/chosen": -292.3299255371094, "logps/ref_chosen": -292.3271789550781, "logps/ref_rejected": -274.20635986328125, "logps/rejected": -274.1815185546875, "loss": 5.5464, "margin_dpo/margin_mean": -0.02755562961101532, "margin_dpo/margin_std": 0.8300514817237854, "step": 21 }, { "KL/chosen_KL_mean": 0.06711769104003906, "KL/mean": -0.007669083774089813, "KL/rejected_KL_mean": -0.08245086669921875, "KL/std": 0.5317828059196472, "epoch": 0.04607329842931937, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1495741754770279, "fcm_dpo/q_t": 0.499626100063324, "grad_norm": 13.725032806396484, "learning_rate": 2.1875e-07, "logits/chosen": 2.115772008895874, "logits/rejected": 2.1254642009735107, "logps/chosen": -253.1990966796875, "logps/ref_chosen": -253.2662353515625, "logps/ref_rejected": -241.4681396484375, "logps/rejected": -241.55059814453125, "loss": 5.5393, "margin_dpo/margin_mean": 0.14957471191883087, "margin_dpo/margin_std": 0.7799807786941528, "step": 22 }, { "KL/chosen_KL_mean": -0.03246879577636719, "KL/mean": -0.02023410052061081, "KL/rejected_KL_mean": -0.00799560546875, "KL/std": 0.6498331427574158, "epoch": 0.048167539267015703, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02447943389415741, "fcm_dpo/q_t": 0.5000612139701843, "grad_norm": 15.345940589904785, "learning_rate": 2.2916666666666663e-07, "logits/chosen": 2.0507888793945312, "logits/rejected": 2.0380196571350098, "logps/chosen": -275.45404052734375, "logps/ref_chosen": -275.4216003417969, "logps/ref_rejected": -264.44024658203125, "logps/rejected": -264.4482727050781, "loss": 5.5462, "margin_dpo/margin_mean": -0.024479106068611145, "margin_dpo/margin_std": 0.8917433023452759, "step": 23 }, { "KL/chosen_KL_mean": 0.10337448120117188, "KL/mean": 0.08301132917404175, "KL/rejected_KL_mean": 0.06264495849609375, "KL/std": 0.6015381813049316, "epoch": 0.050261780104712044, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.040723443031311035, "fcm_dpo/q_t": 0.49989819526672363, "grad_norm": 16.448129653930664, "learning_rate": 2.3958333333333335e-07, "logits/chosen": 1.785954236984253, "logits/rejected": 1.8136179447174072, "logps/chosen": -293.2742004394531, "logps/ref_chosen": -293.3775634765625, "logps/ref_rejected": -301.59014892578125, "logps/rejected": -301.5274963378906, "loss": 5.5436, "margin_dpo/margin_mean": 0.040722861886024475, "margin_dpo/margin_std": 0.8323075771331787, "step": 24 }, { "KL/chosen_KL_mean": 0.0938873291015625, "KL/mean": 0.043157801032066345, "KL/rejected_KL_mean": -0.007564544677734375, "KL/std": 0.5598914623260498, "epoch": 0.05235602094240838, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10144254565238953, "fcm_dpo/q_t": 0.4997463524341583, "grad_norm": 14.537904739379883, "learning_rate": 2.5e-07, "logits/chosen": 1.7452150583267212, "logits/rejected": 1.7821879386901855, "logps/chosen": -240.8397979736328, "logps/ref_chosen": -240.93368530273438, "logps/ref_rejected": -235.3230743408203, "logps/rejected": -235.33062744140625, "loss": 5.5412, "margin_dpo/margin_mean": 0.10144302248954773, "margin_dpo/margin_std": 0.6517617106437683, "step": 25 }, { "KL/chosen_KL_mean": 0.03214263916015625, "KL/mean": 0.013769354671239853, "KL/rejected_KL_mean": -0.004604339599609375, "KL/std": 0.5574186444282532, "epoch": 0.05445026178010471, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03675615042448044, "fcm_dpo/q_t": 0.49990811944007874, "grad_norm": 15.54818344116211, "learning_rate": 2.604166666666667e-07, "logits/chosen": 1.6013331413269043, "logits/rejected": 1.569954752922058, "logps/chosen": -273.5631408691406, "logps/ref_chosen": -273.5953369140625, "logps/ref_rejected": -234.49148559570312, "logps/rejected": -234.49609375, "loss": 5.5438, "margin_dpo/margin_mean": 0.036755383014678955, "margin_dpo/margin_std": 0.6695701479911804, "step": 26 }, { "KL/chosen_KL_mean": 0.028787612915039062, "KL/mean": 0.015575557947158813, "KL/rejected_KL_mean": 0.00238037109375, "KL/std": 0.6904422640800476, "epoch": 0.05654450261780105, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.026397570967674255, "fcm_dpo/q_t": 0.49993401765823364, "grad_norm": 15.199823379516602, "learning_rate": 2.708333333333333e-07, "logits/chosen": 2.25942063331604, "logits/rejected": 2.0365524291992188, "logps/chosen": -314.235107421875, "logps/ref_chosen": -314.26385498046875, "logps/ref_rejected": -242.06735229492188, "logps/rejected": -242.06497192382812, "loss": 5.5442, "margin_dpo/margin_mean": 0.02639685571193695, "margin_dpo/margin_std": 0.8937135934829712, "step": 27 }, { "KL/chosen_KL_mean": 0.0720367431640625, "KL/mean": 0.05425500497221947, "KL/rejected_KL_mean": 0.03647041320800781, "KL/std": 0.5645954012870789, "epoch": 0.05863874345549738, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03556331247091293, "fcm_dpo/q_t": 0.4999110698699951, "grad_norm": 15.47055435180664, "learning_rate": 2.8125e-07, "logits/chosen": 2.1194231510162354, "logits/rejected": 2.1313040256500244, "logps/chosen": -240.66676330566406, "logps/ref_chosen": -240.7388153076172, "logps/ref_rejected": -237.8331298828125, "logps/rejected": -237.79666137695312, "loss": 5.5438, "margin_dpo/margin_mean": 0.035564124584198, "margin_dpo/margin_std": 0.7161281704902649, "step": 28 }, { "KL/chosen_KL_mean": 0.11529350280761719, "KL/mean": 0.05905672162771225, "KL/rejected_KL_mean": 0.002834320068359375, "KL/std": 0.643757164478302, "epoch": 0.060732984293193716, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11245884001255035, "fcm_dpo/q_t": 0.49971887469291687, "grad_norm": 14.707746505737305, "learning_rate": 2.916666666666667e-07, "logits/chosen": 2.1459810733795166, "logits/rejected": 2.06351900100708, "logps/chosen": -307.714111328125, "logps/ref_chosen": -307.82940673828125, "logps/ref_rejected": -281.8109130859375, "logps/rejected": -281.8080749511719, "loss": 5.5408, "margin_dpo/margin_mean": 0.11245732009410858, "margin_dpo/margin_std": 0.9345981478691101, "step": 29 }, { "KL/chosen_KL_mean": 0.024847030639648438, "KL/mean": 0.031680867075920105, "KL/rejected_KL_mean": 0.038501739501953125, "KL/std": 0.6008696556091309, "epoch": 0.06282722513089005, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.013647228479385376, "fcm_dpo/q_t": 0.5000340938568115, "grad_norm": 14.452679634094238, "learning_rate": 3.020833333333333e-07, "logits/chosen": 1.5792551040649414, "logits/rejected": 1.5667495727539062, "logps/chosen": -247.98892211914062, "logps/ref_chosen": -248.01376342773438, "logps/ref_rejected": -259.0014953613281, "logps/rejected": -258.9629821777344, "loss": 5.5458, "margin_dpo/margin_mean": -0.013647839426994324, "margin_dpo/margin_std": 0.7875149250030518, "step": 30 }, { "KL/chosen_KL_mean": 0.11716651916503906, "KL/mean": 0.03221321105957031, "KL/rejected_KL_mean": -0.05274772644042969, "KL/std": 0.6022624969482422, "epoch": 0.06492146596858639, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16990646719932556, "fcm_dpo/q_t": 0.4995752274990082, "grad_norm": 13.671382904052734, "learning_rate": 3.1249999999999997e-07, "logits/chosen": 1.8226486444473267, "logits/rejected": 1.7482107877731323, "logps/chosen": -239.94564819335938, "logps/ref_chosen": -240.0628204345703, "logps/ref_rejected": -201.91021728515625, "logps/rejected": -201.96295166015625, "loss": 5.5384, "margin_dpo/margin_mean": 0.16990603506565094, "margin_dpo/margin_std": 0.7710446119308472, "step": 31 }, { "KL/chosen_KL_mean": 0.1855602264404297, "KL/mean": 0.13113217055797577, "KL/rejected_KL_mean": 0.07670211791992188, "KL/std": 0.6365066170692444, "epoch": 0.06701570680628273, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10885399580001831, "fcm_dpo/q_t": 0.49972790479660034, "grad_norm": 15.790374755859375, "learning_rate": 3.2291666666666666e-07, "logits/chosen": 1.9913053512573242, "logits/rejected": 1.9235936403274536, "logps/chosen": -293.9466857910156, "logps/ref_chosen": -294.1322326660156, "logps/ref_rejected": -265.8150634765625, "logps/rejected": -265.7383728027344, "loss": 5.5409, "margin_dpo/margin_mean": 0.10885459184646606, "margin_dpo/margin_std": 0.7468684911727905, "step": 32 }, { "KL/chosen_KL_mean": 0.29471588134765625, "KL/mean": 0.17659299075603485, "KL/rejected_KL_mean": 0.058460235595703125, "KL/std": 0.698663055896759, "epoch": 0.06910994764397906, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23624789714813232, "fcm_dpo/q_t": 0.49940937757492065, "grad_norm": 14.96553897857666, "learning_rate": 3.333333333333333e-07, "logits/chosen": 1.9445685148239136, "logits/rejected": 1.90474534034729, "logps/chosen": -275.36041259765625, "logps/ref_chosen": -275.6551208496094, "logps/ref_rejected": -253.5972137451172, "logps/rejected": -253.53875732421875, "loss": 5.5358, "margin_dpo/margin_mean": 0.23624709248542786, "margin_dpo/margin_std": 1.0003958940505981, "step": 33 }, { "KL/chosen_KL_mean": 0.15773773193359375, "KL/mean": 0.08135133981704712, "KL/rejected_KL_mean": 0.0049591064453125, "KL/std": 0.7322280406951904, "epoch": 0.0712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15276679396629333, "fcm_dpo/q_t": 0.4996180534362793, "grad_norm": 17.711772918701172, "learning_rate": 3.4375e-07, "logits/chosen": 1.9582128524780273, "logits/rejected": 1.8374905586242676, "logps/chosen": -332.8123474121094, "logps/ref_chosen": -332.9701232910156, "logps/ref_rejected": -278.2213439941406, "logps/rejected": -278.2164306640625, "loss": 5.5392, "margin_dpo/margin_mean": 0.1527663916349411, "margin_dpo/margin_std": 0.99702388048172, "step": 34 }, { "KL/chosen_KL_mean": 0.2559928894042969, "KL/mean": 0.22444842755794525, "KL/rejected_KL_mean": 0.1929035186767578, "KL/std": 0.6824629902839661, "epoch": 0.07329842931937172, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06308795511722565, "fcm_dpo/q_t": 0.49984225630760193, "grad_norm": 13.625845909118652, "learning_rate": 3.541666666666667e-07, "logits/chosen": 1.95424222946167, "logits/rejected": 2.0227391719818115, "logps/chosen": -258.17291259765625, "logps/ref_chosen": -258.42889404296875, "logps/ref_rejected": -259.5119934082031, "logps/rejected": -259.3190612792969, "loss": 5.5427, "margin_dpo/margin_mean": 0.06308749318122864, "margin_dpo/margin_std": 0.8707388043403625, "step": 35 }, { "KL/chosen_KL_mean": 0.3726043701171875, "KL/mean": 0.23814818263053894, "KL/rejected_KL_mean": 0.10369491577148438, "KL/std": 0.7664194107055664, "epoch": 0.07539267015706806, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2689109444618225, "fcm_dpo/q_t": 0.49932774901390076, "grad_norm": 17.59621810913086, "learning_rate": 3.645833333333333e-07, "logits/chosen": 1.8321075439453125, "logits/rejected": 1.7767295837402344, "logps/chosen": -290.24676513671875, "logps/ref_chosen": -290.6193542480469, "logps/ref_rejected": -280.2503967285156, "logps/rejected": -280.1466979980469, "loss": 5.5346, "margin_dpo/margin_mean": 0.26891028881073, "margin_dpo/margin_std": 1.0319002866744995, "step": 36 }, { "KL/chosen_KL_mean": 0.3753223419189453, "KL/mean": 0.2544947862625122, "KL/rejected_KL_mean": 0.13367462158203125, "KL/std": 0.7211363911628723, "epoch": 0.0774869109947644, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24163669347763062, "fcm_dpo/q_t": 0.4993959367275238, "grad_norm": 15.294231414794922, "learning_rate": 3.75e-07, "logits/chosen": 1.7585910558700562, "logits/rejected": 1.740007996559143, "logps/chosen": -303.197509765625, "logps/ref_chosen": -303.57281494140625, "logps/ref_rejected": -294.56878662109375, "logps/rejected": -294.4350891113281, "loss": 5.5356, "margin_dpo/margin_mean": 0.24163639545440674, "margin_dpo/margin_std": 0.9587388038635254, "step": 37 }, { "KL/chosen_KL_mean": 0.30801963806152344, "KL/mean": 0.2135545313358307, "KL/rejected_KL_mean": 0.11908149719238281, "KL/std": 0.7989519238471985, "epoch": 0.07958115183246073, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1889396458864212, "fcm_dpo/q_t": 0.4995276629924774, "grad_norm": 15.346175193786621, "learning_rate": 3.8541666666666665e-07, "logits/chosen": 1.7355865240097046, "logits/rejected": 1.786137580871582, "logps/chosen": -272.2454833984375, "logps/ref_chosen": -272.5534973144531, "logps/ref_rejected": -266.9136047363281, "logps/rejected": -266.7945251464844, "loss": 5.5378, "margin_dpo/margin_mean": 0.18893998861312866, "margin_dpo/margin_std": 1.0406105518341064, "step": 38 }, { "KL/chosen_KL_mean": 0.512542724609375, "KL/mean": 0.30288946628570557, "KL/rejected_KL_mean": 0.09322929382324219, "KL/std": 0.7305359244346619, "epoch": 0.08167539267015707, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4193153977394104, "fcm_dpo/q_t": 0.4989517331123352, "grad_norm": 15.872249603271484, "learning_rate": 3.958333333333333e-07, "logits/chosen": 1.7924615144729614, "logits/rejected": 1.7269222736358643, "logps/chosen": -264.87396240234375, "logps/ref_chosen": -265.38653564453125, "logps/ref_rejected": -244.34774780273438, "logps/rejected": -244.25454711914062, "loss": 5.5285, "margin_dpo/margin_mean": 0.41931572556495667, "margin_dpo/margin_std": 0.9724928140640259, "step": 39 }, { "KL/chosen_KL_mean": 0.5598220825195312, "KL/mean": 0.3143317401409149, "KL/rejected_KL_mean": 0.06884765625, "KL/std": 0.7974763512611389, "epoch": 0.08376963350785341, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.49096524715423584, "fcm_dpo/q_t": 0.49877268075942993, "grad_norm": 14.932929992675781, "learning_rate": 4.0625e-07, "logits/chosen": 1.794917345046997, "logits/rejected": 1.783827543258667, "logps/chosen": -274.333984375, "logps/ref_chosen": -274.893798828125, "logps/ref_rejected": -244.3120880126953, "logps/rejected": -244.2432403564453, "loss": 5.5257, "margin_dpo/margin_mean": 0.4909658432006836, "margin_dpo/margin_std": 1.0364124774932861, "step": 40 }, { "KL/chosen_KL_mean": 0.4529266357421875, "KL/mean": 0.3263741731643677, "KL/rejected_KL_mean": 0.19981765747070312, "KL/std": 0.7767796516418457, "epoch": 0.08586387434554973, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.25309914350509644, "fcm_dpo/q_t": 0.49936726689338684, "grad_norm": 15.412158966064453, "learning_rate": 4.1666666666666667e-07, "logits/chosen": 2.0711989402770996, "logits/rejected": 2.011732816696167, "logps/chosen": -284.3462219238281, "logps/ref_chosen": -284.7991027832031, "logps/ref_rejected": -248.8882598876953, "logps/rejected": -248.68844604492188, "loss": 5.5352, "margin_dpo/margin_mean": 0.25309881567955017, "margin_dpo/margin_std": 1.022508978843689, "step": 41 }, { "KL/chosen_KL_mean": 0.43947601318359375, "KL/mean": 0.2273520678281784, "KL/rejected_KL_mean": 0.015222549438476562, "KL/std": 0.812735378742218, "epoch": 0.08795811518324607, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4242541790008545, "fcm_dpo/q_t": 0.4989393949508667, "grad_norm": 14.70630168914795, "learning_rate": 4.270833333333333e-07, "logits/chosen": 1.9239072799682617, "logits/rejected": 1.9785937070846558, "logps/chosen": -245.4001922607422, "logps/ref_chosen": -245.83966064453125, "logps/ref_rejected": -259.2870788574219, "logps/rejected": -259.2718505859375, "loss": 5.5284, "margin_dpo/margin_mean": 0.42425358295440674, "margin_dpo/margin_std": 1.125490427017212, "step": 42 }, { "KL/chosen_KL_mean": 0.5615425109863281, "KL/mean": 0.32219600677490234, "KL/rejected_KL_mean": 0.08284378051757812, "KL/std": 0.8720331192016602, "epoch": 0.09005235602094241, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4786991775035858, "fcm_dpo/q_t": 0.4988033175468445, "grad_norm": 15.70401668548584, "learning_rate": 4.375e-07, "logits/chosen": 1.87813138961792, "logits/rejected": 1.861478328704834, "logps/chosen": -307.83599853515625, "logps/ref_chosen": -308.39752197265625, "logps/ref_rejected": -270.3797912597656, "logps/rejected": -270.2969665527344, "loss": 5.5262, "margin_dpo/margin_mean": 0.4787002503871918, "margin_dpo/margin_std": 1.0944887399673462, "step": 43 }, { "KL/chosen_KL_mean": 0.6350421905517578, "KL/mean": 0.4450918138027191, "KL/rejected_KL_mean": 0.2551383972167969, "KL/std": 0.9023072719573975, "epoch": 0.09214659685863874, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3798975348472595, "fcm_dpo/q_t": 0.4990502893924713, "grad_norm": 15.863829612731934, "learning_rate": 4.479166666666667e-07, "logits/chosen": 1.9397392272949219, "logits/rejected": 1.9450037479400635, "logps/chosen": -274.7266845703125, "logps/ref_chosen": -275.36175537109375, "logps/ref_rejected": -279.7540588378906, "logps/rejected": -279.49896240234375, "loss": 5.5302, "margin_dpo/margin_mean": 0.3798980116844177, "margin_dpo/margin_std": 1.2212566137313843, "step": 44 }, { "KL/chosen_KL_mean": 0.7438888549804688, "KL/mean": 0.46234309673309326, "KL/rejected_KL_mean": 0.18078994750976562, "KL/std": 1.0001219511032104, "epoch": 0.09424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5631004571914673, "fcm_dpo/q_t": 0.4985923171043396, "grad_norm": 16.84543228149414, "learning_rate": 4.5833333333333327e-07, "logits/chosen": 1.862227439880371, "logits/rejected": 1.8337498903274536, "logps/chosen": -314.3763427734375, "logps/ref_chosen": -315.1202087402344, "logps/ref_rejected": -290.34210205078125, "logps/rejected": -290.1612854003906, "loss": 5.5229, "margin_dpo/margin_mean": 0.5630996227264404, "margin_dpo/margin_std": 1.3207401037216187, "step": 45 }, { "KL/chosen_KL_mean": 0.7039508819580078, "KL/mean": 0.383242130279541, "KL/rejected_KL_mean": 0.06253814697265625, "KL/std": 1.0930442810058594, "epoch": 0.09633507853403141, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.641418993473053, "fcm_dpo/q_t": 0.4983965754508972, "grad_norm": 16.388086318969727, "learning_rate": 4.6874999999999996e-07, "logits/chosen": 1.6258591413497925, "logits/rejected": 1.6258472204208374, "logps/chosen": -292.23663330078125, "logps/ref_chosen": -292.94061279296875, "logps/ref_rejected": -264.0979309082031, "logps/rejected": -264.035400390625, "loss": 5.5198, "margin_dpo/margin_mean": 0.6414193511009216, "margin_dpo/margin_std": 1.4341840744018555, "step": 46 }, { "KL/chosen_KL_mean": 0.6179580688476562, "KL/mean": 0.4481770098209381, "KL/rejected_KL_mean": 0.2783985137939453, "KL/std": 0.9199997186660767, "epoch": 0.09842931937172775, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3395555317401886, "fcm_dpo/q_t": 0.4991511404514313, "grad_norm": 13.547813415527344, "learning_rate": 4.791666666666667e-07, "logits/chosen": 1.856745719909668, "logits/rejected": 1.963684320449829, "logps/chosen": -254.23472595214844, "logps/ref_chosen": -254.85269165039062, "logps/ref_rejected": -255.88380432128906, "logps/rejected": -255.60537719726562, "loss": 5.5318, "margin_dpo/margin_mean": 0.3395553231239319, "margin_dpo/margin_std": 1.2476435899734497, "step": 47 }, { "KL/chosen_KL_mean": 0.7965354919433594, "KL/mean": 0.48358938097953796, "KL/rejected_KL_mean": 0.1706409454345703, "KL/std": 1.1975098848342896, "epoch": 0.10052356020942409, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6258994340896606, "fcm_dpo/q_t": 0.49843546748161316, "grad_norm": 14.78850269317627, "learning_rate": 4.895833333333333e-07, "logits/chosen": 1.9663748741149902, "logits/rejected": 1.955999493598938, "logps/chosen": -261.711181640625, "logps/ref_chosen": -262.5077209472656, "logps/ref_rejected": -239.25320434570312, "logps/rejected": -239.08258056640625, "loss": 5.5204, "margin_dpo/margin_mean": 0.625899076461792, "margin_dpo/margin_std": 1.5170960426330566, "step": 48 }, { "KL/chosen_KL_mean": 0.8709144592285156, "KL/mean": 0.6166729927062988, "KL/rejected_KL_mean": 0.3624401092529297, "KL/std": 1.174827218055725, "epoch": 0.10261780104712041, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5084781646728516, "fcm_dpo/q_t": 0.49872881174087524, "grad_norm": 15.39860725402832, "learning_rate": 5e-07, "logits/chosen": 1.9863102436065674, "logits/rejected": 2.099377393722534, "logps/chosen": -248.65695190429688, "logps/ref_chosen": -249.5278778076172, "logps/ref_rejected": -254.6465301513672, "logps/rejected": -254.2841033935547, "loss": 5.5251, "margin_dpo/margin_mean": 0.5084781646728516, "margin_dpo/margin_std": 1.5813007354736328, "step": 49 }, { "KL/chosen_KL_mean": 0.9026336669921875, "KL/mean": 0.7201584577560425, "KL/rejected_KL_mean": 0.5376853942871094, "KL/std": 1.1079752445220947, "epoch": 0.10471204188481675, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3649483919143677, "fcm_dpo/q_t": 0.49908769130706787, "grad_norm": 14.629812240600586, "learning_rate": 4.999932966293553e-07, "logits/chosen": 1.9577043056488037, "logits/rejected": 1.9702847003936768, "logps/chosen": -291.59906005859375, "logps/ref_chosen": -292.501708984375, "logps/ref_rejected": -294.6496887207031, "logps/rejected": -294.11199951171875, "loss": 5.5308, "margin_dpo/margin_mean": 0.36494964361190796, "margin_dpo/margin_std": 1.3937370777130127, "step": 50 }, { "KL/chosen_KL_mean": 0.9896411895751953, "KL/mean": 0.6506175994873047, "KL/rejected_KL_mean": 0.3115806579589844, "KL/std": 1.320156216621399, "epoch": 0.1068062827225131, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6780645251274109, "fcm_dpo/q_t": 0.49830490350723267, "grad_norm": 14.741464614868164, "learning_rate": 4.999731868769026e-07, "logits/chosen": 1.3491064310073853, "logits/rejected": 1.4821522235870361, "logps/chosen": -258.87225341796875, "logps/ref_chosen": -259.8619079589844, "logps/ref_rejected": -293.1994934082031, "logps/rejected": -292.887939453125, "loss": 5.5185, "margin_dpo/margin_mean": 0.6780642867088318, "margin_dpo/margin_std": 1.7821853160858154, "step": 51 }, { "KL/chosen_KL_mean": 1.2401237487792969, "KL/mean": 0.7772451043128967, "KL/rejected_KL_mean": 0.3143730163574219, "KL/std": 1.382419466972351, "epoch": 0.10890052356020942, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9257485270500183, "fcm_dpo/q_t": 0.497685968875885, "grad_norm": 16.996623992919922, "learning_rate": 4.99939671821067e-07, "logits/chosen": 1.9146828651428223, "logits/rejected": 1.8284406661987305, "logps/chosen": -297.1852111816406, "logps/ref_chosen": -298.42529296875, "logps/ref_rejected": -278.5128479003906, "logps/rejected": -278.1984558105469, "loss": 5.5086, "margin_dpo/margin_mean": 0.9257479906082153, "margin_dpo/margin_std": 1.7723982334136963, "step": 52 }, { "KL/chosen_KL_mean": 1.19000244140625, "KL/mean": 0.8038426637649536, "KL/rejected_KL_mean": 0.4176826477050781, "KL/std": 1.428905725479126, "epoch": 0.11099476439790576, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7723150849342346, "fcm_dpo/q_t": 0.4980694651603699, "grad_norm": 15.993934631347656, "learning_rate": 4.998927532591591e-07, "logits/chosen": 1.8577872514724731, "logits/rejected": 1.927185297012329, "logps/chosen": -274.1184997558594, "logps/ref_chosen": -275.3085021972656, "logps/ref_rejected": -267.97381591796875, "logps/rejected": -267.55615234375, "loss": 5.5148, "margin_dpo/margin_mean": 0.7723162770271301, "margin_dpo/margin_std": 1.8218241930007935, "step": 53 }, { "KL/chosen_KL_mean": 1.3046531677246094, "KL/mean": 0.9638843536376953, "KL/rejected_KL_mean": 0.62310791015625, "KL/std": 1.5169343948364258, "epoch": 0.1130890052356021, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6815524697303772, "fcm_dpo/q_t": 0.4982964098453522, "grad_norm": 14.538448333740234, "learning_rate": 4.998324337072792e-07, "logits/chosen": 1.6982616186141968, "logits/rejected": 1.7266590595245361, "logps/chosen": -296.8426818847656, "logps/ref_chosen": -298.1473388671875, "logps/ref_rejected": -269.7539367675781, "logps/rejected": -269.1308288574219, "loss": 5.5185, "margin_dpo/margin_mean": 0.6815527081489563, "margin_dpo/margin_std": 2.04019832611084, "step": 54 }, { "KL/chosen_KL_mean": 1.1596031188964844, "KL/mean": 0.8441203832626343, "KL/rejected_KL_mean": 0.5286293029785156, "KL/std": 1.3791707754135132, "epoch": 0.11518324607329843, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6309798955917358, "fcm_dpo/q_t": 0.4984227120876312, "grad_norm": 12.82992172241211, "learning_rate": 4.997587164001815e-07, "logits/chosen": 1.7965575456619263, "logits/rejected": 1.8571105003356934, "logps/chosen": -215.5903778076172, "logps/ref_chosen": -216.74998474121094, "logps/ref_rejected": -202.7234649658203, "logps/rejected": -202.19483947753906, "loss": 5.5203, "margin_dpo/margin_mean": 0.6309804320335388, "margin_dpo/margin_std": 1.8771089315414429, "step": 55 }, { "KL/chosen_KL_mean": 1.3335132598876953, "KL/mean": 0.8361304998397827, "KL/rejected_KL_mean": 0.3387489318847656, "KL/std": 1.7761727571487427, "epoch": 0.11727748691099477, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9947671294212341, "fcm_dpo/q_t": 0.4975135326385498, "grad_norm": 14.402202606201172, "learning_rate": 4.996716052911017e-07, "logits/chosen": 1.4954833984375, "logits/rejected": 1.577931523323059, "logps/chosen": -242.35787963867188, "logps/ref_chosen": -243.69140625, "logps/ref_rejected": -234.970947265625, "logps/rejected": -234.63218688964844, "loss": 5.5061, "margin_dpo/margin_mean": 0.9947657585144043, "margin_dpo/margin_std": 2.2741339206695557, "step": 56 }, { "KL/chosen_KL_mean": 1.683126449584961, "KL/mean": 0.9615461230278015, "KL/rejected_KL_mean": 0.23997116088867188, "KL/std": 1.7344322204589844, "epoch": 0.1193717277486911, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4431445598602295, "fcm_dpo/q_t": 0.4963931739330292, "grad_norm": 17.59752655029297, "learning_rate": 4.99571105051544e-07, "logits/chosen": 2.170443296432495, "logits/rejected": 2.078343152999878, "logps/chosen": -309.0632019042969, "logps/ref_chosen": -310.746337890625, "logps/ref_rejected": -270.6407165527344, "logps/rejected": -270.4007263183594, "loss": 5.4882, "margin_dpo/margin_mean": 1.44314444065094, "margin_dpo/margin_std": 2.150846004486084, "step": 57 }, { "KL/chosen_KL_mean": 1.324371337890625, "KL/mean": 0.7948843240737915, "KL/rejected_KL_mean": 0.26540565490722656, "KL/std": 1.598447322845459, "epoch": 0.12146596858638743, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.058971643447876, "fcm_dpo/q_t": 0.497352659702301, "grad_norm": 14.892204284667969, "learning_rate": 4.994572210710314e-07, "logits/chosen": 1.8611407279968262, "logits/rejected": 1.8537086248397827, "logps/chosen": -257.4886169433594, "logps/ref_chosen": -258.8130187988281, "logps/ref_rejected": -268.2101745605469, "logps/rejected": -267.9447937011719, "loss": 5.5034, "margin_dpo/margin_mean": 1.0589708089828491, "margin_dpo/margin_std": 1.9082849025726318, "step": 58 }, { "KL/chosen_KL_mean": 1.4427604675292969, "KL/mean": 1.0179864168167114, "KL/rejected_KL_mean": 0.5932121276855469, "KL/std": 1.8750296831130981, "epoch": 0.12356020942408377, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8495504260063171, "fcm_dpo/q_t": 0.49787670373916626, "grad_norm": 15.2084321975708, "learning_rate": 4.993299594568162e-07, "logits/chosen": 1.5610744953155518, "logits/rejected": 1.4733085632324219, "logps/chosen": -275.7986755371094, "logps/ref_chosen": -277.2414245605469, "logps/ref_rejected": -259.13519287109375, "logps/rejected": -258.5419921875, "loss": 5.512, "margin_dpo/margin_mean": 0.8495496511459351, "margin_dpo/margin_std": 2.5182838439941406, "step": 59 }, { "KL/chosen_KL_mean": 1.5604877471923828, "KL/mean": 0.9622378945350647, "KL/rejected_KL_mean": 0.3640003204345703, "KL/std": 1.7057533264160156, "epoch": 0.1256544502617801, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.196488857269287, "fcm_dpo/q_t": 0.4970093071460724, "grad_norm": 14.753252029418945, "learning_rate": 4.991893270335525e-07, "logits/chosen": 1.9633570909500122, "logits/rejected": 2.014233112335205, "logps/chosen": -249.78536987304688, "logps/ref_chosen": -251.34585571289062, "logps/ref_rejected": -251.4120635986328, "logps/rejected": -251.04806518554688, "loss": 5.4981, "margin_dpo/margin_mean": 1.1964893341064453, "margin_dpo/margin_std": 2.2795042991638184, "step": 60 }, { "KL/chosen_KL_mean": 1.6346378326416016, "KL/mean": 1.08052659034729, "KL/rejected_KL_mean": 0.5264148712158203, "KL/std": 1.824796199798584, "epoch": 0.12774869109947645, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.108229637145996, "fcm_dpo/q_t": 0.49723008275032043, "grad_norm": 15.012340545654297, "learning_rate": 4.990353313429303e-07, "logits/chosen": 1.9480056762695312, "logits/rejected": 1.9169143438339233, "logps/chosen": -267.34796142578125, "logps/ref_chosen": -268.98260498046875, "logps/ref_rejected": -249.34474182128906, "logps/rejected": -248.81832885742188, "loss": 5.5017, "margin_dpo/margin_mean": 1.1082285642623901, "margin_dpo/margin_std": 2.4616637229919434, "step": 61 }, { "KL/chosen_KL_mean": 2.027973175048828, "KL/mean": 1.4072773456573486, "KL/rejected_KL_mean": 0.7865867614746094, "KL/std": 2.0561020374298096, "epoch": 0.12984293193717278, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.241378903388977, "fcm_dpo/q_t": 0.4968973994255066, "grad_norm": 14.663749694824219, "learning_rate": 4.988679806432711e-07, "logits/chosen": 1.9495458602905273, "logits/rejected": 1.9271520376205444, "logps/chosen": -262.1755676269531, "logps/ref_chosen": -264.2034912109375, "logps/ref_rejected": -260.2911376953125, "logps/rejected": -259.50457763671875, "loss": 5.4964, "margin_dpo/margin_mean": 1.2413785457611084, "margin_dpo/margin_std": 2.655601739883423, "step": 62 }, { "KL/chosen_KL_mean": 1.924692153930664, "KL/mean": 1.2763150930404663, "KL/rejected_KL_mean": 0.6279335021972656, "KL/std": 2.1142983436584473, "epoch": 0.1319371727748691, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2967690229415894, "fcm_dpo/q_t": 0.4967590272426605, "grad_norm": 14.749189376831055, "learning_rate": 4.986872839090852e-07, "logits/chosen": 1.6582473516464233, "logits/rejected": 1.6768556833267212, "logps/chosen": -269.5436096191406, "logps/ref_chosen": -271.4682922363281, "logps/ref_rejected": -273.0804748535156, "logps/rejected": -272.4525451660156, "loss": 5.4944, "margin_dpo/margin_mean": 1.2967699766159058, "margin_dpo/margin_std": 2.7775895595550537, "step": 63 }, { "KL/chosen_KL_mean": 2.337747573852539, "KL/mean": 1.6104967594146729, "KL/rejected_KL_mean": 0.8832416534423828, "KL/std": 2.3649282455444336, "epoch": 0.13403141361256546, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4545021057128906, "fcm_dpo/q_t": 0.4963648319244385, "grad_norm": 15.6937255859375, "learning_rate": 4.9849325083059e-07, "logits/chosen": 1.9031604528427124, "logits/rejected": 1.9173095226287842, "logps/chosen": -271.2656555175781, "logps/ref_chosen": -273.6033935546875, "logps/ref_rejected": -259.54107666015625, "logps/rejected": -258.6578369140625, "loss": 5.4882, "margin_dpo/margin_mean": 1.4545015096664429, "margin_dpo/margin_std": 3.0799834728240967, "step": 64 }, { "KL/chosen_KL_mean": 2.314014434814453, "KL/mean": 1.737621784210205, "KL/rejected_KL_mean": 1.1612358093261719, "KL/std": 2.1046664714813232, "epoch": 0.13612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.152785301208496, "fcm_dpo/q_t": 0.4971183240413666, "grad_norm": 14.508123397827148, "learning_rate": 4.982858918131906e-07, "logits/chosen": 1.8851243257522583, "logits/rejected": 1.983131766319275, "logps/chosen": -256.2036437988281, "logps/ref_chosen": -258.5176696777344, "logps/ref_rejected": -264.9944763183594, "logps/rejected": -263.833251953125, "loss": 5.4999, "margin_dpo/margin_mean": 1.152784824371338, "margin_dpo/margin_std": 2.61080265045166, "step": 65 }, { "KL/chosen_KL_mean": 2.190641403198242, "KL/mean": 1.5029757022857666, "KL/rejected_KL_mean": 0.8153171539306641, "KL/std": 2.748277425765991, "epoch": 0.1382198952879581, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3753265142440796, "fcm_dpo/q_t": 0.49656298756599426, "grad_norm": 14.806829452514648, "learning_rate": 4.980652179769217e-07, "logits/chosen": 1.7718381881713867, "logits/rejected": 1.8650543689727783, "logps/chosen": -265.31878662109375, "logps/ref_chosen": -267.5094299316406, "logps/ref_rejected": -277.03228759765625, "logps/rejected": -276.2169494628906, "loss": 5.4917, "margin_dpo/margin_mean": 1.3753262758255005, "margin_dpo/margin_std": 3.608642816543579, "step": 66 }, { "KL/chosen_KL_mean": 2.578306198120117, "KL/mean": 1.970463752746582, "KL/rejected_KL_mean": 1.3626251220703125, "KL/std": 2.6167731285095215, "epoch": 0.14031413612565444, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.215672492980957, "fcm_dpo/q_t": 0.4969613552093506, "grad_norm": 14.19619083404541, "learning_rate": 4.978312411558517e-07, "logits/chosen": 1.9167455434799194, "logits/rejected": 1.9535115957260132, "logps/chosen": -254.88662719726562, "logps/ref_chosen": -257.4649353027344, "logps/ref_rejected": -247.29161071777344, "logps/rejected": -245.9290008544922, "loss": 5.4978, "margin_dpo/margin_mean": 1.2156729698181152, "margin_dpo/margin_std": 3.270585775375366, "step": 67 }, { "KL/chosen_KL_mean": 2.5522289276123047, "KL/mean": 1.6995009183883667, "KL/rejected_KL_mean": 0.8467826843261719, "KL/std": 2.9510626792907715, "epoch": 0.1424083769633508, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7054537534713745, "fcm_dpo/q_t": 0.4957389235496521, "grad_norm": 14.784210205078125, "learning_rate": 4.975839738974473e-07, "logits/chosen": 1.7252765893936157, "logits/rejected": 1.674182653427124, "logps/chosen": -287.93194580078125, "logps/ref_chosen": -290.4841613769531, "logps/ref_rejected": -260.8398132324219, "logps/rejected": -259.9930419921875, "loss": 5.4789, "margin_dpo/margin_mean": 1.7054539918899536, "margin_dpo/margin_std": 3.846391201019287, "step": 68 }, { "KL/chosen_KL_mean": 2.9039573669433594, "KL/mean": 1.8334503173828125, "KL/rejected_KL_mean": 0.7629432678222656, "KL/std": 2.94232439994812, "epoch": 0.14450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.141019821166992, "fcm_dpo/q_t": 0.49464961886405945, "grad_norm": 14.979423522949219, "learning_rate": 4.97323429461901e-07, "logits/chosen": 2.0321993827819824, "logits/rejected": 2.0943429470062256, "logps/chosen": -254.73361206054688, "logps/ref_chosen": -257.6375732421875, "logps/ref_rejected": -235.6666717529297, "logps/rejected": -234.9037322998047, "loss": 5.4615, "margin_dpo/margin_mean": 2.141019344329834, "margin_dpo/margin_std": 3.7320189476013184, "step": 69 }, { "KL/chosen_KL_mean": 3.424154281616211, "KL/mean": 2.3216376304626465, "KL/rejected_KL_mean": 1.2191219329833984, "KL/std": 3.025865077972412, "epoch": 0.14659685863874344, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.2050423622131348, "fcm_dpo/q_t": 0.49449020624160767, "grad_norm": 15.603200912475586, "learning_rate": 4.970496218214204e-07, "logits/chosen": 1.8569802045822144, "logits/rejected": 1.8316940069198608, "logps/chosen": -284.0410461425781, "logps/ref_chosen": -287.4651794433594, "logps/ref_rejected": -253.4770965576172, "logps/rejected": -252.25799560546875, "loss": 5.4589, "margin_dpo/margin_mean": 2.2050423622131348, "margin_dpo/margin_std": 3.663863182067871, "step": 70 }, { "KL/chosen_KL_mean": 3.570659637451172, "KL/mean": 2.601686954498291, "KL/rejected_KL_mean": 1.6327152252197266, "KL/std": 3.250652313232422, "epoch": 0.1486910994764398, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9379527568817139, "fcm_dpo/q_t": 0.4951571822166443, "grad_norm": 14.96021556854248, "learning_rate": 4.967625656594781e-07, "logits/chosen": 1.9369959831237793, "logits/rejected": 1.9424176216125488, "logps/chosen": -277.36474609375, "logps/ref_chosen": -280.9353942871094, "logps/ref_rejected": -275.998779296875, "logps/rejected": -274.3660888671875, "loss": 5.47, "margin_dpo/margin_mean": 1.9379535913467407, "margin_dpo/margin_std": 4.208340644836426, "step": 71 }, { "KL/chosen_KL_mean": 2.9796676635742188, "KL/mean": 1.987913966178894, "KL/rejected_KL_mean": 0.9961509704589844, "KL/std": 3.054412603378296, "epoch": 0.15078534031413612, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.983521580696106, "fcm_dpo/q_t": 0.4950438439846039, "grad_norm": 14.67113971710205, "learning_rate": 4.964622763700252e-07, "logits/chosen": 1.721984624862671, "logits/rejected": 1.8629076480865479, "logps/chosen": -227.16586303710938, "logps/ref_chosen": -230.14553833007812, "logps/ref_rejected": -253.37661743164062, "logps/rejected": -252.3804473876953, "loss": 5.468, "margin_dpo/margin_mean": 1.9835209846496582, "margin_dpo/margin_std": 3.9971396923065186, "step": 72 }, { "KL/chosen_KL_mean": 3.7841663360595703, "KL/mean": 2.8107807636260986, "KL/rejected_KL_mean": 1.8373908996582031, "KL/std": 3.2371246814727783, "epoch": 0.15287958115183245, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9467737674713135, "fcm_dpo/q_t": 0.49513500928878784, "grad_norm": 14.997169494628906, "learning_rate": 4.961487700566646e-07, "logits/chosen": 1.6057504415512085, "logits/rejected": 1.6186208724975586, "logps/chosen": -261.0821228027344, "logps/ref_chosen": -264.86627197265625, "logps/ref_rejected": -245.21304321289062, "logps/rejected": -243.37564086914062, "loss": 5.4698, "margin_dpo/margin_mean": 1.9467735290527344, "margin_dpo/margin_std": 4.460379600524902, "step": 73 }, { "KL/chosen_KL_mean": 3.76385498046875, "KL/mean": 2.8343586921691895, "KL/rejected_KL_mean": 1.9048748016357422, "KL/std": 3.5186123847961426, "epoch": 0.1549738219895288, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.858985185623169, "fcm_dpo/q_t": 0.4953550696372986, "grad_norm": 15.689414024353027, "learning_rate": 4.958220635317885e-07, "logits/chosen": 1.8440401554107666, "logits/rejected": 1.8993223905563354, "logps/chosen": -330.7896728515625, "logps/ref_chosen": -334.55352783203125, "logps/ref_rejected": -325.8403015136719, "logps/rejected": -323.9353942871094, "loss": 5.4735, "margin_dpo/margin_mean": 1.8589857816696167, "margin_dpo/margin_std": 4.583897590637207, "step": 74 }, { "KL/chosen_KL_mean": 4.565900802612305, "KL/mean": 3.20298433303833, "KL/rejected_KL_mean": 1.8400688171386719, "KL/std": 3.6198770999908447, "epoch": 0.15706806282722513, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.7258455753326416, "fcm_dpo/q_t": 0.4931892156600952, "grad_norm": 15.345573425292969, "learning_rate": 4.954821743156767e-07, "logits/chosen": 1.6761469841003418, "logits/rejected": 1.7539714574813843, "logps/chosen": -255.85597229003906, "logps/ref_chosen": -260.421875, "logps/ref_rejected": -273.0882263183594, "logps/rejected": -271.2481689453125, "loss": 5.4389, "margin_dpo/margin_mean": 2.7258448600769043, "margin_dpo/margin_std": 4.2095537185668945, "step": 75 }, { "KL/chosen_KL_mean": 4.54121208190918, "KL/mean": 3.065985918045044, "KL/rejected_KL_mean": 1.590768814086914, "KL/std": 3.8259737491607666, "epoch": 0.15916230366492146, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.950458526611328, "fcm_dpo/q_t": 0.4926307797431946, "grad_norm": 16.780370712280273, "learning_rate": 4.951291206355559e-07, "logits/chosen": 1.8369426727294922, "logits/rejected": 1.8256155252456665, "logps/chosen": -267.1942443847656, "logps/ref_chosen": -271.7354431152344, "logps/ref_rejected": -262.64111328125, "logps/rejected": -261.0503845214844, "loss": 5.4305, "margin_dpo/margin_mean": 2.9504590034484863, "margin_dpo/margin_std": 4.740379333496094, "step": 76 }, { "KL/chosen_KL_mean": 4.897563934326172, "KL/mean": 3.6664695739746094, "KL/rejected_KL_mean": 2.4353809356689453, "KL/std": 4.139870643615723, "epoch": 0.1612565445026178, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.4621849060058594, "fcm_dpo/q_t": 0.4938510060310364, "grad_norm": 14.8322172164917, "learning_rate": 4.947629214246236e-07, "logits/chosen": 1.782356858253479, "logits/rejected": 1.629751443862915, "logps/chosen": -289.7730407714844, "logps/ref_chosen": -294.6706237792969, "logps/ref_rejected": -247.6900177001953, "logps/rejected": -245.25465393066406, "loss": 5.4502, "margin_dpo/margin_mean": 2.4621849060058594, "margin_dpo/margin_std": 5.111874103546143, "step": 77 }, { "KL/chosen_KL_mean": 5.139194488525391, "KL/mean": 3.663858413696289, "KL/rejected_KL_mean": 2.1885147094726562, "KL/std": 3.7755632400512695, "epoch": 0.16335078534031414, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.950676679611206, "fcm_dpo/q_t": 0.49263134598731995, "grad_norm": 14.61767578125, "learning_rate": 4.943835963210323e-07, "logits/chosen": 1.6119264364242554, "logits/rejected": 1.5382490158081055, "logps/chosen": -243.01309204101562, "logps/ref_chosen": -248.1522979736328, "logps/ref_rejected": -209.12808227539062, "logps/rejected": -206.9395294189453, "loss": 5.4309, "margin_dpo/margin_mean": 2.950676441192627, "margin_dpo/margin_std": 5.283993721008301, "step": 78 }, { "KL/chosen_KL_mean": 5.191509246826172, "KL/mean": 3.6148133277893066, "KL/rejected_KL_mean": 2.038118362426758, "KL/std": 4.859566688537598, "epoch": 0.16544502617801046, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.153383493423462, "fcm_dpo/q_t": 0.49211859703063965, "grad_norm": 16.667619705200195, "learning_rate": 4.939911656668361e-07, "logits/chosen": 1.5719172954559326, "logits/rejected": 1.6212188005447388, "logps/chosen": -254.28286743164062, "logps/ref_chosen": -259.4743957519531, "logps/ref_rejected": -246.9627227783203, "logps/rejected": -244.92459106445312, "loss": 5.4248, "margin_dpo/margin_mean": 3.153383255004883, "margin_dpo/margin_std": 6.35499382019043, "step": 79 }, { "KL/chosen_KL_mean": 5.269865036010742, "KL/mean": 3.982539653778076, "KL/rejected_KL_mean": 2.6952075958251953, "KL/std": 4.611552715301514, "epoch": 0.16753926701570682, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.5746498107910156, "fcm_dpo/q_t": 0.49357014894485474, "grad_norm": 16.91059684753418, "learning_rate": 4.935856505068998e-07, "logits/chosen": 1.7765568494796753, "logits/rejected": 1.7734700441360474, "logps/chosen": -280.65667724609375, "logps/ref_chosen": -285.926513671875, "logps/ref_rejected": -253.77996826171875, "logps/rejected": -251.0847625732422, "loss": 5.4465, "margin_dpo/margin_mean": 2.574650287628174, "margin_dpo/margin_std": 5.556662559509277, "step": 80 }, { "KL/chosen_KL_mean": 5.917991638183594, "KL/mean": 4.468988418579102, "KL/rejected_KL_mean": 3.019977569580078, "KL/std": 4.51953125, "epoch": 0.16963350785340314, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.898013114929199, "fcm_dpo/q_t": 0.4927605092525482, "grad_norm": 14.584118843078613, "learning_rate": 4.93167072587771e-07, "logits/chosen": 1.8349915742874146, "logits/rejected": 1.7867443561553955, "logps/chosen": -256.67620849609375, "logps/ref_chosen": -262.5942077636719, "logps/ref_rejected": -236.687744140625, "logps/rejected": -233.66775512695312, "loss": 5.4345, "margin_dpo/margin_mean": 2.898014545440674, "margin_dpo/margin_std": 6.337032794952393, "step": 81 }, { "KL/chosen_KL_mean": 6.365209579467773, "KL/mean": 4.8737077713012695, "KL/rejected_KL_mean": 3.3821964263916016, "KL/std": 4.350286960601807, "epoch": 0.17172774869109947, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.983015298843384, "fcm_dpo/q_t": 0.4925503730773926, "grad_norm": 17.14452362060547, "learning_rate": 4.92735454356513e-07, "logits/chosen": 1.762351632118225, "logits/rejected": 1.7092797756195068, "logps/chosen": -264.677978515625, "logps/ref_chosen": -271.04315185546875, "logps/ref_rejected": -234.03359985351562, "logps/rejected": -230.65138244628906, "loss": 5.4303, "margin_dpo/margin_mean": 2.9830164909362793, "margin_dpo/margin_std": 5.512236595153809, "step": 82 }, { "KL/chosen_KL_mean": 6.469331741333008, "KL/mean": 4.712227821350098, "KL/rejected_KL_mean": 2.9551239013671875, "KL/std": 4.9338226318359375, "epoch": 0.17382198952879582, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.514209032058716, "fcm_dpo/q_t": 0.49122947454452515, "grad_norm": 15.297825813293457, "learning_rate": 4.922908189595017e-07, "logits/chosen": 1.737414836883545, "logits/rejected": 1.8145241737365723, "logps/chosen": -259.8159484863281, "logps/ref_chosen": -266.2852783203125, "logps/ref_rejected": -274.32501220703125, "logps/rejected": -271.369873046875, "loss": 5.4108, "margin_dpo/margin_mean": 3.5142087936401367, "margin_dpo/margin_std": 6.680100440979004, "step": 83 }, { "KL/chosen_KL_mean": 6.714977264404297, "KL/mean": 5.295899391174316, "KL/rejected_KL_mean": 3.876821517944336, "KL/std": 5.049211502075195, "epoch": 0.17591623036649215, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.8381547927856445, "fcm_dpo/q_t": 0.4929097890853882, "grad_norm": 15.236591339111328, "learning_rate": 4.918331902411841e-07, "logits/chosen": 1.9121453762054443, "logits/rejected": 1.904983639717102, "logps/chosen": -280.0656433105469, "logps/ref_chosen": -286.7806091308594, "logps/ref_rejected": -276.1176452636719, "logps/rejected": -272.2408142089844, "loss": 5.4371, "margin_dpo/margin_mean": 2.83815336227417, "margin_dpo/margin_std": 6.488803386688232, "step": 84 }, { "KL/chosen_KL_mean": 6.354665756225586, "KL/mean": 5.3685302734375, "KL/rejected_KL_mean": 4.382389068603516, "KL/std": 4.390091419219971, "epoch": 0.17801047120418848, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.97226881980896, "fcm_dpo/q_t": 0.49507302045822144, "grad_norm": 14.166430473327637, "learning_rate": 4.913625927427995e-07, "logits/chosen": 1.931032419204712, "logits/rejected": 2.031745195388794, "logps/chosen": -228.67518615722656, "logps/ref_chosen": -235.02987670898438, "logps/ref_rejected": -259.80999755859375, "logps/rejected": -255.4276123046875, "loss": 5.4701, "margin_dpo/margin_mean": 1.9722685813903809, "margin_dpo/margin_std": 5.482989311218262, "step": 85 }, { "KL/chosen_KL_mean": 7.849931716918945, "KL/mean": 6.066666603088379, "KL/rejected_KL_mean": 4.28339958190918, "KL/std": 5.507645130157471, "epoch": 0.18010471204188483, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5665531158447266, "fcm_dpo/q_t": 0.4910971522331238, "grad_norm": 16.2667236328125, "learning_rate": 4.908790517010636e-07, "logits/chosen": 1.887448787689209, "logits/rejected": 1.934370994567871, "logps/chosen": -293.46282958984375, "logps/ref_chosen": -301.312744140625, "logps/ref_rejected": -288.8368225097656, "logps/rejected": -284.55340576171875, "loss": 5.4096, "margin_dpo/margin_mean": 3.5665528774261475, "margin_dpo/margin_std": 7.414119720458984, "step": 86 }, { "KL/chosen_KL_mean": 8.08189582824707, "KL/mean": 6.034160137176514, "KL/rejected_KL_mean": 3.986419677734375, "KL/std": 5.5277419090271, "epoch": 0.18219895287958116, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.095483779907227, "fcm_dpo/q_t": 0.48977530002593994, "grad_norm": 15.365828514099121, "learning_rate": 4.903825930468148e-07, "logits/chosen": 1.7935822010040283, "logits/rejected": 1.788144588470459, "logps/chosen": -257.5841064453125, "logps/ref_chosen": -265.6659851074219, "logps/ref_rejected": -242.0697479248047, "logps/rejected": -238.08334350585938, "loss": 5.3875, "margin_dpo/margin_mean": 4.09548282623291, "margin_dpo/margin_std": 6.456631183624268, "step": 87 }, { "KL/chosen_KL_mean": 7.397937774658203, "KL/mean": 5.798396587371826, "KL/rejected_KL_mean": 4.198854446411133, "KL/std": 5.472133636474609, "epoch": 0.18429319371727748, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.199070692062378, "fcm_dpo/q_t": 0.49201327562332153, "grad_norm": 14.061400413513184, "learning_rate": 4.898732434036243e-07, "logits/chosen": 1.8010516166687012, "logits/rejected": 1.820248007774353, "logps/chosen": -253.32351684570312, "logps/ref_chosen": -260.721435546875, "logps/ref_rejected": -262.6593322753906, "logps/rejected": -258.46044921875, "loss": 5.4234, "margin_dpo/margin_mean": 3.1990714073181152, "margin_dpo/margin_std": 6.9819793701171875, "step": 88 }, { "KL/chosen_KL_mean": 9.051471710205078, "KL/mean": 7.091951370239258, "KL/rejected_KL_mean": 5.132427215576172, "KL/std": 5.247347831726074, "epoch": 0.18638743455497384, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.9190514087677, "fcm_dpo/q_t": 0.49021562933921814, "grad_norm": 15.85186767578125, "learning_rate": 4.893510300863676e-07, "logits/chosen": 1.770682454109192, "logits/rejected": 1.7325642108917236, "logps/chosen": -247.95135498046875, "logps/ref_chosen": -257.0028076171875, "logps/ref_rejected": -254.66232299804688, "logps/rejected": -249.52989196777344, "loss": 5.3944, "margin_dpo/margin_mean": 3.9190523624420166, "margin_dpo/margin_std": 6.255563259124756, "step": 89 }, { "KL/chosen_KL_mean": 8.974075317382812, "KL/mean": 7.279847145080566, "KL/rejected_KL_mean": 5.585620880126953, "KL/std": 5.636865139007568, "epoch": 0.18848167539267016, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3884599208831787, "fcm_dpo/q_t": 0.49154117703437805, "grad_norm": 15.255399703979492, "learning_rate": 4.8881598109976e-07, "logits/chosen": 2.067680597305298, "logits/rejected": 2.0169973373413086, "logps/chosen": -289.4363098144531, "logps/ref_chosen": -298.4103698730469, "logps/ref_rejected": -262.8761901855469, "logps/rejected": -257.2905578613281, "loss": 5.4163, "margin_dpo/margin_mean": 3.3884589672088623, "margin_dpo/margin_std": 6.8790998458862305, "step": 90 }, { "KL/chosen_KL_mean": 8.779237747192383, "KL/mean": 7.078490734100342, "KL/rejected_KL_mean": 5.377744674682617, "KL/std": 5.447061538696289, "epoch": 0.1905759162303665, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.40149188041687, "fcm_dpo/q_t": 0.4915134608745575, "grad_norm": 14.696788787841797, "learning_rate": 4.882681251368548e-07, "logits/chosen": 1.5244275331497192, "logits/rejected": 1.557888150215149, "logps/chosen": -218.74464416503906, "logps/ref_chosen": -227.5238800048828, "logps/ref_rejected": -227.11428833007812, "logps/rejected": -221.73654174804688, "loss": 5.4161, "margin_dpo/margin_mean": 3.4014906883239746, "margin_dpo/margin_std": 7.320674896240234, "step": 91 }, { "KL/chosen_KL_mean": 8.950838088989258, "KL/mean": 7.051262378692627, "KL/rejected_KL_mean": 5.151678085327148, "KL/std": 6.589147090911865, "epoch": 0.19267015706806281, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.7991676330566406, "fcm_dpo/q_t": 0.4905143678188324, "grad_norm": 15.557400703430176, "learning_rate": 4.877074915775048e-07, "logits/chosen": 1.8932842016220093, "logits/rejected": 1.7986698150634766, "logps/chosen": -266.431396484375, "logps/ref_chosen": -275.3822326660156, "logps/ref_rejected": -269.8356018066406, "logps/rejected": -264.68389892578125, "loss": 5.401, "margin_dpo/margin_mean": 3.7991671562194824, "margin_dpo/margin_std": 7.670883655548096, "step": 92 }, { "KL/chosen_KL_mean": 9.275632858276367, "KL/mean": 7.499587059020996, "KL/rejected_KL_mean": 5.723550796508789, "KL/std": 5.7577009201049805, "epoch": 0.19476439790575917, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.5520870685577393, "fcm_dpo/q_t": 0.49113625288009644, "grad_norm": 14.2173490524292, "learning_rate": 4.871341104867864e-07, "logits/chosen": 1.7212178707122803, "logits/rejected": 1.702576994895935, "logps/chosen": -218.84075927734375, "logps/ref_chosen": -228.1163787841797, "logps/ref_rejected": -228.26266479492188, "logps/rejected": -222.53909301757812, "loss": 5.41, "margin_dpo/margin_mean": 3.5520858764648438, "margin_dpo/margin_std": 7.237574100494385, "step": 93 }, { "KL/chosen_KL_mean": 10.041501998901367, "KL/mean": 8.051166534423828, "KL/rejected_KL_mean": 6.060827255249023, "KL/std": 6.666971206665039, "epoch": 0.1968586387434555, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.980672597885132, "fcm_dpo/q_t": 0.4900735318660736, "grad_norm": 15.548222541809082, "learning_rate": 4.865480126133871e-07, "logits/chosen": 1.9226418733596802, "logits/rejected": 2.030660629272461, "logps/chosen": -273.9857177734375, "logps/ref_chosen": -284.0272521972656, "logps/ref_rejected": -282.1720275878906, "logps/rejected": -276.1112060546875, "loss": 5.3958, "margin_dpo/margin_mean": 3.980672836303711, "margin_dpo/margin_std": 8.678937911987305, "step": 94 }, { "KL/chosen_KL_mean": 9.779237747192383, "KL/mean": 7.586515426635742, "KL/rejected_KL_mean": 5.393795013427734, "KL/std": 6.788585186004639, "epoch": 0.19895287958115182, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.385440349578857, "fcm_dpo/q_t": 0.48906460404396057, "grad_norm": 15.942317962646484, "learning_rate": 4.859492293879573e-07, "logits/chosen": 1.79828679561615, "logits/rejected": 1.7253429889678955, "logps/chosen": -293.2705993652344, "logps/ref_chosen": -303.0498352050781, "logps/ref_rejected": -259.8980712890625, "logps/rejected": -254.50428771972656, "loss": 5.3798, "margin_dpo/margin_mean": 4.385439395904541, "margin_dpo/margin_std": 8.744900703430176, "step": 95 }, { "KL/chosen_KL_mean": 9.869930267333984, "KL/mean": 7.763474464416504, "KL/rejected_KL_mean": 5.657018661499023, "KL/std": 7.008248329162598, "epoch": 0.20104712041884817, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.212911605834961, "fcm_dpo/q_t": 0.48949211835861206, "grad_norm": 15.15978717803955, "learning_rate": 4.853377929214243e-07, "logits/chosen": 1.7972484827041626, "logits/rejected": 1.7564634084701538, "logps/chosen": -260.7177734375, "logps/ref_chosen": -270.58770751953125, "logps/ref_rejected": -238.90069580078125, "logps/rejected": -233.24366760253906, "loss": 5.3855, "margin_dpo/margin_mean": 4.212912559509277, "margin_dpo/margin_std": 7.945834159851074, "step": 96 }, { "KL/chosen_KL_mean": 9.695920944213867, "KL/mean": 7.339411735534668, "KL/rejected_KL_mean": 4.982898712158203, "KL/std": 6.886740684509277, "epoch": 0.2031413612565445, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.713016033172607, "fcm_dpo/q_t": 0.488258957862854, "grad_norm": 15.31843376159668, "learning_rate": 4.847137360032699e-07, "logits/chosen": 1.8880928754806519, "logits/rejected": 1.7756001949310303, "logps/chosen": -286.4896545410156, "logps/ref_chosen": -296.1855773925781, "logps/ref_rejected": -260.35333251953125, "logps/rejected": -255.3704376220703, "loss": 5.3685, "margin_dpo/margin_mean": 4.713016033172607, "margin_dpo/margin_std": 9.550836563110352, "step": 97 }, { "KL/chosen_KL_mean": 10.064401626586914, "KL/mean": 7.7170209884643555, "KL/rejected_KL_mean": 5.369632720947266, "KL/std": 7.359187126159668, "epoch": 0.20523560209424083, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.694765090942383, "fcm_dpo/q_t": 0.488303005695343, "grad_norm": 15.714393615722656, "learning_rate": 4.84077092099773e-07, "logits/chosen": 2.0168633460998535, "logits/rejected": 2.0064096450805664, "logps/chosen": -267.65093994140625, "logps/ref_chosen": -277.7153625488281, "logps/ref_rejected": -272.9094543457031, "logps/rejected": -267.5398254394531, "loss": 5.3704, "margin_dpo/margin_mean": 4.694764614105225, "margin_dpo/margin_std": 9.518474578857422, "step": 98 }, { "KL/chosen_KL_mean": 10.338481903076172, "KL/mean": 7.729951858520508, "KL/rejected_KL_mean": 5.121423721313477, "KL/std": 7.738285064697266, "epoch": 0.20732984293193718, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.217060089111328, "fcm_dpo/q_t": 0.4869866967201233, "grad_norm": 15.05201530456543, "learning_rate": 4.834278953522137e-07, "logits/chosen": 1.743818759918213, "logits/rejected": 1.7173069715499878, "logps/chosen": -262.1993408203125, "logps/ref_chosen": -272.5378112792969, "logps/ref_rejected": -244.6329345703125, "logps/rejected": -239.51150512695312, "loss": 5.3509, "margin_dpo/margin_mean": 5.21705961227417, "margin_dpo/margin_std": 10.1746244430542, "step": 99 }, { "KL/chosen_KL_mean": 9.348031997680664, "KL/mean": 7.089855670928955, "KL/rejected_KL_mean": 4.831691741943359, "KL/std": 7.804023742675781, "epoch": 0.2094240837696335, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.516352653503418, "fcm_dpo/q_t": 0.48874592781066895, "grad_norm": 15.512487411499023, "learning_rate": 4.827661805750437e-07, "logits/chosen": 1.6178163290023804, "logits/rejected": 1.5342121124267578, "logps/chosen": -280.4643249511719, "logps/ref_chosen": -289.8123474121094, "logps/ref_rejected": -274.29071044921875, "logps/rejected": -269.4590148925781, "loss": 5.376, "margin_dpo/margin_mean": 4.516352653503418, "margin_dpo/margin_std": 9.030591011047363, "step": 100 }, { "KL/chosen_KL_mean": 9.87614631652832, "KL/mean": 7.632713794708252, "KL/rejected_KL_mean": 5.389278411865234, "KL/std": 8.039362907409668, "epoch": 0.21151832460732983, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.486861705780029, "fcm_dpo/q_t": 0.4888134002685547, "grad_norm": 15.381603240966797, "learning_rate": 4.820919832540181e-07, "logits/chosen": 1.7923437356948853, "logits/rejected": 1.9661798477172852, "logps/chosen": -235.83697509765625, "logps/ref_chosen": -245.71311950683594, "logps/ref_rejected": -272.08935546875, "logps/rejected": -266.7000732421875, "loss": 5.3784, "margin_dpo/margin_mean": 4.486861705780029, "margin_dpo/margin_std": 9.73862075805664, "step": 101 }, { "KL/chosen_KL_mean": 10.340808868408203, "KL/mean": 7.144167900085449, "KL/rejected_KL_mean": 3.9475269317626953, "KL/std": 7.482494354248047, "epoch": 0.2136125654450262, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.393280982971191, "fcm_dpo/q_t": 0.48405927419662476, "grad_norm": 15.228653907775879, "learning_rate": 4.814053395442932e-07, "logits/chosen": 1.5331921577453613, "logits/rejected": 1.6188983917236328, "logps/chosen": -199.6016082763672, "logps/ref_chosen": -209.94239807128906, "logps/ref_rejected": -230.66897583007812, "logps/rejected": -226.721435546875, "loss": 5.3023, "margin_dpo/margin_mean": 6.393280506134033, "margin_dpo/margin_std": 8.819887161254883, "step": 102 }, { "KL/chosen_KL_mean": 9.326803207397461, "KL/mean": 6.885707378387451, "KL/rejected_KL_mean": 4.444611549377441, "KL/std": 8.337653160095215, "epoch": 0.2157068062827225, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.882187843322754, "fcm_dpo/q_t": 0.4878406524658203, "grad_norm": 15.703271865844727, "learning_rate": 4.807062862684873e-07, "logits/chosen": 1.6679704189300537, "logits/rejected": 1.7627195119857788, "logps/chosen": -241.9728240966797, "logps/ref_chosen": -251.2996368408203, "logps/ref_rejected": -277.4847412109375, "logps/rejected": -273.0401306152344, "loss": 5.3636, "margin_dpo/margin_mean": 4.882188320159912, "margin_dpo/margin_std": 10.275616645812988, "step": 103 }, { "KL/chosen_KL_mean": 8.242347717285156, "KL/mean": 6.029875755310059, "KL/rejected_KL_mean": 3.8173999786376953, "KL/std": 8.634225845336914, "epoch": 0.21780104712041884, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.424940586090088, "fcm_dpo/q_t": 0.4889841675758362, "grad_norm": 14.384554862976074, "learning_rate": 4.799948609147061e-07, "logits/chosen": 1.7679015398025513, "logits/rejected": 1.8099502325057983, "logps/chosen": -249.53707885742188, "logps/ref_chosen": -257.7794189453125, "logps/ref_rejected": -227.91546630859375, "logps/rejected": -224.09805297851562, "loss": 5.3835, "margin_dpo/margin_mean": 4.42494010925293, "margin_dpo/margin_std": 11.217126846313477, "step": 104 }, { "KL/chosen_KL_mean": 10.295242309570312, "KL/mean": 6.219893455505371, "KL/rejected_KL_mean": 2.1445579528808594, "KL/std": 9.750238418579102, "epoch": 0.2198952879581152, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.1506929397583, "fcm_dpo/q_t": 0.47969570755958557, "grad_norm": 17.80915069580078, "learning_rate": 4.792711016345321e-07, "logits/chosen": 1.8179645538330078, "logits/rejected": 1.78440260887146, "logps/chosen": -301.79876708984375, "logps/ref_chosen": -312.09405517578125, "logps/ref_rejected": -277.97332763671875, "logps/rejected": -275.8287658691406, "loss": 5.2387, "margin_dpo/margin_mean": 8.1506929397583, "margin_dpo/margin_std": 10.811229705810547, "step": 105 }, { "KL/chosen_KL_mean": 6.591794967651367, "KL/mean": 3.870194911956787, "KL/rejected_KL_mean": 1.1485862731933594, "KL/std": 8.947371482849121, "epoch": 0.22198952879581152, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.443210124969482, "fcm_dpo/q_t": 0.48644956946372986, "grad_norm": 28.13164710998535, "learning_rate": 4.785350472409791e-07, "logits/chosen": 1.9726589918136597, "logits/rejected": 2.0139129161834717, "logps/chosen": -278.7745361328125, "logps/ref_chosen": -285.3663330078125, "logps/ref_rejected": -262.2779541015625, "logps/rejected": -261.1293640136719, "loss": 5.3443, "margin_dpo/margin_mean": 5.443212032318115, "margin_dpo/margin_std": 11.273680686950684, "step": 106 }, { "KL/chosen_KL_mean": 9.141887664794922, "KL/mean": 5.479947566986084, "KL/rejected_KL_mean": 1.8180046081542969, "KL/std": 9.478837013244629, "epoch": 0.22408376963350785, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.3238911628723145, "fcm_dpo/q_t": 0.48174697160720825, "grad_norm": 16.708341598510742, "learning_rate": 4.777867372064105e-07, "logits/chosen": 1.8046985864639282, "logits/rejected": 1.8737220764160156, "logps/chosen": -287.1517028808594, "logps/ref_chosen": -296.2935791015625, "logps/ref_rejected": -291.5162353515625, "logps/rejected": -289.6982116699219, "loss": 5.2726, "margin_dpo/margin_mean": 7.323891639709473, "margin_dpo/margin_std": 11.570927619934082, "step": 107 }, { "KL/chosen_KL_mean": 9.301300048828125, "KL/mean": 5.2102484703063965, "KL/rejected_KL_mean": 1.11920166015625, "KL/std": 9.038829803466797, "epoch": 0.2261780104712042, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.182097434997559, "fcm_dpo/q_t": 0.479623943567276, "grad_norm": 16.092702865600586, "learning_rate": 4.770262116604223e-07, "logits/chosen": 1.8490670919418335, "logits/rejected": 1.7061842679977417, "logps/chosen": -278.82305908203125, "logps/ref_chosen": -288.12432861328125, "logps/ref_rejected": -232.35720825195312, "logps/rejected": -231.23802185058594, "loss": 5.2367, "margin_dpo/margin_mean": 8.182095527648926, "margin_dpo/margin_std": 10.616247177124023, "step": 108 }, { "KL/chosen_KL_mean": 7.901805877685547, "KL/mean": 3.661217212677002, "KL/rejected_KL_mean": -0.5793647766113281, "KL/std": 10.176899909973145, "epoch": 0.22827225130890053, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.481165885925293, "fcm_dpo/q_t": 0.47893649339675903, "grad_norm": 17.60049057006836, "learning_rate": 4.7625351138769166e-07, "logits/chosen": 1.7414685487747192, "logits/rejected": 1.6945842504501343, "logps/chosen": -309.29974365234375, "logps/ref_chosen": -317.2015686035156, "logps/ref_rejected": -278.0653381347656, "logps/rejected": -278.6446838378906, "loss": 5.2316, "margin_dpo/margin_mean": 8.481164932250977, "margin_dpo/margin_std": 12.964422225952148, "step": 109 }, { "KL/chosen_KL_mean": 8.356040954589844, "KL/mean": 4.442471027374268, "KL/rejected_KL_mean": 0.5289173126220703, "KL/std": 10.09596061706543, "epoch": 0.23036649214659685, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.827131748199463, "fcm_dpo/q_t": 0.4805219769477844, "grad_norm": 16.00581932067871, "learning_rate": 4.75468677825789e-07, "logits/chosen": 1.8129990100860596, "logits/rejected": 1.8113503456115723, "logps/chosen": -254.58340454101562, "logps/ref_chosen": -262.939453125, "logps/ref_rejected": -257.82275390625, "logps/rejected": -257.2938232421875, "loss": 5.2562, "margin_dpo/margin_mean": 7.827132701873779, "margin_dpo/margin_std": 13.084668159484863, "step": 110 }, { "KL/chosen_KL_mean": 8.985616683959961, "KL/mean": 4.6226325035095215, "KL/rejected_KL_mean": 0.25965309143066406, "KL/std": 11.262866973876953, "epoch": 0.2324607329842932, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.725970268249512, "fcm_dpo/q_t": 0.47832146286964417, "grad_norm": 18.12258529663086, "learning_rate": 4.7467175306295647e-07, "logits/chosen": 1.8897958993911743, "logits/rejected": 1.7945163249969482, "logps/chosen": -310.168212890625, "logps/ref_chosen": -319.15380859375, "logps/ref_rejected": -281.7733154296875, "logps/rejected": -281.513671875, "loss": 5.2221, "margin_dpo/margin_mean": 8.725970268249512, "margin_dpo/margin_std": 12.794700622558594, "step": 111 }, { "KL/chosen_KL_mean": 5.339027404785156, "KL/mean": 3.1850929260253906, "KL/rejected_KL_mean": 1.0311565399169922, "KL/std": 10.531282424926758, "epoch": 0.23455497382198953, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.3078765869140625, "fcm_dpo/q_t": 0.4892743229866028, "grad_norm": 14.627169609069824, "learning_rate": 4.7386277983585053e-07, "logits/chosen": 1.8687463998794556, "logits/rejected": 1.9776508808135986, "logps/chosen": -245.16140747070312, "logps/ref_chosen": -250.50042724609375, "logps/ref_rejected": -268.32635498046875, "logps/rejected": -267.29522705078125, "loss": 5.3983, "margin_dpo/margin_mean": 4.307877063751221, "margin_dpo/margin_std": 14.214605331420898, "step": 112 }, { "KL/chosen_KL_mean": 6.826242446899414, "KL/mean": 1.7497713565826416, "KL/rejected_KL_mean": -3.3266963958740234, "KL/std": 11.603326797485352, "epoch": 0.23664921465968586, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.152936935424805, "fcm_dpo/q_t": 0.4747968018054962, "grad_norm": 16.86988067626953, "learning_rate": 4.7304180152725024e-07, "logits/chosen": 1.5450334548950195, "logits/rejected": 1.574907660484314, "logps/chosen": -272.5753479003906, "logps/ref_chosen": -279.4015808105469, "logps/ref_rejected": -266.6392822265625, "logps/rejected": -269.96600341796875, "loss": 5.1714, "margin_dpo/margin_mean": 10.152936935424805, "margin_dpo/margin_std": 14.223891258239746, "step": 113 }, { "KL/chosen_KL_mean": 5.255594253540039, "KL/mean": 2.662837266921997, "KL/rejected_KL_mean": 0.07008552551269531, "KL/std": 10.680999755859375, "epoch": 0.2387434554973822, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.1855058670043945, "fcm_dpo/q_t": 0.4871106743812561, "grad_norm": 15.769734382629395, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 1.8406256437301636, "logits/rejected": 1.8326904773712158, "logps/chosen": -249.00489807128906, "logps/ref_chosen": -254.26048278808594, "logps/ref_rejected": -241.8769989013672, "logps/rejected": -241.80690002441406, "loss": 5.3594, "margin_dpo/margin_mean": 5.185506343841553, "margin_dpo/margin_std": 13.097881317138672, "step": 114 }, { "KL/chosen_KL_mean": 5.537559509277344, "KL/mean": 2.6346588134765625, "KL/rejected_KL_mean": -0.2682456970214844, "KL/std": 11.89238452911377, "epoch": 0.24083769633507854, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.805802822113037, "fcm_dpo/q_t": 0.4856095016002655, "grad_norm": 15.853104591369629, "learning_rate": 4.7136400641330245e-07, "logits/chosen": 1.6168692111968994, "logits/rejected": 1.6461868286132812, "logps/chosen": -283.2669372558594, "logps/ref_chosen": -288.80450439453125, "logps/ref_rejected": -266.8573913574219, "logps/rejected": -267.1256103515625, "loss": 5.3395, "margin_dpo/margin_mean": 5.805802822113037, "margin_dpo/margin_std": 14.407818794250488, "step": 115 }, { "KL/chosen_KL_mean": 7.151908874511719, "KL/mean": 3.055176258087158, "KL/rejected_KL_mean": -1.0415592193603516, "KL/std": 11.208464622497559, "epoch": 0.24293193717277486, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.193467140197754, "fcm_dpo/q_t": 0.47964489459991455, "grad_norm": 16.368608474731445, "learning_rate": 4.70507279583015e-07, "logits/chosen": 1.656557559967041, "logits/rejected": 1.6314194202423096, "logps/chosen": -265.0380554199219, "logps/ref_chosen": -272.18994140625, "logps/ref_rejected": -253.14334106445312, "logps/rejected": -254.1848907470703, "loss": 5.2458, "margin_dpo/margin_mean": 8.193467140197754, "margin_dpo/margin_std": 14.211151123046875, "step": 116 }, { "KL/chosen_KL_mean": 5.650030136108398, "KL/mean": 1.4515990018844604, "KL/rejected_KL_mean": -2.746824264526367, "KL/std": 11.022621154785156, "epoch": 0.2450261780104712, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.39685344696045, "fcm_dpo/q_t": 0.47912970185279846, "grad_norm": 16.54109764099121, "learning_rate": 4.6963872761652834e-07, "logits/chosen": 1.7056045532226562, "logits/rejected": 1.704630970954895, "logps/chosen": -244.75259399414062, "logps/ref_chosen": -250.4026336669922, "logps/ref_rejected": -229.63853454589844, "logps/rejected": -232.38536071777344, "loss": 5.2352, "margin_dpo/margin_mean": 8.39685344696045, "margin_dpo/margin_std": 12.509329795837402, "step": 117 }, { "KL/chosen_KL_mean": 5.426443099975586, "KL/mean": 0.9376038312911987, "KL/rejected_KL_mean": -3.5512428283691406, "KL/std": 12.780660629272461, "epoch": 0.24712041884816754, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.977682113647461, "fcm_dpo/q_t": 0.47770825028419495, "grad_norm": 20.744915008544922, "learning_rate": 4.687583970916486e-07, "logits/chosen": 1.6222083568572998, "logits/rejected": 1.6639357805252075, "logps/chosen": -252.96945190429688, "logps/ref_chosen": -258.3959045410156, "logps/ref_rejected": -268.84228515625, "logps/rejected": -272.3935546875, "loss": 5.2174, "margin_dpo/margin_mean": 8.977681159973145, "margin_dpo/margin_std": 14.679039001464844, "step": 118 }, { "KL/chosen_KL_mean": 6.003448486328125, "KL/mean": 2.179342269897461, "KL/rejected_KL_mean": -1.6447715759277344, "KL/std": 14.2131986618042, "epoch": 0.24921465968586387, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.648217678070068, "fcm_dpo/q_t": 0.48110345005989075, "grad_norm": 15.949928283691406, "learning_rate": 4.6786633521783005e-07, "logits/chosen": 1.9601776599884033, "logits/rejected": 1.9405962228775024, "logps/chosen": -311.2991027832031, "logps/ref_chosen": -317.30255126953125, "logps/ref_rejected": -300.47552490234375, "logps/rejected": -302.1202697753906, "loss": 5.2785, "margin_dpo/margin_mean": 7.648219108581543, "margin_dpo/margin_std": 16.88101577758789, "step": 119 }, { "KL/chosen_KL_mean": 6.925140380859375, "KL/mean": 2.351628303527832, "KL/rejected_KL_mean": -2.2218856811523438, "KL/std": 13.34640121459961, "epoch": 0.2513089005235602, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.147025108337402, "fcm_dpo/q_t": 0.47732555866241455, "grad_norm": 16.257707595825195, "learning_rate": 4.669625898336438e-07, "logits/chosen": 1.8242895603179932, "logits/rejected": 1.7785429954528809, "logps/chosen": -298.32928466796875, "logps/ref_chosen": -305.2544250488281, "logps/ref_rejected": -259.40716552734375, "logps/rejected": -261.6290283203125, "loss": 5.2168, "margin_dpo/margin_mean": 9.147024154663086, "margin_dpo/margin_std": 16.313743591308594, "step": 120 }, { "KL/chosen_KL_mean": 3.4570655822753906, "KL/mean": 0.7536537647247314, "KL/rejected_KL_mean": -1.9497547149658203, "KL/std": 11.887968063354492, "epoch": 0.2534031413612565, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.4068145751953125, "fcm_dpo/q_t": 0.48658356070518494, "grad_norm": 14.691038131713867, "learning_rate": 4.6604720940421207e-07, "logits/chosen": 1.5685416460037231, "logits/rejected": 1.587632656097412, "logps/chosen": -213.18365478515625, "logps/ref_chosen": -216.64073181152344, "logps/ref_rejected": -224.36422729492188, "logps/rejected": -226.31396484375, "loss": 5.3553, "margin_dpo/margin_mean": 5.406813621520996, "margin_dpo/margin_std": 14.467510223388672, "step": 121 }, { "KL/chosen_KL_mean": 5.674816131591797, "KL/mean": 2.0157768726348877, "KL/rejected_KL_mean": -1.6432666778564453, "KL/std": 14.010313034057617, "epoch": 0.2554973821989529, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.318086624145508, "fcm_dpo/q_t": 0.4818747043609619, "grad_norm": 15.785666465759277, "learning_rate": 4.651202430186092e-07, "logits/chosen": 1.884055495262146, "logits/rejected": 1.871678113937378, "logps/chosen": -263.14398193359375, "logps/ref_chosen": -268.8188171386719, "logps/ref_rejected": -272.55853271484375, "logps/rejected": -274.2017822265625, "loss": 5.2913, "margin_dpo/margin_mean": 7.318086624145508, "margin_dpo/margin_std": 17.51386260986328, "step": 122 }, { "KL/chosen_KL_mean": 8.699848175048828, "KL/mean": 3.4266481399536133, "KL/rejected_KL_mean": -1.8465538024902344, "KL/std": 14.962854385375977, "epoch": 0.25759162303664923, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.546406745910645, "fcm_dpo/q_t": 0.4738994240760803, "grad_norm": 18.218210220336914, "learning_rate": 4.6418174038722924e-07, "logits/chosen": 1.7776902914047241, "logits/rejected": 1.7658780813217163, "logps/chosen": -312.99188232421875, "logps/ref_chosen": -321.6917419433594, "logps/ref_rejected": -279.1517333984375, "logps/rejected": -280.9982604980469, "loss": 5.1692, "margin_dpo/margin_mean": 10.546405792236328, "margin_dpo/margin_std": 18.25002670288086, "step": 123 }, { "KL/chosen_KL_mean": 6.083673477172852, "KL/mean": 1.3417143821716309, "KL/rejected_KL_mean": -3.4002513885498047, "KL/std": 14.426389694213867, "epoch": 0.25968586387434556, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.483942985534668, "fcm_dpo/q_t": 0.47647592425346375, "grad_norm": 16.518327713012695, "learning_rate": 4.6323175183912023e-07, "logits/chosen": 1.7295470237731934, "logits/rejected": 1.6208832263946533, "logps/chosen": -265.591552734375, "logps/ref_chosen": -271.6752014160156, "logps/ref_rejected": -221.85577392578125, "logps/rejected": -225.25601196289062, "loss": 5.2046, "margin_dpo/margin_mean": 9.483943939208984, "margin_dpo/margin_std": 15.952499389648438, "step": 124 }, { "KL/chosen_KL_mean": 5.441370010375977, "KL/mean": 0.8098728656768799, "KL/rejected_KL_mean": -3.821634292602539, "KL/std": 14.032330513000488, "epoch": 0.2617801047120419, "fcm_dpo/beta": 0.009999998845160007, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.263002395629883, "fcm_dpo/q_t": 0.4771071970462799, "grad_norm": 16.489973068237305, "learning_rate": 4.6227032831928483e-07, "logits/chosen": 1.5670815706253052, "logits/rejected": 1.5696477890014648, "logps/chosen": -264.95770263671875, "logps/ref_chosen": -270.3990478515625, "logps/ref_rejected": -264.16925048828125, "logps/rejected": -267.99090576171875, "loss": 5.2188, "margin_dpo/margin_mean": 9.263002395629883, "margin_dpo/margin_std": 18.39293670654297, "step": 125 }, { "KL/chosen_KL_mean": 6.915998458862305, "KL/mean": 1.5826067924499512, "KL/rejected_KL_mean": -3.7507762908935547, "KL/std": 15.615608215332031, "epoch": 0.2638743455497382, "fcm_dpo/beta": 0.010336995124816895, "fcm_dpo/delta": 0.047238241881132126, "fcm_dpo/margin": 10.66677474975586, "fcm_dpo/q_t": 0.4729771614074707, "grad_norm": 17.296894073486328, "learning_rate": 4.612975213859487e-07, "logits/chosen": 1.8302912712097168, "logits/rejected": 1.8179588317871094, "logps/chosen": -306.3352966308594, "logps/ref_chosen": -313.25128173828125, "logps/ref_rejected": -283.0433349609375, "logps/rejected": -286.79412841796875, "loss": 5.1563, "margin_dpo/margin_mean": 10.66677474975586, "margin_dpo/margin_std": 18.422670364379883, "step": 126 }, { "KL/chosen_KL_mean": 8.044687271118164, "KL/mean": 1.8930842876434326, "KL/rejected_KL_mean": -4.258506774902344, "KL/std": 16.056201934814453, "epoch": 0.26596858638743454, "fcm_dpo/beta": 0.010639732703566551, "fcm_dpo/delta": 0.09435366094112396, "fcm_dpo/margin": 12.30319881439209, "fcm_dpo/q_t": 0.46779361367225647, "grad_norm": 57.42796325683594, "learning_rate": 4.603133832077953e-07, "logits/chosen": 1.7370554208755493, "logits/rejected": 1.6627271175384521, "logps/chosen": -290.519287109375, "logps/ref_chosen": -298.5639953613281, "logps/ref_rejected": -274.021728515625, "logps/rejected": -278.2802429199219, "loss": 5.0748, "margin_dpo/margin_mean": 12.303199768066406, "margin_dpo/margin_std": 18.242170333862305, "step": 127 }, { "KL/chosen_KL_mean": 8.76463508605957, "KL/mean": 1.9856010675430298, "KL/rejected_KL_mean": -4.793434143066406, "KL/std": 16.885038375854492, "epoch": 0.2680628272251309, "fcm_dpo/beta": 0.011522623710334301, "fcm_dpo/delta": 0.04720144718885422, "fcm_dpo/margin": 13.558062553405762, "fcm_dpo/q_t": 0.4617130160331726, "grad_norm": 18.26601219177246, "learning_rate": 4.5931796656116837e-07, "logits/chosen": 1.786920189857483, "logits/rejected": 1.8387939929962158, "logps/chosen": -248.36624145507812, "logps/ref_chosen": -257.130859375, "logps/ref_rejected": -247.71636962890625, "logps/rejected": -252.5098114013672, "loss": 4.9995, "margin_dpo/margin_mean": 13.558062553405762, "margin_dpo/margin_std": 20.108369827270508, "step": 128 }, { "KL/chosen_KL_mean": 9.020750045776367, "KL/mean": 2.7810397148132324, "KL/rejected_KL_mean": -3.4586639404296875, "KL/std": 16.992502212524414, "epoch": 0.27015706806282724, "fcm_dpo/beta": 0.012070320546627045, "fcm_dpo/delta": 0.09296167641878128, "fcm_dpo/margin": 12.47940444946289, "fcm_dpo/q_t": 0.4633902609348297, "grad_norm": 19.298423767089844, "learning_rate": 4.5831132482724193e-07, "logits/chosen": 1.652014970779419, "logits/rejected": 1.6925582885742188, "logps/chosen": -285.9397888183594, "logps/ref_chosen": -294.9605712890625, "logps/ref_rejected": -272.6249694824219, "logps/rejected": -276.0836181640625, "loss": 5.0286, "margin_dpo/margin_mean": 12.47940444946289, "margin_dpo/margin_std": 20.265625, "step": 129 }, { "KL/chosen_KL_mean": 7.594112396240234, "KL/mean": 1.8055014610290527, "KL/rejected_KL_mean": -3.9831180572509766, "KL/std": 15.501653671264648, "epoch": 0.27225130890052357, "fcm_dpo/beta": 0.012528286315500736, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 11.577238082885742, "fcm_dpo/q_t": 0.4642757475376129, "grad_norm": 25.295881271362305, "learning_rate": 4.5729351198915705e-07, "logits/chosen": 1.7699538469314575, "logits/rejected": 1.8514341115951538, "logps/chosen": -268.8481140136719, "logps/ref_chosen": -276.4422302246094, "logps/ref_rejected": -290.241455078125, "logps/rejected": -294.2245788574219, "loss": 5.0455, "margin_dpo/margin_mean": 11.577238082885742, "margin_dpo/margin_std": 19.18193244934082, "step": 130 }, { "KL/chosen_KL_mean": 4.91627311706543, "KL/mean": 0.08451282978057861, "KL/rejected_KL_mean": -4.747249603271484, "KL/std": 16.430450439453125, "epoch": 0.2743455497382199, "fcm_dpo/beta": 0.012808036059141159, "fcm_dpo/delta": 0.04387928918004036, "fcm_dpo/margin": 9.663520812988281, "fcm_dpo/q_t": 0.47001218795776367, "grad_norm": 22.026689529418945, "learning_rate": 4.5626458262912735e-07, "logits/chosen": 1.614654779434204, "logits/rejected": 1.6121958494186401, "logps/chosen": -299.9084167480469, "logps/ref_chosen": -304.82470703125, "logps/ref_rejected": -289.9726867675781, "logps/rejected": -294.7199401855469, "loss": 5.1431, "margin_dpo/margin_mean": 9.663520812988281, "margin_dpo/margin_std": 21.14334487915039, "step": 131 }, { "KL/chosen_KL_mean": 7.009241104125977, "KL/mean": 0.5452959537506104, "KL/rejected_KL_mean": -5.91864013671875, "KL/std": 18.429950714111328, "epoch": 0.2764397905759162, "fcm_dpo/beta": 0.013177389279007912, "fcm_dpo/delta": 0.0732303261756897, "fcm_dpo/margin": 12.927886009216309, "fcm_dpo/q_t": 0.45908477902412415, "grad_norm": 22.78095054626465, "learning_rate": 4.5522459192551166e-07, "logits/chosen": 1.464105486869812, "logits/rejected": 1.5743765830993652, "logps/chosen": -253.30551147460938, "logps/ref_chosen": -260.31475830078125, "logps/ref_rejected": -280.93756103515625, "logps/rejected": -286.856201171875, "loss": 4.992, "margin_dpo/margin_mean": 12.927887916564941, "margin_dpo/margin_std": 23.27849006652832, "step": 132 }, { "KL/chosen_KL_mean": 7.630144119262695, "KL/mean": 0.963181734085083, "KL/rejected_KL_mean": -5.70378303527832, "KL/std": 17.05446434020996, "epoch": 0.27853403141361255, "fcm_dpo/beta": 0.014581998810172081, "fcm_dpo/delta": 0.12239662557840347, "fcm_dpo/margin": 13.333932876586914, "fcm_dpo/q_t": 0.45295092463493347, "grad_norm": 22.359704971313477, "learning_rate": 4.541735956498554e-07, "logits/chosen": 1.6703652143478394, "logits/rejected": 1.6475379467010498, "logps/chosen": -271.4904479980469, "logps/ref_chosen": -279.12060546875, "logps/ref_rejected": -247.22982788085938, "logps/rejected": -252.93360900878906, "loss": 4.9045, "margin_dpo/margin_mean": 13.33393383026123, "margin_dpo/margin_std": 20.930585861206055, "step": 133 }, { "KL/chosen_KL_mean": 1.8229942321777344, "KL/mean": -2.830925703048706, "KL/rejected_KL_mean": -7.484846115112305, "KL/std": 17.411767959594727, "epoch": 0.2806282722513089, "fcm_dpo/beta": 0.01576686091721058, "fcm_dpo/delta": 0.08350040018558502, "fcm_dpo/margin": 9.307854652404785, "fcm_dpo/q_t": 0.46470966935157776, "grad_norm": 33.37110137939453, "learning_rate": 4.5311165016389914e-07, "logits/chosen": 1.7755942344665527, "logits/rejected": 1.8094536066055298, "logps/chosen": -313.78106689453125, "logps/ref_chosen": -315.6040954589844, "logps/ref_rejected": -291.45635986328125, "logps/rejected": -298.9411926269531, "loss": 5.0808, "margin_dpo/margin_mean": 9.307854652404785, "margin_dpo/margin_std": 19.93376350402832, "step": 134 }, { "KL/chosen_KL_mean": 5.3419647216796875, "KL/mean": -1.218588948249817, "KL/rejected_KL_mean": -7.779151916503906, "KL/std": 17.860780715942383, "epoch": 0.28272251308900526, "fcm_dpo/beta": 0.017457405105233192, "fcm_dpo/delta": 0.13837897777557373, "fcm_dpo/margin": 13.12112045288086, "fcm_dpo/q_t": 0.4456590414047241, "grad_norm": 27.369003295898438, "learning_rate": 4.520388124165564e-07, "logits/chosen": 1.5128746032714844, "logits/rejected": 1.5043995380401611, "logps/chosen": -277.531005859375, "logps/ref_chosen": -282.87298583984375, "logps/ref_rejected": -265.0682373046875, "logps/rejected": -272.84735107421875, "loss": 4.8075, "margin_dpo/margin_mean": 13.121119499206543, "margin_dpo/margin_std": 19.0527400970459, "step": 135 }, { "KL/chosen_KL_mean": 4.886213302612305, "KL/mean": -1.2634704113006592, "KL/rejected_KL_mean": -7.4131622314453125, "KL/std": 17.349163055419922, "epoch": 0.2848167539267016, "fcm_dpo/beta": 0.020285794511437416, "fcm_dpo/delta": 0.1794380247592926, "fcm_dpo/margin": 12.299372673034668, "fcm_dpo/q_t": 0.44218307733535767, "grad_norm": 34.979557037353516, "learning_rate": 4.5095513994085974e-07, "logits/chosen": 1.5496782064437866, "logits/rejected": 1.7162466049194336, "logps/chosen": -258.5556945800781, "logps/ref_chosen": -263.4419250488281, "logps/ref_rejected": -251.43150329589844, "logps/rejected": -258.8446350097656, "loss": 4.8138, "margin_dpo/margin_mean": 12.299372673034668, "margin_dpo/margin_std": 21.40203094482422, "step": 136 }, { "KL/chosen_KL_mean": 3.0768299102783203, "KL/mean": -2.479499101638794, "KL/rejected_KL_mean": -8.035833358764648, "KL/std": 18.846830368041992, "epoch": 0.2869109947643979, "fcm_dpo/beta": 0.02238212712109089, "fcm_dpo/delta": 0.16478125751018524, "fcm_dpo/margin": 11.112651824951172, "fcm_dpo/q_t": 0.44189003109931946, "grad_norm": 36.73350524902344, "learning_rate": 4.498606908508753e-07, "logits/chosen": 1.6065609455108643, "logits/rejected": 1.597914695739746, "logps/chosen": -292.19390869140625, "logps/ref_chosen": -295.2707214355469, "logps/ref_rejected": -283.9851989746094, "logps/rejected": -292.02105712890625, "loss": 4.8242, "margin_dpo/margin_mean": 11.112653732299805, "margin_dpo/margin_std": 20.317659378051758, "step": 137 }, { "KL/chosen_KL_mean": 1.7046375274658203, "KL/mean": -3.7809078693389893, "KL/rejected_KL_mean": -9.266460418701172, "KL/std": 18.058706283569336, "epoch": 0.28900523560209423, "fcm_dpo/beta": 0.026084531098604202, "fcm_dpo/delta": 0.20727840065956116, "fcm_dpo/margin": 10.971098899841309, "fcm_dpo/q_t": 0.4376677870750427, "grad_norm": 40.71832275390625, "learning_rate": 4.487555238385862e-07, "logits/chosen": 1.6645891666412354, "logits/rejected": 1.663257360458374, "logps/chosen": -280.8532409667969, "logps/ref_chosen": -282.5578918457031, "logps/ref_rejected": -262.4772644042969, "logps/rejected": -271.74371337890625, "loss": 4.8585, "margin_dpo/margin_mean": 10.971098899841309, "margin_dpo/margin_std": 22.973066329956055, "step": 138 }, { "KL/chosen_KL_mean": 2.924623489379883, "KL/mean": -1.1854338645935059, "KL/rejected_KL_mean": -5.295494079589844, "KL/std": 18.88577651977539, "epoch": 0.29109947643979056, "fcm_dpo/beta": 0.03053542599081993, "fcm_dpo/delta": 0.18156927824020386, "fcm_dpo/margin": 8.220112800598145, "fcm_dpo/q_t": 0.44587525725364685, "grad_norm": 49.938262939453125, "learning_rate": 4.476396981707453e-07, "logits/chosen": 1.568600058555603, "logits/rejected": 1.6356700658798218, "logps/chosen": -260.7261657714844, "logps/ref_chosen": -263.65081787109375, "logps/ref_rejected": -266.2420959472656, "logps/rejected": -271.53759765625, "loss": 5.1004, "margin_dpo/margin_mean": 8.220112800598145, "margin_dpo/margin_std": 23.391185760498047, "step": 139 }, { "KL/chosen_KL_mean": 6.302621841430664, "KL/mean": -1.1070319414138794, "KL/rejected_KL_mean": -8.516683578491211, "KL/std": 18.66071319580078, "epoch": 0.2931937172774869, "fcm_dpo/beta": 0.03383221849799156, "fcm_dpo/delta": 0.05534835532307625, "fcm_dpo/margin": 14.81930923461914, "fcm_dpo/q_t": 0.39205098152160645, "grad_norm": 47.0718994140625, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 1.5065391063690186, "logits/rejected": 1.4494394063949585, "logps/chosen": -266.0798034667969, "logps/ref_chosen": -272.3824462890625, "logps/ref_rejected": -248.96444702148438, "logps/rejected": -257.48114013671875, "loss": 4.2848, "margin_dpo/margin_mean": 14.819307327270508, "margin_dpo/margin_std": 20.897306442260742, "step": 140 }, { "KL/chosen_KL_mean": 3.9792404174804688, "KL/mean": -3.0592873096466064, "KL/rejected_KL_mean": -10.097808837890625, "KL/std": 19.82098388671875, "epoch": 0.29528795811518327, "fcm_dpo/beta": 0.03550390154123306, "fcm_dpo/delta": 0.10487215965986252, "fcm_dpo/margin": 14.077037811279297, "fcm_dpo/q_t": 0.393840491771698, "grad_norm": 75.06885528564453, "learning_rate": 4.453763107901675e-07, "logits/chosen": 1.6357208490371704, "logits/rejected": 1.6846797466278076, "logps/chosen": -284.0918273925781, "logps/ref_chosen": -288.071044921875, "logps/ref_rejected": -274.83477783203125, "logps/rejected": -284.93255615234375, "loss": 4.4699, "margin_dpo/margin_mean": 14.077035903930664, "margin_dpo/margin_std": 23.960243225097656, "step": 141 }, { "KL/chosen_KL_mean": 6.928815841674805, "KL/mean": 0.08577963709831238, "KL/rejected_KL_mean": -6.757253646850586, "KL/std": 19.40366554260254, "epoch": 0.2973821989528796, "fcm_dpo/beta": 0.03750036656856537, "fcm_dpo/delta": 0.029122523963451385, "fcm_dpo/margin": 13.686077117919922, "fcm_dpo/q_t": 0.3947056531906128, "grad_norm": 54.780887603759766, "learning_rate": 4.4422887045602674e-07, "logits/chosen": 1.6604958772659302, "logits/rejected": 1.5612449645996094, "logps/chosen": -282.06304931640625, "logps/ref_chosen": -288.9918518066406, "logps/ref_rejected": -228.26071166992188, "logps/rejected": -235.01795959472656, "loss": 4.4883, "margin_dpo/margin_mean": 13.686078071594238, "margin_dpo/margin_std": 23.471479415893555, "step": 142 }, { "KL/chosen_KL_mean": 6.831939697265625, "KL/mean": -1.064931869506836, "KL/rejected_KL_mean": -8.961799621582031, "KL/std": 20.378860473632812, "epoch": 0.2994764397905759, "fcm_dpo/beta": 0.03808540105819702, "fcm_dpo/delta": -0.003338230773806572, "fcm_dpo/margin": 15.793725967407227, "fcm_dpo/q_t": 0.3775142729282379, "grad_norm": 60.79389953613281, "learning_rate": 4.4307101421701755e-07, "logits/chosen": 1.5778990983963013, "logits/rejected": 1.6198242902755737, "logps/chosen": -282.6425476074219, "logps/ref_chosen": -289.4744873046875, "logps/ref_rejected": -265.130615234375, "logps/rejected": -274.0924377441406, "loss": 4.3248, "margin_dpo/margin_mean": 15.793725967407227, "margin_dpo/margin_std": 25.55742835998535, "step": 143 }, { "KL/chosen_KL_mean": 8.030105590820312, "KL/mean": 1.6460278034210205, "KL/rejected_KL_mean": -4.7380523681640625, "KL/std": 19.515775680541992, "epoch": 0.30157068062827225, "fcm_dpo/beta": 0.036904361099004745, "fcm_dpo/delta": 0.008406475186347961, "fcm_dpo/margin": 12.768148422241211, "fcm_dpo/q_t": 0.4033263921737671, "grad_norm": 57.53997802734375, "learning_rate": 4.419028041654559e-07, "logits/chosen": 1.6351606845855713, "logits/rejected": 1.612158179283142, "logps/chosen": -282.5562744140625, "logps/ref_chosen": -290.5863342285156, "logps/ref_rejected": -263.41046142578125, "logps/rejected": -268.1485290527344, "loss": 4.5598, "margin_dpo/margin_mean": 12.768148422241211, "margin_dpo/margin_std": 23.036344528198242, "step": 144 }, { "KL/chosen_KL_mean": 8.423906326293945, "KL/mean": 0.10684564709663391, "KL/rejected_KL_mean": -8.210210800170898, "KL/std": 18.5434627532959, "epoch": 0.3036649214659686, "fcm_dpo/beta": 0.0359405092895031, "fcm_dpo/delta": -0.07233719527721405, "fcm_dpo/margin": 16.634119033813477, "fcm_dpo/q_t": 0.37615370750427246, "grad_norm": 46.92060470581055, "learning_rate": 4.4072430294890166e-07, "logits/chosen": 1.5900194644927979, "logits/rejected": 1.497979760169983, "logps/chosen": -256.1163330078125, "logps/ref_chosen": -264.5402526855469, "logps/ref_rejected": -214.7958221435547, "logps/rejected": -223.0060272216797, "loss": 4.1937, "margin_dpo/margin_mean": 16.634119033813477, "margin_dpo/margin_std": 23.260997772216797, "step": 145 }, { "KL/chosen_KL_mean": 4.434247970581055, "KL/mean": -1.973922610282898, "KL/rejected_KL_mean": -8.382087707519531, "KL/std": 18.502222061157227, "epoch": 0.3057591623036649, "fcm_dpo/beta": 0.03668835386633873, "fcm_dpo/delta": 0.06929504871368408, "fcm_dpo/margin": 12.816330909729004, "fcm_dpo/q_t": 0.4019925892353058, "grad_norm": 77.25863647460938, "learning_rate": 4.395355737667985e-07, "logits/chosen": 1.4586389064788818, "logits/rejected": 1.5658501386642456, "logps/chosen": -265.74554443359375, "logps/ref_chosen": -270.17974853515625, "logps/ref_rejected": -261.2323913574219, "logps/rejected": -269.6144714355469, "loss": 4.5585, "margin_dpo/margin_mean": 12.816332817077637, "margin_dpo/margin_std": 22.71735191345215, "step": 146 }, { "KL/chosen_KL_mean": 3.596536636352539, "KL/mean": -1.9176825284957886, "KL/rejected_KL_mean": -7.431909561157227, "KL/std": 20.64834213256836, "epoch": 0.3078534031413613, "fcm_dpo/beta": 0.03882746770977974, "fcm_dpo/delta": 0.024022063240408897, "fcm_dpo/margin": 11.028444290161133, "fcm_dpo/q_t": 0.41412436962127686, "grad_norm": 59.746368408203125, "learning_rate": 4.3833668036708483e-07, "logits/chosen": 1.4037699699401855, "logits/rejected": 1.367514729499817, "logps/chosen": -279.1840515136719, "logps/ref_chosen": -282.7806091308594, "logps/ref_rejected": -249.23399353027344, "logps/rejected": -256.6659240722656, "loss": 4.9488, "margin_dpo/margin_mean": 11.028443336486816, "margin_dpo/margin_std": 25.061433792114258, "step": 147 }, { "KL/chosen_KL_mean": 4.780019760131836, "KL/mean": -1.0210793018341064, "KL/rejected_KL_mean": -6.822177886962891, "KL/std": 18.859333038330078, "epoch": 0.3099476439790576, "fcm_dpo/beta": 0.03977343067526817, "fcm_dpo/delta": 0.08418260514736176, "fcm_dpo/margin": 11.602202415466309, "fcm_dpo/q_t": 0.40774232149124146, "grad_norm": 69.44275665283203, "learning_rate": 4.3712768704277524e-07, "logits/chosen": 1.4985826015472412, "logits/rejected": 1.509657621383667, "logps/chosen": -224.13729858398438, "logps/ref_chosen": -228.9173126220703, "logps/ref_rejected": -226.81222534179688, "logps/rejected": -233.63441467285156, "loss": 4.7155, "margin_dpo/margin_mean": 11.602203369140625, "margin_dpo/margin_std": 23.362445831298828, "step": 148 }, { "KL/chosen_KL_mean": 6.897031784057617, "KL/mean": -0.17732220888137817, "KL/rejected_KL_mean": -7.251680374145508, "KL/std": 19.505613327026367, "epoch": 0.31204188481675393, "fcm_dpo/beta": 0.04087590053677559, "fcm_dpo/delta": 0.021851126104593277, "fcm_dpo/margin": 14.14870548248291, "fcm_dpo/q_t": 0.3834710717201233, "grad_norm": 63.1515998840332, "learning_rate": 4.3590865862851263e-07, "logits/chosen": 1.524064302444458, "logits/rejected": 1.4874056577682495, "logps/chosen": -304.18890380859375, "logps/ref_chosen": -311.0859375, "logps/ref_rejected": -303.225341796875, "logps/rejected": -310.47705078125, "loss": 4.3311, "margin_dpo/margin_mean": 14.14870548248291, "margin_dpo/margin_std": 22.75583267211914, "step": 149 }, { "KL/chosen_KL_mean": 5.022129058837891, "KL/mean": -1.7816307544708252, "KL/rejected_KL_mean": -8.585397720336914, "KL/std": 18.76820182800293, "epoch": 0.31413612565445026, "fcm_dpo/beta": 0.042426105588674545, "fcm_dpo/delta": 0.022796878591179848, "fcm_dpo/margin": 13.607522964477539, "fcm_dpo/q_t": 0.3835679888725281, "grad_norm": 65.22611999511719, "learning_rate": 4.346796604970912e-07, "logits/chosen": 1.5500192642211914, "logits/rejected": 1.561482310295105, "logps/chosen": -265.4613342285156, "logps/ref_chosen": -270.48345947265625, "logps/ref_rejected": -269.0491638183594, "logps/rejected": -277.634521484375, "loss": 4.3796, "margin_dpo/margin_mean": 13.607524871826172, "margin_dpo/margin_std": 22.5236759185791, "step": 150 }, { "KL/chosen_KL_mean": 8.34054183959961, "KL/mean": -1.2480887174606323, "KL/rejected_KL_mean": -10.836723327636719, "KL/std": 20.108816146850586, "epoch": 0.3162303664921466, "fcm_dpo/beta": 0.03923875465989113, "fcm_dpo/delta": -0.16742047667503357, "fcm_dpo/margin": 19.177261352539062, "fcm_dpo/q_t": 0.3483484983444214, "grad_norm": 89.0177001953125, "learning_rate": 4.3344075855595097e-07, "logits/chosen": 1.5356732606887817, "logits/rejected": 1.5543012619018555, "logps/chosen": -279.8251647949219, "logps/ref_chosen": -288.16571044921875, "logps/ref_rejected": -264.7412414550781, "logps/rejected": -275.5779724121094, "loss": 3.9405, "margin_dpo/margin_mean": 19.177261352539062, "margin_dpo/margin_std": 25.098867416381836, "step": 151 }, { "KL/chosen_KL_mean": 7.048564910888672, "KL/mean": -0.2566535472869873, "KL/rejected_KL_mean": -7.561862945556641, "KL/std": 20.26637077331543, "epoch": 0.3183246073298429, "fcm_dpo/beta": 0.03661992773413658, "fcm_dpo/delta": -0.01711050420999527, "fcm_dpo/margin": 14.61042594909668, "fcm_dpo/q_t": 0.3898727297782898, "grad_norm": 71.59041595458984, "learning_rate": 4.3219201924364323e-07, "logits/chosen": 1.3534401655197144, "logits/rejected": 1.4261000156402588, "logps/chosen": -252.88995361328125, "logps/ref_chosen": -259.9385070800781, "logps/ref_rejected": -266.2453308105469, "logps/rejected": -273.8072509765625, "loss": 4.45, "margin_dpo/margin_mean": 14.610424995422363, "margin_dpo/margin_std": 24.417827606201172, "step": 152 }, { "KL/chosen_KL_mean": 5.933012008666992, "KL/mean": -5.1869964599609375, "KL/rejected_KL_mean": -16.3070125579834, "KL/std": 21.712013244628906, "epoch": 0.3204188481675393, "fcm_dpo/beta": 0.03363032639026642, "fcm_dpo/delta": -0.16363948583602905, "fcm_dpo/margin": 22.240018844604492, "fcm_dpo/q_t": 0.34008270502090454, "grad_norm": 49.01840591430664, "learning_rate": 4.309335095262675e-07, "logits/chosen": 1.5906455516815186, "logits/rejected": 1.6551276445388794, "logps/chosen": -259.6440124511719, "logps/ref_chosen": -265.5769958496094, "logps/ref_rejected": -277.6246032714844, "logps/rejected": -293.931640625, "loss": 3.637, "margin_dpo/margin_mean": 22.240018844604492, "margin_dpo/margin_std": 23.46230697631836, "step": 153 }, { "KL/chosen_KL_mean": 3.613248825073242, "KL/mean": -4.316822052001953, "KL/rejected_KL_mean": -12.246877670288086, "KL/std": 22.21419906616211, "epoch": 0.3225130890052356, "fcm_dpo/beta": 0.032934922724962234, "fcm_dpo/delta": 0.02667096257209778, "fcm_dpo/margin": 15.860122680664062, "fcm_dpo/q_t": 0.38882002234458923, "grad_norm": 52.11634063720703, "learning_rate": 4.2966529689388064e-07, "logits/chosen": 1.4579464197158813, "logits/rejected": 1.4288320541381836, "logps/chosen": -278.66015625, "logps/ref_chosen": -282.2734069824219, "logps/ref_rejected": -261.5135498046875, "logps/rejected": -273.76043701171875, "loss": 4.4732, "margin_dpo/margin_mean": 15.860122680664062, "margin_dpo/margin_std": 26.72602081298828, "step": 154 }, { "KL/chosen_KL_mean": 1.7544536590576172, "KL/mean": -4.108672142028809, "KL/rejected_KL_mean": -9.971803665161133, "KL/std": 20.209108352661133, "epoch": 0.32460732984293195, "fcm_dpo/beta": 0.034309010952711105, "fcm_dpo/delta": 0.07300734519958496, "fcm_dpo/margin": 11.726253509521484, "fcm_dpo/q_t": 0.4163626432418823, "grad_norm": 60.788536071777344, "learning_rate": 4.2838744935687716e-07, "logits/chosen": 1.4265573024749756, "logits/rejected": 1.5197659730911255, "logps/chosen": -276.9571228027344, "logps/ref_chosen": -278.7115783691406, "logps/ref_rejected": -298.0945739746094, "logps/rejected": -308.06640625, "loss": 4.8173, "margin_dpo/margin_mean": 11.7262544631958, "margin_dpo/margin_std": 24.825592041015625, "step": 155 }, { "KL/chosen_KL_mean": 2.966684341430664, "KL/mean": -6.782113075256348, "KL/rejected_KL_mean": -16.53091049194336, "KL/std": 22.180252075195312, "epoch": 0.3267015706806283, "fcm_dpo/beta": 0.03426394239068031, "fcm_dpo/delta": -0.07360708713531494, "fcm_dpo/margin": 19.497600555419922, "fcm_dpo/q_t": 0.3636919856071472, "grad_norm": 49.924827575683594, "learning_rate": 4.271000354423425e-07, "logits/chosen": 1.6021648645401, "logits/rejected": 1.6253923177719116, "logps/chosen": -265.38446044921875, "logps/ref_chosen": -268.35113525390625, "logps/ref_rejected": -241.9660186767578, "logps/rejected": -258.4969482421875, "loss": 4.1912, "margin_dpo/margin_mean": 19.49759864807129, "margin_dpo/margin_std": 29.38343048095703, "step": 156 }, { "KL/chosen_KL_mean": 1.2138385772705078, "KL/mean": -5.53639554977417, "KL/rejected_KL_mean": -12.286630630493164, "KL/std": 21.31039047241211, "epoch": 0.3287958115183246, "fcm_dpo/beta": 0.033198121935129166, "fcm_dpo/delta": 0.017839806154370308, "fcm_dpo/margin": 13.50045394897461, "fcm_dpo/q_t": 0.40580445528030396, "grad_norm": 58.3088264465332, "learning_rate": 4.258031241903777e-07, "logits/chosen": 1.3480119705200195, "logits/rejected": 1.4023487567901611, "logps/chosen": -264.1897888183594, "logps/ref_chosen": -265.40362548828125, "logps/ref_rejected": -257.82440185546875, "logps/rejected": -270.11102294921875, "loss": 4.6207, "margin_dpo/margin_mean": 13.50045394897461, "margin_dpo/margin_std": 24.77008056640625, "step": 157 }, { "KL/chosen_KL_mean": 1.459920883178711, "KL/mean": -5.981760501861572, "KL/rejected_KL_mean": -13.423439025878906, "KL/std": 20.990612030029297, "epoch": 0.3308900523560209, "fcm_dpo/beta": 0.0347001887857914, "fcm_dpo/delta": 0.019363276660442352, "fcm_dpo/margin": 14.8833646774292, "fcm_dpo/q_t": 0.39081138372421265, "grad_norm": 54.962223052978516, "learning_rate": 4.2449678515039743e-07, "logits/chosen": 1.6265257596969604, "logits/rejected": 1.6831825971603394, "logps/chosen": -279.7023620605469, "logps/ref_chosen": -281.1622619628906, "logps/ref_rejected": -268.2899475097656, "logps/rejected": -281.71337890625, "loss": 4.4528, "margin_dpo/margin_mean": 14.883365631103516, "margin_dpo/margin_std": 24.7315616607666, "step": 158 }, { "KL/chosen_KL_mean": 0.3786163330078125, "KL/mean": -5.6010661125183105, "KL/rejected_KL_mean": -11.580739974975586, "KL/std": 22.01244354248047, "epoch": 0.33298429319371725, "fcm_dpo/beta": 0.03370116651058197, "fcm_dpo/delta": -0.017574762925505638, "fcm_dpo/margin": 11.95935344696045, "fcm_dpo/q_t": 0.4201620817184448, "grad_norm": 61.2664794921875, "learning_rate": 4.2318108837739986e-07, "logits/chosen": 1.600152611732483, "logits/rejected": 1.492041826248169, "logps/chosen": -310.9463806152344, "logps/ref_chosen": -311.32501220703125, "logps/ref_rejected": -248.85350036621094, "logps/rejected": -260.4342346191406, "loss": 5.0339, "margin_dpo/margin_mean": 11.95935344696045, "margin_dpo/margin_std": 29.016633987426758, "step": 159 }, { "KL/chosen_KL_mean": 3.298036575317383, "KL/mean": -5.638082027435303, "KL/rejected_KL_mean": -14.574199676513672, "KL/std": 21.566913604736328, "epoch": 0.33507853403141363, "fcm_dpo/beta": 0.03293953090906143, "fcm_dpo/delta": -0.03199426829814911, "fcm_dpo/margin": 17.872236251831055, "fcm_dpo/q_t": 0.3772663474082947, "grad_norm": 53.90055847167969, "learning_rate": 4.218561044282098e-07, "logits/chosen": 1.5303587913513184, "logits/rejected": 1.5337125062942505, "logps/chosen": -263.8109130859375, "logps/ref_chosen": -267.10894775390625, "logps/ref_rejected": -258.66162109375, "logps/rejected": -273.2358093261719, "loss": 4.161, "margin_dpo/margin_mean": 17.872238159179688, "margin_dpo/margin_std": 24.7597713470459, "step": 160 }, { "KL/chosen_KL_mean": 3.1708030700683594, "KL/mean": -6.072520732879639, "KL/rejected_KL_mean": -15.315837860107422, "KL/std": 22.220367431640625, "epoch": 0.33717277486910996, "fcm_dpo/beta": 0.033242467790842056, "fcm_dpo/delta": -0.01598743163049221, "fcm_dpo/margin": 18.486648559570312, "fcm_dpo/q_t": 0.37375712394714355, "grad_norm": 54.25385665893555, "learning_rate": 4.2052190435769554e-07, "logits/chosen": 1.8315788507461548, "logits/rejected": 1.737775206565857, "logps/chosen": -298.7215881347656, "logps/ref_chosen": -301.89239501953125, "logps/ref_rejected": -247.645263671875, "logps/rejected": -262.9610900878906, "loss": 4.2261, "margin_dpo/margin_mean": 18.486648559570312, "margin_dpo/margin_std": 27.791015625, "step": 161 }, { "KL/chosen_KL_mean": 4.625587463378906, "KL/mean": -3.6662158966064453, "KL/rejected_KL_mean": -11.95802116394043, "KL/std": 21.571430206298828, "epoch": 0.3392670157068063, "fcm_dpo/beta": 0.03174804151058197, "fcm_dpo/delta": 0.019307700917124748, "fcm_dpo/margin": 16.583608627319336, "fcm_dpo/q_t": 0.3874986171722412, "grad_norm": 53.21702575683594, "learning_rate": 4.1917855971495763e-07, "logits/chosen": 1.5175679922103882, "logits/rejected": 1.4912818670272827, "logps/chosen": -283.60748291015625, "logps/ref_chosen": -288.2330627441406, "logps/ref_rejected": -252.5148162841797, "logps/rejected": -264.47283935546875, "loss": 4.3715, "margin_dpo/margin_mean": 16.583608627319336, "margin_dpo/margin_std": 25.756649017333984, "step": 162 }, { "KL/chosen_KL_mean": 2.9671707153320312, "KL/mean": -6.372570991516113, "KL/rejected_KL_mean": -15.712316513061523, "KL/std": 20.523710250854492, "epoch": 0.3413612565445026, "fcm_dpo/beta": 0.03297853097319603, "fcm_dpo/delta": -0.0184785183519125, "fcm_dpo/margin": 18.67947769165039, "fcm_dpo/q_t": 0.3669191002845764, "grad_norm": 81.9054183959961, "learning_rate": 4.1782614253949255e-07, "logits/chosen": 1.7574456930160522, "logits/rejected": 1.7493940591812134, "logps/chosen": -283.796630859375, "logps/ref_chosen": -286.7638244628906, "logps/ref_rejected": -262.8302917480469, "logps/rejected": -278.5425720214844, "loss": 3.9888, "margin_dpo/margin_mean": 18.67947769165039, "margin_dpo/margin_std": 22.76688575744629, "step": 163 }, { "KL/chosen_KL_mean": -2.706052780151367, "KL/mean": -10.915844917297363, "KL/rejected_KL_mean": -19.125640869140625, "KL/std": 21.754802703857422, "epoch": 0.34345549738219894, "fcm_dpo/beta": 0.032983679324388504, "fcm_dpo/delta": 0.002447988837957382, "fcm_dpo/margin": 16.41958236694336, "fcm_dpo/q_t": 0.3856901526451111, "grad_norm": 52.84718322753906, "learning_rate": 4.164647253573289e-07, "logits/chosen": 1.7213536500930786, "logits/rejected": 1.680013656616211, "logps/chosen": -262.9845275878906, "logps/ref_chosen": -260.2784729003906, "logps/ref_rejected": -217.10556030273438, "logps/rejected": -236.231201171875, "loss": 4.2968, "margin_dpo/margin_mean": 16.41958236694336, "margin_dpo/margin_std": 24.709409713745117, "step": 164 }, { "KL/chosen_KL_mean": 1.5090866088867188, "KL/mean": -5.225419521331787, "KL/rejected_KL_mean": -11.95993423461914, "KL/std": 20.277297973632812, "epoch": 0.34554973821989526, "fcm_dpo/beta": 0.034100648015737534, "fcm_dpo/delta": 0.061964474618434906, "fcm_dpo/margin": 13.469009399414062, "fcm_dpo/q_t": 0.4033600986003876, "grad_norm": 61.006935119628906, "learning_rate": 4.1509438117713863e-07, "logits/chosen": 1.6673741340637207, "logits/rejected": 1.7128586769104004, "logps/chosen": -269.04522705078125, "logps/ref_chosen": -270.5543212890625, "logps/ref_rejected": -259.80218505859375, "logps/rejected": -271.7620849609375, "loss": 4.5383, "margin_dpo/margin_mean": 13.469008445739746, "margin_dpo/margin_std": 23.338573455810547, "step": 165 }, { "KL/chosen_KL_mean": 0.41426753997802734, "KL/mean": -5.925065517425537, "KL/rejected_KL_mean": -12.26440143585205, "KL/std": 22.263172149658203, "epoch": 0.34764397905759165, "fcm_dpo/beta": 0.036087341606616974, "fcm_dpo/delta": 0.15015417337417603, "fcm_dpo/margin": 12.678671836853027, "fcm_dpo/q_t": 0.4126741290092468, "grad_norm": 66.5831298828125, "learning_rate": 4.137151834863213e-07, "logits/chosen": 1.5135771036148071, "logits/rejected": 1.4846245050430298, "logps/chosen": -269.88018798828125, "logps/ref_chosen": -270.29443359375, "logps/ref_rejected": -230.40151977539062, "logps/rejected": -242.66590881347656, "loss": 4.9011, "margin_dpo/margin_mean": 12.678671836853027, "margin_dpo/margin_std": 28.675046920776367, "step": 166 }, { "KL/chosen_KL_mean": 0.8034381866455078, "KL/mean": -6.434817790985107, "KL/rejected_KL_mean": -13.673067092895508, "KL/std": 21.16307830810547, "epoch": 0.34973821989528797, "fcm_dpo/beta": 0.03901813179254532, "fcm_dpo/delta": -0.016899898648262024, "fcm_dpo/margin": 14.476500511169434, "fcm_dpo/q_t": 0.38619351387023926, "grad_norm": 68.85694122314453, "learning_rate": 4.123272062470633e-07, "logits/chosen": 1.7194454669952393, "logits/rejected": 1.6758253574371338, "logps/chosen": -272.65716552734375, "logps/ref_chosen": -273.4605712890625, "logps/ref_rejected": -248.86997985839844, "logps/rejected": -262.5430603027344, "loss": 4.4764, "margin_dpo/margin_mean": 14.476500511169434, "margin_dpo/margin_std": 25.108524322509766, "step": 167 }, { "KL/chosen_KL_mean": 4.459650039672852, "KL/mean": -4.971283912658691, "KL/rejected_KL_mean": -14.402233123779297, "KL/std": 23.291791915893555, "epoch": 0.3518324607329843, "fcm_dpo/beta": 0.035170651972293854, "fcm_dpo/delta": -0.1423185020685196, "fcm_dpo/margin": 18.861879348754883, "fcm_dpo/q_t": 0.36044758558273315, "grad_norm": 66.56237030029297, "learning_rate": 4.1093052389237174e-07, "logits/chosen": 1.5125325918197632, "logits/rejected": 1.4943667650222778, "logps/chosen": -301.2650451660156, "logps/ref_chosen": -305.7247009277344, "logps/ref_rejected": -290.16650390625, "logps/rejected": -304.5687561035156, "loss": 4.0711, "margin_dpo/margin_mean": 18.86187744140625, "margin_dpo/margin_std": 25.238630294799805, "step": 168 }, { "KL/chosen_KL_mean": 3.7789669036865234, "KL/mean": -7.170663833618164, "KL/rejected_KL_mean": -18.12029457092285, "KL/std": 22.5257568359375, "epoch": 0.3539267015706806, "fcm_dpo/beta": 0.03189728409051895, "fcm_dpo/delta": -0.1565765142440796, "fcm_dpo/margin": 21.899253845214844, "fcm_dpo/q_t": 0.35491833090782166, "grad_norm": 41.584468841552734, "learning_rate": 4.0952521132208267e-07, "logits/chosen": 1.3543460369110107, "logits/rejected": 1.3774365186691284, "logps/chosen": -248.39129638671875, "logps/ref_chosen": -252.1702880859375, "logps/ref_rejected": -233.34750366210938, "logps/rejected": -251.46780395507812, "loss": 3.866, "margin_dpo/margin_mean": 21.89925765991211, "margin_dpo/margin_std": 25.406164169311523, "step": 169 }, { "KL/chosen_KL_mean": 4.566259384155273, "KL/mean": -4.576227188110352, "KL/rejected_KL_mean": -13.718713760375977, "KL/std": 21.150253295898438, "epoch": 0.35602094240837695, "fcm_dpo/beta": 0.030278891324996948, "fcm_dpo/delta": 0.049296632409095764, "fcm_dpo/margin": 18.284975051879883, "fcm_dpo/q_t": 0.3811044692993164, "grad_norm": 54.09820556640625, "learning_rate": 4.081113438988443e-07, "logits/chosen": 1.4744322299957275, "logits/rejected": 1.5083649158477783, "logps/chosen": -297.9121398925781, "logps/ref_chosen": -302.4783935546875, "logps/ref_rejected": -259.40777587890625, "logps/rejected": -273.1264953613281, "loss": 4.1314, "margin_dpo/margin_mean": 18.284975051879883, "margin_dpo/margin_std": 24.58019256591797, "step": 170 }, { "KL/chosen_KL_mean": 4.070655822753906, "KL/mean": -5.274190902709961, "KL/rejected_KL_mean": -14.619039535522461, "KL/std": 21.39913558959961, "epoch": 0.3581151832460733, "fcm_dpo/beta": 0.03019765205681324, "fcm_dpo/delta": -0.03337910771369934, "fcm_dpo/margin": 18.689701080322266, "fcm_dpo/q_t": 0.3792933225631714, "grad_norm": 53.6861572265625, "learning_rate": 4.0668899744407567e-07, "logits/chosen": 1.3581851720809937, "logits/rejected": 1.465482234954834, "logps/chosen": -246.57606506347656, "logps/ref_chosen": -250.646728515625, "logps/ref_rejected": -259.1659851074219, "logps/rejected": -273.7850341796875, "loss": 4.1222, "margin_dpo/margin_mean": 18.689701080322266, "margin_dpo/margin_std": 23.99151611328125, "step": 171 }, { "KL/chosen_KL_mean": 4.425405502319336, "KL/mean": -1.8122390508651733, "KL/rejected_KL_mean": -8.049894332885742, "KL/std": 22.412261962890625, "epoch": 0.36020942408376966, "fcm_dpo/beta": 0.032538753002882004, "fcm_dpo/delta": 0.1456703394651413, "fcm_dpo/margin": 12.475287437438965, "fcm_dpo/q_t": 0.41149380803108215, "grad_norm": 57.02654266357422, "learning_rate": 4.0525824823390043e-07, "logits/chosen": 1.8623565435409546, "logits/rejected": 1.883276343345642, "logps/chosen": -324.59954833984375, "logps/ref_chosen": -329.02490234375, "logps/ref_rejected": -293.0215759277344, "logps/rejected": -301.0714416503906, "loss": 4.7384, "margin_dpo/margin_mean": 12.475286483764648, "margin_dpo/margin_std": 25.173187255859375, "step": 172 }, { "KL/chosen_KL_mean": 3.771291732788086, "KL/mean": -4.03558349609375, "KL/rejected_KL_mean": -11.84245491027832, "KL/std": 21.920373916625977, "epoch": 0.362303664921466, "fcm_dpo/beta": 0.03458525240421295, "fcm_dpo/delta": 0.0016709789633750916, "fcm_dpo/margin": 15.613749504089355, "fcm_dpo/q_t": 0.3865172863006592, "grad_norm": 50.34199523925781, "learning_rate": 4.0381917299505686e-07, "logits/chosen": 1.5832735300064087, "logits/rejected": 1.5554523468017578, "logps/chosen": -286.36932373046875, "logps/ref_chosen": -290.140625, "logps/ref_rejected": -271.474853515625, "logps/rejected": -283.3172912597656, "loss": 4.3357, "margin_dpo/margin_mean": 15.613749504089355, "margin_dpo/margin_std": 24.040199279785156, "step": 173 }, { "KL/chosen_KL_mean": 3.915590286254883, "KL/mean": -5.95166540145874, "KL/rejected_KL_mean": -15.818925857543945, "KL/std": 21.474803924560547, "epoch": 0.3643979057591623, "fcm_dpo/beta": 0.033181384205818176, "fcm_dpo/delta": -0.06058121472597122, "fcm_dpo/margin": 19.734516143798828, "fcm_dpo/q_t": 0.3638980984687805, "grad_norm": 54.13778305053711, "learning_rate": 4.0237184890078243e-07, "logits/chosen": 1.6420636177062988, "logits/rejected": 1.568307876586914, "logps/chosen": -325.39166259765625, "logps/ref_chosen": -329.3072509765625, "logps/ref_rejected": -256.6839599609375, "logps/rejected": -272.50286865234375, "loss": 3.9372, "margin_dpo/margin_mean": 19.734516143798828, "margin_dpo/margin_std": 24.167448043823242, "step": 174 }, { "KL/chosen_KL_mean": 1.5434303283691406, "KL/mean": -6.360366344451904, "KL/rejected_KL_mean": -14.264167785644531, "KL/std": 22.346118927001953, "epoch": 0.36649214659685864, "fcm_dpo/beta": 0.03351647034287453, "fcm_dpo/delta": 0.07235424220561981, "fcm_dpo/margin": 15.807592391967773, "fcm_dpo/q_t": 0.38753843307495117, "grad_norm": 66.53447723388672, "learning_rate": 4.00916353566676e-07, "logits/chosen": 1.6045417785644531, "logits/rejected": 1.592195987701416, "logps/chosen": -274.35882568359375, "logps/ref_chosen": -275.9023132324219, "logps/ref_rejected": -281.4163513183594, "logps/rejected": -295.6805114746094, "loss": 4.4513, "margin_dpo/margin_mean": 15.807592391967773, "margin_dpo/margin_std": 26.658302307128906, "step": 175 }, { "KL/chosen_KL_mean": -0.5240001678466797, "KL/mean": -6.366352081298828, "KL/rejected_KL_mean": -12.208709716796875, "KL/std": 21.937408447265625, "epoch": 0.36858638743455496, "fcm_dpo/beta": 0.035578951239585876, "fcm_dpo/delta": 0.00773615762591362, "fcm_dpo/margin": 11.684707641601562, "fcm_dpo/q_t": 0.41137951612472534, "grad_norm": 64.17352294921875, "learning_rate": 3.994527650465352e-07, "logits/chosen": 1.6549088954925537, "logits/rejected": 1.727834939956665, "logps/chosen": -244.08819580078125, "logps/ref_chosen": -243.56423950195312, "logps/ref_rejected": -239.12905883789062, "logps/rejected": -251.33773803710938, "loss": 4.9509, "margin_dpo/margin_mean": 11.684707641601562, "margin_dpo/margin_std": 26.69280433654785, "step": 176 }, { "KL/chosen_KL_mean": -0.5834159851074219, "KL/mean": -6.346553802490234, "KL/rejected_KL_mean": -12.109701156616211, "KL/std": 22.373661041259766, "epoch": 0.3706806282722513, "fcm_dpo/beta": 0.035139717161655426, "fcm_dpo/delta": 0.015670977532863617, "fcm_dpo/margin": 11.526307106018066, "fcm_dpo/q_t": 0.4153884947299957, "grad_norm": 86.48933410644531, "learning_rate": 3.979811618281705e-07, "logits/chosen": 1.7052347660064697, "logits/rejected": 1.6299563646316528, "logps/chosen": -293.8870849609375, "logps/ref_chosen": -293.30364990234375, "logps/ref_rejected": -292.293701171875, "logps/rejected": -304.4034118652344, "loss": 5.0891, "margin_dpo/margin_mean": 11.52630615234375, "margin_dpo/margin_std": 28.712135314941406, "step": 177 }, { "KL/chosen_KL_mean": 2.7671356201171875, "KL/mean": -6.616493225097656, "KL/rejected_KL_mean": -16.000125885009766, "KL/std": 23.80596923828125, "epoch": 0.37277486910994767, "fcm_dpo/beta": 0.033524710685014725, "fcm_dpo/delta": -0.08717905730009079, "fcm_dpo/margin": 18.767263412475586, "fcm_dpo/q_t": 0.37235215306282043, "grad_norm": 50.03008270263672, "learning_rate": 3.9650162282919654e-07, "logits/chosen": 1.4354071617126465, "logits/rejected": 1.439162254333496, "logps/chosen": -273.3591003417969, "logps/ref_chosen": -276.126220703125, "logps/ref_rejected": -240.31179809570312, "logps/rejected": -256.3119201660156, "loss": 4.1963, "margin_dpo/margin_mean": 18.767263412475586, "margin_dpo/margin_std": 28.032089233398438, "step": 178 }, { "KL/chosen_KL_mean": 2.1550731658935547, "KL/mean": -4.787572860717773, "KL/rejected_KL_mean": -11.730218887329102, "KL/std": 22.734161376953125, "epoch": 0.374869109947644, "fcm_dpo/beta": 0.03260195255279541, "fcm_dpo/delta": 0.021392395719885826, "fcm_dpo/margin": 13.885309219360352, "fcm_dpo/q_t": 0.40605348348617554, "grad_norm": 64.03956604003906, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 1.4492707252502441, "logits/rejected": 1.559048056602478, "logps/chosen": -247.70736694335938, "logps/ref_chosen": -249.8624267578125, "logps/ref_rejected": -271.18603515625, "logps/rejected": -282.9162902832031, "loss": 4.6721, "margin_dpo/margin_mean": 13.885309219360352, "margin_dpo/margin_std": 26.38463592529297, "step": 179 }, { "KL/chosen_KL_mean": 2.6901912689208984, "KL/mean": -6.234161376953125, "KL/rejected_KL_mean": -15.158517837524414, "KL/std": 22.213472366333008, "epoch": 0.3769633507853403, "fcm_dpo/beta": 0.03353189304471016, "fcm_dpo/delta": 0.0003486163914203644, "fcm_dpo/margin": 17.848716735839844, "fcm_dpo/q_t": 0.3777158558368683, "grad_norm": 54.19392395019531, "learning_rate": 3.935190552834828e-07, "logits/chosen": 1.423414707183838, "logits/rejected": 1.4377055168151855, "logps/chosen": -257.9789123535156, "logps/ref_chosen": -260.66912841796875, "logps/ref_rejected": -229.25491333007812, "logps/rejected": -244.41343688964844, "loss": 4.1846, "margin_dpo/margin_mean": 17.848716735839844, "margin_dpo/margin_std": 25.64480972290039, "step": 180 }, { "KL/chosen_KL_mean": 3.9794750213623047, "KL/mean": -4.322617530822754, "KL/rejected_KL_mean": -12.624713897705078, "KL/std": 22.12684440612793, "epoch": 0.37905759162303665, "fcm_dpo/beta": 0.0333593524992466, "fcm_dpo/delta": 0.047833461314439774, "fcm_dpo/margin": 16.604196548461914, "fcm_dpo/q_t": 0.38562846183776855, "grad_norm": 54.91704559326172, "learning_rate": 3.920161866827889e-07, "logits/chosen": 1.6338317394256592, "logits/rejected": 1.6056936979293823, "logps/chosen": -286.90911865234375, "logps/ref_chosen": -290.8885803222656, "logps/ref_rejected": -275.1922302246094, "logps/rejected": -287.81695556640625, "loss": 4.4188, "margin_dpo/margin_mean": 16.60419273376465, "margin_dpo/margin_std": 27.110607147216797, "step": 181 }, { "KL/chosen_KL_mean": 2.504199981689453, "KL/mean": -7.788280010223389, "KL/rejected_KL_mean": -18.080759048461914, "KL/std": 22.3680362701416, "epoch": 0.381151832460733, "fcm_dpo/beta": 0.033523909747600555, "fcm_dpo/delta": -0.09699434787034988, "fcm_dpo/margin": 20.58496856689453, "fcm_dpo/q_t": 0.3516695499420166, "grad_norm": 48.38198471069336, "learning_rate": 3.90505702185e-07, "logits/chosen": 1.529535174369812, "logits/rejected": 1.5425692796707153, "logps/chosen": -268.0626525878906, "logps/ref_chosen": -270.56683349609375, "logps/ref_rejected": -260.3321533203125, "logps/rejected": -278.41290283203125, "loss": 3.8327, "margin_dpo/margin_mean": 20.58496856689453, "margin_dpo/margin_std": 23.530336380004883, "step": 182 }, { "KL/chosen_KL_mean": 3.2648563385009766, "KL/mean": -5.201539039611816, "KL/rejected_KL_mean": -13.667932510375977, "KL/std": 22.122039794921875, "epoch": 0.3832460732984293, "fcm_dpo/beta": 0.03249736875295639, "fcm_dpo/delta": 0.05231393501162529, "fcm_dpo/margin": 16.932781219482422, "fcm_dpo/q_t": 0.3850289285182953, "grad_norm": 52.90995407104492, "learning_rate": 3.889876827928156e-07, "logits/chosen": 1.479565978050232, "logits/rejected": 1.5102664232254028, "logps/chosen": -257.76312255859375, "logps/ref_chosen": -261.0279846191406, "logps/ref_rejected": -240.72361755371094, "logps/rejected": -254.39157104492188, "loss": 4.3147, "margin_dpo/margin_mean": 16.932781219482422, "margin_dpo/margin_std": 26.4365291595459, "step": 183 }, { "KL/chosen_KL_mean": 7.062267303466797, "KL/mean": -5.364647388458252, "KL/rejected_KL_mean": -17.791555404663086, "KL/std": 24.86423110961914, "epoch": 0.38534031413612563, "fcm_dpo/beta": 0.03018895350396633, "fcm_dpo/delta": -0.1675216257572174, "fcm_dpo/margin": 24.85382652282715, "fcm_dpo/q_t": 0.3426292836666107, "grad_norm": 55.49262619018555, "learning_rate": 3.874622099130087e-07, "logits/chosen": 1.6249607801437378, "logits/rejected": 1.5996079444885254, "logps/chosen": -302.0216064453125, "logps/ref_chosen": -309.0838317871094, "logps/ref_rejected": -263.6435241699219, "logps/rejected": -281.43511962890625, "loss": 3.7688, "margin_dpo/margin_mean": 24.85382652282715, "margin_dpo/margin_std": 29.232576370239258, "step": 184 }, { "KL/chosen_KL_mean": 1.7405548095703125, "KL/mean": -6.731338977813721, "KL/rejected_KL_mean": -15.203241348266602, "KL/std": 23.19679832458496, "epoch": 0.387434554973822, "fcm_dpo/beta": 0.02973821386694908, "fcm_dpo/delta": 0.045723918825387955, "fcm_dpo/margin": 16.943790435791016, "fcm_dpo/q_t": 0.39501094818115234, "grad_norm": 51.1673583984375, "learning_rate": 3.859293653520604e-07, "logits/chosen": 1.5313293933868408, "logits/rejected": 1.5998423099517822, "logps/chosen": -263.9611511230469, "logps/ref_chosen": -265.70172119140625, "logps/ref_rejected": -257.5074157714844, "logps/rejected": -272.7106628417969, "loss": 4.4361, "margin_dpo/margin_mean": 16.943788528442383, "margin_dpo/margin_std": 27.48980140686035, "step": 185 }, { "KL/chosen_KL_mean": -0.06787872314453125, "KL/mean": -8.355184555053711, "KL/rejected_KL_mean": -16.64249038696289, "KL/std": 22.052623748779297, "epoch": 0.38952879581151834, "fcm_dpo/beta": 0.031189240515232086, "fcm_dpo/delta": 0.0864519476890564, "fcm_dpo/margin": 16.57461929321289, "fcm_dpo/q_t": 0.39101552963256836, "grad_norm": 57.871768951416016, "learning_rate": 3.8438923131177237e-07, "logits/chosen": 1.5475683212280273, "logits/rejected": 1.5632997751235962, "logps/chosen": -291.7928466796875, "logps/ref_chosen": -291.7249755859375, "logps/ref_rejected": -273.54010009765625, "logps/rejected": -290.1826171875, "loss": 4.3362, "margin_dpo/margin_mean": 16.574621200561523, "margin_dpo/margin_std": 25.190021514892578, "step": 186 }, { "KL/chosen_KL_mean": 0.42089271545410156, "KL/mean": -8.181295394897461, "KL/rejected_KL_mean": -16.783475875854492, "KL/std": 24.009702682495117, "epoch": 0.39162303664921466, "fcm_dpo/beta": 0.03242059797048569, "fcm_dpo/delta": 0.044073861092329025, "fcm_dpo/margin": 17.204360961914062, "fcm_dpo/q_t": 0.3884267807006836, "grad_norm": 65.55802917480469, "learning_rate": 3.828418903848593e-07, "logits/chosen": 1.5783560276031494, "logits/rejected": 1.5967649221420288, "logps/chosen": -321.2809753417969, "logps/ref_chosen": -321.70184326171875, "logps/ref_rejected": -261.8386535644531, "logps/rejected": -278.62213134765625, "loss": 4.5248, "margin_dpo/margin_mean": 17.204364776611328, "margin_dpo/margin_std": 30.568544387817383, "step": 187 }, { "KL/chosen_KL_mean": -1.0099678039550781, "KL/mean": -9.896193504333496, "KL/rejected_KL_mean": -18.782432556152344, "KL/std": 23.633604049682617, "epoch": 0.393717277486911, "fcm_dpo/beta": 0.03295998275279999, "fcm_dpo/delta": 0.013461939990520477, "fcm_dpo/margin": 17.772464752197266, "fcm_dpo/q_t": 0.3803802728652954, "grad_norm": 51.01662826538086, "learning_rate": 3.812874255505191e-07, "logits/chosen": 1.4776414632797241, "logits/rejected": 1.45059335231781, "logps/chosen": -256.9253845214844, "logps/ref_chosen": -255.91542053222656, "logps/ref_rejected": -245.1119842529297, "logps/rejected": -263.8944091796875, "loss": 4.4022, "margin_dpo/margin_mean": 17.772464752197266, "margin_dpo/margin_std": 29.367900848388672, "step": 188 }, { "KL/chosen_KL_mean": 4.5848846435546875, "KL/mean": -5.746577739715576, "KL/rejected_KL_mean": -16.078039169311523, "KL/std": 22.845165252685547, "epoch": 0.3958115183246073, "fcm_dpo/beta": 0.03189515694975853, "fcm_dpo/delta": -0.12400280684232712, "fcm_dpo/margin": 20.662931442260742, "fcm_dpo/q_t": 0.3588719069957733, "grad_norm": 48.38937759399414, "learning_rate": 3.797259201699833e-07, "logits/chosen": 1.689395785331726, "logits/rejected": 1.7200841903686523, "logps/chosen": -261.4336853027344, "logps/ref_chosen": -266.0185546875, "logps/ref_rejected": -272.862548828125, "logps/rejected": -288.9405822753906, "loss": 3.8595, "margin_dpo/margin_mean": 20.662931442260742, "margin_dpo/margin_std": 23.503387451171875, "step": 189 }, { "KL/chosen_KL_mean": 3.0656661987304688, "KL/mean": -6.925785064697266, "KL/rejected_KL_mean": -16.917236328125, "KL/std": 22.68586540222168, "epoch": 0.39790575916230364, "fcm_dpo/beta": 0.03057757578790188, "fcm_dpo/delta": -0.012582660652697086, "fcm_dpo/margin": 19.982908248901367, "fcm_dpo/q_t": 0.37064746022224426, "grad_norm": 48.570960998535156, "learning_rate": 3.781574579820464e-07, "logits/chosen": 1.4705171585083008, "logits/rejected": 1.4011236429214478, "logps/chosen": -247.08529663085938, "logps/ref_chosen": -250.15098571777344, "logps/ref_rejected": -223.34393310546875, "logps/rejected": -240.26116943359375, "loss": 4.1479, "margin_dpo/margin_mean": 19.982908248901367, "margin_dpo/margin_std": 28.228897094726562, "step": 190 }, { "KL/chosen_KL_mean": 3.550985336303711, "KL/mean": -4.874011516571045, "KL/rejected_KL_mean": -13.299015045166016, "KL/std": 23.36663055419922, "epoch": 0.4, "fcm_dpo/beta": 0.030532926321029663, "fcm_dpo/delta": 0.032537586987018585, "fcm_dpo/margin": 16.849985122680664, "fcm_dpo/q_t": 0.39333850145339966, "grad_norm": 63.48810958862305, "learning_rate": 3.765821230985757e-07, "logits/chosen": 1.395258903503418, "logits/rejected": 1.4522275924682617, "logps/chosen": -234.8408203125, "logps/ref_chosen": -238.3917999267578, "logps/ref_rejected": -243.70272827148438, "logps/rejected": -257.0017395019531, "loss": 4.4708, "margin_dpo/margin_mean": 16.84998321533203, "margin_dpo/margin_std": 28.85344696044922, "step": 191 }, { "KL/chosen_KL_mean": 0.2355213165283203, "KL/mean": -7.245480060577393, "KL/rejected_KL_mean": -14.726480484008789, "KL/std": 22.562896728515625, "epoch": 0.40209424083769635, "fcm_dpo/beta": 0.03104531019926071, "fcm_dpo/delta": 0.005957015790045261, "fcm_dpo/margin": 14.96200180053711, "fcm_dpo/q_t": 0.40129604935646057, "grad_norm": 58.2281379699707, "learning_rate": 3.75e-07, "logits/chosen": 1.4725991487503052, "logits/rejected": 1.5391660928726196, "logps/chosen": -256.4014587402344, "logps/ref_chosen": -256.6369934082031, "logps/ref_rejected": -256.8894348144531, "logps/rejected": -271.61590576171875, "loss": 4.6748, "margin_dpo/margin_mean": 14.962002754211426, "margin_dpo/margin_std": 28.23845100402832, "step": 192 }, { "KL/chosen_KL_mean": -1.5852489471435547, "KL/mean": -9.660032272338867, "KL/rejected_KL_mean": -17.73480796813965, "KL/std": 24.343469619750977, "epoch": 0.4041884816753927, "fcm_dpo/beta": 0.03241005539894104, "fcm_dpo/delta": 0.08015380799770355, "fcm_dpo/margin": 16.149559020996094, "fcm_dpo/q_t": 0.3922501802444458, "grad_norm": 71.72376251220703, "learning_rate": 3.734111735307796e-07, "logits/chosen": 1.6393442153930664, "logits/rejected": 1.7344868183135986, "logps/chosen": -274.3849182128906, "logps/ref_chosen": -272.7996826171875, "logps/ref_rejected": -289.2391052246094, "logps/rejected": -306.973876953125, "loss": 4.463, "margin_dpo/margin_mean": 16.149559020996094, "margin_dpo/margin_std": 27.722713470458984, "step": 193 }, { "KL/chosen_KL_mean": -0.7121067047119141, "KL/mean": -6.904905319213867, "KL/rejected_KL_mean": -13.097705841064453, "KL/std": 24.412935256958008, "epoch": 0.406282722513089, "fcm_dpo/beta": 0.03421565517783165, "fcm_dpo/delta": 0.06331188976764679, "fcm_dpo/margin": 12.385600090026855, "fcm_dpo/q_t": 0.4166790843009949, "grad_norm": 57.3293571472168, "learning_rate": 3.7181572889485623e-07, "logits/chosen": 1.179120659828186, "logits/rejected": 1.2023894786834717, "logps/chosen": -282.0211486816406, "logps/ref_chosen": -281.30902099609375, "logps/ref_rejected": -246.3243408203125, "logps/rejected": -259.42205810546875, "loss": 4.8153, "margin_dpo/margin_mean": 12.385601043701172, "margin_dpo/margin_std": 26.814218521118164, "step": 194 }, { "KL/chosen_KL_mean": -1.6187248229980469, "KL/mean": -7.541670799255371, "KL/rejected_KL_mean": -13.464611053466797, "KL/std": 24.115158081054688, "epoch": 0.4083769633507853, "fcm_dpo/beta": 0.0353362001478672, "fcm_dpo/delta": 0.02736041322350502, "fcm_dpo/margin": 11.845905303955078, "fcm_dpo/q_t": 0.4158301055431366, "grad_norm": 78.81611633300781, "learning_rate": 3.7021375165108377e-07, "logits/chosen": 1.621145248413086, "logits/rejected": 1.7222257852554321, "logps/chosen": -268.476806640625, "logps/ref_chosen": -266.8581237792969, "logps/ref_rejected": -276.0855712890625, "logps/rejected": -289.5501708984375, "loss": 4.9263, "margin_dpo/margin_mean": 11.845905303955078, "margin_dpo/margin_std": 27.449527740478516, "step": 195 }, { "KL/chosen_KL_mean": 3.640165328979492, "KL/mean": -5.483175277709961, "KL/rejected_KL_mean": -14.606521606445312, "KL/std": 22.341976165771484, "epoch": 0.41047120418848165, "fcm_dpo/beta": 0.03500073403120041, "fcm_dpo/delta": -0.04595329985022545, "fcm_dpo/margin": 18.246681213378906, "fcm_dpo/q_t": 0.3699175715446472, "grad_norm": 57.45576477050781, "learning_rate": 3.6860532770864005e-07, "logits/chosen": 1.2707561254501343, "logits/rejected": 1.3572243452072144, "logps/chosen": -263.35479736328125, "logps/ref_chosen": -266.99493408203125, "logps/ref_rejected": -248.60513305664062, "logps/rejected": -263.211669921875, "loss": 4.1055, "margin_dpo/margin_mean": 18.246681213378906, "margin_dpo/margin_std": 24.88081932067871, "step": 196 }, { "KL/chosen_KL_mean": 5.81505012512207, "KL/mean": -6.082913398742676, "KL/rejected_KL_mean": -17.98088264465332, "KL/std": 24.73003387451172, "epoch": 0.41256544502617803, "fcm_dpo/beta": 0.03207242116332054, "fcm_dpo/delta": -0.2244684398174286, "fcm_dpo/margin": 23.795928955078125, "fcm_dpo/q_t": 0.3399311900138855, "grad_norm": 54.1199836730957, "learning_rate": 3.6699054332241985e-07, "logits/chosen": 1.4284682273864746, "logits/rejected": 1.386784315109253, "logps/chosen": -297.17474365234375, "logps/ref_chosen": -302.98980712890625, "logps/ref_rejected": -262.8002624511719, "logps/rejected": -280.7811279296875, "loss": 3.7765, "margin_dpo/margin_mean": 23.795928955078125, "margin_dpo/margin_std": 26.546613693237305, "step": 197 }, { "KL/chosen_KL_mean": 3.993558883666992, "KL/mean": -5.0717973709106445, "KL/rejected_KL_mean": -14.137165069580078, "KL/std": 24.779373168945312, "epoch": 0.41465968586387436, "fcm_dpo/beta": 0.028974642977118492, "fcm_dpo/delta": 0.0227762870490551, "fcm_dpo/margin": 18.130712509155273, "fcm_dpo/q_t": 0.38926345109939575, "grad_norm": 48.55694580078125, "learning_rate": 3.653694850884091e-07, "logits/chosen": 1.6023445129394531, "logits/rejected": 1.686340570449829, "logps/chosen": -292.2182312011719, "logps/ref_chosen": -296.2117614746094, "logps/ref_rejected": -289.8357238769531, "logps/rejected": -303.972900390625, "loss": 4.4946, "margin_dpo/margin_mean": 18.130712509155273, "margin_dpo/margin_std": 31.369131088256836, "step": 198 }, { "KL/chosen_KL_mean": 3.8633079528808594, "KL/mean": -5.587414741516113, "KL/rejected_KL_mean": -15.03813362121582, "KL/std": 23.215652465820312, "epoch": 0.4167539267015707, "fcm_dpo/beta": 0.02959287539124489, "fcm_dpo/delta": -0.016490591689944267, "fcm_dpo/margin": 18.901432037353516, "fcm_dpo/q_t": 0.3825532793998718, "grad_norm": 49.97153854370117, "learning_rate": 3.6374223993904124e-07, "logits/chosen": 1.3078348636627197, "logits/rejected": 1.2245980501174927, "logps/chosen": -251.4293212890625, "logps/ref_chosen": -255.29263305664062, "logps/ref_rejected": -212.69761657714844, "logps/rejected": -227.73574829101562, "loss": 4.2515, "margin_dpo/margin_mean": 18.901432037353516, "margin_dpo/margin_std": 28.002967834472656, "step": 199 }, { "KL/chosen_KL_mean": 5.087657928466797, "KL/mean": -3.0331709384918213, "KL/rejected_KL_mean": -11.153993606567383, "KL/std": 24.184293746948242, "epoch": 0.418848167539267, "fcm_dpo/beta": 0.03048442304134369, "fcm_dpo/delta": 0.10826665163040161, "fcm_dpo/margin": 16.24165916442871, "fcm_dpo/q_t": 0.40207067131996155, "grad_norm": 62.7784538269043, "learning_rate": 3.621088951385353e-07, "logits/chosen": 1.7090070247650146, "logits/rejected": 1.6965177059173584, "logps/chosen": -308.4487609863281, "logps/ref_chosen": -313.5364074707031, "logps/ref_rejected": -274.349609375, "logps/rejected": -285.5035400390625, "loss": 4.7548, "margin_dpo/margin_mean": 16.241661071777344, "margin_dpo/margin_std": 33.140193939208984, "step": 200 }, { "KL/chosen_KL_mean": 2.934652328491211, "KL/mean": -6.4366374015808105, "KL/rejected_KL_mean": -15.807924270629883, "KL/std": 24.043773651123047, "epoch": 0.42094240837696334, "fcm_dpo/beta": 0.03153421729803085, "fcm_dpo/delta": -0.04986376315355301, "fcm_dpo/margin": 18.742576599121094, "fcm_dpo/q_t": 0.37566325068473816, "grad_norm": 48.46647262573242, "learning_rate": 3.604695382782159e-07, "logits/chosen": 1.3615880012512207, "logits/rejected": 1.3875739574432373, "logps/chosen": -259.77252197265625, "logps/ref_chosen": -262.7071838378906, "logps/ref_rejected": -243.85879516601562, "logps/rejected": -259.6667175292969, "loss": 4.21, "margin_dpo/margin_mean": 18.742576599121094, "margin_dpo/margin_std": 26.73308563232422, "step": 201 }, { "KL/chosen_KL_mean": 1.704813003540039, "KL/mean": -7.445696830749512, "KL/rejected_KL_mean": -16.596208572387695, "KL/std": 24.027362823486328, "epoch": 0.42303664921465967, "fcm_dpo/beta": 0.030500078573822975, "fcm_dpo/delta": -0.014350108802318573, "fcm_dpo/margin": 18.301023483276367, "fcm_dpo/q_t": 0.3851291239261627, "grad_norm": 65.09416961669922, "learning_rate": 3.588242572718162e-07, "logits/chosen": 1.4482910633087158, "logits/rejected": 1.4086792469024658, "logps/chosen": -262.5819396972656, "logps/ref_chosen": -264.2867431640625, "logps/ref_rejected": -231.6894989013672, "logps/rejected": -248.2857208251953, "loss": 4.4414, "margin_dpo/margin_mean": 18.301021575927734, "margin_dpo/margin_std": 30.050230026245117, "step": 202 }, { "KL/chosen_KL_mean": 1.4567317962646484, "KL/mean": -5.678568363189697, "KL/rejected_KL_mean": -12.81386947631836, "KL/std": 23.273643493652344, "epoch": 0.42513089005235605, "fcm_dpo/beta": 0.031557291746139526, "fcm_dpo/delta": 0.10629908740520477, "fcm_dpo/margin": 14.270602226257324, "fcm_dpo/q_t": 0.4037012755870819, "grad_norm": 58.54388427734375, "learning_rate": 3.571731403507635e-07, "logits/chosen": 1.5311411619186401, "logits/rejected": 1.4974974393844604, "logps/chosen": -269.82745361328125, "logps/ref_chosen": -271.2842102050781, "logps/ref_rejected": -247.95462036132812, "logps/rejected": -260.76849365234375, "loss": 4.4926, "margin_dpo/margin_mean": 14.27060317993164, "margin_dpo/margin_std": 23.92316246032715, "step": 203 }, { "KL/chosen_KL_mean": 4.097650527954102, "KL/mean": -6.428679466247559, "KL/rejected_KL_mean": -16.954998016357422, "KL/std": 24.10330581665039, "epoch": 0.4272251308900524, "fcm_dpo/beta": 0.031438861042261124, "fcm_dpo/delta": -0.06763069331645966, "fcm_dpo/margin": 21.05264663696289, "fcm_dpo/q_t": 0.3640082776546478, "grad_norm": 48.709930419921875, "learning_rate": 3.5551627605944746e-07, "logits/chosen": 1.4213337898254395, "logits/rejected": 1.3685266971588135, "logps/chosen": -306.0328063964844, "logps/ref_chosen": -310.13043212890625, "logps/ref_rejected": -267.3905334472656, "logps/rejected": -284.34552001953125, "loss": 3.9993, "margin_dpo/margin_mean": 21.05264663696289, "margin_dpo/margin_std": 28.11899185180664, "step": 204 }, { "KL/chosen_KL_mean": 5.760053634643555, "KL/mean": -4.147031784057617, "KL/rejected_KL_mean": -14.054115295410156, "KL/std": 23.153827667236328, "epoch": 0.4293193717277487, "fcm_dpo/beta": 0.031499359756708145, "fcm_dpo/delta": -0.02703755721449852, "fcm_dpo/margin": 19.81418228149414, "fcm_dpo/q_t": 0.3702598810195923, "grad_norm": 52.241336822509766, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 1.4928886890411377, "logits/rejected": 1.487642765045166, "logps/chosen": -271.4006042480469, "logps/ref_chosen": -277.16064453125, "logps/ref_rejected": -292.91815185546875, "logps/rejected": -306.9722900390625, "loss": 4.1179, "margin_dpo/margin_mean": 19.814180374145508, "margin_dpo/margin_std": 26.955835342407227, "step": 205 }, { "KL/chosen_KL_mean": 5.546255111694336, "KL/mean": -3.0547800064086914, "KL/rejected_KL_mean": -11.655815124511719, "KL/std": 22.979106903076172, "epoch": 0.431413612565445, "fcm_dpo/beta": 0.030934946611523628, "fcm_dpo/delta": 0.06948360800743103, "fcm_dpo/margin": 17.202072143554688, "fcm_dpo/q_t": 0.3871549367904663, "grad_norm": 49.26683807373047, "learning_rate": 3.5218566107988867e-07, "logits/chosen": 1.606423020362854, "logits/rejected": 1.7115347385406494, "logps/chosen": -281.236328125, "logps/ref_chosen": -286.7825927734375, "logps/ref_rejected": -289.08270263671875, "logps/rejected": -300.738525390625, "loss": 4.2983, "margin_dpo/margin_mean": 17.202072143554688, "margin_dpo/margin_std": 25.05809783935547, "step": 206 }, { "KL/chosen_KL_mean": 3.277751922607422, "KL/mean": -4.28781270980835, "KL/rejected_KL_mean": -11.853385925292969, "KL/std": 24.55677604675293, "epoch": 0.43350785340314135, "fcm_dpo/beta": 0.03269320726394653, "fcm_dpo/delta": 0.04207862913608551, "fcm_dpo/margin": 15.13114070892334, "fcm_dpo/q_t": 0.40015873312950134, "grad_norm": 55.763126373291016, "learning_rate": 3.505120890024195e-07, "logits/chosen": 1.4150789976119995, "logits/rejected": 1.5169289112091064, "logps/chosen": -263.2544860839844, "logps/ref_chosen": -266.5322570800781, "logps/ref_rejected": -275.20672607421875, "logps/rejected": -287.06011962890625, "loss": 4.774, "margin_dpo/margin_mean": 15.13114070892334, "margin_dpo/margin_std": 31.045963287353516, "step": 207 }, { "KL/chosen_KL_mean": 3.792125701904297, "KL/mean": -5.186357498168945, "KL/rejected_KL_mean": -14.164838790893555, "KL/std": 23.937915802001953, "epoch": 0.4356020942408377, "fcm_dpo/beta": 0.03304613381624222, "fcm_dpo/delta": -0.04791893810033798, "fcm_dpo/margin": 17.956958770751953, "fcm_dpo/q_t": 0.37966203689575195, "grad_norm": 57.23706817626953, "learning_rate": 3.4883312676665534e-07, "logits/chosen": 1.6336067914962769, "logits/rejected": 1.6375012397766113, "logps/chosen": -269.00921630859375, "logps/ref_chosen": -272.80133056640625, "logps/ref_rejected": -288.4167785644531, "logps/rejected": -302.58160400390625, "loss": 4.2962, "margin_dpo/margin_mean": 17.956960678100586, "margin_dpo/margin_std": 28.082860946655273, "step": 208 }, { "KL/chosen_KL_mean": 1.9714374542236328, "KL/mean": -5.760955333709717, "KL/rejected_KL_mean": -13.493354797363281, "KL/std": 23.7287540435791, "epoch": 0.437696335078534, "fcm_dpo/beta": 0.03246966749429703, "fcm_dpo/delta": 0.029187699779868126, "fcm_dpo/margin": 15.46480941772461, "fcm_dpo/q_t": 0.3972010016441345, "grad_norm": 54.213714599609375, "learning_rate": 3.4714886441024573e-07, "logits/chosen": 1.59073805809021, "logits/rejected": 1.4431016445159912, "logps/chosen": -308.7079772949219, "logps/ref_chosen": -310.679443359375, "logps/ref_rejected": -259.3792419433594, "logps/rejected": -272.8725891113281, "loss": 4.55, "margin_dpo/margin_mean": 15.464810371398926, "margin_dpo/margin_std": 26.98141860961914, "step": 209 }, { "KL/chosen_KL_mean": 4.238682746887207, "KL/mean": -4.264247894287109, "KL/rejected_KL_mean": -12.767183303833008, "KL/std": 24.33123207092285, "epoch": 0.4397905759162304, "fcm_dpo/beta": 0.03341270983219147, "fcm_dpo/delta": 0.030814504250884056, "fcm_dpo/margin": 17.005861282348633, "fcm_dpo/q_t": 0.3825717568397522, "grad_norm": 50.88881301879883, "learning_rate": 3.454593922550693e-07, "logits/chosen": 1.312711238861084, "logits/rejected": 1.3513705730438232, "logps/chosen": -270.7294616699219, "logps/ref_chosen": -274.9681701660156, "logps/ref_rejected": -283.346923828125, "logps/rejected": -296.1141357421875, "loss": 4.3537, "margin_dpo/margin_mean": 17.005861282348633, "margin_dpo/margin_std": 27.453842163085938, "step": 210 }, { "KL/chosen_KL_mean": 7.164724349975586, "KL/mean": -4.725644588470459, "KL/rejected_KL_mean": -16.61601448059082, "KL/std": 24.55815315246582, "epoch": 0.4418848167539267, "fcm_dpo/beta": 0.03139013797044754, "fcm_dpo/delta": -0.16069211065769196, "fcm_dpo/margin": 23.780738830566406, "fcm_dpo/q_t": 0.3442986011505127, "grad_norm": 44.99844741821289, "learning_rate": 3.4376480090239047e-07, "logits/chosen": 1.4098219871520996, "logits/rejected": 1.3897714614868164, "logps/chosen": -259.1148681640625, "logps/ref_chosen": -266.2795715332031, "logps/ref_rejected": -249.7120819091797, "logps/rejected": -266.328125, "loss": 3.7348, "margin_dpo/margin_mean": 23.78073501586914, "margin_dpo/margin_std": 26.240310668945312, "step": 211 }, { "KL/chosen_KL_mean": 5.324275970458984, "KL/mean": -2.900439739227295, "KL/rejected_KL_mean": -11.125146865844727, "KL/std": 22.97644805908203, "epoch": 0.44397905759162304, "fcm_dpo/beta": 0.029926881194114685, "fcm_dpo/delta": 0.05482662841677666, "fcm_dpo/margin": 16.449420928955078, "fcm_dpo/q_t": 0.3944656550884247, "grad_norm": 44.68327331542969, "learning_rate": 3.4206518122800055e-07, "logits/chosen": 1.4912159442901611, "logits/rejected": 1.5235073566436768, "logps/chosen": -259.42852783203125, "logps/ref_chosen": -264.7528076171875, "logps/ref_rejected": -241.02842712402344, "logps/rejected": -252.153564453125, "loss": 4.4221, "margin_dpo/margin_mean": 16.449420928955078, "margin_dpo/margin_std": 26.83086585998535, "step": 212 }, { "KL/chosen_KL_mean": 3.322385787963867, "KL/mean": -4.5970659255981445, "KL/rejected_KL_mean": -12.51651382446289, "KL/std": 23.645183563232422, "epoch": 0.44607329842931936, "fcm_dpo/beta": 0.031889475882053375, "fcm_dpo/delta": 0.09982357919216156, "fcm_dpo/margin": 15.838902473449707, "fcm_dpo/q_t": 0.3998870551586151, "grad_norm": 55.86567306518555, "learning_rate": 3.403606243773448e-07, "logits/chosen": 1.6490623950958252, "logits/rejected": 1.7675503492355347, "logps/chosen": -290.81622314453125, "logps/ref_chosen": -294.13861083984375, "logps/ref_rejected": -303.11669921875, "logps/rejected": -315.63323974609375, "loss": 4.4696, "margin_dpo/margin_mean": 15.838902473449707, "margin_dpo/margin_std": 27.366634368896484, "step": 213 }, { "KL/chosen_KL_mean": -1.0719566345214844, "KL/mean": -9.300503730773926, "KL/rejected_KL_mean": -17.529054641723633, "KL/std": 24.037235260009766, "epoch": 0.4481675392670157, "fcm_dpo/beta": 0.03453436866402626, "fcm_dpo/delta": 0.03112040087580681, "fcm_dpo/margin": 16.457096099853516, "fcm_dpo/q_t": 0.3864825665950775, "grad_norm": 65.27860260009766, "learning_rate": 3.3865122176063385e-07, "logits/chosen": 1.5908783674240112, "logits/rejected": 1.7395843267440796, "logps/chosen": -267.6487121582031, "logps/ref_chosen": -266.5767517089844, "logps/ref_rejected": -291.2264709472656, "logps/rejected": -308.75555419921875, "loss": 4.403, "margin_dpo/margin_mean": 16.457094192504883, "margin_dpo/margin_std": 27.721385955810547, "step": 214 }, { "KL/chosen_KL_mean": 1.1925544738769531, "KL/mean": -6.978704929351807, "KL/rejected_KL_mean": -15.149971008300781, "KL/std": 25.942358016967773, "epoch": 0.450261780104712, "fcm_dpo/beta": 0.03314455598592758, "fcm_dpo/delta": -0.054589588195085526, "fcm_dpo/margin": 16.3425235748291, "fcm_dpo/q_t": 0.3942345380783081, "grad_norm": 55.492122650146484, "learning_rate": 3.3693706504794243e-07, "logits/chosen": 1.6796056032180786, "logits/rejected": 1.6907001733779907, "logps/chosen": -283.84820556640625, "logps/ref_chosen": -285.040771484375, "logps/ref_rejected": -258.0254821777344, "logps/rejected": -273.1754455566406, "loss": 4.5728, "margin_dpo/margin_mean": 16.3425235748291, "margin_dpo/margin_std": 29.976999282836914, "step": 215 }, { "KL/chosen_KL_mean": 3.9657115936279297, "KL/mean": -5.047689437866211, "KL/rejected_KL_mean": -14.06108283996582, "KL/std": 24.962993621826172, "epoch": 0.4523560209424084, "fcm_dpo/beta": 0.03226002678275108, "fcm_dpo/delta": 0.018793560564517975, "fcm_dpo/margin": 18.02680015563965, "fcm_dpo/q_t": 0.37889283895492554, "grad_norm": 59.685245513916016, "learning_rate": 3.3521824616429284e-07, "logits/chosen": 1.8646385669708252, "logits/rejected": 1.8905611038208008, "logps/chosen": -328.7856750488281, "logps/ref_chosen": -332.7513732910156, "logps/ref_rejected": -302.9167785644531, "logps/rejected": -316.9778137207031, "loss": 4.3924, "margin_dpo/margin_mean": 18.02680206298828, "margin_dpo/margin_std": 29.033185958862305, "step": 216 }, { "KL/chosen_KL_mean": 0.4787712097167969, "KL/mean": -11.578567504882812, "KL/rejected_KL_mean": -23.635900497436523, "KL/std": 26.74181365966797, "epoch": 0.4544502617801047, "fcm_dpo/beta": 0.03131940960884094, "fcm_dpo/delta": -0.1656835675239563, "fcm_dpo/margin": 24.114669799804688, "fcm_dpo/q_t": 0.3509507179260254, "grad_norm": 47.78200149536133, "learning_rate": 3.334948572847253e-07, "logits/chosen": 1.795779824256897, "logits/rejected": 1.7963271141052246, "logps/chosen": -285.9891662597656, "logps/ref_chosen": -286.46795654296875, "logps/ref_rejected": -253.99298095703125, "logps/rejected": -277.6288757324219, "loss": 3.9438, "margin_dpo/margin_mean": 24.114669799804688, "margin_dpo/margin_std": 32.39649200439453, "step": 217 }, { "KL/chosen_KL_mean": 4.0630035400390625, "KL/mean": -5.699796676635742, "KL/rejected_KL_mean": -15.462593078613281, "KL/std": 25.529905319213867, "epoch": 0.45654450261780105, "fcm_dpo/beta": 0.02972830832004547, "fcm_dpo/delta": 0.019646622240543365, "fcm_dpo/margin": 19.525588989257812, "fcm_dpo/q_t": 0.3794595003128052, "grad_norm": 48.438018798828125, "learning_rate": 3.317669908293554e-07, "logits/chosen": 1.6519176959991455, "logits/rejected": 1.7010029554367065, "logps/chosen": -307.3155212402344, "logps/ref_chosen": -311.3785400390625, "logps/ref_rejected": -290.3092956542969, "logps/rejected": -305.77191162109375, "loss": 4.2268, "margin_dpo/margin_mean": 19.525588989257812, "margin_dpo/margin_std": 29.03296661376953, "step": 218 }, { "KL/chosen_KL_mean": 3.8179988861083984, "KL/mean": -6.301147937774658, "KL/rejected_KL_mean": -16.42029571533203, "KL/std": 23.378128051757812, "epoch": 0.4586387434554974, "fcm_dpo/beta": 0.02980736270546913, "fcm_dpo/delta": -0.006483299657702446, "fcm_dpo/margin": 20.238300323486328, "fcm_dpo/q_t": 0.37255263328552246, "grad_norm": 42.6064338684082, "learning_rate": 3.300347394584172e-07, "logits/chosen": 1.4093046188354492, "logits/rejected": 1.4627782106399536, "logps/chosen": -255.51490783691406, "logps/ref_chosen": -259.33294677734375, "logps/ref_rejected": -261.7661437988281, "logps/rejected": -278.1864013671875, "loss": 4.1426, "margin_dpo/margin_mean": 20.23830223083496, "margin_dpo/margin_std": 27.70301628112793, "step": 219 }, { "KL/chosen_KL_mean": 3.390085220336914, "KL/mean": -7.234885215759277, "KL/rejected_KL_mean": -17.859859466552734, "KL/std": 25.234189987182617, "epoch": 0.4607329842931937, "fcm_dpo/beta": 0.029233120381832123, "fcm_dpo/delta": -0.02283724956214428, "fcm_dpo/margin": 21.249942779541016, "fcm_dpo/q_t": 0.3688524663448334, "grad_norm": 46.714271545410156, "learning_rate": 3.2829819606729477e-07, "logits/chosen": 1.8225642442703247, "logits/rejected": 1.7747106552124023, "logps/chosen": -302.8143005371094, "logps/ref_chosen": -306.2043762207031, "logps/ref_rejected": -256.7926330566406, "logps/rejected": -274.6524658203125, "loss": 4.1038, "margin_dpo/margin_mean": 21.24994468688965, "margin_dpo/margin_std": 29.586055755615234, "step": 220 }, { "KL/chosen_KL_mean": 1.3392181396484375, "KL/mean": -6.753597259521484, "KL/rejected_KL_mean": -14.84642219543457, "KL/std": 25.3695125579834, "epoch": 0.46282722513089003, "fcm_dpo/beta": 0.0307632889598608, "fcm_dpo/delta": 0.10439582169055939, "fcm_dpo/margin": 16.185644149780273, "fcm_dpo/q_t": 0.39704880118370056, "grad_norm": 55.72863006591797, "learning_rate": 3.265574537815398e-07, "logits/chosen": 1.3814107179641724, "logits/rejected": 1.4919143915176392, "logps/chosen": -291.99273681640625, "logps/ref_chosen": -293.3319396972656, "logps/ref_rejected": -282.7599182128906, "logps/rejected": -297.6063232421875, "loss": 4.5227, "margin_dpo/margin_mean": 16.18564224243164, "margin_dpo/margin_std": 28.6278076171875, "step": 221 }, { "KL/chosen_KL_mean": 2.481769561767578, "KL/mean": -7.462711811065674, "KL/rejected_KL_mean": -17.407188415527344, "KL/std": 24.014915466308594, "epoch": 0.4649214659685864, "fcm_dpo/beta": 0.030535133555531502, "fcm_dpo/delta": -0.011182472109794617, "fcm_dpo/margin": 19.888965606689453, "fcm_dpo/q_t": 0.3723604679107666, "grad_norm": 50.793792724609375, "learning_rate": 3.248126059518784e-07, "logits/chosen": 1.49371337890625, "logits/rejected": 1.3759980201721191, "logps/chosen": -284.5243835449219, "logps/ref_chosen": -287.0061340332031, "logps/ref_rejected": -233.23130798339844, "logps/rejected": -250.63851928710938, "loss": 4.1239, "margin_dpo/margin_mean": 19.888965606689453, "margin_dpo/margin_std": 26.840036392211914, "step": 222 }, { "KL/chosen_KL_mean": 1.7029399871826172, "KL/mean": -9.730170249938965, "KL/rejected_KL_mean": -21.16328239440918, "KL/std": 27.028404235839844, "epoch": 0.46701570680628274, "fcm_dpo/beta": 0.02971070259809494, "fcm_dpo/delta": -0.08874759078025818, "fcm_dpo/margin": 22.866230010986328, "fcm_dpo/q_t": 0.3580406904220581, "grad_norm": 43.08765411376953, "learning_rate": 3.230637461492043e-07, "logits/chosen": 1.57514488697052, "logits/rejected": 1.4774196147918701, "logps/chosen": -274.11236572265625, "logps/ref_chosen": -275.8153076171875, "logps/ref_rejected": -238.54286193847656, "logps/rejected": -259.70611572265625, "loss": 3.9717, "margin_dpo/margin_mean": 22.866230010986328, "margin_dpo/margin_std": 29.15939712524414, "step": 223 }, { "KL/chosen_KL_mean": 0.9415683746337891, "KL/mean": -10.042745590209961, "KL/rejected_KL_mean": -21.027055740356445, "KL/std": 25.473079681396484, "epoch": 0.46910994764397906, "fcm_dpo/beta": 0.02928377129137516, "fcm_dpo/delta": -0.04776962101459503, "fcm_dpo/margin": 21.9686222076416, "fcm_dpo/q_t": 0.3629598915576935, "grad_norm": 43.5782470703125, "learning_rate": 3.213109681595612e-07, "logits/chosen": 1.6065367460250854, "logits/rejected": 1.6727735996246338, "logps/chosen": -238.1588134765625, "logps/ref_chosen": -239.1003875732422, "logps/ref_rejected": -230.31900024414062, "logps/rejected": -251.34605407714844, "loss": 3.9708, "margin_dpo/margin_mean": 21.9686222076416, "margin_dpo/margin_std": 26.808597564697266, "step": 224 }, { "KL/chosen_KL_mean": -2.3414173126220703, "KL/mean": -10.983441352844238, "KL/rejected_KL_mean": -19.625471115112305, "KL/std": 27.00665855407715, "epoch": 0.4712041884816754, "fcm_dpo/beta": 0.027854181826114655, "fcm_dpo/delta": 0.051747214049100876, "fcm_dpo/margin": 17.28403663635254, "fcm_dpo/q_t": 0.4044538736343384, "grad_norm": 50.141746520996094, "learning_rate": 3.1955436597911315e-07, "logits/chosen": 1.6040029525756836, "logits/rejected": 1.6216011047363281, "logps/chosen": -302.6707763671875, "logps/ref_chosen": -300.3293762207031, "logps/ref_rejected": -332.132568359375, "logps/rejected": -351.758056640625, "loss": 4.6892, "margin_dpo/margin_mean": 17.284034729003906, "margin_dpo/margin_std": 33.72834777832031, "step": 225 }, { "KL/chosen_KL_mean": -1.0356464385986328, "KL/mean": -8.687429428100586, "KL/rejected_KL_mean": -16.339204788208008, "KL/std": 24.889102935791016, "epoch": 0.4732984293193717, "fcm_dpo/beta": 0.03033391945064068, "fcm_dpo/delta": 0.0942978709936142, "fcm_dpo/margin": 15.30355453491211, "fcm_dpo/q_t": 0.40447574853897095, "grad_norm": 55.16220474243164, "learning_rate": 3.1779403380910425e-07, "logits/chosen": 1.3315622806549072, "logits/rejected": 1.3884360790252686, "logps/chosen": -246.4477996826172, "logps/ref_chosen": -245.41213989257812, "logps/ref_rejected": -251.72999572753906, "logps/rejected": -268.0691833496094, "loss": 4.6364, "margin_dpo/margin_mean": 15.30355453491211, "margin_dpo/margin_std": 29.265268325805664, "step": 226 }, { "KL/chosen_KL_mean": -0.8187618255615234, "KL/mean": -11.41288948059082, "KL/rejected_KL_mean": -22.00701332092285, "KL/std": 26.464872360229492, "epoch": 0.47539267015706804, "fcm_dpo/beta": 0.03090839646756649, "fcm_dpo/delta": -0.05948437750339508, "fcm_dpo/margin": 21.188251495361328, "fcm_dpo/q_t": 0.36408165097236633, "grad_norm": 51.02029800415039, "learning_rate": 3.160300660508064e-07, "logits/chosen": 1.3621947765350342, "logits/rejected": 1.3967113494873047, "logps/chosen": -277.694580078125, "logps/ref_chosen": -276.87579345703125, "logps/ref_rejected": -259.0455017089844, "logps/rejected": -281.052490234375, "loss": 4.1226, "margin_dpo/margin_mean": 21.18825340270996, "margin_dpo/margin_std": 29.894878387451172, "step": 227 }, { "KL/chosen_KL_mean": 0.9791069030761719, "KL/mean": -10.024959564208984, "KL/rejected_KL_mean": -21.029029846191406, "KL/std": 25.749013900756836, "epoch": 0.4774869109947644, "fcm_dpo/beta": 0.0297478549182415, "fcm_dpo/delta": -0.05890221893787384, "fcm_dpo/margin": 22.008134841918945, "fcm_dpo/q_t": 0.3642002046108246, "grad_norm": 55.73063659667969, "learning_rate": 3.1426255730045695e-07, "logits/chosen": 1.686366319656372, "logits/rejected": 1.6875087022781372, "logps/chosen": -302.991943359375, "logps/ref_chosen": -303.9710693359375, "logps/ref_rejected": -255.62033081054688, "logps/rejected": -276.64935302734375, "loss": 4.0223, "margin_dpo/margin_mean": 22.00813865661621, "margin_dpo/margin_std": 28.885726928710938, "step": 228 }, { "KL/chosen_KL_mean": 0.8257102966308594, "KL/mean": -12.805261611938477, "KL/rejected_KL_mean": -26.436227798461914, "KL/std": 26.923622131347656, "epoch": 0.47958115183246075, "fcm_dpo/beta": 0.02663799375295639, "fcm_dpo/delta": -0.13948528468608856, "fcm_dpo/margin": 27.261943817138672, "fcm_dpo/q_t": 0.3465552031993866, "grad_norm": 56.9528694152832, "learning_rate": 3.1249160234418644e-07, "logits/chosen": 1.4259783029556274, "logits/rejected": 1.3962115049362183, "logps/chosen": -280.8702392578125, "logps/ref_chosen": -281.6959533691406, "logps/ref_rejected": -258.93353271484375, "logps/rejected": -285.3697814941406, "loss": 3.7582, "margin_dpo/margin_mean": 27.261943817138672, "margin_dpo/margin_std": 30.23539161682129, "step": 229 }, { "KL/chosen_KL_mean": -0.2526702880859375, "KL/mean": -11.950700759887695, "KL/rejected_KL_mean": -23.64872932434082, "KL/std": 26.261003494262695, "epoch": 0.4816753926701571, "fcm_dpo/beta": 0.02576565369963646, "fcm_dpo/delta": -0.005254525691270828, "fcm_dpo/margin": 23.39605140686035, "fcm_dpo/q_t": 0.37064623832702637, "grad_norm": 47.49397277832031, "learning_rate": 3.1071729615293424e-07, "logits/chosen": 1.2236592769622803, "logits/rejected": 1.302830696105957, "logps/chosen": -224.20921325683594, "logps/ref_chosen": -223.95655822753906, "logps/ref_rejected": -238.95187377929688, "logps/rejected": -262.6005859375, "loss": 4.1101, "margin_dpo/margin_mean": 23.39605140686035, "margin_dpo/margin_std": 31.021326065063477, "step": 230 }, { "KL/chosen_KL_mean": -2.974292755126953, "KL/mean": -12.331928253173828, "KL/rejected_KL_mean": -21.68956184387207, "KL/std": 25.90277671813965, "epoch": 0.4837696335078534, "fcm_dpo/beta": 0.026006178930401802, "fcm_dpo/delta": 0.07183073461055756, "fcm_dpo/margin": 18.71527099609375, "fcm_dpo/q_t": 0.3939741849899292, "grad_norm": 40.720008850097656, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 1.339698314666748, "logits/rejected": 1.3909707069396973, "logps/chosen": -317.8988952636719, "logps/ref_chosen": -314.92462158203125, "logps/ref_rejected": -279.6347351074219, "logps/rejected": -301.32427978515625, "loss": 4.3163, "margin_dpo/margin_mean": 18.715272903442383, "margin_dpo/margin_std": 27.25091552734375, "step": 231 }, { "KL/chosen_KL_mean": -0.09930038452148438, "KL/mean": -10.04248046875, "KL/rejected_KL_mean": -19.98565673828125, "KL/std": 27.593305587768555, "epoch": 0.48586387434554973, "fcm_dpo/beta": 0.026716448366642, "fcm_dpo/delta": -0.0027984483167529106, "fcm_dpo/margin": 19.886362075805664, "fcm_dpo/q_t": 0.3892497420310974, "grad_norm": 57.157737731933594, "learning_rate": 3.071590108427243e-07, "logits/chosen": 1.283789873123169, "logits/rejected": 1.3314770460128784, "logps/chosen": -263.41888427734375, "logps/ref_chosen": -263.3196105957031, "logps/ref_rejected": -247.13612365722656, "logps/rejected": -267.12176513671875, "loss": 4.3119, "margin_dpo/margin_mean": 19.886363983154297, "margin_dpo/margin_std": 29.738393783569336, "step": 232 }, { "KL/chosen_KL_mean": 0.6233539581298828, "KL/mean": -10.353264808654785, "KL/rejected_KL_mean": -21.32988929748535, "KL/std": 26.082984924316406, "epoch": 0.48795811518324606, "fcm_dpo/beta": 0.02723000943660736, "fcm_dpo/delta": 0.0005163960158824921, "fcm_dpo/margin": 21.95323371887207, "fcm_dpo/q_t": 0.37384089827537537, "grad_norm": 48.244415283203125, "learning_rate": 3.05375222543809e-07, "logits/chosen": 1.3779985904693604, "logits/rejected": 1.439375877380371, "logps/chosen": -278.7388610839844, "logps/ref_chosen": -279.36224365234375, "logps/ref_rejected": -264.46112060546875, "logps/rejected": -285.791015625, "loss": 4.1144, "margin_dpo/margin_mean": 21.953237533569336, "margin_dpo/margin_std": 29.3866024017334, "step": 233 }, { "KL/chosen_KL_mean": 0.3291606903076172, "KL/mean": -11.041275978088379, "KL/rejected_KL_mean": -22.411699295043945, "KL/std": 27.928560256958008, "epoch": 0.4900523560209424, "fcm_dpo/beta": 0.027022581547498703, "fcm_dpo/delta": -0.015877969563007355, "fcm_dpo/margin": 22.74085807800293, "fcm_dpo/q_t": 0.3736668825149536, "grad_norm": 44.880882263183594, "learning_rate": 3.035884646397637e-07, "logits/chosen": 1.440163254737854, "logits/rejected": 1.5084267854690552, "logps/chosen": -286.1199645996094, "logps/ref_chosen": -286.4491271972656, "logps/ref_rejected": -294.5469665527344, "logps/rejected": -316.95867919921875, "loss": 4.2289, "margin_dpo/margin_mean": 22.740859985351562, "margin_dpo/margin_std": 34.04836654663086, "step": 234 }, { "KL/chosen_KL_mean": -2.050537109375, "KL/mean": -12.448473930358887, "KL/rejected_KL_mean": -22.84640884399414, "KL/std": 29.29195785522461, "epoch": 0.49214659685863876, "fcm_dpo/beta": 0.027660487219691277, "fcm_dpo/delta": 0.023279912769794464, "fcm_dpo/margin": 20.79585838317871, "fcm_dpo/q_t": 0.380502313375473, "grad_norm": 52.81705856323242, "learning_rate": 3.017988329489923e-07, "logits/chosen": 1.4513448476791382, "logits/rejected": 1.4381203651428223, "logps/chosen": -284.4106140136719, "logps/ref_chosen": -282.36004638671875, "logps/ref_rejected": -243.79052734375, "logps/rejected": -266.6369323730469, "loss": 4.3436, "margin_dpo/margin_mean": 20.79585838317871, "margin_dpo/margin_std": 32.966270446777344, "step": 235 }, { "KL/chosen_KL_mean": 1.3165740966796875, "KL/mean": -10.353586196899414, "KL/rejected_KL_mean": -22.023738861083984, "KL/std": 27.25334930419922, "epoch": 0.4942408376963351, "fcm_dpo/beta": 0.027351150289177895, "fcm_dpo/delta": -0.04152906313538551, "fcm_dpo/margin": 23.340309143066406, "fcm_dpo/q_t": 0.36344993114471436, "grad_norm": 42.070194244384766, "learning_rate": 3.000064234440111e-07, "logits/chosen": 1.2549378871917725, "logits/rejected": 1.2370033264160156, "logps/chosen": -277.8228454589844, "logps/ref_chosen": -279.139404296875, "logps/ref_rejected": -240.5850067138672, "logps/rejected": -262.6087646484375, "loss": 4.068, "margin_dpo/margin_mean": 23.340309143066406, "margin_dpo/margin_std": 30.97328758239746, "step": 236 }, { "KL/chosen_KL_mean": -1.8713130950927734, "KL/mean": -11.904808044433594, "KL/rejected_KL_mean": -21.938297271728516, "KL/std": 24.791580200195312, "epoch": 0.4963350785340314, "fcm_dpo/beta": 0.02666321024298668, "fcm_dpo/delta": -0.00504470057785511, "fcm_dpo/margin": 20.06699562072754, "fcm_dpo/q_t": 0.38698601722717285, "grad_norm": 51.159584045410156, "learning_rate": 2.9821133224630223e-07, "logits/chosen": 1.4548025131225586, "logits/rejected": 1.5232490301132202, "logps/chosen": -260.6828308105469, "logps/ref_chosen": -258.8114929199219, "logps/ref_rejected": -266.3898010253906, "logps/rejected": -288.32806396484375, "loss": 4.3179, "margin_dpo/margin_mean": 20.06699562072754, "margin_dpo/margin_std": 30.49276351928711, "step": 237 }, { "KL/chosen_KL_mean": -2.533510208129883, "KL/mean": -13.915240287780762, "KL/rejected_KL_mean": -25.296966552734375, "KL/std": 28.526103973388672, "epoch": 0.49842931937172774, "fcm_dpo/beta": 0.025130130350589752, "fcm_dpo/delta": -0.050658006221055984, "fcm_dpo/margin": 22.763446807861328, "fcm_dpo/q_t": 0.3809536397457123, "grad_norm": 44.547672271728516, "learning_rate": 2.964136556211588e-07, "logits/chosen": 1.3475905656814575, "logits/rejected": 1.2551782131195068, "logps/chosen": -304.5997009277344, "logps/ref_chosen": -302.0661926269531, "logps/ref_rejected": -265.5518798828125, "logps/rejected": -290.8488464355469, "loss": 4.2063, "margin_dpo/margin_mean": 22.763446807861328, "margin_dpo/margin_std": 32.277366638183594, "step": 238 }, { "KL/chosen_KL_mean": 0.11390304565429688, "KL/mean": -9.70567798614502, "KL/rejected_KL_mean": -19.525257110595703, "KL/std": 27.92169761657715, "epoch": 0.5005235602094241, "fcm_dpo/beta": 0.025815239176154137, "fcm_dpo/delta": 0.09881044924259186, "fcm_dpo/margin": 19.639163970947266, "fcm_dpo/q_t": 0.3956132233142853, "grad_norm": 50.240867614746094, "learning_rate": 2.946134899725226e-07, "logits/chosen": 1.5159615278244019, "logits/rejected": 1.6065120697021484, "logps/chosen": -258.88446044921875, "logps/ref_chosen": -258.99835205078125, "logps/ref_rejected": -273.7834777832031, "logps/rejected": -293.3087463378906, "loss": 4.4926, "margin_dpo/margin_mean": 19.639163970947266, "margin_dpo/margin_std": 34.4343147277832, "step": 239 }, { "KL/chosen_KL_mean": 0.9601726531982422, "KL/mean": -11.192224502563477, "KL/rejected_KL_mean": -23.34463119506836, "KL/std": 27.793094635009766, "epoch": 0.5026178010471204, "fcm_dpo/beta": 0.025913547724485397, "fcm_dpo/delta": -0.08534112572669983, "fcm_dpo/margin": 24.3048095703125, "fcm_dpo/q_t": 0.36542585492134094, "grad_norm": 50.47843933105469, "learning_rate": 2.9281093183781403e-07, "logits/chosen": 1.328792691230774, "logits/rejected": 1.3966224193572998, "logps/chosen": -274.5628967285156, "logps/ref_chosen": -275.5230407714844, "logps/ref_rejected": -255.50784301757812, "logps/rejected": -278.85247802734375, "loss": 3.9814, "margin_dpo/margin_mean": 24.3048095703125, "margin_dpo/margin_std": 30.22945213317871, "step": 240 }, { "KL/chosen_KL_mean": -2.142576217651367, "KL/mean": -11.555323600769043, "KL/rejected_KL_mean": -20.968074798583984, "KL/std": 28.288204193115234, "epoch": 0.5047120418848168, "fcm_dpo/beta": 0.02569686621427536, "fcm_dpo/delta": 0.06423062831163406, "fcm_dpo/margin": 18.825511932373047, "fcm_dpo/q_t": 0.40052130818367004, "grad_norm": 43.8680419921875, "learning_rate": 2.910060778827554e-07, "logits/chosen": 1.573228120803833, "logits/rejected": 1.6145110130310059, "logps/chosen": -255.50607299804688, "logps/ref_chosen": -253.36349487304688, "logps/ref_rejected": -247.1916961669922, "logps/rejected": -268.1597595214844, "loss": 4.5423, "margin_dpo/margin_mean": 18.825511932373047, "margin_dpo/margin_std": 33.42321014404297, "step": 241 }, { "KL/chosen_KL_mean": -0.8480720520019531, "KL/mean": -12.20622444152832, "KL/rejected_KL_mean": -23.564376831054688, "KL/std": 27.829221725463867, "epoch": 0.506806282722513, "fcm_dpo/beta": 0.026592141017317772, "fcm_dpo/delta": -0.006941572763025761, "fcm_dpo/margin": 22.71630096435547, "fcm_dpo/q_t": 0.3716428279876709, "grad_norm": 40.12123107910156, "learning_rate": 2.891990248961871e-07, "logits/chosen": 1.5100977420806885, "logits/rejected": 1.4358429908752441, "logps/chosen": -265.8361511230469, "logps/ref_chosen": -264.9880676269531, "logps/ref_rejected": -243.07838439941406, "logps/rejected": -266.64276123046875, "loss": 4.0944, "margin_dpo/margin_mean": 22.71630096435547, "margin_dpo/margin_std": 29.884212493896484, "step": 242 }, { "KL/chosen_KL_mean": 1.9819889068603516, "KL/mean": -11.120399475097656, "KL/rejected_KL_mean": -24.22279930114746, "KL/std": 27.46685791015625, "epoch": 0.5089005235602094, "fcm_dpo/beta": 0.025906052440404892, "fcm_dpo/delta": -0.08735494315624237, "fcm_dpo/margin": 26.204782485961914, "fcm_dpo/q_t": 0.35590317845344543, "grad_norm": 55.49812316894531, "learning_rate": 2.873898697848762e-07, "logits/chosen": 1.5263855457305908, "logits/rejected": 1.5749152898788452, "logps/chosen": -313.8887023925781, "logps/ref_chosen": -315.8706970214844, "logps/ref_rejected": -304.3729248046875, "logps/rejected": -328.595703125, "loss": 3.8786, "margin_dpo/margin_mean": 26.204784393310547, "margin_dpo/margin_std": 29.490520477294922, "step": 243 }, { "KL/chosen_KL_mean": 0.8133888244628906, "KL/mean": -12.799525260925293, "KL/rejected_KL_mean": -26.412450790405273, "KL/std": 28.79197120666504, "epoch": 0.5109947643979058, "fcm_dpo/beta": 0.023789148777723312, "fcm_dpo/delta": -0.05430053174495697, "fcm_dpo/margin": 27.22583770751953, "fcm_dpo/q_t": 0.36135512590408325, "grad_norm": 37.841976165771484, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 1.4552251100540161, "logits/rejected": 1.4061833620071411, "logps/chosen": -310.2837829589844, "logps/ref_chosen": -311.09716796875, "logps/ref_rejected": -266.01959228515625, "logps/rejected": -292.4320373535156, "loss": 3.9512, "margin_dpo/margin_mean": 27.22583770751953, "margin_dpo/margin_std": 32.62396240234375, "step": 244 }, { "KL/chosen_KL_mean": 1.0648555755615234, "KL/mean": -12.222169876098633, "KL/rejected_KL_mean": -25.509199142456055, "KL/std": 28.52521514892578, "epoch": 0.5130890052356021, "fcm_dpo/beta": 0.023074612021446228, "fcm_dpo/delta": -0.08860162645578384, "fcm_dpo/margin": 26.574054718017578, "fcm_dpo/q_t": 0.36784839630126953, "grad_norm": 45.462127685546875, "learning_rate": 2.837656413735479e-07, "logits/chosen": 1.5822170972824097, "logits/rejected": 1.4620615243911743, "logps/chosen": -283.67559814453125, "logps/ref_chosen": -284.740478515625, "logps/ref_rejected": -237.64117431640625, "logps/rejected": -263.150390625, "loss": 4.0302, "margin_dpo/margin_mean": 26.574054718017578, "margin_dpo/margin_std": 32.760372161865234, "step": 245 }, { "KL/chosen_KL_mean": -5.959909439086914, "KL/mean": -14.731524467468262, "KL/rejected_KL_mean": -23.503145217895508, "KL/std": 28.082195281982422, "epoch": 0.5151832460732985, "fcm_dpo/beta": 0.02217438444495201, "fcm_dpo/delta": 0.033019039779901505, "fcm_dpo/margin": 17.543224334716797, "fcm_dpo/q_t": 0.4182347059249878, "grad_norm": 46.06009292602539, "learning_rate": 2.8195076242990116e-07, "logits/chosen": 1.49590265750885, "logits/rejected": 1.5044995546340942, "logps/chosen": -278.57281494140625, "logps/ref_chosen": -272.6128845214844, "logps/ref_rejected": -249.99725341796875, "logps/rejected": -273.50042724609375, "loss": 4.7293, "margin_dpo/margin_mean": 17.54322624206543, "margin_dpo/margin_std": 34.45924758911133, "step": 246 }, { "KL/chosen_KL_mean": -1.1401596069335938, "KL/mean": -12.122604370117188, "KL/rejected_KL_mean": -23.10504722595215, "KL/std": 27.030519485473633, "epoch": 0.5172774869109947, "fcm_dpo/beta": 0.023736296221613884, "fcm_dpo/delta": 0.08034541457891464, "fcm_dpo/margin": 21.964893341064453, "fcm_dpo/q_t": 0.3879540264606476, "grad_norm": 40.82952880859375, "learning_rate": 2.801341700638307e-07, "logits/chosen": 1.4822200536727905, "logits/rejected": 1.4732378721237183, "logps/chosen": -275.5759582519531, "logps/ref_chosen": -274.435791015625, "logps/ref_rejected": -257.8125305175781, "logps/rejected": -280.9175720214844, "loss": 4.261, "margin_dpo/margin_mean": 21.964893341064453, "margin_dpo/margin_std": 31.085176467895508, "step": 247 }, { "KL/chosen_KL_mean": -2.6777801513671875, "KL/mean": -12.569023132324219, "KL/rejected_KL_mean": -22.46026611328125, "KL/std": 27.558944702148438, "epoch": 0.5193717277486911, "fcm_dpo/beta": 0.02531389333307743, "fcm_dpo/delta": 0.10261654108762741, "fcm_dpo/margin": 19.782489776611328, "fcm_dpo/q_t": 0.39476096630096436, "grad_norm": 46.36613845825195, "learning_rate": 2.7831596169367227e-07, "logits/chosen": 1.4401912689208984, "logits/rejected": 1.4732575416564941, "logps/chosen": -263.9891052246094, "logps/ref_chosen": -261.3113098144531, "logps/ref_rejected": -231.152587890625, "logps/rejected": -253.61285400390625, "loss": 4.4323, "margin_dpo/margin_mean": 19.782487869262695, "margin_dpo/margin_std": 32.60723876953125, "step": 248 }, { "KL/chosen_KL_mean": -3.753652572631836, "KL/mean": -14.500252723693848, "KL/rejected_KL_mean": -25.246868133544922, "KL/std": 29.098094940185547, "epoch": 0.5214659685863874, "fcm_dpo/beta": 0.02699856646358967, "fcm_dpo/delta": 0.018965115770697594, "fcm_dpo/margin": 21.49321746826172, "fcm_dpo/q_t": 0.3801358938217163, "grad_norm": 43.70502471923828, "learning_rate": 2.7649623482442274e-07, "logits/chosen": 1.4656652212142944, "logits/rejected": 1.406292200088501, "logps/chosen": -272.22088623046875, "logps/ref_chosen": -268.46722412109375, "logps/ref_rejected": -240.27597045898438, "logps/rejected": -265.5228271484375, "loss": 4.2513, "margin_dpo/margin_mean": 21.49321746826172, "margin_dpo/margin_std": 32.12462615966797, "step": 249 }, { "KL/chosen_KL_mean": -3.9061927795410156, "KL/mean": -14.475398063659668, "KL/rejected_KL_mean": -25.04460334777832, "KL/std": 29.591577529907227, "epoch": 0.5235602094240838, "fcm_dpo/beta": 0.02638782374560833, "fcm_dpo/delta": 0.04420147091150284, "fcm_dpo/margin": 21.13842010498047, "fcm_dpo/q_t": 0.3843858242034912, "grad_norm": 46.38828659057617, "learning_rate": 2.7467508704251135e-07, "logits/chosen": 1.5810189247131348, "logits/rejected": 1.6897047758102417, "logps/chosen": -289.01141357421875, "logps/ref_chosen": -285.1051940917969, "logps/ref_rejected": -281.2331848144531, "logps/rejected": -306.2778015136719, "loss": 4.4161, "margin_dpo/margin_mean": 21.13842010498047, "margin_dpo/margin_std": 34.87960433959961, "step": 250 }, { "KL/chosen_KL_mean": -1.2985305786132812, "KL/mean": -12.381001472473145, "KL/rejected_KL_mean": -23.463476181030273, "KL/std": 27.799116134643555, "epoch": 0.5256544502617801, "fcm_dpo/beta": 0.02752423658967018, "fcm_dpo/delta": -0.06013893708586693, "fcm_dpo/margin": 22.164939880371094, "fcm_dpo/q_t": 0.373883992433548, "grad_norm": 53.40812301635742, "learning_rate": 2.7285261601056697e-07, "logits/chosen": 1.5160138607025146, "logits/rejected": 1.4690115451812744, "logps/chosen": -275.4111633300781, "logps/ref_chosen": -274.1126403808594, "logps/ref_rejected": -256.7730407714844, "logps/rejected": -280.2365417480469, "loss": 4.1646, "margin_dpo/margin_mean": 22.16494369506836, "margin_dpo/margin_std": 31.104576110839844, "step": 251 }, { "KL/chosen_KL_mean": -5.993812561035156, "KL/mean": -17.212871551513672, "KL/rejected_KL_mean": -28.431922912597656, "KL/std": 29.184667587280273, "epoch": 0.5277486910994764, "fcm_dpo/beta": 0.025960441678762436, "fcm_dpo/delta": 0.015204920433461666, "fcm_dpo/margin": 22.438114166259766, "fcm_dpo/q_t": 0.37905487418174744, "grad_norm": 48.171077728271484, "learning_rate": 2.7102891946217994e-07, "logits/chosen": 1.4628584384918213, "logits/rejected": 1.422957181930542, "logps/chosen": -295.5595397949219, "logps/ref_chosen": -289.56573486328125, "logps/ref_rejected": -277.9891662597656, "logps/rejected": -306.42108154296875, "loss": 4.3431, "margin_dpo/margin_mean": 22.438114166259766, "margin_dpo/margin_std": 34.84325408935547, "step": 252 }, { "KL/chosen_KL_mean": -5.769189834594727, "KL/mean": -15.143193244934082, "KL/rejected_KL_mean": -24.517194747924805, "KL/std": 29.447696685791016, "epoch": 0.5298429319371728, "fcm_dpo/beta": 0.026602119207382202, "fcm_dpo/delta": -0.0013262918218970299, "fcm_dpo/margin": 18.748003005981445, "fcm_dpo/q_t": 0.3968650698661804, "grad_norm": 63.81908416748047, "learning_rate": 2.692040951966617e-07, "logits/chosen": 1.346488118171692, "logits/rejected": 1.3190160989761353, "logps/chosen": -275.953125, "logps/ref_chosen": -270.1839294433594, "logps/ref_rejected": -245.86949157714844, "logps/rejected": -270.38665771484375, "loss": 4.538, "margin_dpo/margin_mean": 18.748003005981445, "margin_dpo/margin_std": 33.28840255737305, "step": 253 }, { "KL/chosen_KL_mean": -2.1069259643554688, "KL/mean": -13.89623737335205, "KL/rejected_KL_mean": -25.685544967651367, "KL/std": 29.125656127929688, "epoch": 0.5319371727748691, "fcm_dpo/beta": 0.027346130460500717, "fcm_dpo/delta": -0.049901798367500305, "fcm_dpo/margin": 23.5786190032959, "fcm_dpo/q_t": 0.36554965376853943, "grad_norm": 62.93083572387695, "learning_rate": 2.6737824107379947e-07, "logits/chosen": 1.3603527545928955, "logits/rejected": 1.3280959129333496, "logps/chosen": -262.7933654785156, "logps/ref_chosen": -260.6864318847656, "logps/ref_rejected": -245.99508666992188, "logps/rejected": -271.6806335449219, "loss": 4.0305, "margin_dpo/margin_mean": 23.57862091064453, "margin_dpo/margin_std": 29.09947967529297, "step": 254 }, { "KL/chosen_KL_mean": -0.32198143005371094, "KL/mean": -13.63015365600586, "KL/rejected_KL_mean": -26.938322067260742, "KL/std": 29.864192962646484, "epoch": 0.5340314136125655, "fcm_dpo/beta": 0.025404341518878937, "fcm_dpo/delta": -0.080487459897995, "fcm_dpo/margin": 26.616352081298828, "fcm_dpo/q_t": 0.3620305359363556, "grad_norm": 40.981224060058594, "learning_rate": 2.655514550086086e-07, "logits/chosen": 1.6588364839553833, "logits/rejected": 1.5612589120864868, "logps/chosen": -298.23583984375, "logps/ref_chosen": -297.9138488769531, "logps/ref_rejected": -253.39952087402344, "logps/rejected": -280.33782958984375, "loss": 4.0428, "margin_dpo/margin_mean": 26.616350173950195, "margin_dpo/margin_std": 36.11050796508789, "step": 255 }, { "KL/chosen_KL_mean": 0.7054767608642578, "KL/mean": -12.810953140258789, "KL/rejected_KL_mean": -26.32737922668457, "KL/std": 31.35456085205078, "epoch": 0.5361256544502618, "fcm_dpo/beta": 0.024050066247582436, "fcm_dpo/delta": -0.055629149079322815, "fcm_dpo/margin": 27.032861709594727, "fcm_dpo/q_t": 0.36533451080322266, "grad_norm": 44.06679916381836, "learning_rate": 2.6372383496608186e-07, "logits/chosen": 1.5450178384780884, "logits/rejected": 1.4958531856536865, "logps/chosen": -309.22088623046875, "logps/ref_chosen": -309.9263916015625, "logps/ref_rejected": -253.452392578125, "logps/rejected": -279.77978515625, "loss": 4.1981, "margin_dpo/margin_mean": 27.032861709594727, "margin_dpo/margin_std": 38.51383590698242, "step": 256 }, { "KL/chosen_KL_mean": -2.0746002197265625, "KL/mean": -13.949150085449219, "KL/rejected_KL_mean": -25.823705673217773, "KL/std": 30.354991912841797, "epoch": 0.5382198952879581, "fcm_dpo/beta": 0.023041877895593643, "fcm_dpo/delta": 0.055370353162288666, "fcm_dpo/margin": 23.749109268188477, "fcm_dpo/q_t": 0.38487961888313293, "grad_norm": 57.64748764038086, "learning_rate": 2.618954789559356e-07, "logits/chosen": 1.1348298788070679, "logits/rejected": 1.152570128440857, "logps/chosen": -261.8882141113281, "logps/ref_chosen": -259.8136291503906, "logps/ref_rejected": -245.84947204589844, "logps/rejected": -271.6731872558594, "loss": 4.1992, "margin_dpo/margin_mean": 23.749109268188477, "margin_dpo/margin_std": 33.00218200683594, "step": 257 }, { "KL/chosen_KL_mean": -1.9968814849853516, "KL/mean": -14.562836647033691, "KL/rejected_KL_mean": -27.128786087036133, "KL/std": 30.502214431762695, "epoch": 0.5403141361256545, "fcm_dpo/beta": 0.023513339459896088, "fcm_dpo/delta": 0.00881824642419815, "fcm_dpo/margin": 25.13190269470215, "fcm_dpo/q_t": 0.3715367615222931, "grad_norm": 46.1146354675293, "learning_rate": 2.600664850273538e-07, "logits/chosen": 1.6273773908615112, "logits/rejected": 1.7062067985534668, "logps/chosen": -267.3505554199219, "logps/ref_chosen": -265.35369873046875, "logps/ref_rejected": -247.9174346923828, "logps/rejected": -275.04620361328125, "loss": 4.0437, "margin_dpo/margin_mean": 25.13190269470215, "margin_dpo/margin_std": 31.523521423339844, "step": 258 }, { "KL/chosen_KL_mean": 0.11016464233398438, "KL/mean": -12.010380744934082, "KL/rejected_KL_mean": -24.13092041015625, "KL/std": 30.323959350585938, "epoch": 0.5424083769633508, "fcm_dpo/beta": 0.024552009999752045, "fcm_dpo/delta": 0.004387339577078819, "fcm_dpo/margin": 24.241092681884766, "fcm_dpo/q_t": 0.3753139078617096, "grad_norm": 51.9947395324707, "learning_rate": 2.582369512637302e-07, "logits/chosen": 1.5842348337173462, "logits/rejected": 1.5799920558929443, "logps/chosen": -288.1130676269531, "logps/ref_chosen": -288.2232360839844, "logps/ref_rejected": -270.7300109863281, "logps/rejected": -294.8609619140625, "loss": 4.1724, "margin_dpo/margin_mean": 24.241092681884766, "margin_dpo/margin_std": 35.039058685302734, "step": 259 }, { "KL/chosen_KL_mean": -3.2845516204833984, "KL/mean": -9.023161888122559, "KL/rejected_KL_mean": -14.761768341064453, "KL/std": 28.911399841308594, "epoch": 0.5445026178010471, "fcm_dpo/beta": 0.02567979507148266, "fcm_dpo/delta": 0.1519329994916916, "fcm_dpo/margin": 11.477216720581055, "fcm_dpo/q_t": 0.43951642513275146, "grad_norm": 71.44678497314453, "learning_rate": 2.5640697577740815e-07, "logits/chosen": 1.722202181816101, "logits/rejected": 1.764743447303772, "logps/chosen": -284.9042663574219, "logps/ref_chosen": -281.61968994140625, "logps/ref_rejected": -271.03973388671875, "logps/rejected": -285.801513671875, "loss": 5.1871, "margin_dpo/margin_mean": 11.477216720581055, "margin_dpo/margin_std": 33.64004135131836, "step": 260 }, { "KL/chosen_KL_mean": -6.1820526123046875, "KL/mean": -16.529911041259766, "KL/rejected_KL_mean": -26.877765655517578, "KL/std": 29.89157485961914, "epoch": 0.5465968586387434, "fcm_dpo/beta": 0.027530504390597343, "fcm_dpo/delta": 0.029914624989032745, "fcm_dpo/margin": 20.69570541381836, "fcm_dpo/q_t": 0.3858029246330261, "grad_norm": 57.450679779052734, "learning_rate": 2.5457665670441937e-07, "logits/chosen": 1.2852181196212769, "logits/rejected": 1.4253342151641846, "logps/chosen": -248.6452178955078, "logps/ref_chosen": -242.4631805419922, "logps/ref_rejected": -244.0020751953125, "logps/rejected": -270.87982177734375, "loss": 4.4957, "margin_dpo/margin_mean": 20.69570541381836, "margin_dpo/margin_std": 36.37822723388672, "step": 261 }, { "KL/chosen_KL_mean": -0.7161979675292969, "KL/mean": -13.208702087402344, "KL/rejected_KL_mean": -25.701202392578125, "KL/std": 30.572877883911133, "epoch": 0.5486910994764398, "fcm_dpo/beta": 0.027186822146177292, "fcm_dpo/delta": -0.08552050590515137, "fcm_dpo/margin": 24.984996795654297, "fcm_dpo/q_t": 0.36049196124076843, "grad_norm": 50.54449462890625, "learning_rate": 2.527460921992209e-07, "logits/chosen": 1.580787181854248, "logits/rejected": 1.5897417068481445, "logps/chosen": -292.704833984375, "logps/ref_chosen": -291.98858642578125, "logps/ref_rejected": -273.35955810546875, "logps/rejected": -299.060791015625, "loss": 3.9733, "margin_dpo/margin_mean": 24.984994888305664, "margin_dpo/margin_std": 32.54281997680664, "step": 262 }, { "KL/chosen_KL_mean": -4.733024597167969, "KL/mean": -15.698508262634277, "KL/rejected_KL_mean": -26.663997650146484, "KL/std": 30.50117301940918, "epoch": 0.5507853403141362, "fcm_dpo/beta": 0.025977104902267456, "fcm_dpo/delta": -0.03878769278526306, "fcm_dpo/margin": 21.930967330932617, "fcm_dpo/q_t": 0.3833428621292114, "grad_norm": 55.72367858886719, "learning_rate": 2.509153804294318e-07, "logits/chosen": 1.263526439666748, "logits/rejected": 1.2350153923034668, "logps/chosen": -279.72149658203125, "logps/ref_chosen": -274.98846435546875, "logps/ref_rejected": -254.75782775878906, "logps/rejected": -281.4218444824219, "loss": 4.3527, "margin_dpo/margin_mean": 21.930967330932617, "margin_dpo/margin_std": 35.15817642211914, "step": 263 }, { "KL/chosen_KL_mean": 0.7876987457275391, "KL/mean": -12.341998100280762, "KL/rejected_KL_mean": -25.4716854095459, "KL/std": 30.41027069091797, "epoch": 0.5528795811518324, "fcm_dpo/beta": 0.02479763887822628, "fcm_dpo/delta": -0.056634265929460526, "fcm_dpo/margin": 26.259387969970703, "fcm_dpo/q_t": 0.36550551652908325, "grad_norm": 40.78193664550781, "learning_rate": 2.4908461957056825e-07, "logits/chosen": 1.1997872591018677, "logits/rejected": 1.2514468431472778, "logps/chosen": -250.83847045898438, "logps/ref_chosen": -251.62615966796875, "logps/ref_rejected": -251.9549560546875, "logps/rejected": -277.4266357421875, "loss": 4.0046, "margin_dpo/margin_mean": 26.259389877319336, "margin_dpo/margin_std": 34.020729064941406, "step": 264 }, { "KL/chosen_KL_mean": -3.2062511444091797, "KL/mean": -15.04905891418457, "KL/rejected_KL_mean": -26.89187240600586, "KL/std": 31.76069450378418, "epoch": 0.5549738219895288, "fcm_dpo/beta": 0.02372550778090954, "fcm_dpo/delta": -0.01052355207502842, "fcm_dpo/margin": 23.685611724853516, "fcm_dpo/q_t": 0.38212448358535767, "grad_norm": 45.85237121582031, "learning_rate": 2.4725390780077905e-07, "logits/chosen": 1.3538559675216675, "logits/rejected": 1.3851525783538818, "logps/chosen": -281.1428527832031, "logps/ref_chosen": -277.9366149902344, "logps/ref_rejected": -262.689453125, "logps/rejected": -289.5813293457031, "loss": 4.2632, "margin_dpo/margin_mean": 23.685611724853516, "margin_dpo/margin_std": 34.29460144042969, "step": 265 }, { "KL/chosen_KL_mean": 2.9426498413085938, "KL/mean": -9.96561336517334, "KL/rejected_KL_mean": -22.873876571655273, "KL/std": 30.54420280456543, "epoch": 0.5570680628272251, "fcm_dpo/beta": 0.02399318292737007, "fcm_dpo/delta": -0.022777840495109558, "fcm_dpo/margin": 25.816530227661133, "fcm_dpo/q_t": 0.3668290376663208, "grad_norm": 43.122013092041016, "learning_rate": 2.454233432955807e-07, "logits/chosen": 1.5934802293777466, "logits/rejected": 1.5779345035552979, "logps/chosen": -267.50225830078125, "logps/ref_chosen": -270.4449157714844, "logps/ref_rejected": -259.5942077636719, "logps/rejected": -282.4681091308594, "loss": 4.0291, "margin_dpo/margin_mean": 25.816532135009766, "margin_dpo/margin_std": 32.49258041381836, "step": 266 }, { "KL/chosen_KL_mean": -2.7025794982910156, "KL/mean": -13.08747386932373, "KL/rejected_KL_mean": -23.472375869750977, "KL/std": 30.725170135498047, "epoch": 0.5591623036649215, "fcm_dpo/beta": 0.02411152422428131, "fcm_dpo/delta": 0.051255080848932266, "fcm_dpo/margin": 20.769792556762695, "fcm_dpo/q_t": 0.3949718773365021, "grad_norm": 46.152915954589844, "learning_rate": 2.435930242225919e-07, "logits/chosen": 1.2272932529449463, "logits/rejected": 1.2689601182937622, "logps/chosen": -272.54248046875, "logps/ref_chosen": -269.83990478515625, "logps/ref_rejected": -246.33746337890625, "logps/rejected": -269.8098449707031, "loss": 4.5142, "margin_dpo/margin_mean": 20.769792556762695, "margin_dpo/margin_std": 35.88163757324219, "step": 267 }, { "KL/chosen_KL_mean": 0.5022964477539062, "KL/mean": -12.046917915344238, "KL/rejected_KL_mean": -24.59613037109375, "KL/std": 31.01312828063965, "epoch": 0.5612565445026177, "fcm_dpo/beta": 0.02405945211648941, "fcm_dpo/delta": -0.05520617961883545, "fcm_dpo/margin": 25.09842872619629, "fcm_dpo/q_t": 0.3740164637565613, "grad_norm": 44.2439079284668, "learning_rate": 2.4176304873626984e-07, "logits/chosen": 1.409890055656433, "logits/rejected": 1.375052809715271, "logps/chosen": -297.0316467285156, "logps/ref_chosen": -297.533935546875, "logps/ref_rejected": -270.3407897949219, "logps/rejected": -294.9368896484375, "loss": 4.1233, "margin_dpo/margin_mean": 25.09842872619629, "margin_dpo/margin_std": 34.37728500366211, "step": 268 }, { "KL/chosen_KL_mean": -1.2247867584228516, "KL/mean": -11.996273040771484, "KL/rejected_KL_mean": -22.76775550842285, "KL/std": 30.386701583862305, "epoch": 0.5633507853403141, "fcm_dpo/beta": 0.02467486262321472, "fcm_dpo/delta": 0.070290707051754, "fcm_dpo/margin": 21.5429744720459, "fcm_dpo/q_t": 0.3866848051548004, "grad_norm": 84.861572265625, "learning_rate": 2.399335149726463e-07, "logits/chosen": 1.200810432434082, "logits/rejected": 1.2272669076919556, "logps/chosen": -241.5636444091797, "logps/ref_chosen": -240.33883666992188, "logps/ref_rejected": -220.71717834472656, "logps/rejected": -243.48492431640625, "loss": 4.4276, "margin_dpo/margin_mean": 21.542972564697266, "margin_dpo/margin_std": 35.83372497558594, "step": 269 }, { "KL/chosen_KL_mean": 0.8291587829589844, "KL/mean": -11.934760093688965, "KL/rejected_KL_mean": -24.69868278503418, "KL/std": 31.864492416381836, "epoch": 0.5654450261780105, "fcm_dpo/beta": 0.024463558569550514, "fcm_dpo/delta": -0.027698107063770294, "fcm_dpo/margin": 25.527835845947266, "fcm_dpo/q_t": 0.36969655752182007, "grad_norm": 49.83897018432617, "learning_rate": 2.381045210440644e-07, "logits/chosen": 1.6544300317764282, "logits/rejected": 1.634549617767334, "logps/chosen": -308.06396484375, "logps/ref_chosen": -308.8930969238281, "logps/ref_rejected": -278.47869873046875, "logps/rejected": -303.1773986816406, "loss": 4.2113, "margin_dpo/margin_mean": 25.527835845947266, "margin_dpo/margin_std": 37.18598556518555, "step": 270 }, { "KL/chosen_KL_mean": -0.1355876922607422, "KL/mean": -12.542990684509277, "KL/rejected_KL_mean": -24.950380325317383, "KL/std": 30.933940887451172, "epoch": 0.5675392670157068, "fcm_dpo/beta": 0.02482183650135994, "fcm_dpo/delta": -0.018352961167693138, "fcm_dpo/margin": 24.814781188964844, "fcm_dpo/q_t": 0.3741310238838196, "grad_norm": 50.110206604003906, "learning_rate": 2.3627616503391812e-07, "logits/chosen": 1.38817298412323, "logits/rejected": 1.3879042863845825, "logps/chosen": -275.5638122558594, "logps/ref_chosen": -275.42822265625, "logps/ref_rejected": -251.50906372070312, "logps/rejected": -276.45941162109375, "loss": 4.132, "margin_dpo/margin_mean": 24.814781188964844, "margin_dpo/margin_std": 34.804603576660156, "step": 271 }, { "KL/chosen_KL_mean": 1.4148082733154297, "KL/mean": -9.921167373657227, "KL/rejected_KL_mean": -21.25714874267578, "KL/std": 30.931760787963867, "epoch": 0.5696335078534032, "fcm_dpo/beta": 0.023737920448184013, "fcm_dpo/delta": -0.0019292645156383514, "fcm_dpo/margin": 22.671958923339844, "fcm_dpo/q_t": 0.386726975440979, "grad_norm": 82.14847564697266, "learning_rate": 2.344485449913914e-07, "logits/chosen": 1.6356867551803589, "logits/rejected": 1.651943564414978, "logps/chosen": -287.96795654296875, "logps/ref_chosen": -289.38275146484375, "logps/ref_rejected": -274.895751953125, "logps/rejected": -296.15289306640625, "loss": 4.3293, "margin_dpo/margin_mean": 22.671958923339844, "margin_dpo/margin_std": 34.74875259399414, "step": 272 }, { "KL/chosen_KL_mean": 0.9697151184082031, "KL/mean": -11.799010276794434, "KL/rejected_KL_mean": -24.567733764648438, "KL/std": 30.73468589782715, "epoch": 0.5717277486910994, "fcm_dpo/beta": 0.02284211292862892, "fcm_dpo/delta": -0.036032721400260925, "fcm_dpo/margin": 25.53744888305664, "fcm_dpo/q_t": 0.3772345781326294, "grad_norm": 41.545528411865234, "learning_rate": 2.3262175892620062e-07, "logits/chosen": 1.272435188293457, "logits/rejected": 1.365143060684204, "logps/chosen": -284.01873779296875, "logps/ref_chosen": -284.98846435546875, "logps/ref_rejected": -273.5077819824219, "logps/rejected": -298.0755310058594, "loss": 4.2081, "margin_dpo/margin_mean": 25.53744888305664, "margin_dpo/margin_std": 36.59501266479492, "step": 273 }, { "KL/chosen_KL_mean": 0.7551822662353516, "KL/mean": -13.36454963684082, "KL/rejected_KL_mean": -27.48428726196289, "KL/std": 32.58087921142578, "epoch": 0.5738219895287958, "fcm_dpo/beta": 0.022201525047421455, "fcm_dpo/delta": -0.09407390654087067, "fcm_dpo/margin": 28.23946189880371, "fcm_dpo/q_t": 0.36471086740493774, "grad_norm": 37.03623580932617, "learning_rate": 2.3079590480333827e-07, "logits/chosen": 1.5402204990386963, "logits/rejected": 1.5999386310577393, "logps/chosen": -261.9244079589844, "logps/ref_chosen": -262.6795959472656, "logps/ref_rejected": -236.13792419433594, "logps/rejected": -263.62225341796875, "loss": 4.0284, "margin_dpo/margin_mean": 28.239459991455078, "margin_dpo/margin_std": 35.92232131958008, "step": 274 }, { "KL/chosen_KL_mean": 2.9862890243530273, "KL/mean": -12.728757858276367, "KL/rejected_KL_mean": -28.443809509277344, "KL/std": 31.11075210571289, "epoch": 0.5759162303664922, "fcm_dpo/beta": 0.021001681685447693, "fcm_dpo/delta": -0.06485743820667267, "fcm_dpo/margin": 31.430097579956055, "fcm_dpo/q_t": 0.3577304482460022, "grad_norm": 36.05902099609375, "learning_rate": 2.2897108053782e-07, "logits/chosen": 1.327195167541504, "logits/rejected": 1.4250489473342896, "logps/chosen": -243.4132080078125, "logps/ref_chosen": -246.39950561523438, "logps/ref_rejected": -247.4836883544922, "logps/rejected": -275.927490234375, "loss": 3.8239, "margin_dpo/margin_mean": 31.430099487304688, "margin_dpo/margin_std": 34.86501693725586, "step": 275 }, { "KL/chosen_KL_mean": 4.6192169189453125, "KL/mean": -9.579551696777344, "KL/rejected_KL_mean": -23.778339385986328, "KL/std": 30.969295501708984, "epoch": 0.5780104712041885, "fcm_dpo/beta": 0.0204936396330595, "fcm_dpo/delta": 0.018703483045101166, "fcm_dpo/margin": 28.39756202697754, "fcm_dpo/q_t": 0.37401020526885986, "grad_norm": 38.20977020263672, "learning_rate": 2.2714738398943308e-07, "logits/chosen": 1.6414986848831177, "logits/rejected": 1.7540438175201416, "logps/chosen": -284.30084228515625, "logps/ref_chosen": -288.9200439453125, "logps/ref_rejected": -291.69305419921875, "logps/rejected": -315.47137451171875, "loss": 4.0103, "margin_dpo/margin_mean": 28.397560119628906, "margin_dpo/margin_std": 34.412315368652344, "step": 276 }, { "KL/chosen_KL_mean": -2.7718963623046875, "KL/mean": -13.085480690002441, "KL/rejected_KL_mean": -23.399076461791992, "KL/std": 30.761445999145508, "epoch": 0.5801047120418849, "fcm_dpo/beta": 0.02186032012104988, "fcm_dpo/delta": 0.10023684799671173, "fcm_dpo/margin": 20.627185821533203, "fcm_dpo/q_t": 0.4030647277832031, "grad_norm": 40.561344146728516, "learning_rate": 2.2532491295748865e-07, "logits/chosen": 1.16107177734375, "logits/rejected": 1.1822755336761475, "logps/chosen": -259.8568115234375, "logps/ref_chosen": -257.08489990234375, "logps/ref_rejected": -251.61532592773438, "logps/rejected": -275.014404296875, "loss": 4.6091, "margin_dpo/margin_mean": 20.62718391418457, "margin_dpo/margin_std": 37.93412780761719, "step": 277 }, { "KL/chosen_KL_mean": -5.834524154663086, "KL/mean": -15.804929733276367, "KL/rejected_KL_mean": -25.77533531188965, "KL/std": 32.02848434448242, "epoch": 0.5821989528795811, "fcm_dpo/beta": 0.022726019844412804, "fcm_dpo/delta": 0.009016614407300949, "fcm_dpo/margin": 19.940826416015625, "fcm_dpo/q_t": 0.40428292751312256, "grad_norm": 43.55221176147461, "learning_rate": 2.2350376517557726e-07, "logits/chosen": 1.2224705219268799, "logits/rejected": 1.1500102281570435, "logps/chosen": -266.5758972167969, "logps/ref_chosen": -260.7413635253906, "logps/ref_rejected": -227.4325408935547, "logps/rejected": -253.2078857421875, "loss": 4.7198, "margin_dpo/margin_mean": 19.940826416015625, "margin_dpo/margin_std": 39.22321701049805, "step": 278 }, { "KL/chosen_KL_mean": -0.06951332092285156, "KL/mean": -15.247830390930176, "KL/rejected_KL_mean": -30.4261474609375, "KL/std": 34.06287384033203, "epoch": 0.5842931937172775, "fcm_dpo/beta": 0.02161788009107113, "fcm_dpo/delta": -0.06343191862106323, "fcm_dpo/margin": 30.356632232666016, "fcm_dpo/q_t": 0.3606010377407074, "grad_norm": 39.08658981323242, "learning_rate": 2.2168403830632769e-07, "logits/chosen": 1.1724772453308105, "logits/rejected": 1.2419006824493408, "logps/chosen": -303.01702880859375, "logps/ref_chosen": -302.947509765625, "logps/ref_rejected": -294.1771545410156, "logps/rejected": -324.6033020019531, "loss": 3.9275, "margin_dpo/margin_mean": 30.35663414001465, "margin_dpo/margin_std": 36.632137298583984, "step": 279 }, { "KL/chosen_KL_mean": -0.2930927276611328, "KL/mean": -13.874507904052734, "KL/rejected_KL_mean": -27.45592498779297, "KL/std": 30.778343200683594, "epoch": 0.5863874345549738, "fcm_dpo/beta": 0.02132745087146759, "fcm_dpo/delta": 0.021288638934493065, "fcm_dpo/margin": 27.16282844543457, "fcm_dpo/q_t": 0.37750864028930664, "grad_norm": 44.36606216430664, "learning_rate": 2.1986582993616925e-07, "logits/chosen": 1.2394180297851562, "logits/rejected": 1.3001892566680908, "logps/chosen": -258.191162109375, "logps/ref_chosen": -257.8981018066406, "logps/ref_rejected": -244.9084014892578, "logps/rejected": -272.36431884765625, "loss": 4.1394, "margin_dpo/margin_mean": 27.162826538085938, "margin_dpo/margin_std": 36.861446380615234, "step": 280 }, { "KL/chosen_KL_mean": -6.789348602294922, "KL/mean": -19.637094497680664, "KL/rejected_KL_mean": -32.484840393066406, "KL/std": 33.552734375, "epoch": 0.5884816753926702, "fcm_dpo/beta": 0.021715322509407997, "fcm_dpo/delta": -0.008483527228236198, "fcm_dpo/margin": 25.695491790771484, "fcm_dpo/q_t": 0.38245102763175964, "grad_norm": 43.89140319824219, "learning_rate": 2.1804923757009882e-07, "logits/chosen": 1.3251450061798096, "logits/rejected": 1.377968430519104, "logps/chosen": -296.9840393066406, "logps/ref_chosen": -290.1946716308594, "logps/ref_rejected": -290.98883056640625, "logps/rejected": -323.47369384765625, "loss": 4.1787, "margin_dpo/margin_mean": 25.695493698120117, "margin_dpo/margin_std": 35.247562408447266, "step": 281 }, { "KL/chosen_KL_mean": -4.626255035400391, "KL/mean": -17.162841796875, "KL/rejected_KL_mean": -29.699420928955078, "KL/std": 33.78520965576172, "epoch": 0.5905759162303665, "fcm_dpo/beta": 0.02181394025683403, "fcm_dpo/delta": 0.05608820542693138, "fcm_dpo/margin": 25.07317352294922, "fcm_dpo/q_t": 0.3867621123790741, "grad_norm": 48.58553695678711, "learning_rate": 2.1623435862645205e-07, "logits/chosen": 1.5871410369873047, "logits/rejected": 1.6240686178207397, "logps/chosen": -313.543701171875, "logps/ref_chosen": -308.91741943359375, "logps/ref_rejected": -272.2177734375, "logps/rejected": -301.91717529296875, "loss": 4.351, "margin_dpo/margin_mean": 25.07317352294922, "margin_dpo/margin_std": 39.92859649658203, "step": 282 }, { "KL/chosen_KL_mean": -7.476833343505859, "KL/mean": -20.217012405395508, "KL/rejected_KL_mean": -32.95720672607422, "KL/std": 31.2985782623291, "epoch": 0.5926701570680628, "fcm_dpo/beta": 0.02321285754442215, "fcm_dpo/delta": 0.005616554990410805, "fcm_dpo/margin": 25.48038673400879, "fcm_dpo/q_t": 0.3762415945529938, "grad_norm": 44.006996154785156, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 1.5656588077545166, "logits/rejected": 1.6084039211273193, "logps/chosen": -255.4988250732422, "logps/ref_chosen": -248.02200317382812, "logps/ref_rejected": -234.78541564941406, "logps/rejected": -267.7426452636719, "loss": 4.2182, "margin_dpo/margin_mean": 25.48038673400879, "margin_dpo/margin_std": 36.498069763183594, "step": 283 }, { "KL/chosen_KL_mean": -7.681129455566406, "KL/mean": -20.3056697845459, "KL/rejected_KL_mean": -32.930213928222656, "KL/std": 34.0436897277832, "epoch": 0.5947643979057592, "fcm_dpo/beta": 0.023079728707671165, "fcm_dpo/delta": 0.015980251133441925, "fcm_dpo/margin": 25.24908447265625, "fcm_dpo/q_t": 0.38073068857192993, "grad_norm": 45.13376235961914, "learning_rate": 2.1261013021512378e-07, "logits/chosen": 1.3176496028900146, "logits/rejected": 1.2986741065979004, "logps/chosen": -272.1703796386719, "logps/ref_chosen": -264.4892578125, "logps/ref_rejected": -255.43092346191406, "logps/rejected": -288.36114501953125, "loss": 4.4246, "margin_dpo/margin_mean": 25.249080657958984, "margin_dpo/margin_std": 42.03435134887695, "step": 284 }, { "KL/chosen_KL_mean": -6.775789260864258, "KL/mean": -16.943470001220703, "KL/rejected_KL_mean": -27.11114501953125, "KL/std": 31.593013763427734, "epoch": 0.5968586387434555, "fcm_dpo/beta": 0.023795977234840393, "fcm_dpo/delta": 0.00033229123800992966, "fcm_dpo/margin": 20.335365295410156, "fcm_dpo/q_t": 0.3984742760658264, "grad_norm": 57.09181594848633, "learning_rate": 2.1080097510381294e-07, "logits/chosen": 1.5144256353378296, "logits/rejected": 1.507455825805664, "logps/chosen": -308.2445068359375, "logps/ref_chosen": -301.46868896484375, "logps/ref_rejected": -274.2380676269531, "logps/rejected": -301.3492431640625, "loss": 4.5294, "margin_dpo/margin_mean": 20.335365295410156, "margin_dpo/margin_std": 34.23255920410156, "step": 285 }, { "KL/chosen_KL_mean": -8.32232666015625, "KL/mean": -19.552255630493164, "KL/rejected_KL_mean": -30.782180786132812, "KL/std": 32.217063903808594, "epoch": 0.5989528795811518, "fcm_dpo/beta": 0.023625340312719345, "fcm_dpo/delta": 0.07382149249315262, "fcm_dpo/margin": 22.4598445892334, "fcm_dpo/q_t": 0.3923597037792206, "grad_norm": 51.8724365234375, "learning_rate": 2.089939221172446e-07, "logits/chosen": 1.5711992979049683, "logits/rejected": 1.6472262144088745, "logps/chosen": -274.8813781738281, "logps/ref_chosen": -266.5590515136719, "logps/ref_rejected": -280.12066650390625, "logps/rejected": -310.90289306640625, "loss": 4.5259, "margin_dpo/margin_mean": 22.4598445892334, "margin_dpo/margin_std": 40.209190368652344, "step": 286 }, { "KL/chosen_KL_mean": -7.606317520141602, "KL/mean": -19.478515625, "KL/rejected_KL_mean": -31.35072135925293, "KL/std": 30.595090866088867, "epoch": 0.6010471204188481, "fcm_dpo/beta": 0.024595845490694046, "fcm_dpo/delta": 0.015402635559439659, "fcm_dpo/margin": 23.74439811706543, "fcm_dpo/q_t": 0.37745368480682373, "grad_norm": 49.05549240112305, "learning_rate": 2.0718906816218595e-07, "logits/chosen": 1.2915270328521729, "logits/rejected": 1.335796594619751, "logps/chosen": -277.54266357421875, "logps/ref_chosen": -269.93634033203125, "logps/ref_rejected": -232.32330322265625, "logps/rejected": -263.6740417480469, "loss": 4.3272, "margin_dpo/margin_mean": 23.744396209716797, "margin_dpo/margin_std": 37.42576217651367, "step": 287 }, { "KL/chosen_KL_mean": -7.191789627075195, "KL/mean": -18.660404205322266, "KL/rejected_KL_mean": -30.129026412963867, "KL/std": 32.191680908203125, "epoch": 0.6031413612565445, "fcm_dpo/beta": 0.02573678456246853, "fcm_dpo/delta": 0.008166726678609848, "fcm_dpo/margin": 22.937244415283203, "fcm_dpo/q_t": 0.3816714882850647, "grad_norm": 52.09629821777344, "learning_rate": 2.053865100274774e-07, "logits/chosen": 1.4072273969650269, "logits/rejected": 1.3486778736114502, "logps/chosen": -291.2373046875, "logps/ref_chosen": -284.04547119140625, "logps/ref_rejected": -261.19378662109375, "logps/rejected": -291.3227844238281, "loss": 4.3388, "margin_dpo/margin_mean": 22.937246322631836, "margin_dpo/margin_std": 37.41172790527344, "step": 288 }, { "KL/chosen_KL_mean": -10.13895034790039, "KL/mean": -18.886932373046875, "KL/rejected_KL_mean": -27.63491439819336, "KL/std": 33.31560134887695, "epoch": 0.6052356020942409, "fcm_dpo/beta": 0.025546494871377945, "fcm_dpo/delta": 0.05048464983701706, "fcm_dpo/margin": 17.4959774017334, "fcm_dpo/q_t": 0.40983152389526367, "grad_norm": 51.257930755615234, "learning_rate": 2.035863443788411e-07, "logits/chosen": 1.4110479354858398, "logits/rejected": 1.36525297164917, "logps/chosen": -333.4201965332031, "logps/ref_chosen": -323.28125, "logps/ref_rejected": -274.2093200683594, "logps/rejected": -301.84423828125, "loss": 4.8117, "margin_dpo/margin_mean": 17.4959774017334, "margin_dpo/margin_std": 37.651920318603516, "step": 289 }, { "KL/chosen_KL_mean": -6.435283660888672, "KL/mean": -17.43682861328125, "KL/rejected_KL_mean": -28.43837547302246, "KL/std": 33.041744232177734, "epoch": 0.6073298429319371, "fcm_dpo/beta": 0.025500264018774033, "fcm_dpo/delta": -0.022538959980010986, "fcm_dpo/margin": 22.00309944152832, "fcm_dpo/q_t": 0.38546958565711975, "grad_norm": 56.410152435302734, "learning_rate": 2.0178866775369774e-07, "logits/chosen": 1.3780235052108765, "logits/rejected": 1.3338419198989868, "logps/chosen": -323.8706970214844, "logps/ref_chosen": -317.4353942871094, "logps/ref_rejected": -309.32403564453125, "logps/rejected": -337.7624206542969, "loss": 4.38, "margin_dpo/margin_mean": 22.00309944152832, "margin_dpo/margin_std": 35.39474868774414, "step": 290 }, { "KL/chosen_KL_mean": -5.390897750854492, "KL/mean": -20.070323944091797, "KL/rejected_KL_mean": -34.749759674072266, "KL/std": 34.942203521728516, "epoch": 0.6094240837696335, "fcm_dpo/beta": 0.02390497922897339, "fcm_dpo/delta": -0.11430876702070236, "fcm_dpo/margin": 29.35886001586914, "fcm_dpo/q_t": 0.35525354743003845, "grad_norm": 39.238685607910156, "learning_rate": 1.9999357655598891e-07, "logits/chosen": 1.2701756954193115, "logits/rejected": 1.4179400205612183, "logps/chosen": -270.8991394042969, "logps/ref_chosen": -265.50823974609375, "logps/ref_rejected": -275.1380310058594, "logps/rejected": -309.8877868652344, "loss": 3.9931, "margin_dpo/margin_mean": 29.35886001586914, "margin_dpo/margin_std": 38.15861511230469, "step": 291 }, { "KL/chosen_KL_mean": -2.940328598022461, "KL/mean": -14.973896980285645, "KL/rejected_KL_mean": -27.007455825805664, "KL/std": 32.84165573120117, "epoch": 0.6115183246073298, "fcm_dpo/beta": 0.02307257242500782, "fcm_dpo/delta": -0.007151174359023571, "fcm_dpo/margin": 24.06713104248047, "fcm_dpo/q_t": 0.3814610242843628, "grad_norm": 52.42905044555664, "learning_rate": 1.9820116705100775e-07, "logits/chosen": 1.338476538658142, "logits/rejected": 1.4540705680847168, "logps/chosen": -252.84165954589844, "logps/ref_chosen": -249.9013214111328, "logps/ref_rejected": -274.58245849609375, "logps/rejected": -301.5899353027344, "loss": 4.2864, "margin_dpo/margin_mean": 24.067129135131836, "margin_dpo/margin_std": 35.604591369628906, "step": 292 }, { "KL/chosen_KL_mean": -8.426515579223633, "KL/mean": -20.50110626220703, "KL/rejected_KL_mean": -32.57570266723633, "KL/std": 34.23207092285156, "epoch": 0.6136125654450262, "fcm_dpo/beta": 0.023404065519571304, "fcm_dpo/delta": -0.01296854019165039, "fcm_dpo/margin": 24.149171829223633, "fcm_dpo/q_t": 0.38174688816070557, "grad_norm": 54.628761291503906, "learning_rate": 1.9641153536023642e-07, "logits/chosen": 1.5635805130004883, "logits/rejected": 1.5213494300842285, "logps/chosen": -304.73248291015625, "logps/ref_chosen": -296.30596923828125, "logps/ref_rejected": -268.0640869140625, "logps/rejected": -300.6397705078125, "loss": 4.2688, "margin_dpo/margin_mean": 24.149169921875, "margin_dpo/margin_std": 36.250484466552734, "step": 293 }, { "KL/chosen_KL_mean": -5.804834365844727, "KL/mean": -16.724685668945312, "KL/rejected_KL_mean": -27.64453125, "KL/std": 32.72875213623047, "epoch": 0.6157068062827226, "fcm_dpo/beta": 0.023987282067537308, "fcm_dpo/delta": 0.07841254025697708, "fcm_dpo/margin": 21.839691162109375, "fcm_dpo/q_t": 0.3923855125904083, "grad_norm": 58.72978210449219, "learning_rate": 1.9462477745619106e-07, "logits/chosen": 1.5025861263275146, "logits/rejected": 1.5393353700637817, "logps/chosen": -239.68780517578125, "logps/ref_chosen": -233.8829803466797, "logps/ref_rejected": -224.14334106445312, "logps/rejected": -251.78787231445312, "loss": 4.4377, "margin_dpo/margin_mean": 21.839691162109375, "margin_dpo/margin_std": 36.5875129699707, "step": 294 }, { "KL/chosen_KL_mean": -6.276914596557617, "KL/mean": -18.98217010498047, "KL/rejected_KL_mean": -31.687421798706055, "KL/std": 33.36238479614258, "epoch": 0.6178010471204188, "fcm_dpo/beta": 0.02413392812013626, "fcm_dpo/delta": -0.015403367578983307, "fcm_dpo/margin": 25.410507202148438, "fcm_dpo/q_t": 0.37668105959892273, "grad_norm": 41.531402587890625, "learning_rate": 1.928409891572757e-07, "logits/chosen": 1.3584163188934326, "logits/rejected": 1.348545789718628, "logps/chosen": -250.38824462890625, "logps/ref_chosen": -244.111328125, "logps/ref_rejected": -240.18373107910156, "logps/rejected": -271.8711242675781, "loss": 4.28, "margin_dpo/margin_mean": 25.410507202148438, "margin_dpo/margin_std": 39.370628356933594, "step": 295 }, { "KL/chosen_KL_mean": -4.065727233886719, "KL/mean": -21.36711311340332, "KL/rejected_KL_mean": -38.66850662231445, "KL/std": 35.191795349121094, "epoch": 0.6198952879581152, "fcm_dpo/beta": 0.021722082048654556, "fcm_dpo/delta": -0.21751227974891663, "fcm_dpo/margin": 34.602783203125, "fcm_dpo/q_t": 0.340170294046402, "grad_norm": 43.19810485839844, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 1.4622397422790527, "logits/rejected": 1.6147329807281494, "logps/chosen": -288.74627685546875, "logps/ref_chosen": -284.6805725097656, "logps/ref_rejected": -282.1923828125, "logps/rejected": -320.86083984375, "loss": 3.7668, "margin_dpo/margin_mean": 34.602779388427734, "margin_dpo/margin_std": 39.81512451171875, "step": 296 }, { "KL/chosen_KL_mean": -5.419075012207031, "KL/mean": -18.290597915649414, "KL/rejected_KL_mean": -31.16211700439453, "KL/std": 34.01791763305664, "epoch": 0.6219895287958115, "fcm_dpo/beta": 0.020490456372499466, "fcm_dpo/delta": 0.07718978822231293, "fcm_dpo/margin": 25.743038177490234, "fcm_dpo/q_t": 0.3893577754497528, "grad_norm": 42.12018585205078, "learning_rate": 1.8928270384706582e-07, "logits/chosen": 1.5777825117111206, "logits/rejected": 1.5727683305740356, "logps/chosen": -285.647705078125, "logps/ref_chosen": -280.2286376953125, "logps/ref_rejected": -280.8172912597656, "logps/rejected": -311.97943115234375, "loss": 4.3216, "margin_dpo/margin_mean": 25.743038177490234, "margin_dpo/margin_std": 38.29811096191406, "step": 297 }, { "KL/chosen_KL_mean": -0.35713958740234375, "KL/mean": -15.070883750915527, "KL/rejected_KL_mean": -29.784626007080078, "KL/std": 34.838706970214844, "epoch": 0.6240837696335079, "fcm_dpo/beta": 0.02112782560288906, "fcm_dpo/delta": -0.07453294098377228, "fcm_dpo/margin": 29.427486419677734, "fcm_dpo/q_t": 0.3718750774860382, "grad_norm": 42.121124267578125, "learning_rate": 1.875083976558136e-07, "logits/chosen": 1.0546501874923706, "logits/rejected": 1.023137092590332, "logps/chosen": -297.3574523925781, "logps/ref_chosen": -297.0003356933594, "logps/ref_rejected": -279.1561279296875, "logps/rejected": -308.94073486328125, "loss": 4.1183, "margin_dpo/margin_mean": 29.4274845123291, "margin_dpo/margin_std": 40.24848937988281, "step": 298 }, { "KL/chosen_KL_mean": -7.39764404296875, "KL/mean": -19.16098403930664, "KL/rejected_KL_mean": -30.9243106842041, "KL/std": 34.038780212402344, "epoch": 0.6261780104712041, "fcm_dpo/beta": 0.0208900086581707, "fcm_dpo/delta": 0.04286790266633034, "fcm_dpo/margin": 23.52667236328125, "fcm_dpo/q_t": 0.3937390148639679, "grad_norm": 47.02155685424805, "learning_rate": 1.8573744269954297e-07, "logits/chosen": 1.3947925567626953, "logits/rejected": 1.4756568670272827, "logps/chosen": -278.86907958984375, "logps/ref_chosen": -271.471435546875, "logps/ref_rejected": -268.0870056152344, "logps/rejected": -299.0113220214844, "loss": 4.3195, "margin_dpo/margin_mean": 23.52667236328125, "margin_dpo/margin_std": 34.02577209472656, "step": 299 }, { "KL/chosen_KL_mean": -6.763256072998047, "KL/mean": -18.935020446777344, "KL/rejected_KL_mean": -31.10677719116211, "KL/std": 32.86475372314453, "epoch": 0.6282722513089005, "fcm_dpo/beta": 0.021149903535842896, "fcm_dpo/delta": 0.032501671463251114, "fcm_dpo/margin": 24.343521118164062, "fcm_dpo/q_t": 0.39046284556388855, "grad_norm": 61.73127365112305, "learning_rate": 1.839699339491937e-07, "logits/chosen": 1.5065898895263672, "logits/rejected": 1.4867640733718872, "logps/chosen": -313.12115478515625, "logps/ref_chosen": -306.35784912109375, "logps/ref_rejected": -268.64776611328125, "logps/rejected": -299.7545471191406, "loss": 4.3839, "margin_dpo/margin_mean": 24.343521118164062, "margin_dpo/margin_std": 38.673377990722656, "step": 300 }, { "KL/chosen_KL_mean": -7.070320129394531, "KL/mean": -19.201602935791016, "KL/rejected_KL_mean": -31.3328800201416, "KL/std": 34.259525299072266, "epoch": 0.6303664921465969, "fcm_dpo/beta": 0.022351747378706932, "fcm_dpo/delta": 0.06047248840332031, "fcm_dpo/margin": 24.26255226135254, "fcm_dpo/q_t": 0.387367308139801, "grad_norm": 47.29561996459961, "learning_rate": 1.8220596619089573e-07, "logits/chosen": 1.300453782081604, "logits/rejected": 1.3295258283615112, "logps/chosen": -277.06280517578125, "logps/ref_chosen": -269.9924621582031, "logps/ref_rejected": -271.3135681152344, "logps/rejected": -302.6464538574219, "loss": 4.3893, "margin_dpo/margin_mean": 24.26255226135254, "margin_dpo/margin_std": 39.784507751464844, "step": 301 }, { "KL/chosen_KL_mean": -5.97479248046875, "KL/mean": -20.132221221923828, "KL/rejected_KL_mean": -34.289649963378906, "KL/std": 33.1053466796875, "epoch": 0.6324607329842932, "fcm_dpo/beta": 0.021658364683389664, "fcm_dpo/delta": -0.06815872341394424, "fcm_dpo/margin": 28.314838409423828, "fcm_dpo/q_t": 0.3689638376235962, "grad_norm": 42.739051818847656, "learning_rate": 1.8044563402088682e-07, "logits/chosen": 1.5012693405151367, "logits/rejected": 1.5727123022079468, "logps/chosen": -266.43548583984375, "logps/ref_chosen": -260.460693359375, "logps/ref_rejected": -254.2005615234375, "logps/rejected": -288.4902038574219, "loss": 4.118, "margin_dpo/margin_mean": 28.31484031677246, "margin_dpo/margin_std": 38.24860763549805, "step": 302 }, { "KL/chosen_KL_mean": -4.667455673217773, "KL/mean": -17.669416427612305, "KL/rejected_KL_mean": -30.671361923217773, "KL/std": 34.337013244628906, "epoch": 0.6345549738219896, "fcm_dpo/beta": 0.021459730342030525, "fcm_dpo/delta": -0.02537161111831665, "fcm_dpo/margin": 26.00390625, "fcm_dpo/q_t": 0.3800218105316162, "grad_norm": 50.90971374511719, "learning_rate": 1.7868903184043885e-07, "logits/chosen": 1.248102068901062, "logits/rejected": 1.2776561975479126, "logps/chosen": -300.76373291015625, "logps/ref_chosen": -296.09625244140625, "logps/ref_rejected": -264.7806701660156, "logps/rejected": -295.4520263671875, "loss": 4.3216, "margin_dpo/margin_mean": 26.003908157348633, "margin_dpo/margin_std": 39.529457092285156, "step": 303 }, { "KL/chosen_KL_mean": -11.185628890991211, "KL/mean": -23.168046951293945, "KL/rejected_KL_mean": -35.15047073364258, "KL/std": 36.77131271362305, "epoch": 0.6366492146596858, "fcm_dpo/beta": 0.021864380687475204, "fcm_dpo/delta": 0.07957626134157181, "fcm_dpo/margin": 23.964828491210938, "fcm_dpo/q_t": 0.3897435963153839, "grad_norm": 56.1390495300293, "learning_rate": 1.7693625385079574e-07, "logits/chosen": 1.4579769372940063, "logits/rejected": 1.455704689025879, "logps/chosen": -293.03509521484375, "logps/ref_chosen": -281.8494873046875, "logps/ref_rejected": -234.3673858642578, "logps/rejected": -269.5178527832031, "loss": 4.3434, "margin_dpo/margin_mean": 23.964828491210938, "margin_dpo/margin_std": 37.54193878173828, "step": 304 }, { "KL/chosen_KL_mean": -1.231353759765625, "KL/mean": -18.501249313354492, "KL/rejected_KL_mean": -35.77115249633789, "KL/std": 35.91838455200195, "epoch": 0.6387434554973822, "fcm_dpo/beta": 0.021032236516475677, "fcm_dpo/delta": -0.13967712223529816, "fcm_dpo/margin": 34.539798736572266, "fcm_dpo/q_t": 0.34484899044036865, "grad_norm": 44.405479431152344, "learning_rate": 1.7518739404812155e-07, "logits/chosen": 1.2939680814743042, "logits/rejected": 1.388938069343567, "logps/chosen": -249.2741241455078, "logps/ref_chosen": -248.0427703857422, "logps/ref_rejected": -263.464599609375, "logps/rejected": -299.23577880859375, "loss": 3.7795, "margin_dpo/margin_mean": 34.53980255126953, "margin_dpo/margin_std": 38.691078186035156, "step": 305 }, { "KL/chosen_KL_mean": -0.8278446197509766, "KL/mean": -15.201208114624023, "KL/rejected_KL_mean": -29.57457160949707, "KL/std": 33.9945068359375, "epoch": 0.6408376963350786, "fcm_dpo/beta": 0.02030482515692711, "fcm_dpo/delta": 0.01582380384206772, "fcm_dpo/margin": 28.746726989746094, "fcm_dpo/q_t": 0.37564003467559814, "grad_norm": 42.810768127441406, "learning_rate": 1.7344254621846017e-07, "logits/chosen": 1.4623794555664062, "logits/rejected": 1.3773338794708252, "logps/chosen": -312.11376953125, "logps/ref_chosen": -311.2859191894531, "logps/ref_rejected": -268.4949645996094, "logps/rejected": -298.06951904296875, "loss": 4.142, "margin_dpo/margin_mean": 28.746726989746094, "margin_dpo/margin_std": 38.66185760498047, "step": 306 }, { "KL/chosen_KL_mean": -5.657934188842773, "KL/mean": -21.340206146240234, "KL/rejected_KL_mean": -37.022483825683594, "KL/std": 36.54871368408203, "epoch": 0.6429319371727749, "fcm_dpo/beta": 0.01990652084350586, "fcm_dpo/delta": -0.09277225285768509, "fcm_dpo/margin": 31.364559173583984, "fcm_dpo/q_t": 0.3639976978302002, "grad_norm": 36.270023345947266, "learning_rate": 1.717018039327053e-07, "logits/chosen": 1.384634017944336, "logits/rejected": 1.3476641178131104, "logps/chosen": -275.87042236328125, "logps/ref_chosen": -270.21246337890625, "logps/ref_rejected": -239.1932830810547, "logps/rejected": -276.2157287597656, "loss": 4.0124, "margin_dpo/margin_mean": 31.36455535888672, "margin_dpo/margin_std": 38.58887481689453, "step": 307 }, { "KL/chosen_KL_mean": -7.683645248413086, "KL/mean": -19.247112274169922, "KL/rejected_KL_mean": -30.810577392578125, "KL/std": 35.05336380004883, "epoch": 0.6450261780104712, "fcm_dpo/beta": 0.019985979422926903, "fcm_dpo/delta": 0.14615751802921295, "fcm_dpo/margin": 23.126928329467773, "fcm_dpo/q_t": 0.4005933403968811, "grad_norm": 38.301239013671875, "learning_rate": 1.699652605415828e-07, "logits/chosen": 1.4254227876663208, "logits/rejected": 1.4125336408615112, "logps/chosen": -296.20648193359375, "logps/ref_chosen": -288.5228271484375, "logps/ref_rejected": -253.90597534179688, "logps/rejected": -284.7165832519531, "loss": 4.4693, "margin_dpo/margin_mean": 23.126930236816406, "margin_dpo/margin_std": 38.16487121582031, "step": 308 }, { "KL/chosen_KL_mean": -7.721963882446289, "KL/mean": -22.947174072265625, "KL/rejected_KL_mean": -38.1723747253418, "KL/std": 35.69746780395508, "epoch": 0.6471204188481675, "fcm_dpo/beta": 0.02104841358959675, "fcm_dpo/delta": -0.04495348036289215, "fcm_dpo/margin": 30.45040512084961, "fcm_dpo/q_t": 0.36615273356437683, "grad_norm": 41.06772994995117, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 1.2092869281768799, "logits/rejected": 1.278046727180481, "logps/chosen": -279.61669921875, "logps/ref_chosen": -271.8947448730469, "logps/ref_rejected": -260.5709228515625, "logps/rejected": -298.7432556152344, "loss": 4.0536, "margin_dpo/margin_mean": 30.45040512084961, "margin_dpo/margin_std": 40.20072937011719, "step": 309 }, { "KL/chosen_KL_mean": -10.000167846679688, "KL/mean": -23.129484176635742, "KL/rejected_KL_mean": -36.25880432128906, "KL/std": 33.86246871948242, "epoch": 0.6492146596858639, "fcm_dpo/beta": 0.0207273680716753, "fcm_dpo/delta": 0.0043011680245399475, "fcm_dpo/margin": 26.258644104003906, "fcm_dpo/q_t": 0.38430681824684143, "grad_norm": 46.07544708251953, "learning_rate": 1.6650514271527465e-07, "logits/chosen": 1.467308759689331, "logits/rejected": 1.573281168937683, "logps/chosen": -280.1756286621094, "logps/ref_chosen": -270.17547607421875, "logps/ref_rejected": -257.6781311035156, "logps/rejected": -293.9369201660156, "loss": 4.2085, "margin_dpo/margin_mean": 26.258644104003906, "margin_dpo/margin_std": 36.735416412353516, "step": 310 }, { "KL/chosen_KL_mean": -9.258365631103516, "KL/mean": -23.174116134643555, "KL/rejected_KL_mean": -37.08986282348633, "KL/std": 35.249542236328125, "epoch": 0.6513089005235602, "fcm_dpo/beta": 0.02109854482114315, "fcm_dpo/delta": 0.011416468769311905, "fcm_dpo/margin": 27.831506729125977, "fcm_dpo/q_t": 0.376702219247818, "grad_norm": 49.128501892089844, "learning_rate": 1.647817538357072e-07, "logits/chosen": 1.146061658859253, "logits/rejected": 1.2810291051864624, "logps/chosen": -273.0206298828125, "logps/ref_chosen": -263.76226806640625, "logps/ref_rejected": -263.8758850097656, "logps/rejected": -300.9657287597656, "loss": 4.2775, "margin_dpo/margin_mean": 27.83150863647461, "margin_dpo/margin_std": 42.079673767089844, "step": 311 }, { "KL/chosen_KL_mean": -8.129560470581055, "KL/mean": -21.88031005859375, "KL/rejected_KL_mean": -35.631072998046875, "KL/std": 33.595943450927734, "epoch": 0.6534031413612565, "fcm_dpo/beta": 0.02058618888258934, "fcm_dpo/delta": 0.034953583031892776, "fcm_dpo/margin": 27.501510620117188, "fcm_dpo/q_t": 0.37825676798820496, "grad_norm": 49.96211624145508, "learning_rate": 1.6306293495205755e-07, "logits/chosen": 1.4143263101577759, "logits/rejected": 1.37110435962677, "logps/chosen": -282.8019714355469, "logps/ref_chosen": -274.67242431640625, "logps/ref_rejected": -242.1553955078125, "logps/rejected": -277.7864685058594, "loss": 4.2139, "margin_dpo/margin_mean": 27.50151252746582, "margin_dpo/margin_std": 38.7227668762207, "step": 312 }, { "KL/chosen_KL_mean": -9.555313110351562, "KL/mean": -23.033031463623047, "KL/rejected_KL_mean": -36.5107536315918, "KL/std": 37.531429290771484, "epoch": 0.6554973821989529, "fcm_dpo/beta": 0.021746179088950157, "fcm_dpo/delta": -0.046679336577653885, "fcm_dpo/margin": 26.955429077148438, "fcm_dpo/q_t": 0.3799072802066803, "grad_norm": 41.02532958984375, "learning_rate": 1.6134877823936607e-07, "logits/chosen": 1.40608811378479, "logits/rejected": 1.4458882808685303, "logps/chosen": -304.7431640625, "logps/ref_chosen": -295.1878662109375, "logps/ref_rejected": -271.3431091308594, "logps/rejected": -307.8538818359375, "loss": 4.3841, "margin_dpo/margin_mean": 26.955429077148438, "margin_dpo/margin_std": 42.24140930175781, "step": 313 }, { "KL/chosen_KL_mean": -7.515888214111328, "KL/mean": -20.866962432861328, "KL/rejected_KL_mean": -34.218040466308594, "KL/std": 35.264766693115234, "epoch": 0.6575916230366492, "fcm_dpo/beta": 0.02145400270819664, "fcm_dpo/delta": 0.0263986699283123, "fcm_dpo/margin": 26.702152252197266, "fcm_dpo/q_t": 0.3787587881088257, "grad_norm": 43.04774856567383, "learning_rate": 1.5963937562265522e-07, "logits/chosen": 1.3249597549438477, "logits/rejected": 1.352759599685669, "logps/chosen": -297.2837219238281, "logps/ref_chosen": -289.767822265625, "logps/ref_rejected": -267.0824279785156, "logps/rejected": -301.3004455566406, "loss": 4.2972, "margin_dpo/margin_mean": 26.702152252197266, "margin_dpo/margin_std": 40.670169830322266, "step": 314 }, { "KL/chosen_KL_mean": -6.553337097167969, "KL/mean": -21.8732967376709, "KL/rejected_KL_mean": -37.193267822265625, "KL/std": 35.742828369140625, "epoch": 0.6596858638743456, "fcm_dpo/beta": 0.020826663821935654, "fcm_dpo/delta": -0.04218269884586334, "fcm_dpo/margin": 30.639928817749023, "fcm_dpo/q_t": 0.3655908703804016, "grad_norm": 39.36012649536133, "learning_rate": 1.5793481877199943e-07, "logits/chosen": 1.5957554578781128, "logits/rejected": 1.5931099653244019, "logps/chosen": -304.9723815917969, "logps/ref_chosen": -298.4190673828125, "logps/ref_rejected": -266.64447021484375, "logps/rejected": -303.83770751953125, "loss": 4.0497, "margin_dpo/margin_mean": 30.63992691040039, "margin_dpo/margin_std": 40.569557189941406, "step": 315 }, { "KL/chosen_KL_mean": -8.842336654663086, "KL/mean": -25.09406089782715, "KL/rejected_KL_mean": -41.345787048339844, "KL/std": 36.73515701293945, "epoch": 0.6617801047120419, "fcm_dpo/beta": 0.019872888922691345, "fcm_dpo/delta": -0.050521984696388245, "fcm_dpo/margin": 32.50345230102539, "fcm_dpo/q_t": 0.36559754610061646, "grad_norm": 41.435325622558594, "learning_rate": 1.562351990976095e-07, "logits/chosen": 1.4114371538162231, "logits/rejected": 1.3870769739151, "logps/chosen": -311.0722351074219, "logps/ref_chosen": -302.2298583984375, "logps/ref_rejected": -272.7162170410156, "logps/rejected": -314.0619812011719, "loss": 4.0318, "margin_dpo/margin_mean": 32.50345230102539, "margin_dpo/margin_std": 43.02135467529297, "step": 316 }, { "KL/chosen_KL_mean": -5.404819488525391, "KL/mean": -19.728431701660156, "KL/rejected_KL_mean": -34.052059173583984, "KL/std": 34.26416015625, "epoch": 0.6638743455497382, "fcm_dpo/beta": 0.019800491631031036, "fcm_dpo/delta": 0.03399087116122246, "fcm_dpo/margin": 28.647233963012695, "fcm_dpo/q_t": 0.3793562948703766, "grad_norm": 45.80251693725586, "learning_rate": 1.5454060774493065e-07, "logits/chosen": 1.3171772956848145, "logits/rejected": 1.286457896232605, "logps/chosen": -252.25823974609375, "logps/ref_chosen": -246.85342407226562, "logps/ref_rejected": -216.5264892578125, "logps/rejected": -250.5785369873047, "loss": 4.1583, "margin_dpo/margin_mean": 28.647232055664062, "margin_dpo/margin_std": 39.641090393066406, "step": 317 }, { "KL/chosen_KL_mean": -7.649684906005859, "KL/mean": -21.736515045166016, "KL/rejected_KL_mean": -35.82334899902344, "KL/std": 35.691532135009766, "epoch": 0.6659685863874345, "fcm_dpo/beta": 0.02003423683345318, "fcm_dpo/delta": 0.03660472482442856, "fcm_dpo/margin": 28.173673629760742, "fcm_dpo/q_t": 0.3774641156196594, "grad_norm": 44.10676956176758, "learning_rate": 1.5285113558975427e-07, "logits/chosen": 1.3463513851165771, "logits/rejected": 1.4206820726394653, "logps/chosen": -270.29931640625, "logps/ref_chosen": -262.649658203125, "logps/ref_rejected": -251.24945068359375, "logps/rejected": -287.0727844238281, "loss": 4.1704, "margin_dpo/margin_mean": 28.17367172241211, "margin_dpo/margin_std": 38.388057708740234, "step": 318 }, { "KL/chosen_KL_mean": -7.615129470825195, "KL/mean": -22.933155059814453, "KL/rejected_KL_mean": -38.25117492675781, "KL/std": 35.312164306640625, "epoch": 0.6680628272251309, "fcm_dpo/beta": 0.02006690390408039, "fcm_dpo/delta": -0.017564807087183, "fcm_dpo/margin": 30.636045455932617, "fcm_dpo/q_t": 0.36597752571105957, "grad_norm": 37.150978088378906, "learning_rate": 1.5116687323334464e-07, "logits/chosen": 1.3580387830734253, "logits/rejected": 1.3618606328964233, "logps/chosen": -301.00146484375, "logps/ref_chosen": -293.3863220214844, "logps/ref_rejected": -238.86026000976562, "logps/rejected": -277.1114501953125, "loss": 3.9492, "margin_dpo/margin_mean": 30.636043548583984, "margin_dpo/margin_std": 36.14667510986328, "step": 319 }, { "KL/chosen_KL_mean": -5.44903564453125, "KL/mean": -20.269977569580078, "KL/rejected_KL_mean": -35.090904235839844, "KL/std": 35.45115280151367, "epoch": 0.6701570680628273, "fcm_dpo/beta": 0.0197887159883976, "fcm_dpo/delta": -0.052189748734235764, "fcm_dpo/margin": 29.64186668395996, "fcm_dpo/q_t": 0.37496477365493774, "grad_norm": 42.01469421386719, "learning_rate": 1.4948791099758052e-07, "logits/chosen": 1.458266019821167, "logits/rejected": 1.4709018468856812, "logps/chosen": -300.95281982421875, "logps/ref_chosen": -295.5038146972656, "logps/ref_rejected": -278.524169921875, "logps/rejected": -313.6151123046875, "loss": 4.2415, "margin_dpo/margin_mean": 29.641864776611328, "margin_dpo/margin_std": 43.217124938964844, "step": 320 }, { "KL/chosen_KL_mean": -8.362350463867188, "KL/mean": -21.00874137878418, "KL/rejected_KL_mean": -33.65511703491211, "KL/std": 35.1718635559082, "epoch": 0.6722513089005235, "fcm_dpo/beta": 0.020625794306397438, "fcm_dpo/delta": 0.08006928116083145, "fcm_dpo/margin": 25.292766571044922, "fcm_dpo/q_t": 0.3896982669830322, "grad_norm": 42.11892318725586, "learning_rate": 1.478143389201113e-07, "logits/chosen": 1.6907308101654053, "logits/rejected": 1.6073836088180542, "logps/chosen": -289.11773681640625, "logps/ref_chosen": -280.7553405761719, "logps/ref_rejected": -237.47286987304688, "logps/rejected": -271.12799072265625, "loss": 4.3342, "margin_dpo/margin_mean": 25.292766571044922, "margin_dpo/margin_std": 39.51212692260742, "step": 321 }, { "KL/chosen_KL_mean": -7.008272171020508, "KL/mean": -22.74313735961914, "KL/rejected_KL_mean": -38.478004455566406, "KL/std": 35.99267578125, "epoch": 0.6743455497382199, "fcm_dpo/beta": 0.020769845694303513, "fcm_dpo/delta": -0.05772664397954941, "fcm_dpo/margin": 31.4697265625, "fcm_dpo/q_t": 0.3622688949108124, "grad_norm": 46.31185531616211, "learning_rate": 1.461462467495284e-07, "logits/chosen": 1.26900053024292, "logits/rejected": 1.255014181137085, "logps/chosen": -306.0995788574219, "logps/ref_chosen": -299.09130859375, "logps/ref_rejected": -268.9637756347656, "logps/rejected": -307.4417419433594, "loss": 3.9975, "margin_dpo/margin_mean": 31.4697265625, "margin_dpo/margin_std": 40.3599739074707, "step": 322 }, { "KL/chosen_KL_mean": -12.073066711425781, "KL/mean": -22.89596939086914, "KL/rejected_KL_mean": -33.7188720703125, "KL/std": 33.74138641357422, "epoch": 0.6764397905759162, "fcm_dpo/beta": 0.020806610584259033, "fcm_dpo/delta": 0.08319887518882751, "fcm_dpo/margin": 21.645793914794922, "fcm_dpo/q_t": 0.40604478120803833, "grad_norm": 50.52162551879883, "learning_rate": 1.4448372394055246e-07, "logits/chosen": 1.5834410190582275, "logits/rejected": 1.522539496421814, "logps/chosen": -288.76910400390625, "logps/ref_chosen": -276.696044921875, "logps/ref_rejected": -225.78790283203125, "logps/rejected": -259.50677490234375, "loss": 4.69, "margin_dpo/margin_mean": 21.645793914794922, "margin_dpo/margin_std": 42.407405853271484, "step": 323 }, { "KL/chosen_KL_mean": -5.5897064208984375, "KL/mean": -22.75788688659668, "KL/rejected_KL_mean": -39.926063537597656, "KL/std": 35.74304962158203, "epoch": 0.6785340314136126, "fcm_dpo/beta": 0.01992855593562126, "fcm_dpo/delta": -0.09492021054029465, "fcm_dpo/margin": 34.33635330200195, "fcm_dpo/q_t": 0.3534626364707947, "grad_norm": 59.69330978393555, "learning_rate": 1.428268596492364e-07, "logits/chosen": 1.3824721574783325, "logits/rejected": 1.4464294910430908, "logps/chosen": -236.17471313476562, "logps/ref_chosen": -230.58499145507812, "logps/ref_rejected": -229.8074493408203, "logps/rejected": -269.7335205078125, "loss": 3.7786, "margin_dpo/margin_mean": 34.33634948730469, "margin_dpo/margin_std": 37.06605911254883, "step": 324 }, { "KL/chosen_KL_mean": -6.523805618286133, "KL/mean": -21.514896392822266, "KL/rejected_KL_mean": -36.50599670410156, "KL/std": 38.17298889160156, "epoch": 0.680628272251309, "fcm_dpo/beta": 0.019193103536963463, "fcm_dpo/delta": -0.02855522558093071, "fcm_dpo/margin": 29.982200622558594, "fcm_dpo/q_t": 0.383538156747818, "grad_norm": 46.019588470458984, "learning_rate": 1.4117574272818386e-07, "logits/chosen": 1.2470699548721313, "logits/rejected": 1.259156584739685, "logps/chosen": -277.0798034667969, "logps/ref_chosen": -270.5559997558594, "logps/ref_rejected": -266.328369140625, "logps/rejected": -302.8343505859375, "loss": 4.3168, "margin_dpo/margin_mean": 29.982200622558594, "margin_dpo/margin_std": 47.193885803222656, "step": 325 }, { "KL/chosen_KL_mean": -6.474815368652344, "KL/mean": -20.72551727294922, "KL/rejected_KL_mean": -34.97622299194336, "KL/std": 33.760223388671875, "epoch": 0.6827225130890052, "fcm_dpo/beta": 0.019825132563710213, "fcm_dpo/delta": 0.035527441650629044, "fcm_dpo/margin": 28.501415252685547, "fcm_dpo/q_t": 0.3799126148223877, "grad_norm": 61.182281494140625, "learning_rate": 1.3953046172178413e-07, "logits/chosen": 1.3199985027313232, "logits/rejected": 1.396640419960022, "logps/chosen": -239.71340942382812, "logps/ref_chosen": -233.2386016845703, "logps/ref_rejected": -258.154541015625, "logps/rejected": -293.1307678222656, "loss": 4.2604, "margin_dpo/margin_mean": 28.501415252685547, "margin_dpo/margin_std": 42.30944061279297, "step": 326 }, { "KL/chosen_KL_mean": -0.09135627746582031, "KL/mean": -16.629085540771484, "KL/rejected_KL_mean": -33.166831970214844, "KL/std": 35.447776794433594, "epoch": 0.6848167539267016, "fcm_dpo/beta": 0.019445110112428665, "fcm_dpo/delta": -0.048530131578445435, "fcm_dpo/margin": 33.075462341308594, "fcm_dpo/q_t": 0.3634200990200043, "grad_norm": 36.06775665283203, "learning_rate": 1.3789110486146468e-07, "logits/chosen": 1.5129821300506592, "logits/rejected": 1.5304069519042969, "logps/chosen": -268.6375427246094, "logps/ref_chosen": -268.5461730957031, "logps/ref_rejected": -266.5316467285156, "logps/rejected": -299.698486328125, "loss": 4.0516, "margin_dpo/margin_mean": 33.075462341308594, "margin_dpo/margin_std": 43.234737396240234, "step": 327 }, { "KL/chosen_KL_mean": -8.440298080444336, "KL/mean": -22.394554138183594, "KL/rejected_KL_mean": -36.34880828857422, "KL/std": 35.926353454589844, "epoch": 0.6869109947643979, "fcm_dpo/beta": 0.019331524148583412, "fcm_dpo/delta": 0.06380513310432434, "fcm_dpo/margin": 27.90850830078125, "fcm_dpo/q_t": 0.38404858112335205, "grad_norm": 35.696510314941406, "learning_rate": 1.362577600609588e-07, "logits/chosen": 1.454308271408081, "logits/rejected": 1.4678572416305542, "logps/chosen": -303.31707763671875, "logps/ref_chosen": -294.8768005371094, "logps/ref_rejected": -281.09771728515625, "logps/rejected": -317.446533203125, "loss": 4.2428, "margin_dpo/margin_mean": 27.908506393432617, "margin_dpo/margin_std": 39.95448684692383, "step": 328 }, { "KL/chosen_KL_mean": -11.517646789550781, "KL/mean": -24.1033878326416, "KL/rejected_KL_mean": -36.68913269042969, "KL/std": 38.94247817993164, "epoch": 0.6890052356020943, "fcm_dpo/beta": 0.020308827981352806, "fcm_dpo/delta": 0.041645098477602005, "fcm_dpo/margin": 25.171483993530273, "fcm_dpo/q_t": 0.3962605595588684, "grad_norm": 48.052101135253906, "learning_rate": 1.3463051491159093e-07, "logits/chosen": 1.7028543949127197, "logits/rejected": 1.7223467826843262, "logps/chosen": -323.8154296875, "logps/ref_chosen": -312.2977600097656, "logps/ref_rejected": -304.6536560058594, "logps/rejected": -341.3427429199219, "loss": 4.5731, "margin_dpo/margin_mean": 25.171485900878906, "margin_dpo/margin_std": 45.91475296020508, "step": 329 }, { "KL/chosen_KL_mean": -9.744510650634766, "KL/mean": -21.173860549926758, "KL/rejected_KL_mean": -32.60321807861328, "KL/std": 36.04499053955078, "epoch": 0.6910994764397905, "fcm_dpo/beta": 0.021884959191083908, "fcm_dpo/delta": 0.10345837473869324, "fcm_dpo/margin": 22.858707427978516, "fcm_dpo/q_t": 0.3985418379306793, "grad_norm": 49.91687774658203, "learning_rate": 1.3300945667758012e-07, "logits/chosen": 1.309699535369873, "logits/rejected": 1.3560664653778076, "logps/chosen": -303.9486999511719, "logps/ref_chosen": -294.2042236328125, "logps/ref_rejected": -296.2102355957031, "logps/rejected": -328.8134765625, "loss": 4.5652, "margin_dpo/margin_mean": 22.858707427978516, "margin_dpo/margin_std": 42.745452880859375, "step": 330 }, { "KL/chosen_KL_mean": -5.785097122192383, "KL/mean": -20.0535945892334, "KL/rejected_KL_mean": -34.32209777832031, "KL/std": 36.29338455200195, "epoch": 0.6931937172774869, "fcm_dpo/beta": 0.021633943542838097, "fcm_dpo/delta": -0.0712338387966156, "fcm_dpo/margin": 28.536991119384766, "fcm_dpo/q_t": 0.3712262809276581, "grad_norm": 46.72981643676758, "learning_rate": 1.3139467229135998e-07, "logits/chosen": 1.5535022020339966, "logits/rejected": 1.564803957939148, "logps/chosen": -274.0977478027344, "logps/ref_chosen": -268.3126525878906, "logps/ref_rejected": -255.5543212890625, "logps/rejected": -289.87640380859375, "loss": 4.164, "margin_dpo/margin_mean": 28.5369930267334, "margin_dpo/margin_std": 40.84954071044922, "step": 331 }, { "KL/chosen_KL_mean": -7.356256484985352, "KL/mean": -23.277719497680664, "KL/rejected_KL_mean": -39.19919204711914, "KL/std": 35.934471130371094, "epoch": 0.6952879581151833, "fcm_dpo/beta": 0.020195266231894493, "fcm_dpo/delta": -0.04919080436229706, "fcm_dpo/margin": 31.84294319152832, "fcm_dpo/q_t": 0.3655468225479126, "grad_norm": 40.15316390991211, "learning_rate": 1.2978624834891626e-07, "logits/chosen": 1.53472900390625, "logits/rejected": 1.5672580003738403, "logps/chosen": -271.106689453125, "logps/ref_chosen": -263.75042724609375, "logps/ref_rejected": -234.5144500732422, "logps/rejected": -273.713623046875, "loss": 4.0945, "margin_dpo/margin_mean": 31.842941284179688, "margin_dpo/margin_std": 43.65056228637695, "step": 332 }, { "KL/chosen_KL_mean": -4.462230682373047, "KL/mean": -17.031757354736328, "KL/rejected_KL_mean": -29.601285934448242, "KL/std": 34.89313888549805, "epoch": 0.6973821989528796, "fcm_dpo/beta": 0.02129294164478779, "fcm_dpo/delta": 0.06581726670265198, "fcm_dpo/margin": 25.139057159423828, "fcm_dpo/q_t": 0.3857528865337372, "grad_norm": 52.96217346191406, "learning_rate": 1.281842711051438e-07, "logits/chosen": 1.63423490524292, "logits/rejected": 1.6159865856170654, "logps/chosen": -291.2786560058594, "logps/ref_chosen": -286.81646728515625, "logps/ref_rejected": -260.4169616699219, "logps/rejected": -290.0182189941406, "loss": 4.3227, "margin_dpo/margin_mean": 25.139057159423828, "margin_dpo/margin_std": 38.40645980834961, "step": 333 }, { "KL/chosen_KL_mean": -3.3436641693115234, "KL/mean": -18.017654418945312, "KL/rejected_KL_mean": -32.691654205322266, "KL/std": 36.4597053527832, "epoch": 0.6994764397905759, "fcm_dpo/beta": 0.021236978471279144, "fcm_dpo/delta": -0.07299379259347916, "fcm_dpo/margin": 29.347986221313477, "fcm_dpo/q_t": 0.3713444471359253, "grad_norm": 48.177833557128906, "learning_rate": 1.2658882646922033e-07, "logits/chosen": 1.4887720346450806, "logits/rejected": 1.4807991981506348, "logps/chosen": -296.654541015625, "logps/ref_chosen": -293.3109130859375, "logps/ref_rejected": -264.1449279785156, "logps/rejected": -296.8365783691406, "loss": 4.2622, "margin_dpo/margin_mean": 29.34798812866211, "margin_dpo/margin_std": 43.27864456176758, "step": 334 }, { "KL/chosen_KL_mean": -11.716907501220703, "KL/mean": -24.610498428344727, "KL/rejected_KL_mean": -37.50409698486328, "KL/std": 35.383934020996094, "epoch": 0.7015706806282722, "fcm_dpo/beta": 0.020372921600937843, "fcm_dpo/delta": 0.014315152540802956, "fcm_dpo/margin": 25.787187576293945, "fcm_dpo/q_t": 0.39156395196914673, "grad_norm": 52.540733337402344, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 1.2842155694961548, "logits/rejected": 1.3597443103790283, "logps/chosen": -281.4732666015625, "logps/ref_chosen": -269.75634765625, "logps/ref_rejected": -241.08438110351562, "logps/rejected": -278.5884704589844, "loss": 4.5266, "margin_dpo/margin_mean": 25.787187576293945, "margin_dpo/margin_std": 44.98386764526367, "step": 335 }, { "KL/chosen_KL_mean": -6.840425491333008, "KL/mean": -19.60683822631836, "KL/rejected_KL_mean": -32.373252868652344, "KL/std": 35.83504104614258, "epoch": 0.7036649214659686, "fcm_dpo/beta": 0.020670827478170395, "fcm_dpo/delta": 0.019132599234580994, "fcm_dpo/margin": 25.532833099365234, "fcm_dpo/q_t": 0.3904283046722412, "grad_norm": 41.105438232421875, "learning_rate": 1.2341787690142435e-07, "logits/chosen": 1.408745527267456, "logits/rejected": 1.4292922019958496, "logps/chosen": -244.21221923828125, "logps/ref_chosen": -237.37179565429688, "logps/ref_rejected": -253.5463409423828, "logps/rejected": -285.9195861816406, "loss": 4.3834, "margin_dpo/margin_mean": 25.532833099365234, "margin_dpo/margin_std": 41.5406494140625, "step": 336 }, { "KL/chosen_KL_mean": -9.320682525634766, "KL/mean": -23.700389862060547, "KL/rejected_KL_mean": -38.08009338378906, "KL/std": 35.18950653076172, "epoch": 0.7057591623036649, "fcm_dpo/beta": 0.020788341760635376, "fcm_dpo/delta": -0.04489980638027191, "fcm_dpo/margin": 28.759424209594727, "fcm_dpo/q_t": 0.3769844174385071, "grad_norm": 43.92770767211914, "learning_rate": 1.2184254201795363e-07, "logits/chosen": 1.1415191888809204, "logits/rejected": 1.1761301755905151, "logps/chosen": -270.62835693359375, "logps/ref_chosen": -261.3076477050781, "logps/ref_rejected": -252.3170166015625, "logps/rejected": -290.3971252441406, "loss": 4.2185, "margin_dpo/margin_mean": 28.759424209594727, "margin_dpo/margin_std": 42.25171661376953, "step": 337 }, { "KL/chosen_KL_mean": 0.043670654296875, "KL/mean": -16.341350555419922, "KL/rejected_KL_mean": -32.72637176513672, "KL/std": 36.32640075683594, "epoch": 0.7078534031413612, "fcm_dpo/beta": 0.019904684275388718, "fcm_dpo/delta": -0.05567874759435654, "fcm_dpo/margin": 32.770042419433594, "fcm_dpo/q_t": 0.36414873600006104, "grad_norm": 44.52647018432617, "learning_rate": 1.202740798300168e-07, "logits/chosen": 1.5777091979980469, "logits/rejected": 1.5485655069351196, "logps/chosen": -268.72808837890625, "logps/ref_chosen": -268.7717590332031, "logps/ref_rejected": -232.3352813720703, "logps/rejected": -265.0616455078125, "loss": 4.0027, "margin_dpo/margin_mean": 32.77003860473633, "margin_dpo/margin_std": 43.178646087646484, "step": 338 }, { "KL/chosen_KL_mean": -3.705251693725586, "KL/mean": -20.399065017700195, "KL/rejected_KL_mean": -37.0928840637207, "KL/std": 37.70600128173828, "epoch": 0.7099476439790576, "fcm_dpo/beta": 0.01890597492456436, "fcm_dpo/delta": -0.0346045047044754, "fcm_dpo/margin": 33.387630462646484, "fcm_dpo/q_t": 0.364393949508667, "grad_norm": 40.38267135620117, "learning_rate": 1.1871257444948096e-07, "logits/chosen": 1.378730297088623, "logits/rejected": 1.336645245552063, "logps/chosen": -297.3183898925781, "logps/ref_chosen": -293.6131286621094, "logps/ref_rejected": -281.87738037109375, "logps/rejected": -318.97027587890625, "loss": 4.0497, "margin_dpo/margin_mean": 33.387630462646484, "margin_dpo/margin_std": 44.12839889526367, "step": 339 }, { "KL/chosen_KL_mean": -7.231061935424805, "KL/mean": -21.675649642944336, "KL/rejected_KL_mean": -36.12025451660156, "KL/std": 37.74037170410156, "epoch": 0.7120418848167539, "fcm_dpo/beta": 0.019060026854276657, "fcm_dpo/delta": 0.05121763423085213, "fcm_dpo/margin": 28.889205932617188, "fcm_dpo/q_t": 0.38518282771110535, "grad_norm": 41.96944808959961, "learning_rate": 1.1715810961514072e-07, "logits/chosen": 1.042682409286499, "logits/rejected": 1.1461615562438965, "logps/chosen": -258.9881286621094, "logps/ref_chosen": -251.75704956054688, "logps/ref_rejected": -255.76441955566406, "logps/rejected": -291.8846740722656, "loss": 4.2907, "margin_dpo/margin_mean": 28.88920783996582, "margin_dpo/margin_std": 43.76124572753906, "step": 340 }, { "KL/chosen_KL_mean": -14.124200820922852, "KL/mean": -25.866802215576172, "KL/rejected_KL_mean": -37.60939025878906, "KL/std": 39.57828903198242, "epoch": 0.7141361256544503, "fcm_dpo/beta": 0.020194221287965775, "fcm_dpo/delta": 0.058979712426662445, "fcm_dpo/margin": 23.485197067260742, "fcm_dpo/q_t": 0.40257659554481506, "grad_norm": 52.897464752197266, "learning_rate": 1.1561076868822755e-07, "logits/chosen": 1.408077359199524, "logits/rejected": 1.5080537796020508, "logps/chosen": -320.7718200683594, "logps/ref_chosen": -306.64764404296875, "logps/ref_rejected": -301.4248352050781, "logps/rejected": -339.03424072265625, "loss": 4.6646, "margin_dpo/margin_mean": 23.485197067260742, "margin_dpo/margin_std": 45.1025505065918, "step": 341 }, { "KL/chosen_KL_mean": -4.896259307861328, "KL/mean": -19.20393943786621, "KL/rejected_KL_mean": -33.511627197265625, "KL/std": 37.52045440673828, "epoch": 0.7162303664921466, "fcm_dpo/beta": 0.020969020202755928, "fcm_dpo/delta": -0.0023633725941181183, "fcm_dpo/margin": 28.615371704101562, "fcm_dpo/q_t": 0.3745231032371521, "grad_norm": 43.30937576293945, "learning_rate": 1.1407063464793965e-07, "logits/chosen": 1.527671456336975, "logits/rejected": 1.570663332939148, "logps/chosen": -267.565185546875, "logps/ref_chosen": -262.6689147949219, "logps/ref_rejected": -260.8409118652344, "logps/rejected": -294.3525390625, "loss": 4.1659, "margin_dpo/margin_mean": 28.615371704101562, "margin_dpo/margin_std": 40.10309600830078, "step": 342 }, { "KL/chosen_KL_mean": -4.125505447387695, "KL/mean": -17.500391006469727, "KL/rejected_KL_mean": -30.875286102294922, "KL/std": 37.22210693359375, "epoch": 0.7183246073298429, "fcm_dpo/beta": 0.020785322412848473, "fcm_dpo/delta": 0.04561643302440643, "fcm_dpo/margin": 26.749792098999023, "fcm_dpo/q_t": 0.3863937258720398, "grad_norm": 49.587791442871094, "learning_rate": 1.125377900869913e-07, "logits/chosen": 1.5034934282302856, "logits/rejected": 1.4729253053665161, "logps/chosen": -303.75146484375, "logps/ref_chosen": -299.6259765625, "logps/ref_rejected": -261.2701416015625, "logps/rejected": -292.14544677734375, "loss": 4.3603, "margin_dpo/margin_mean": 26.749792098999023, "margin_dpo/margin_std": 42.81586456298828, "step": 343 }, { "KL/chosen_KL_mean": -5.473409652709961, "KL/mean": -20.547622680664062, "KL/rejected_KL_mean": -35.62184143066406, "KL/std": 37.833587646484375, "epoch": 0.7204188481675393, "fcm_dpo/beta": 0.021515903994441032, "fcm_dpo/delta": -0.053787894546985626, "fcm_dpo/margin": 30.14841079711914, "fcm_dpo/q_t": 0.3678366541862488, "grad_norm": 40.71405029296875, "learning_rate": 1.110123172071844e-07, "logits/chosen": 1.3965474367141724, "logits/rejected": 1.421886920928955, "logps/chosen": -296.423828125, "logps/ref_chosen": -290.950439453125, "logps/ref_rejected": -255.15374755859375, "logps/rejected": -290.77557373046875, "loss": 4.1411, "margin_dpo/margin_mean": 30.14841079711914, "margin_dpo/margin_std": 42.23366165161133, "step": 344 }, { "KL/chosen_KL_mean": -1.812255859375, "KL/mean": -15.496662139892578, "KL/rejected_KL_mean": -29.181068420410156, "KL/std": 35.286991119384766, "epoch": 0.7225130890052356, "fcm_dpo/beta": 0.02078464813530445, "fcm_dpo/delta": 0.030701331794261932, "fcm_dpo/margin": 27.368812561035156, "fcm_dpo/q_t": 0.37889885902404785, "grad_norm": 40.63283157348633, "learning_rate": 1.09494297815e-07, "logits/chosen": 1.4443119764328003, "logits/rejected": 1.5055794715881348, "logps/chosen": -275.078125, "logps/ref_chosen": -273.2658996582031, "logps/ref_rejected": -263.9664306640625, "logps/rejected": -293.1474914550781, "loss": 4.126, "margin_dpo/margin_mean": 27.368812561035156, "margin_dpo/margin_std": 36.180908203125, "step": 345 }, { "KL/chosen_KL_mean": -9.687026977539062, "KL/mean": -24.494407653808594, "KL/rejected_KL_mean": -39.30180358886719, "KL/std": 36.424896240234375, "epoch": 0.724607329842932, "fcm_dpo/beta": 0.020185664296150208, "fcm_dpo/delta": -0.05673644691705704, "fcm_dpo/margin": 29.61478042602539, "fcm_dpo/q_t": 0.3723044991493225, "grad_norm": 41.669403076171875, "learning_rate": 1.0798381331721107e-07, "logits/chosen": 1.423187255859375, "logits/rejected": 1.4670082330703735, "logps/chosen": -272.36920166015625, "logps/ref_chosen": -262.6821594238281, "logps/ref_rejected": -226.46859741210938, "logps/rejected": -265.7704162597656, "loss": 4.1397, "margin_dpo/margin_mean": 29.614784240722656, "margin_dpo/margin_std": 38.93079376220703, "step": 346 }, { "KL/chosen_KL_mean": -8.110563278198242, "KL/mean": -20.27287483215332, "KL/rejected_KL_mean": -32.43518829345703, "KL/std": 35.65875244140625, "epoch": 0.7267015706806282, "fcm_dpo/beta": 0.02065792679786682, "fcm_dpo/delta": 0.10232961177825928, "fcm_dpo/margin": 24.324626922607422, "fcm_dpo/q_t": 0.39234477281570435, "grad_norm": 47.63564682006836, "learning_rate": 1.0648094471651722e-07, "logits/chosen": 1.4029858112335205, "logits/rejected": 1.4183018207550049, "logps/chosen": -271.9351806640625, "logps/ref_chosen": -263.8246154785156, "logps/ref_rejected": -237.9121551513672, "logps/rejected": -270.34735107421875, "loss": 4.4389, "margin_dpo/margin_mean": 24.324626922607422, "margin_dpo/margin_std": 40.04618453979492, "step": 347 }, { "KL/chosen_KL_mean": -3.905031204223633, "KL/mean": -15.264150619506836, "KL/rejected_KL_mean": -26.623271942138672, "KL/std": 36.13999938964844, "epoch": 0.7287958115183246, "fcm_dpo/beta": 0.02141868695616722, "fcm_dpo/delta": 0.012950234115123749, "fcm_dpo/margin": 22.718244552612305, "fcm_dpo/q_t": 0.39967650175094604, "grad_norm": 42.38212585449219, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 1.3707566261291504, "logits/rejected": 1.397828459739685, "logps/chosen": -282.058349609375, "logps/ref_chosen": -278.1533203125, "logps/ref_rejected": -263.39691162109375, "logps/rejected": -290.02020263671875, "loss": 4.4869, "margin_dpo/margin_mean": 22.71824073791504, "margin_dpo/margin_std": 38.25540542602539, "step": 348 }, { "KL/chosen_KL_mean": -6.222209930419922, "KL/mean": -21.949100494384766, "KL/rejected_KL_mean": -37.67598342895508, "KL/std": 38.69367218017578, "epoch": 0.7308900523560209, "fcm_dpo/beta": 0.020623190328478813, "fcm_dpo/delta": -0.11218712478876114, "fcm_dpo/margin": 31.453781127929688, "fcm_dpo/q_t": 0.3656691014766693, "grad_norm": 50.22072982788086, "learning_rate": 1.0349837717080347e-07, "logits/chosen": 1.647545576095581, "logits/rejected": 1.678118109703064, "logps/chosen": -324.6772766113281, "logps/ref_chosen": -318.455078125, "logps/ref_rejected": -288.5691833496094, "logps/rejected": -326.2451171875, "loss": 4.0761, "margin_dpo/margin_mean": 31.45378303527832, "margin_dpo/margin_std": 42.22328186035156, "step": 349 }, { "KL/chosen_KL_mean": -8.401691436767578, "KL/mean": -22.730972290039062, "KL/rejected_KL_mean": -37.06025695800781, "KL/std": 38.17472839355469, "epoch": 0.7329842931937173, "fcm_dpo/beta": 0.02000611647963524, "fcm_dpo/delta": -0.04766222834587097, "fcm_dpo/margin": 28.6585693359375, "fcm_dpo/q_t": 0.3795121908187866, "grad_norm": 44.429786682128906, "learning_rate": 1.0201883817182949e-07, "logits/chosen": 1.4076790809631348, "logits/rejected": 1.3850817680358887, "logps/chosen": -292.59088134765625, "logps/ref_chosen": -284.1891784667969, "logps/ref_rejected": -247.94589233398438, "logps/rejected": -285.0061340332031, "loss": 4.3099, "margin_dpo/margin_mean": 28.6585693359375, "margin_dpo/margin_std": 43.448463439941406, "step": 350 }, { "KL/chosen_KL_mean": -5.480752944946289, "KL/mean": -16.321317672729492, "KL/rejected_KL_mean": -27.161882400512695, "KL/std": 36.58054733276367, "epoch": 0.7350785340314137, "fcm_dpo/beta": 0.020766399800777435, "fcm_dpo/delta": 0.10397867113351822, "fcm_dpo/margin": 21.68113136291504, "fcm_dpo/q_t": 0.4100274443626404, "grad_norm": 53.084224700927734, "learning_rate": 1.0054723495346482e-07, "logits/chosen": 1.6622663736343384, "logits/rejected": 1.6453075408935547, "logps/chosen": -308.7888488769531, "logps/ref_chosen": -303.30810546875, "logps/ref_rejected": -260.1866455078125, "logps/rejected": -287.3485107421875, "loss": 4.7544, "margin_dpo/margin_mean": 21.681129455566406, "margin_dpo/margin_std": 44.741676330566406, "step": 351 }, { "KL/chosen_KL_mean": -5.815118789672852, "KL/mean": -21.60936737060547, "KL/rejected_KL_mean": -37.40361785888672, "KL/std": 37.99953842163086, "epoch": 0.7371727748691099, "fcm_dpo/beta": 0.020391924306750298, "fcm_dpo/delta": -0.050486717373132706, "fcm_dpo/margin": 31.588489532470703, "fcm_dpo/q_t": 0.36532890796661377, "grad_norm": 39.152435302734375, "learning_rate": 9.908364643332398e-08, "logits/chosen": 1.3815817832946777, "logits/rejected": 1.5531269311904907, "logps/chosen": -252.62222290039062, "logps/ref_chosen": -246.80709838867188, "logps/ref_rejected": -253.13223266601562, "logps/rejected": -290.5358581542969, "loss": 4.1302, "margin_dpo/margin_mean": 31.58849334716797, "margin_dpo/margin_std": 42.74152755737305, "step": 352 }, { "KL/chosen_KL_mean": -6.554887771606445, "KL/mean": -22.182954788208008, "KL/rejected_KL_mean": -37.81101989746094, "KL/std": 37.12267303466797, "epoch": 0.7392670157068063, "fcm_dpo/beta": 0.019203264266252518, "fcm_dpo/delta": -0.060051385313272476, "fcm_dpo/margin": 31.256126403808594, "fcm_dpo/q_t": 0.37281614542007446, "grad_norm": 45.9362678527832, "learning_rate": 9.76281510992176e-08, "logits/chosen": 1.3499311208724976, "logits/rejected": 1.4172637462615967, "logps/chosen": -268.1563720703125, "logps/ref_chosen": -261.6014709472656, "logps/ref_rejected": -262.74761962890625, "logps/rejected": -300.5586242675781, "loss": 4.1085, "margin_dpo/margin_mean": 31.256126403808594, "margin_dpo/margin_std": 41.039066314697266, "step": 353 }, { "KL/chosen_KL_mean": -7.3160295486450195, "KL/mean": -16.680030822753906, "KL/rejected_KL_mean": -26.044034957885742, "KL/std": 37.433170318603516, "epoch": 0.7413612565445026, "fcm_dpo/beta": 0.01969905197620392, "fcm_dpo/delta": 0.08830844610929489, "fcm_dpo/margin": 18.72800064086914, "fcm_dpo/q_t": 0.4253513813018799, "grad_norm": 47.81275177001953, "learning_rate": 9.618082700494318e-08, "logits/chosen": 1.524624228477478, "logits/rejected": 1.634684681892395, "logps/chosen": -259.4644470214844, "logps/ref_chosen": -252.14840698242188, "logps/ref_rejected": -244.1017608642578, "logps/rejected": -270.1457824707031, "loss": 4.984, "margin_dpo/margin_mean": 18.72800064086914, "margin_dpo/margin_std": 46.1493034362793, "step": 354 }, { "KL/chosen_KL_mean": -6.15156364440918, "KL/mean": -22.054759979248047, "KL/rejected_KL_mean": -37.95793533325195, "KL/std": 38.03178787231445, "epoch": 0.743455497382199, "fcm_dpo/beta": 0.019542938098311424, "fcm_dpo/delta": -0.025795362889766693, "fcm_dpo/margin": 31.806377410888672, "fcm_dpo/q_t": 0.36854490637779236, "grad_norm": 38.87739562988281, "learning_rate": 9.474175176609956e-08, "logits/chosen": 1.4897949695587158, "logits/rejected": 1.593730092048645, "logps/chosen": -292.3140869140625, "logps/ref_chosen": -286.16253662109375, "logps/ref_rejected": -272.3275146484375, "logps/rejected": -310.2854919433594, "loss": 4.1449, "margin_dpo/margin_mean": 31.806377410888672, "margin_dpo/margin_std": 44.38508224487305, "step": 355 }, { "KL/chosen_KL_mean": -4.26123046875, "KL/mean": -16.71814727783203, "KL/rejected_KL_mean": -29.175060272216797, "KL/std": 34.76670837402344, "epoch": 0.7455497382198953, "fcm_dpo/beta": 0.020574284717440605, "fcm_dpo/delta": 0.04235248267650604, "fcm_dpo/margin": 24.913829803466797, "fcm_dpo/q_t": 0.39130815863609314, "grad_norm": 41.14774703979492, "learning_rate": 9.331100255592436e-08, "logits/chosen": 1.5562423467636108, "logits/rejected": 1.603265643119812, "logps/chosen": -202.41400146484375, "logps/ref_chosen": -198.1527862548828, "logps/ref_rejected": -209.80569458007812, "logps/rejected": -238.9807586669922, "loss": 4.3745, "margin_dpo/margin_mean": 24.91383171081543, "margin_dpo/margin_std": 38.90260314941406, "step": 356 }, { "KL/chosen_KL_mean": -7.882879257202148, "KL/mean": -22.27526092529297, "KL/rejected_KL_mean": -36.66762924194336, "KL/std": 36.296688079833984, "epoch": 0.7476439790575916, "fcm_dpo/beta": 0.02032068371772766, "fcm_dpo/delta": -0.040481653064489365, "fcm_dpo/margin": 28.784751892089844, "fcm_dpo/q_t": 0.3821459114551544, "grad_norm": 38.99663543701172, "learning_rate": 9.18886561011557e-08, "logits/chosen": 1.5026752948760986, "logits/rejected": 1.5520050525665283, "logps/chosen": -264.3900451660156, "logps/ref_chosen": -256.50714111328125, "logps/ref_rejected": -237.40769958496094, "logps/rejected": -274.0753479003906, "loss": 4.3032, "margin_dpo/margin_mean": 28.784751892089844, "margin_dpo/margin_std": 44.479881286621094, "step": 357 }, { "KL/chosen_KL_mean": -1.4076976776123047, "KL/mean": -17.4775390625, "KL/rejected_KL_mean": -33.547386169433594, "KL/std": 37.1981315612793, "epoch": 0.749738219895288, "fcm_dpo/beta": 0.0198576208204031, "fcm_dpo/delta": -0.04260528087615967, "fcm_dpo/margin": 32.13968276977539, "fcm_dpo/q_t": 0.36835816502571106, "grad_norm": 38.77471160888672, "learning_rate": 9.047478867791731e-08, "logits/chosen": 1.4969563484191895, "logits/rejected": 1.4219486713409424, "logps/chosen": -293.85498046875, "logps/ref_chosen": -292.447265625, "logps/ref_rejected": -255.47872924804688, "logps/rejected": -289.026123046875, "loss": 4.17, "margin_dpo/margin_mean": 32.139686584472656, "margin_dpo/margin_std": 45.9384765625, "step": 358 }, { "KL/chosen_KL_mean": -1.8805828094482422, "KL/mean": -16.736221313476562, "KL/rejected_KL_mean": -31.591859817504883, "KL/std": 36.91073989868164, "epoch": 0.7518324607329843, "fcm_dpo/beta": 0.0197370033711195, "fcm_dpo/delta": 0.012806244194507599, "fcm_dpo/margin": 29.71127700805664, "fcm_dpo/q_t": 0.37310200929641724, "grad_norm": 46.65188217163086, "learning_rate": 8.906947610762825e-08, "logits/chosen": 1.6143345832824707, "logits/rejected": 1.5761593580245972, "logps/chosen": -295.64056396484375, "logps/ref_chosen": -293.7599792480469, "logps/ref_rejected": -259.4142761230469, "logps/rejected": -291.0061340332031, "loss": 4.1267, "margin_dpo/margin_mean": 29.71127700805664, "margin_dpo/margin_std": 39.605995178222656, "step": 359 }, { "KL/chosen_KL_mean": -6.4589996337890625, "KL/mean": -19.369590759277344, "KL/rejected_KL_mean": -32.280174255371094, "KL/std": 36.81121826171875, "epoch": 0.7539267015706806, "fcm_dpo/beta": 0.01992819271981716, "fcm_dpo/delta": 0.03362187370657921, "fcm_dpo/margin": 25.8211669921875, "fcm_dpo/q_t": 0.389692485332489, "grad_norm": 50.656654357910156, "learning_rate": 8.76727937529367e-08, "logits/chosen": 1.5480406284332275, "logits/rejected": 1.4931936264038086, "logps/chosen": -307.37274169921875, "logps/ref_chosen": -300.91375732421875, "logps/ref_rejected": -256.4080810546875, "logps/rejected": -288.688232421875, "loss": 4.3402, "margin_dpo/margin_mean": 25.821170806884766, "margin_dpo/margin_std": 39.488887786865234, "step": 360 }, { "KL/chosen_KL_mean": -0.2387371063232422, "KL/mean": -17.319400787353516, "KL/rejected_KL_mean": -34.40007019042969, "KL/std": 38.262062072753906, "epoch": 0.7560209424083769, "fcm_dpo/beta": 0.019601240754127502, "fcm_dpo/delta": -0.0743849128484726, "fcm_dpo/margin": 34.16133117675781, "fcm_dpo/q_t": 0.3612736463546753, "grad_norm": 38.9226188659668, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.9703270792961121, "logits/rejected": 1.04444420337677, "logps/chosen": -256.799072265625, "logps/ref_chosen": -256.5603332519531, "logps/ref_rejected": -271.043701171875, "logps/rejected": -305.4437561035156, "loss": 4.0002, "margin_dpo/margin_mean": 34.16133499145508, "margin_dpo/margin_std": 45.310611724853516, "step": 361 }, { "KL/chosen_KL_mean": -2.9542789459228516, "KL/mean": -16.490875244140625, "KL/rejected_KL_mean": -30.02747344970703, "KL/std": 34.13772964477539, "epoch": 0.7581151832460733, "fcm_dpo/beta": 0.019504547119140625, "fcm_dpo/delta": 0.07467053830623627, "fcm_dpo/margin": 27.073196411132812, "fcm_dpo/q_t": 0.38740891218185425, "grad_norm": 51.599544525146484, "learning_rate": 8.490561882286135e-08, "logits/chosen": 1.5528841018676758, "logits/rejected": 1.4546035528182983, "logps/chosen": -297.65472412109375, "logps/ref_chosen": -294.700439453125, "logps/ref_rejected": -239.67922973632812, "logps/rejected": -269.70672607421875, "loss": 4.2945, "margin_dpo/margin_mean": 27.073198318481445, "margin_dpo/margin_std": 39.47699737548828, "step": 362 }, { "KL/chosen_KL_mean": -3.676485061645508, "KL/mean": -19.368915557861328, "KL/rejected_KL_mean": -35.06134796142578, "KL/std": 37.644344329833984, "epoch": 0.7602094240837697, "fcm_dpo/beta": 0.019930703565478325, "fcm_dpo/delta": -0.02877102419734001, "fcm_dpo/margin": 31.38487434387207, "fcm_dpo/q_t": 0.36726850271224976, "grad_norm": 46.586761474609375, "learning_rate": 8.353527464267104e-08, "logits/chosen": 1.4351568222045898, "logits/rejected": 1.4231568574905396, "logps/chosen": -297.3523254394531, "logps/ref_chosen": -293.67584228515625, "logps/ref_rejected": -260.85675048828125, "logps/rejected": -295.9181213378906, "loss": 4.0882, "margin_dpo/margin_mean": 31.38487434387207, "margin_dpo/margin_std": 42.509033203125, "step": 363 }, { "KL/chosen_KL_mean": -14.330312728881836, "KL/mean": -26.858896255493164, "KL/rejected_KL_mean": -39.38748550415039, "KL/std": 37.47501754760742, "epoch": 0.762303664921466, "fcm_dpo/beta": 0.019709203392267227, "fcm_dpo/delta": 0.023300671949982643, "fcm_dpo/margin": 25.057165145874023, "fcm_dpo/q_t": 0.3976060748100281, "grad_norm": 42.29460906982422, "learning_rate": 8.217385746050742e-08, "logits/chosen": 1.3992136716842651, "logits/rejected": 1.460357666015625, "logps/chosen": -293.2257080078125, "logps/ref_chosen": -278.8953857421875, "logps/ref_rejected": -283.8898620605469, "logps/rejected": -323.2773132324219, "loss": 4.5456, "margin_dpo/margin_mean": 25.057165145874023, "margin_dpo/margin_std": 44.332950592041016, "step": 364 }, { "KL/chosen_KL_mean": -4.5511627197265625, "KL/mean": -19.906902313232422, "KL/rejected_KL_mean": -35.26264190673828, "KL/std": 38.66453552246094, "epoch": 0.7643979057591623, "fcm_dpo/beta": 0.02006695233285427, "fcm_dpo/delta": -0.018149226903915405, "fcm_dpo/margin": 30.71148681640625, "fcm_dpo/q_t": 0.37343525886535645, "grad_norm": 61.094390869140625, "learning_rate": 8.082144028504231e-08, "logits/chosen": 1.3744169473648071, "logits/rejected": 1.406067132949829, "logps/chosen": -270.195068359375, "logps/ref_chosen": -265.6439208984375, "logps/ref_rejected": -255.40138244628906, "logps/rejected": -290.66400146484375, "loss": 4.1269, "margin_dpo/margin_mean": 30.71148681640625, "margin_dpo/margin_std": 43.33749008178711, "step": 365 }, { "KL/chosen_KL_mean": -6.680112838745117, "KL/mean": -22.339298248291016, "KL/rejected_KL_mean": -37.99848937988281, "KL/std": 38.10138702392578, "epoch": 0.7664921465968586, "fcm_dpo/beta": 0.019372303038835526, "fcm_dpo/delta": -0.008491618558764458, "fcm_dpo/margin": 31.318374633789062, "fcm_dpo/q_t": 0.37076860666275024, "grad_norm": 36.94200134277344, "learning_rate": 7.947809564230445e-08, "logits/chosen": 1.5797892808914185, "logits/rejected": 1.5851740837097168, "logps/chosen": -286.90460205078125, "logps/ref_chosen": -280.2244873046875, "logps/ref_rejected": -248.72366333007812, "logps/rejected": -286.72210693359375, "loss": 4.1119, "margin_dpo/margin_mean": 31.31837272644043, "margin_dpo/margin_std": 42.21375274658203, "step": 366 }, { "KL/chosen_KL_mean": -4.263193130493164, "KL/mean": -20.494478225708008, "KL/rejected_KL_mean": -36.72576904296875, "KL/std": 36.97572326660156, "epoch": 0.768586387434555, "fcm_dpo/beta": 0.019131647422909737, "fcm_dpo/delta": -0.023691326379776, "fcm_dpo/margin": 32.46257400512695, "fcm_dpo/q_t": 0.36672520637512207, "grad_norm": 44.760868072509766, "learning_rate": 7.814389557179016e-08, "logits/chosen": 1.3935260772705078, "logits/rejected": 1.3774563074111938, "logps/chosen": -297.1998596191406, "logps/ref_chosen": -292.9366760253906, "logps/ref_rejected": -261.88763427734375, "logps/rejected": -298.6133728027344, "loss": 3.9551, "margin_dpo/margin_mean": 32.46257400512695, "margin_dpo/margin_std": 39.45927810668945, "step": 367 }, { "KL/chosen_KL_mean": -0.7215766906738281, "KL/mean": -19.261903762817383, "KL/rejected_KL_mean": -37.802223205566406, "KL/std": 38.32929229736328, "epoch": 0.7706806282722513, "fcm_dpo/beta": 0.018251920118927956, "fcm_dpo/delta": -0.08612095564603806, "fcm_dpo/margin": 37.08063888549805, "fcm_dpo/q_t": 0.35398417711257935, "grad_norm": 38.82743453979492, "learning_rate": 7.681891162260015e-08, "logits/chosen": 1.420930027961731, "logits/rejected": 1.4985603094100952, "logps/chosen": -285.6273193359375, "logps/ref_chosen": -284.90576171875, "logps/ref_rejected": -256.59197998046875, "logps/rejected": -294.3941955566406, "loss": 3.8001, "margin_dpo/margin_mean": 37.08063888549805, "margin_dpo/margin_std": 40.02778244018555, "step": 368 }, { "KL/chosen_KL_mean": -5.350444793701172, "KL/mean": -18.499868392944336, "KL/rejected_KL_mean": -31.64929962158203, "KL/std": 36.598114013671875, "epoch": 0.7727748691099476, "fcm_dpo/beta": 0.018250327557325363, "fcm_dpo/delta": 0.07484041899442673, "fcm_dpo/margin": 26.298860549926758, "fcm_dpo/q_t": 0.397874653339386, "grad_norm": 49.15440368652344, "learning_rate": 7.550321484960251e-08, "logits/chosen": 1.309835433959961, "logits/rejected": 1.3136308193206787, "logps/chosen": -281.266357421875, "logps/ref_chosen": -275.91595458984375, "logps/ref_rejected": -264.79736328125, "logps/rejected": -296.4466552734375, "loss": 4.4474, "margin_dpo/margin_mean": 26.298860549926758, "margin_dpo/margin_std": 42.30354309082031, "step": 369 }, { "KL/chosen_KL_mean": -4.719293594360352, "KL/mean": -20.991464614868164, "KL/rejected_KL_mean": -37.263633728027344, "KL/std": 36.66276550292969, "epoch": 0.774869109947644, "fcm_dpo/beta": 0.01837090589106083, "fcm_dpo/delta": -0.046606432646512985, "fcm_dpo/margin": 32.544342041015625, "fcm_dpo/q_t": 0.370233952999115, "grad_norm": 34.6463623046875, "learning_rate": 7.419687580962222e-08, "logits/chosen": 1.2520381212234497, "logits/rejected": 1.2990646362304688, "logps/chosen": -249.66131591796875, "logps/ref_chosen": -244.9420166015625, "logps/ref_rejected": -235.61978149414062, "logps/rejected": -272.8834228515625, "loss": 4.0898, "margin_dpo/margin_mean": 32.544342041015625, "margin_dpo/margin_std": 43.14802551269531, "step": 370 }, { "KL/chosen_KL_mean": -7.0436248779296875, "KL/mean": -20.212116241455078, "KL/rejected_KL_mean": -33.38059997558594, "KL/std": 37.32386016845703, "epoch": 0.7769633507853403, "fcm_dpo/beta": 0.019493352621793747, "fcm_dpo/delta": 0.08880892395973206, "fcm_dpo/margin": 26.336973190307617, "fcm_dpo/q_t": 0.39040279388427734, "grad_norm": 55.44318771362305, "learning_rate": 7.289996455765748e-08, "logits/chosen": 1.2644152641296387, "logits/rejected": 1.2208240032196045, "logps/chosen": -296.3934631347656, "logps/ref_chosen": -289.3498229980469, "logps/ref_rejected": -243.70205688476562, "logps/rejected": -277.0826416015625, "loss": 4.3441, "margin_dpo/margin_mean": 26.33697509765625, "margin_dpo/margin_std": 40.09697723388672, "step": 371 }, { "KL/chosen_KL_mean": -3.991708755493164, "KL/mean": -23.316816329956055, "KL/rejected_KL_mean": -42.64191436767578, "KL/std": 39.0753173828125, "epoch": 0.7790575916230367, "fcm_dpo/beta": 0.018297212198376656, "fcm_dpo/delta": -0.11904658377170563, "fcm_dpo/margin": 38.65019607543945, "fcm_dpo/q_t": 0.3469123840332031, "grad_norm": 38.49943161010742, "learning_rate": 7.161255064312283e-08, "logits/chosen": 1.6757351160049438, "logits/rejected": 1.6532821655273438, "logps/chosen": -325.6452331542969, "logps/ref_chosen": -321.65350341796875, "logps/ref_rejected": -282.73773193359375, "logps/rejected": -325.379638671875, "loss": 3.7615, "margin_dpo/margin_mean": 38.65019226074219, "margin_dpo/margin_std": 41.94559097290039, "step": 372 }, { "KL/chosen_KL_mean": -4.463361740112305, "KL/mean": -20.560094833374023, "KL/rejected_KL_mean": -36.656829833984375, "KL/std": 39.52842330932617, "epoch": 0.7811518324607329, "fcm_dpo/beta": 0.01794876903295517, "fcm_dpo/delta": 0.0224030502140522, "fcm_dpo/margin": 32.19346618652344, "fcm_dpo/q_t": 0.3757041096687317, "grad_norm": 38.73172378540039, "learning_rate": 7.033470310611945e-08, "logits/chosen": 1.400538682937622, "logits/rejected": 1.3460829257965088, "logps/chosen": -314.2138366699219, "logps/ref_chosen": -309.75048828125, "logps/ref_rejected": -267.41925048828125, "logps/rejected": -304.0760803222656, "loss": 4.113, "margin_dpo/margin_mean": 32.19346618652344, "margin_dpo/margin_std": 42.02708435058594, "step": 373 }, { "KL/chosen_KL_mean": -5.112339019775391, "KL/mean": -17.980426788330078, "KL/rejected_KL_mean": -30.84852409362793, "KL/std": 37.83140182495117, "epoch": 0.7832460732984293, "fcm_dpo/beta": 0.019059795886278152, "fcm_dpo/delta": 0.11390713602304459, "fcm_dpo/margin": 25.736186981201172, "fcm_dpo/q_t": 0.39781954884529114, "grad_norm": 41.872093200683594, "learning_rate": 6.906649047373245e-08, "logits/chosen": 1.4691405296325684, "logits/rejected": 1.4243155717849731, "logps/chosen": -316.1064453125, "logps/ref_chosen": -310.9941101074219, "logps/ref_rejected": -279.94915771484375, "logps/rejected": -310.7976989746094, "loss": 4.4453, "margin_dpo/margin_mean": 25.736183166503906, "margin_dpo/margin_std": 41.93350601196289, "step": 374 }, { "KL/chosen_KL_mean": -5.18425178527832, "KL/mean": -19.81911277770996, "KL/rejected_KL_mean": -34.45397186279297, "KL/std": 35.83353805541992, "epoch": 0.7853403141361257, "fcm_dpo/beta": 0.020486168563365936, "fcm_dpo/delta": -0.002023853361606598, "fcm_dpo/margin": 29.269716262817383, "fcm_dpo/q_t": 0.3779732584953308, "grad_norm": 38.118839263916016, "learning_rate": 6.780798075635675e-08, "logits/chosen": 1.664709210395813, "logits/rejected": 1.5848511457443237, "logps/chosen": -314.3677062988281, "logps/ref_chosen": -309.18341064453125, "logps/ref_rejected": -256.9963073730469, "logps/rejected": -291.45025634765625, "loss": 4.2414, "margin_dpo/margin_mean": 29.26971435546875, "margin_dpo/margin_std": 44.088584899902344, "step": 375 }, { "KL/chosen_KL_mean": -10.31795883178711, "KL/mean": -24.29334831237793, "KL/rejected_KL_mean": -38.26873779296875, "KL/std": 38.56963348388672, "epoch": 0.787434554973822, "fcm_dpo/beta": 0.019764304161071777, "fcm_dpo/delta": -0.0034585334360599518, "fcm_dpo/margin": 27.95077133178711, "fcm_dpo/q_t": 0.38577285408973694, "grad_norm": 43.306644439697266, "learning_rate": 6.655924144404906e-08, "logits/chosen": 1.6048816442489624, "logits/rejected": 1.6655864715576172, "logps/chosen": -292.5931396484375, "logps/ref_chosen": -282.275146484375, "logps/ref_rejected": -255.17941284179688, "logps/rejected": -293.4481506347656, "loss": 4.3495, "margin_dpo/margin_mean": 27.950769424438477, "margin_dpo/margin_std": 43.145328521728516, "step": 376 }, { "KL/chosen_KL_mean": -7.029672622680664, "KL/mean": -20.04590606689453, "KL/rejected_KL_mean": -33.06214141845703, "KL/std": 36.110069274902344, "epoch": 0.7895287958115184, "fcm_dpo/beta": 0.020612318068742752, "fcm_dpo/delta": 0.065596804022789, "fcm_dpo/margin": 26.03246307373047, "fcm_dpo/q_t": 0.39077773690223694, "grad_norm": 40.35654067993164, "learning_rate": 6.532033950290885e-08, "logits/chosen": 1.4133380651474, "logits/rejected": 1.431910514831543, "logps/chosen": -299.9068603515625, "logps/ref_chosen": -292.8772277832031, "logps/ref_rejected": -266.78509521484375, "logps/rejected": -299.84722900390625, "loss": 4.3946, "margin_dpo/margin_mean": 26.03246307373047, "margin_dpo/margin_std": 42.76789855957031, "step": 377 }, { "KL/chosen_KL_mean": -7.506763458251953, "KL/mean": -20.087665557861328, "KL/rejected_KL_mean": -32.6685791015625, "KL/std": 38.545738220214844, "epoch": 0.7916230366492146, "fcm_dpo/beta": 0.021401086822152138, "fcm_dpo/delta": 0.00895449798554182, "fcm_dpo/margin": 25.16181755065918, "fcm_dpo/q_t": 0.38900816440582275, "grad_norm": 48.65238571166992, "learning_rate": 6.409134137148736e-08, "logits/chosen": 1.5526601076126099, "logits/rejected": 1.6031973361968994, "logps/chosen": -280.40045166015625, "logps/ref_chosen": -272.8937072753906, "logps/ref_rejected": -293.058349609375, "logps/rejected": -325.7269287109375, "loss": 4.3716, "margin_dpo/margin_mean": 25.16181755065918, "margin_dpo/margin_std": 40.151939392089844, "step": 378 }, { "KL/chosen_KL_mean": -5.767313003540039, "KL/mean": -19.887914657592773, "KL/rejected_KL_mean": -34.00851821899414, "KL/std": 35.92317581176758, "epoch": 0.793717277486911, "fcm_dpo/beta": 0.02095981501042843, "fcm_dpo/delta": 0.0069956425577402115, "fcm_dpo/margin": 28.241214752197266, "fcm_dpo/q_t": 0.37797582149505615, "grad_norm": 52.0435905456543, "learning_rate": 6.28723129572247e-08, "logits/chosen": 1.3321895599365234, "logits/rejected": 1.3269301652908325, "logps/chosen": -264.9089050292969, "logps/ref_chosen": -259.1416015625, "logps/ref_rejected": -229.64996337890625, "logps/rejected": -263.6584777832031, "loss": 4.2885, "margin_dpo/margin_mean": 28.2412166595459, "margin_dpo/margin_std": 43.94401168823242, "step": 379 }, { "KL/chosen_KL_mean": -9.226726531982422, "KL/mean": -23.88406753540039, "KL/rejected_KL_mean": -38.541419982910156, "KL/std": 39.329925537109375, "epoch": 0.7958115183246073, "fcm_dpo/beta": 0.020642951130867004, "fcm_dpo/delta": -0.07479599863290787, "fcm_dpo/margin": 29.31468963623047, "fcm_dpo/q_t": 0.3779948353767395, "grad_norm": 53.2426872253418, "learning_rate": 6.166331963291519e-08, "logits/chosen": 1.5265839099884033, "logits/rejected": 1.4460859298706055, "logps/chosen": -305.5646667480469, "logps/ref_chosen": -296.33795166015625, "logps/ref_rejected": -283.7359924316406, "logps/rejected": -322.27740478515625, "loss": 4.2068, "margin_dpo/margin_mean": 29.31468963623047, "margin_dpo/margin_std": 43.51792907714844, "step": 380 }, { "KL/chosen_KL_mean": -8.992172241210938, "KL/mean": -23.937061309814453, "KL/rejected_KL_mean": -38.8819465637207, "KL/std": 38.82854461669922, "epoch": 0.7979057591623037, "fcm_dpo/beta": 0.019958244636654854, "fcm_dpo/delta": 0.002718057483434677, "fcm_dpo/margin": 29.889772415161133, "fcm_dpo/q_t": 0.376090943813324, "grad_norm": 59.369956970214844, "learning_rate": 6.046442623320145e-08, "logits/chosen": 1.263857126235962, "logits/rejected": 1.29478120803833, "logps/chosen": -257.96539306640625, "logps/ref_chosen": -248.97320556640625, "logps/ref_rejected": -259.5140075683594, "logps/rejected": -298.3959655761719, "loss": 4.1502, "margin_dpo/margin_mean": 29.889774322509766, "margin_dpo/margin_std": 42.35132598876953, "step": 381 }, { "KL/chosen_KL_mean": -1.8784332275390625, "KL/mean": -19.92194175720215, "KL/rejected_KL_mean": -37.9654541015625, "KL/std": 37.75444793701172, "epoch": 0.8, "fcm_dpo/beta": 0.019227033481001854, "fcm_dpo/delta": -0.10280030220746994, "fcm_dpo/margin": 36.087013244628906, "fcm_dpo/q_t": 0.35315173864364624, "grad_norm": 46.13385009765625, "learning_rate": 5.9275697051098275e-08, "logits/chosen": 1.3950705528259277, "logits/rejected": 1.5298354625701904, "logps/chosen": -282.55072021484375, "logps/ref_chosen": -280.67230224609375, "logps/ref_rejected": -272.4359130859375, "logps/rejected": -310.4013366699219, "loss": 3.8681, "margin_dpo/margin_mean": 36.087013244628906, "margin_dpo/margin_std": 42.34565734863281, "step": 382 }, { "KL/chosen_KL_mean": -5.869083404541016, "KL/mean": -20.48516082763672, "KL/rejected_KL_mean": -35.10124206542969, "KL/std": 36.95631790161133, "epoch": 0.8020942408376963, "fcm_dpo/beta": 0.018535105511546135, "fcm_dpo/delta": 0.006197173148393631, "fcm_dpo/margin": 29.232160568237305, "fcm_dpo/q_t": 0.38371843099594116, "grad_norm": 52.805259704589844, "learning_rate": 5.809719583454414e-08, "logits/chosen": 0.9901968240737915, "logits/rejected": 1.0240157842636108, "logps/chosen": -273.5275573730469, "logps/ref_chosen": -267.658447265625, "logps/ref_rejected": -262.9810791015625, "logps/rejected": -298.0823059082031, "loss": 4.2197, "margin_dpo/margin_mean": 29.232160568237305, "margin_dpo/margin_std": 40.99141311645508, "step": 383 }, { "KL/chosen_KL_mean": -7.576007843017578, "KL/mean": -19.12158203125, "KL/rejected_KL_mean": -30.667158126831055, "KL/std": 35.436187744140625, "epoch": 0.8041884816753927, "fcm_dpo/beta": 0.02009403146803379, "fcm_dpo/delta": 0.139180526137352, "fcm_dpo/margin": 23.091142654418945, "fcm_dpo/q_t": 0.40392279624938965, "grad_norm": 56.3973388671875, "learning_rate": 5.6928985782982524e-08, "logits/chosen": 1.4375354051589966, "logits/rejected": 1.6076993942260742, "logps/chosen": -273.82257080078125, "logps/ref_chosen": -266.2465515136719, "logps/ref_rejected": -281.1450500488281, "logps/rejected": -311.8122253417969, "loss": 4.614, "margin_dpo/margin_mean": 23.091140747070312, "margin_dpo/margin_std": 43.37619400024414, "step": 384 }, { "KL/chosen_KL_mean": -6.043983459472656, "KL/mean": -20.263090133666992, "KL/rejected_KL_mean": -34.482177734375, "KL/std": 37.30935287475586, "epoch": 0.806282722513089, "fcm_dpo/beta": 0.02078263834118843, "fcm_dpo/delta": 0.007318824529647827, "fcm_dpo/margin": 28.43819808959961, "fcm_dpo/q_t": 0.3783205449581146, "grad_norm": 44.51754379272461, "learning_rate": 5.57711295439732e-08, "logits/chosen": 1.384292721748352, "logits/rejected": 1.3526872396469116, "logps/chosen": -280.349853515625, "logps/ref_chosen": -274.30584716796875, "logps/ref_rejected": -242.94247436523438, "logps/rejected": -277.4246520996094, "loss": 4.2147, "margin_dpo/margin_mean": 28.43819808959961, "margin_dpo/margin_std": 41.831268310546875, "step": 385 }, { "KL/chosen_KL_mean": -3.1711044311523438, "KL/mean": -20.52104949951172, "KL/rejected_KL_mean": -37.871002197265625, "KL/std": 39.688323974609375, "epoch": 0.8083769633507853, "fcm_dpo/beta": 0.019427498802542686, "fcm_dpo/delta": -0.0865735337138176, "fcm_dpo/margin": 34.69987869262695, "fcm_dpo/q_t": 0.3566838204860687, "grad_norm": 38.12848663330078, "learning_rate": 5.4623689209832484e-08, "logits/chosen": 1.7019646167755127, "logits/rejected": 1.7641416788101196, "logps/chosen": -312.43756103515625, "logps/ref_chosen": -309.2664489746094, "logps/ref_rejected": -298.4182434082031, "logps/rejected": -336.2892150878906, "loss": 3.9511, "margin_dpo/margin_mean": 34.69988250732422, "margin_dpo/margin_std": 42.274879455566406, "step": 386 }, { "KL/chosen_KL_mean": -3.9953250885009766, "KL/mean": -20.024505615234375, "KL/rejected_KL_mean": -36.05369567871094, "KL/std": 38.43357849121094, "epoch": 0.8104712041884817, "fcm_dpo/beta": 0.019006196409463882, "fcm_dpo/delta": -0.06764067709445953, "fcm_dpo/margin": 32.05836486816406, "fcm_dpo/q_t": 0.37299811840057373, "grad_norm": 38.89561462402344, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 1.4557772874832153, "logits/rejected": 1.4827194213867188, "logps/chosen": -268.0162353515625, "logps/ref_chosen": -264.0209045410156, "logps/ref_rejected": -264.3447570800781, "logps/rejected": -300.3984375, "loss": 4.1598, "margin_dpo/margin_mean": 32.05836486816406, "margin_dpo/margin_std": 44.533687591552734, "step": 387 }, { "KL/chosen_KL_mean": -11.501548767089844, "KL/mean": -23.445556640625, "KL/rejected_KL_mean": -35.38957214355469, "KL/std": 35.58702087402344, "epoch": 0.812565445026178, "fcm_dpo/beta": 0.018847376108169556, "fcm_dpo/delta": 0.03600002080202103, "fcm_dpo/margin": 23.88802719116211, "fcm_dpo/q_t": 0.4046207666397095, "grad_norm": 59.93172073364258, "learning_rate": 5.2360301829254745e-08, "logits/chosen": 1.3926148414611816, "logits/rejected": 1.42686128616333, "logps/chosen": -274.72039794921875, "logps/ref_chosen": -263.2188415527344, "logps/ref_rejected": -236.08546447753906, "logps/rejected": -271.4750061035156, "loss": 4.5808, "margin_dpo/margin_mean": 23.88802719116211, "margin_dpo/margin_std": 42.49546813964844, "step": 388 }, { "KL/chosen_KL_mean": -5.861154556274414, "KL/mean": -21.513050079345703, "KL/rejected_KL_mean": -37.16495132446289, "KL/std": 35.916481018066406, "epoch": 0.8146596858638744, "fcm_dpo/beta": 0.018980950117111206, "fcm_dpo/delta": 0.004103680606931448, "fcm_dpo/margin": 31.303802490234375, "fcm_dpo/q_t": 0.37250933051109314, "grad_norm": 39.85160827636719, "learning_rate": 5.1244476161413806e-08, "logits/chosen": 1.433082103729248, "logits/rejected": 1.3986594676971436, "logps/chosen": -279.6590270996094, "logps/ref_chosen": -273.79791259765625, "logps/ref_rejected": -242.4090118408203, "logps/rejected": -279.5739440917969, "loss": 4.114, "margin_dpo/margin_mean": 31.303802490234375, "margin_dpo/margin_std": 41.43454360961914, "step": 389 }, { "KL/chosen_KL_mean": -5.90693473815918, "KL/mean": -21.321962356567383, "KL/rejected_KL_mean": -36.73699188232422, "KL/std": 35.705467224121094, "epoch": 0.8167539267015707, "fcm_dpo/beta": 0.019459852948784828, "fcm_dpo/delta": -0.0016894396394491196, "fcm_dpo/margin": 30.830060958862305, "fcm_dpo/q_t": 0.3736165165901184, "grad_norm": 46.194740295410156, "learning_rate": 5.013930914912476e-08, "logits/chosen": 1.3219387531280518, "logits/rejected": 1.3668385744094849, "logps/chosen": -281.22210693359375, "logps/ref_chosen": -275.3151550292969, "logps/ref_rejected": -278.1514587402344, "logps/rejected": -314.88848876953125, "loss": 4.2208, "margin_dpo/margin_mean": 30.830059051513672, "margin_dpo/margin_std": 44.96910858154297, "step": 390 }, { "KL/chosen_KL_mean": -10.438993453979492, "KL/mean": -25.486530303955078, "KL/rejected_KL_mean": -40.53406524658203, "KL/std": 38.74292755126953, "epoch": 0.818848167539267, "fcm_dpo/beta": 0.01855631358921528, "fcm_dpo/delta": -0.060978908091783524, "fcm_dpo/margin": 30.095077514648438, "fcm_dpo/q_t": 0.3802896738052368, "grad_norm": 35.211151123046875, "learning_rate": 4.904486005914027e-08, "logits/chosen": 1.5178322792053223, "logits/rejected": 1.4987064599990845, "logps/chosen": -285.6383361816406, "logps/ref_chosen": -275.1993408203125, "logps/ref_rejected": -261.32122802734375, "logps/rejected": -301.85528564453125, "loss": 4.1482, "margin_dpo/margin_mean": 30.095077514648438, "margin_dpo/margin_std": 39.87967300415039, "step": 391 }, { "KL/chosen_KL_mean": -5.631750106811523, "KL/mean": -25.026447296142578, "KL/rejected_KL_mean": -44.4211540222168, "KL/std": 38.97100830078125, "epoch": 0.8209424083769633, "fcm_dpo/beta": 0.016711238771677017, "fcm_dpo/delta": -0.11873180419206619, "fcm_dpo/margin": 38.78941345214844, "fcm_dpo/q_t": 0.3590206503868103, "grad_norm": 31.533557891845703, "learning_rate": 4.796118758344353e-08, "logits/chosen": 1.291164755821228, "logits/rejected": 1.2510210275650024, "logps/chosen": -305.45953369140625, "logps/ref_chosen": -299.8277587890625, "logps/ref_rejected": -251.24798583984375, "logps/rejected": -295.66912841796875, "loss": 3.8347, "margin_dpo/margin_mean": 38.78941345214844, "margin_dpo/margin_std": 41.62689208984375, "step": 392 }, { "KL/chosen_KL_mean": -5.230253219604492, "KL/mean": -19.421533584594727, "KL/rejected_KL_mean": -33.612815856933594, "KL/std": 36.66984558105469, "epoch": 0.8230366492146597, "fcm_dpo/beta": 0.01709682308137417, "fcm_dpo/delta": 0.062248699367046356, "fcm_dpo/margin": 28.382564544677734, "fcm_dpo/q_t": 0.3945763111114502, "grad_norm": 41.22100067138672, "learning_rate": 4.688834983610082e-08, "logits/chosen": 1.3987441062927246, "logits/rejected": 1.3628212213516235, "logps/chosen": -285.69140625, "logps/ref_chosen": -280.4610900878906, "logps/ref_rejected": -228.34823608398438, "logps/rejected": -261.9610290527344, "loss": 4.385, "margin_dpo/margin_mean": 28.382564544677734, "margin_dpo/margin_std": 43.90695571899414, "step": 393 }, { "KL/chosen_KL_mean": -3.360300064086914, "KL/mean": -16.546672821044922, "KL/rejected_KL_mean": -29.73303985595703, "KL/std": 35.980621337890625, "epoch": 0.8251308900523561, "fcm_dpo/beta": 0.01769283041357994, "fcm_dpo/delta": 0.07207678258419037, "fcm_dpo/margin": 26.37274169921875, "fcm_dpo/q_t": 0.40133777260780334, "grad_norm": 36.066593170166016, "learning_rate": 4.582640435014459e-08, "logits/chosen": 1.5340707302093506, "logits/rejected": 1.6380658149719238, "logps/chosen": -322.96038818359375, "logps/ref_chosen": -319.60009765625, "logps/ref_rejected": -316.42822265625, "logps/rejected": -346.1612548828125, "loss": 4.413, "margin_dpo/margin_mean": 26.37274169921875, "margin_dpo/margin_std": 40.14125061035156, "step": 394 }, { "KL/chosen_KL_mean": -7.686824798583984, "KL/mean": -22.12656021118164, "KL/rejected_KL_mean": -36.56629943847656, "KL/std": 36.45963668823242, "epoch": 0.8272251308900523, "fcm_dpo/beta": 0.018843896687030792, "fcm_dpo/delta": 0.05734879896044731, "fcm_dpo/margin": 28.87946319580078, "fcm_dpo/q_t": 0.38550975918769836, "grad_norm": 43.15156173706055, "learning_rate": 4.477540807448832e-08, "logits/chosen": 1.1463429927825928, "logits/rejected": 1.2481884956359863, "logps/chosen": -268.10467529296875, "logps/ref_chosen": -260.4178466796875, "logps/ref_rejected": -270.04498291015625, "logps/rejected": -306.6112976074219, "loss": 4.2021, "margin_dpo/margin_mean": 28.87946319580078, "margin_dpo/margin_std": 39.44015121459961, "step": 395 }, { "KL/chosen_KL_mean": -6.388545989990234, "KL/mean": -21.564329147338867, "KL/rejected_KL_mean": -36.740116119384766, "KL/std": 37.587928771972656, "epoch": 0.8293193717277487, "fcm_dpo/beta": 0.019035927951335907, "fcm_dpo/delta": -0.04120335355401039, "fcm_dpo/margin": 30.351564407348633, "fcm_dpo/q_t": 0.3766162097454071, "grad_norm": 39.945011138916016, "learning_rate": 4.373541737087263e-08, "logits/chosen": 1.628139853477478, "logits/rejected": 1.6172528266906738, "logps/chosen": -287.0915832519531, "logps/ref_chosen": -280.7030029296875, "logps/ref_rejected": -250.76385498046875, "logps/rejected": -287.50396728515625, "loss": 4.1762, "margin_dpo/margin_mean": 30.351564407348633, "margin_dpo/margin_std": 42.127288818359375, "step": 396 }, { "KL/chosen_KL_mean": -4.6651458740234375, "KL/mean": -19.02102279663086, "KL/rejected_KL_mean": -33.37690734863281, "KL/std": 38.191673278808594, "epoch": 0.831413612565445, "fcm_dpo/beta": 0.018389301374554634, "fcm_dpo/delta": -0.04833086580038071, "fcm_dpo/margin": 28.711769104003906, "fcm_dpo/q_t": 0.3868543803691864, "grad_norm": 37.401527404785156, "learning_rate": 4.270648801084295e-08, "logits/chosen": 1.2988146543502808, "logits/rejected": 1.3936009407043457, "logps/chosen": -305.5516052246094, "logps/ref_chosen": -300.8864440917969, "logps/ref_rejected": -288.5682067871094, "logps/rejected": -321.94512939453125, "loss": 4.333, "margin_dpo/margin_mean": 28.711769104003906, "margin_dpo/margin_std": 42.40093231201172, "step": 397 }, { "KL/chosen_KL_mean": -12.974161148071289, "KL/mean": -24.25926971435547, "KL/rejected_KL_mean": -35.544368743896484, "KL/std": 38.720703125, "epoch": 0.8335078534031414, "fcm_dpo/beta": 0.018865050747990608, "fcm_dpo/delta": 0.12429942190647125, "fcm_dpo/margin": 22.570209503173828, "fcm_dpo/q_t": 0.41304242610931396, "grad_norm": 73.18061828613281, "learning_rate": 4.168867517275806e-08, "logits/chosen": 1.156326174736023, "logits/rejected": 1.1686561107635498, "logps/chosen": -306.2765197753906, "logps/ref_chosen": -293.3023681640625, "logps/ref_rejected": -270.0457763671875, "logps/rejected": -305.59014892578125, "loss": 4.8132, "margin_dpo/margin_mean": 22.570213317871094, "margin_dpo/margin_std": 48.60316467285156, "step": 398 }, { "KL/chosen_KL_mean": -8.889374732971191, "KL/mean": -22.127668380737305, "KL/rejected_KL_mean": -35.365962982177734, "KL/std": 39.10053634643555, "epoch": 0.8356020942408376, "fcm_dpo/beta": 0.020086301490664482, "fcm_dpo/delta": 0.07246292382478714, "fcm_dpo/margin": 26.476594924926758, "fcm_dpo/q_t": 0.3878498375415802, "grad_norm": 46.15391540527344, "learning_rate": 4.0682033438831584e-08, "logits/chosen": 1.3159246444702148, "logits/rejected": 1.37637197971344, "logps/chosen": -292.28863525390625, "logps/ref_chosen": -283.39923095703125, "logps/ref_rejected": -264.5092468261719, "logps/rejected": -299.8752136230469, "loss": 4.3662, "margin_dpo/margin_mean": 26.476594924926758, "margin_dpo/margin_std": 41.488059997558594, "step": 399 }, { "KL/chosen_KL_mean": -6.484638214111328, "KL/mean": -21.498777389526367, "KL/rejected_KL_mean": -36.51289749145508, "KL/std": 35.018836975097656, "epoch": 0.837696335078534, "fcm_dpo/beta": 0.020130477845668793, "fcm_dpo/delta": -0.05539043992757797, "fcm_dpo/margin": 30.028249740600586, "fcm_dpo/q_t": 0.37194401025772095, "grad_norm": 43.100982666015625, "learning_rate": 3.968661679220467e-08, "logits/chosen": 1.3445234298706055, "logits/rejected": 1.40453040599823, "logps/chosen": -260.95245361328125, "logps/ref_chosen": -254.4678192138672, "logps/ref_rejected": -252.00326538085938, "logps/rejected": -288.5161437988281, "loss": 4.1465, "margin_dpo/margin_mean": 30.028249740600586, "margin_dpo/margin_std": 40.52297592163086, "step": 400 }, { "KL/chosen_KL_mean": -5.151699066162109, "KL/mean": -23.097524642944336, "KL/rejected_KL_mean": -41.04334259033203, "KL/std": 39.70207595825195, "epoch": 0.8397905759162304, "fcm_dpo/beta": 0.019138243049383163, "fcm_dpo/delta": -0.0941125750541687, "fcm_dpo/margin": 35.891639709472656, "fcm_dpo/q_t": 0.3543274402618408, "grad_norm": 55.37236022949219, "learning_rate": 3.8702478614051345e-08, "logits/chosen": 1.364419937133789, "logits/rejected": 1.390592098236084, "logps/chosen": -315.7512512207031, "logps/ref_chosen": -310.5995788574219, "logps/ref_rejected": -292.7754821777344, "logps/rejected": -333.8188171386719, "loss": 3.8402, "margin_dpo/margin_mean": 35.891639709472656, "margin_dpo/margin_std": 41.956520080566406, "step": 401 }, { "KL/chosen_KL_mean": -3.8576431274414062, "KL/mean": -19.410980224609375, "KL/rejected_KL_mean": -34.964324951171875, "KL/std": 38.21479034423828, "epoch": 0.8418848167539267, "fcm_dpo/beta": 0.01879115216434002, "fcm_dpo/delta": 0.014545461162924767, "fcm_dpo/margin": 31.106664657592773, "fcm_dpo/q_t": 0.3764471113681793, "grad_norm": 35.57643127441406, "learning_rate": 3.772967168071517e-08, "logits/chosen": 1.4840364456176758, "logits/rejected": 1.4013035297393799, "logps/chosen": -303.708984375, "logps/ref_chosen": -299.8513488769531, "logps/ref_rejected": -278.5664367675781, "logps/rejected": -313.53076171875, "loss": 4.1945, "margin_dpo/margin_mean": 31.106664657592773, "margin_dpo/margin_std": 44.161434173583984, "step": 402 }, { "KL/chosen_KL_mean": -3.828258514404297, "KL/mean": -24.283838272094727, "KL/rejected_KL_mean": -44.73942565917969, "KL/std": 39.80998992919922, "epoch": 0.8439790575916231, "fcm_dpo/beta": 0.017841363325715065, "fcm_dpo/delta": -0.13944655656814575, "fcm_dpo/margin": 40.911163330078125, "fcm_dpo/q_t": 0.3421737253665924, "grad_norm": 37.96006393432617, "learning_rate": 3.676824816087978e-08, "logits/chosen": 1.5281907320022583, "logits/rejected": 1.5547796487808228, "logps/chosen": -306.2598876953125, "logps/ref_chosen": -302.4316101074219, "logps/ref_rejected": -270.951171875, "logps/rejected": -315.6905822753906, "loss": 3.6257, "margin_dpo/margin_mean": 40.91115951538086, "margin_dpo/margin_std": 40.92106628417969, "step": 403 }, { "KL/chosen_KL_mean": -9.52946662902832, "KL/mean": -22.904190063476562, "KL/rejected_KL_mean": -36.278907775878906, "KL/std": 38.89113235473633, "epoch": 0.8460732984293193, "fcm_dpo/beta": 0.01735624298453331, "fcm_dpo/delta": 0.07848434150218964, "fcm_dpo/margin": 26.749435424804688, "fcm_dpo/q_t": 0.4002022445201874, "grad_norm": 47.30289077758789, "learning_rate": 3.581825961277074e-08, "logits/chosen": 1.5369975566864014, "logits/rejected": 1.5787920951843262, "logps/chosen": -299.4325256347656, "logps/ref_chosen": -289.9031066894531, "logps/ref_rejected": -252.84918212890625, "logps/rejected": -289.1280822753906, "loss": 4.5215, "margin_dpo/margin_mean": 26.749435424804688, "margin_dpo/margin_std": 44.79439163208008, "step": 404 }, { "KL/chosen_KL_mean": -6.036563873291016, "KL/mean": -22.505189895629883, "KL/rejected_KL_mean": -38.97380828857422, "KL/std": 39.04482650756836, "epoch": 0.8481675392670157, "fcm_dpo/beta": 0.018190890550613403, "fcm_dpo/delta": -0.0007007038220763206, "fcm_dpo/margin": 32.93724822998047, "fcm_dpo/q_t": 0.3718109130859375, "grad_norm": 32.787559509277344, "learning_rate": 3.487975698139084e-08, "logits/chosen": 1.4501793384552002, "logits/rejected": 1.6205589771270752, "logps/chosen": -254.21060180664062, "logps/ref_chosen": -248.17404174804688, "logps/ref_rejected": -252.23764038085938, "logps/rejected": -291.2114562988281, "loss": 4.0347, "margin_dpo/margin_mean": 32.93724822998047, "margin_dpo/margin_std": 41.04658889770508, "step": 405 }, { "KL/chosen_KL_mean": -10.721895217895508, "KL/mean": -23.478830337524414, "KL/rejected_KL_mean": -36.23577880859375, "KL/std": 37.815269470214844, "epoch": 0.8502617801047121, "fcm_dpo/beta": 0.018215851858258247, "fcm_dpo/delta": 0.03488771989941597, "fcm_dpo/margin": 25.51388168334961, "fcm_dpo/q_t": 0.4009677767753601, "grad_norm": 48.44990921020508, "learning_rate": 3.3952790595787986e-08, "logits/chosen": 1.2857542037963867, "logits/rejected": 1.3423209190368652, "logps/chosen": -287.1380310058594, "logps/ref_chosen": -276.4161376953125, "logps/ref_rejected": -264.0082702636719, "logps/rejected": -300.2440185546875, "loss": 4.5071, "margin_dpo/margin_mean": 25.513879776000977, "margin_dpo/margin_std": 43.34572219848633, "step": 406 }, { "KL/chosen_KL_mean": -13.202856063842773, "KL/mean": -28.707468032836914, "KL/rejected_KL_mean": -44.21208190917969, "KL/std": 39.666770935058594, "epoch": 0.8523560209424084, "fcm_dpo/beta": 0.01855405420064926, "fcm_dpo/delta": 0.02565898559987545, "fcm_dpo/margin": 31.009233474731445, "fcm_dpo/q_t": 0.3793700337409973, "grad_norm": 40.636619567871094, "learning_rate": 3.303741016635614e-08, "logits/chosen": 1.6501725912094116, "logits/rejected": 1.6834803819656372, "logps/chosen": -270.64019775390625, "logps/ref_chosen": -257.4373474121094, "logps/ref_rejected": -219.66390991210938, "logps/rejected": -263.8759765625, "loss": 4.1759, "margin_dpo/margin_mean": 31.00923728942871, "margin_dpo/margin_std": 43.9052734375, "step": 407 }, { "KL/chosen_KL_mean": -7.150575637817383, "KL/mean": -22.323060989379883, "KL/rejected_KL_mean": -37.49555206298828, "KL/std": 37.51593780517578, "epoch": 0.8544502617801047, "fcm_dpo/beta": 0.01921917498111725, "fcm_dpo/delta": 0.014848966151475906, "fcm_dpo/margin": 30.344974517822266, "fcm_dpo/q_t": 0.37587589025497437, "grad_norm": 41.318260192871094, "learning_rate": 3.2133664782169944e-08, "logits/chosen": 1.4881930351257324, "logits/rejected": 1.6108531951904297, "logps/chosen": -295.8039245605469, "logps/ref_chosen": -288.6533508300781, "logps/ref_rejected": -291.54107666015625, "logps/rejected": -329.03662109375, "loss": 4.171, "margin_dpo/margin_mean": 30.344972610473633, "margin_dpo/margin_std": 41.09373474121094, "step": 408 }, { "KL/chosen_KL_mean": -8.33292007446289, "KL/mean": -24.719064712524414, "KL/rejected_KL_mean": -41.1052131652832, "KL/std": 38.697723388671875, "epoch": 0.856544502617801, "fcm_dpo/beta": 0.018651580438017845, "fcm_dpo/delta": -0.013694172725081444, "fcm_dpo/margin": 32.77228546142578, "fcm_dpo/q_t": 0.3731399178504944, "grad_norm": 43.54973602294922, "learning_rate": 3.12416029083514e-08, "logits/chosen": 1.5189157724380493, "logits/rejected": 1.6089897155761719, "logps/chosen": -286.0703125, "logps/ref_chosen": -277.7373962402344, "logps/ref_rejected": -272.988525390625, "logps/rejected": -314.0937194824219, "loss": 4.1718, "margin_dpo/margin_mean": 32.77228546142578, "margin_dpo/margin_std": 47.268409729003906, "step": 409 }, { "KL/chosen_KL_mean": -6.488674163818359, "KL/mean": -20.94354820251465, "KL/rejected_KL_mean": -35.39842224121094, "KL/std": 38.69110870361328, "epoch": 0.8586387434554974, "fcm_dpo/beta": 0.01897762157022953, "fcm_dpo/delta": -0.00017729774117469788, "fcm_dpo/margin": 28.909753799438477, "fcm_dpo/q_t": 0.38531434535980225, "grad_norm": 39.235260009765625, "learning_rate": 3.036127238347164e-08, "logits/chosen": 1.6496421098709106, "logits/rejected": 1.6191202402114868, "logps/chosen": -280.70977783203125, "logps/ref_chosen": -274.2210998535156, "logps/ref_rejected": -264.33465576171875, "logps/rejected": -299.73309326171875, "loss": 4.3552, "margin_dpo/margin_mean": 28.90974998474121, "margin_dpo/margin_std": 45.983367919921875, "step": 410 }, { "KL/chosen_KL_mean": -5.183387756347656, "KL/mean": -21.776199340820312, "KL/rejected_KL_mean": -38.3690071105957, "KL/std": 39.54701232910156, "epoch": 0.8607329842931937, "fcm_dpo/beta": 0.018332835286855698, "fcm_dpo/delta": -0.0687853991985321, "fcm_dpo/margin": 33.185630798339844, "fcm_dpo/q_t": 0.3696379065513611, "grad_norm": 40.225460052490234, "learning_rate": 2.9492720416985e-08, "logits/chosen": 1.442059874534607, "logits/rejected": 1.4909590482711792, "logps/chosen": -278.0513916015625, "logps/ref_chosen": -272.86798095703125, "logps/ref_rejected": -250.35772705078125, "logps/rejected": -288.72674560546875, "loss": 4.0559, "margin_dpo/margin_mean": 33.185630798339844, "margin_dpo/margin_std": 41.469581604003906, "step": 411 }, { "KL/chosen_KL_mean": -10.896099090576172, "KL/mean": -25.94062042236328, "KL/rejected_KL_mean": -40.985145568847656, "KL/std": 39.2274169921875, "epoch": 0.86282722513089, "fcm_dpo/beta": 0.017277266830205917, "fcm_dpo/delta": -0.023805608972907066, "fcm_dpo/margin": 30.089038848876953, "fcm_dpo/q_t": 0.3888762593269348, "grad_norm": 36.48346710205078, "learning_rate": 2.863599358669755e-08, "logits/chosen": 1.4429807662963867, "logits/rejected": 1.544088363647461, "logps/chosen": -278.8153076171875, "logps/ref_chosen": -267.919189453125, "logps/ref_rejected": -271.73162841796875, "logps/rejected": -312.7167663574219, "loss": 4.3299, "margin_dpo/margin_mean": 30.089040756225586, "margin_dpo/margin_std": 45.334686279296875, "step": 412 }, { "KL/chosen_KL_mean": -7.543354034423828, "KL/mean": -20.802610397338867, "KL/rejected_KL_mean": -34.06187057495117, "KL/std": 37.07807922363281, "epoch": 0.8649214659685864, "fcm_dpo/beta": 0.018553953617811203, "fcm_dpo/delta": 0.11212041229009628, "fcm_dpo/margin": 26.518516540527344, "fcm_dpo/q_t": 0.3955715298652649, "grad_norm": 47.803489685058594, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 1.1849263906478882, "logits/rejected": 1.2352192401885986, "logps/chosen": -272.50140380859375, "logps/ref_chosen": -264.95806884765625, "logps/ref_rejected": -267.40093994140625, "logps/rejected": -301.46282958984375, "loss": 4.3189, "margin_dpo/margin_mean": 26.518516540527344, "margin_dpo/margin_std": 39.869110107421875, "step": 413 }, { "KL/chosen_KL_mean": -10.233379364013672, "KL/mean": -25.646608352661133, "KL/rejected_KL_mean": -41.059837341308594, "KL/std": 39.12742614746094, "epoch": 0.8670157068062827, "fcm_dpo/beta": 0.01886744238436222, "fcm_dpo/delta": -0.043764229863882065, "fcm_dpo/margin": 30.826452255249023, "fcm_dpo/q_t": 0.3794119358062744, "grad_norm": 66.53843688964844, "learning_rate": 2.6958198472749717e-08, "logits/chosen": 1.5521750450134277, "logits/rejected": 1.6116161346435547, "logps/chosen": -300.2391357421875, "logps/ref_chosen": -290.00579833984375, "logps/ref_rejected": -266.8810119628906, "logps/rejected": -307.94085693359375, "loss": 4.2611, "margin_dpo/margin_mean": 30.82645034790039, "margin_dpo/margin_std": 46.020320892333984, "step": 414 }, { "KL/chosen_KL_mean": -5.656030654907227, "KL/mean": -23.226139068603516, "KL/rejected_KL_mean": -40.796260833740234, "KL/std": 38.863365173339844, "epoch": 0.8691099476439791, "fcm_dpo/beta": 0.018389534205198288, "fcm_dpo/delta": -0.049901485443115234, "fcm_dpo/margin": 35.140228271484375, "fcm_dpo/q_t": 0.36399033665657043, "grad_norm": 39.664424896240234, "learning_rate": 2.613722016414943e-08, "logits/chosen": 1.4949822425842285, "logits/rejected": 1.5791767835617065, "logps/chosen": -295.467041015625, "logps/ref_chosen": -289.81103515625, "logps/ref_rejected": -278.2758483886719, "logps/rejected": -319.0721435546875, "loss": 3.9743, "margin_dpo/margin_mean": 35.140228271484375, "margin_dpo/margin_std": 44.041748046875, "step": 415 }, { "KL/chosen_KL_mean": -4.292598724365234, "KL/mean": -21.877098083496094, "KL/rejected_KL_mean": -39.46160125732422, "KL/std": 38.75770568847656, "epoch": 0.8712041884816754, "fcm_dpo/beta": 0.016837069764733315, "fcm_dpo/delta": -0.04827772453427315, "fcm_dpo/margin": 35.16900634765625, "fcm_dpo/q_t": 0.37268009781837463, "grad_norm": 36.53273391723633, "learning_rate": 2.5328246937043525e-08, "logits/chosen": 1.546844482421875, "logits/rejected": 1.51033616065979, "logps/chosen": -308.94140625, "logps/ref_chosen": -304.6488037109375, "logps/ref_rejected": -264.2703857421875, "logps/rejected": -303.73199462890625, "loss": 4.0881, "margin_dpo/margin_mean": 35.16900634765625, "margin_dpo/margin_std": 45.07542419433594, "step": 416 }, { "KL/chosen_KL_mean": -6.947269439697266, "KL/mean": -22.328693389892578, "KL/rejected_KL_mean": -37.71012878417969, "KL/std": 37.7503662109375, "epoch": 0.8732984293193717, "fcm_dpo/beta": 0.01717413403093815, "fcm_dpo/delta": 0.016851838678121567, "fcm_dpo/margin": 30.76287078857422, "fcm_dpo/q_t": 0.3861393928527832, "grad_norm": 41.19669723510742, "learning_rate": 2.4531322174210973e-08, "logits/chosen": 1.467063069343567, "logits/rejected": 1.5336133241653442, "logps/chosen": -308.836181640625, "logps/ref_chosen": -301.8889465332031, "logps/ref_rejected": -275.0347595214844, "logps/rejected": -312.744873046875, "loss": 4.2633, "margin_dpo/margin_mean": 30.762866973876953, "margin_dpo/margin_std": 43.28641891479492, "step": 417 }, { "KL/chosen_KL_mean": -12.334508895874023, "KL/mean": -26.311321258544922, "KL/rejected_KL_mean": -40.288143157958984, "KL/std": 39.47275161743164, "epoch": 0.875392670157068, "fcm_dpo/beta": 0.017377493903040886, "fcm_dpo/delta": -0.01183069497346878, "fcm_dpo/margin": 27.953641891479492, "fcm_dpo/q_t": 0.39519989490509033, "grad_norm": 50.294490814208984, "learning_rate": 2.3746488612308295e-08, "logits/chosen": 1.4132051467895508, "logits/rejected": 1.4223443269729614, "logps/chosen": -281.58184814453125, "logps/ref_chosen": -269.2473449707031, "logps/ref_rejected": -275.5192565917969, "logps/rejected": -315.8074035644531, "loss": 4.3995, "margin_dpo/margin_mean": 27.953643798828125, "margin_dpo/margin_std": 42.52241897583008, "step": 418 }, { "KL/chosen_KL_mean": -11.854555130004883, "KL/mean": -30.271154403686523, "KL/rejected_KL_mean": -48.68775939941406, "KL/std": 41.765953063964844, "epoch": 0.8774869109947644, "fcm_dpo/beta": 0.016464462503790855, "fcm_dpo/delta": -0.05605008453130722, "fcm_dpo/margin": 36.83320999145508, "fcm_dpo/q_t": 0.36880791187286377, "grad_norm": 34.323795318603516, "learning_rate": 2.297378833957761e-08, "logits/chosen": 1.4475170373916626, "logits/rejected": 1.3895591497421265, "logps/chosen": -303.01495361328125, "logps/ref_chosen": -291.1604309082031, "logps/ref_rejected": -245.18325805664062, "logps/rejected": -293.8710021972656, "loss": 4.075, "margin_dpo/margin_mean": 36.83320999145508, "margin_dpo/margin_std": 47.66911697387695, "step": 419 }, { "KL/chosen_KL_mean": -10.157136917114258, "KL/mean": -27.602710723876953, "KL/rejected_KL_mean": -45.048274993896484, "KL/std": 40.69925308227539, "epoch": 0.8795811518324608, "fcm_dpo/beta": 0.016606658697128296, "fcm_dpo/delta": 0.020674100145697594, "fcm_dpo/margin": 34.891143798828125, "fcm_dpo/q_t": 0.37389612197875977, "grad_norm": 38.67104721069336, "learning_rate": 2.2213262793589482e-08, "logits/chosen": 1.411975622177124, "logits/rejected": 1.364816427230835, "logps/chosen": -268.6269836425781, "logps/ref_chosen": -258.4698181152344, "logps/ref_rejected": -242.916015625, "logps/rejected": -287.96429443359375, "loss": 4.1057, "margin_dpo/margin_mean": 34.891143798828125, "margin_dpo/margin_std": 45.79709243774414, "step": 420 }, { "KL/chosen_KL_mean": -3.941652297973633, "KL/mean": -20.879179000854492, "KL/rejected_KL_mean": -37.816707611083984, "KL/std": 37.67058563232422, "epoch": 0.881675392670157, "fcm_dpo/beta": 0.01673482172191143, "fcm_dpo/delta": 0.034025367349386215, "fcm_dpo/margin": 33.87507247924805, "fcm_dpo/q_t": 0.37542274594306946, "grad_norm": 33.99753189086914, "learning_rate": 2.1464952759020856e-08, "logits/chosen": 1.2181888818740845, "logits/rejected": 1.1769070625305176, "logps/chosen": -294.0824279785156, "logps/ref_chosen": -290.1407775878906, "logps/ref_rejected": -239.65676879882812, "logps/rejected": -277.47344970703125, "loss": 4.0449, "margin_dpo/margin_mean": 33.87507247924805, "margin_dpo/margin_std": 40.482669830322266, "step": 421 }, { "KL/chosen_KL_mean": -8.990995407104492, "KL/mean": -22.929763793945312, "KL/rejected_KL_mean": -36.8685302734375, "KL/std": 37.68763732910156, "epoch": 0.8837696335078534, "fcm_dpo/beta": 0.018041063100099564, "fcm_dpo/delta": 0.10089747607707977, "fcm_dpo/margin": 27.877517700195312, "fcm_dpo/q_t": 0.392340749502182, "grad_norm": 46.466827392578125, "learning_rate": 2.07288983654679e-08, "logits/chosen": 1.280303716659546, "logits/rejected": 1.326034426689148, "logps/chosen": -285.9270324707031, "logps/ref_chosen": -276.93603515625, "logps/ref_rejected": -250.398681640625, "logps/rejected": -287.2672119140625, "loss": 4.4266, "margin_dpo/margin_mean": 27.877519607543945, "margin_dpo/margin_std": 45.0127067565918, "step": 422 }, { "KL/chosen_KL_mean": -6.769617080688477, "KL/mean": -24.917064666748047, "KL/rejected_KL_mean": -43.06451416015625, "KL/std": 40.1577262878418, "epoch": 0.8858638743455497, "fcm_dpo/beta": 0.018019232898950577, "fcm_dpo/delta": -0.05869268625974655, "fcm_dpo/margin": 36.29489517211914, "fcm_dpo/q_t": 0.3627795875072479, "grad_norm": 34.83606719970703, "learning_rate": 2.0005139085293942e-08, "logits/chosen": 1.4147975444793701, "logits/rejected": 1.4877537488937378, "logps/chosen": -292.19970703125, "logps/ref_chosen": -285.43011474609375, "logps/ref_rejected": -258.0743408203125, "logps/rejected": -301.13885498046875, "loss": 3.9663, "margin_dpo/margin_mean": 36.294891357421875, "margin_dpo/margin_std": 45.899147033691406, "step": 423 }, { "KL/chosen_KL_mean": -4.655523300170898, "KL/mean": -21.756614685058594, "KL/rejected_KL_mean": -38.85771179199219, "KL/std": 37.40180969238281, "epoch": 0.8879581151832461, "fcm_dpo/beta": 0.017788197845220566, "fcm_dpo/delta": -0.010666869580745697, "fcm_dpo/margin": 34.20218276977539, "fcm_dpo/q_t": 0.3682326078414917, "grad_norm": 37.221805572509766, "learning_rate": 1.9293713731512673e-08, "logits/chosen": 1.4349040985107422, "logits/rejected": 1.4128334522247314, "logps/chosen": -304.0005187988281, "logps/ref_chosen": -299.3450012207031, "logps/ref_rejected": -244.60169982910156, "logps/rejected": -283.45941162109375, "loss": 4.0036, "margin_dpo/margin_mean": 34.202178955078125, "margin_dpo/margin_std": 41.48379898071289, "step": 424 }, { "KL/chosen_KL_mean": -8.957204818725586, "KL/mean": -23.96082305908203, "KL/rejected_KL_mean": -38.96443176269531, "KL/std": 39.549564361572266, "epoch": 0.8900523560209425, "fcm_dpo/beta": 0.017491746693849564, "fcm_dpo/delta": -0.010492067784070969, "fcm_dpo/margin": 30.007234573364258, "fcm_dpo/q_t": 0.38694730401039124, "grad_norm": 35.75682830810547, "learning_rate": 1.8594660455706763e-08, "logits/chosen": 1.5854737758636475, "logits/rejected": 1.6753469705581665, "logps/chosen": -281.9069519042969, "logps/ref_chosen": -272.9497375488281, "logps/ref_rejected": -274.6335754394531, "logps/rejected": -313.5980224609375, "loss": 4.2994, "margin_dpo/margin_mean": 30.007234573364258, "margin_dpo/margin_std": 42.59223175048828, "step": 425 }, { "KL/chosen_KL_mean": -5.635072708129883, "KL/mean": -22.328262329101562, "KL/rejected_KL_mean": -39.02144241333008, "KL/std": 39.1007194519043, "epoch": 0.8921465968586387, "fcm_dpo/beta": 0.01741386204957962, "fcm_dpo/delta": -0.023887064307928085, "fcm_dpo/margin": 33.386375427246094, "fcm_dpo/q_t": 0.37471699714660645, "grad_norm": 38.75654983520508, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 1.4561810493469238, "logits/rejected": 1.490870475769043, "logps/chosen": -287.5766296386719, "logps/ref_chosen": -281.9415588378906, "logps/ref_rejected": -253.20486450195312, "logps/rejected": -292.2262878417969, "loss": 4.0885, "margin_dpo/margin_mean": 33.386375427246094, "margin_dpo/margin_std": 43.17636489868164, "step": 426 }, { "KL/chosen_KL_mean": -4.920648574829102, "KL/mean": -26.763418197631836, "KL/rejected_KL_mean": -48.6061897277832, "KL/std": 40.561283111572266, "epoch": 0.8942408376963351, "fcm_dpo/beta": 0.01672692783176899, "fcm_dpo/delta": -0.1363137662410736, "fcm_dpo/margin": 43.68552780151367, "fcm_dpo/q_t": 0.34173884987831116, "grad_norm": 33.15166473388672, "learning_rate": 1.7233819424956247e-08, "logits/chosen": 1.4568595886230469, "logits/rejected": 1.4481300115585327, "logps/chosen": -285.5372619628906, "logps/ref_chosen": -280.6166076660156, "logps/ref_rejected": -263.7532958984375, "logps/rejected": -312.3595275878906, "loss": 3.6473, "margin_dpo/margin_mean": 43.68552780151367, "margin_dpo/margin_std": 42.516300201416016, "step": 427 }, { "KL/chosen_KL_mean": -7.886140823364258, "KL/mean": -28.404512405395508, "KL/rejected_KL_mean": -48.922889709472656, "KL/std": 40.87632369995117, "epoch": 0.8963350785340314, "fcm_dpo/beta": 0.015319863334298134, "fcm_dpo/delta": -0.03134823217988014, "fcm_dpo/margin": 41.036746978759766, "fcm_dpo/q_t": 0.36050668358802795, "grad_norm": 36.520240783691406, "learning_rate": 1.6572104647786245e-08, "logits/chosen": 1.235212802886963, "logits/rejected": 1.3253754377365112, "logps/chosen": -294.0483703613281, "logps/ref_chosen": -286.1622314453125, "logps/ref_rejected": -273.3803405761719, "logps/rejected": -322.30322265625, "loss": 3.8141, "margin_dpo/margin_mean": 41.036746978759766, "margin_dpo/margin_std": 41.604248046875, "step": 428 }, { "KL/chosen_KL_mean": -8.069477081298828, "KL/mean": -24.538747787475586, "KL/rejected_KL_mean": -41.008018493652344, "KL/std": 38.877811431884766, "epoch": 0.8984293193717278, "fcm_dpo/beta": 0.015415645204484463, "fcm_dpo/delta": 0.04732373356819153, "fcm_dpo/margin": 32.93855667114258, "fcm_dpo/q_t": 0.3912946581840515, "grad_norm": 29.93174934387207, "learning_rate": 1.5922907900227017e-08, "logits/chosen": 1.491369605064392, "logits/rejected": 1.491215705871582, "logps/chosen": -274.40606689453125, "logps/ref_chosen": -266.3365783691406, "logps/ref_rejected": -255.01956176757812, "logps/rejected": -296.027587890625, "loss": 4.3693, "margin_dpo/margin_mean": 32.93855667114258, "margin_dpo/margin_std": 50.389549255371094, "step": 429 }, { "KL/chosen_KL_mean": -7.625219345092773, "KL/mean": -22.037118911743164, "KL/rejected_KL_mean": -36.44902038574219, "KL/std": 37.850013732910156, "epoch": 0.900523560209424, "fcm_dpo/beta": 0.016457416117191315, "fcm_dpo/delta": 0.06396263837814331, "fcm_dpo/margin": 28.82379722595215, "fcm_dpo/q_t": 0.39826422929763794, "grad_norm": 66.31047821044922, "learning_rate": 1.5286263996730026e-08, "logits/chosen": 1.4523224830627441, "logits/rejected": 1.4776817560195923, "logps/chosen": -289.776611328125, "logps/ref_chosen": -282.1513977050781, "logps/ref_rejected": -263.4792175292969, "logps/rejected": -299.9282531738281, "loss": 4.4131, "margin_dpo/margin_mean": 28.823795318603516, "margin_dpo/margin_std": 44.25785827636719, "step": 430 }, { "KL/chosen_KL_mean": -9.346113204956055, "KL/mean": -23.709529876708984, "KL/rejected_KL_mean": -38.072940826416016, "KL/std": 39.246543884277344, "epoch": 0.9026178010471204, "fcm_dpo/beta": 0.017523720860481262, "fcm_dpo/delta": 0.09749704599380493, "fcm_dpo/margin": 28.726835250854492, "fcm_dpo/q_t": 0.3940257728099823, "grad_norm": 41.90385437011719, "learning_rate": 1.4662207078575684e-08, "logits/chosen": 1.5903630256652832, "logits/rejected": 1.750954270362854, "logps/chosen": -278.0592346191406, "logps/ref_chosen": -268.713134765625, "logps/ref_rejected": -266.13446044921875, "logps/rejected": -304.2074279785156, "loss": 4.345, "margin_dpo/margin_mean": 28.726835250854492, "margin_dpo/margin_std": 44.24424362182617, "step": 431 }, { "KL/chosen_KL_mean": -5.156131744384766, "KL/mean": -21.788583755493164, "KL/rejected_KL_mean": -38.42102813720703, "KL/std": 38.79159164428711, "epoch": 0.9047120418848168, "fcm_dpo/beta": 0.017326056957244873, "fcm_dpo/delta": 0.02436162531375885, "fcm_dpo/margin": 33.264896392822266, "fcm_dpo/q_t": 0.37820175290107727, "grad_norm": 36.16777801513672, "learning_rate": 1.40507706120426e-08, "logits/chosen": 1.6775932312011719, "logits/rejected": 1.7190947532653809, "logps/chosen": -289.1403503417969, "logps/ref_chosen": -283.9842224121094, "logps/ref_rejected": -253.9568634033203, "logps/rejected": -292.3779296875, "loss": 4.1458, "margin_dpo/margin_mean": 33.26490020751953, "margin_dpo/margin_std": 45.536964416503906, "step": 432 }, { "KL/chosen_KL_mean": -13.55865478515625, "KL/mean": -27.595563888549805, "KL/rejected_KL_mean": -41.632476806640625, "KL/std": 38.84959411621094, "epoch": 0.9068062827225131, "fcm_dpo/beta": 0.01837783120572567, "fcm_dpo/delta": 0.08972224593162537, "fcm_dpo/margin": 28.07382583618164, "fcm_dpo/q_t": 0.38659048080444336, "grad_norm": 41.5974235534668, "learning_rate": 1.345198738661285e-08, "logits/chosen": 1.3054968118667603, "logits/rejected": 1.3207252025604248, "logps/chosen": -256.0018310546875, "logps/ref_chosen": -242.4431915283203, "logps/ref_rejected": -252.87979125976562, "logps/rejected": -294.51226806640625, "loss": 4.3669, "margin_dpo/margin_mean": 28.073823928833008, "margin_dpo/margin_std": 43.10464096069336, "step": 433 }, { "KL/chosen_KL_mean": -10.705320358276367, "KL/mean": -25.866989135742188, "KL/rejected_KL_mean": -41.028648376464844, "KL/std": 38.6884651184082, "epoch": 0.9089005235602095, "fcm_dpo/beta": 0.019349027425050735, "fcm_dpo/delta": 0.012435603886842728, "fcm_dpo/margin": 30.323333740234375, "fcm_dpo/q_t": 0.378314733505249, "grad_norm": 42.35694885253906, "learning_rate": 1.2865889513213628e-08, "logits/chosen": 1.5045528411865234, "logits/rejected": 1.4233698844909668, "logps/chosen": -298.989501953125, "logps/ref_chosen": -288.2841796875, "logps/ref_rejected": -252.9123992919922, "logps/rejected": -293.9410400390625, "loss": 4.2264, "margin_dpo/margin_mean": 30.323333740234375, "margin_dpo/margin_std": 44.75054931640625, "step": 434 }, { "KL/chosen_KL_mean": -5.947370529174805, "KL/mean": -20.233585357666016, "KL/rejected_KL_mean": -34.519805908203125, "KL/std": 38.588722229003906, "epoch": 0.9109947643979057, "fcm_dpo/beta": 0.01993248052895069, "fcm_dpo/delta": 0.029382452368736267, "fcm_dpo/margin": 28.572433471679688, "fcm_dpo/q_t": 0.3792368769645691, "grad_norm": 58.921566009521484, "learning_rate": 1.2292508422495157e-08, "logits/chosen": 1.4650075435638428, "logits/rejected": 1.547285795211792, "logps/chosen": -254.9622039794922, "logps/ref_chosen": -249.01483154296875, "logps/ref_rejected": -243.81192016601562, "logps/rejected": -278.3316955566406, "loss": 4.2578, "margin_dpo/margin_mean": 28.572433471679688, "margin_dpo/margin_std": 41.97202682495117, "step": 435 }, { "KL/chosen_KL_mean": -10.389013290405273, "KL/mean": -21.403196334838867, "KL/rejected_KL_mean": -32.417381286621094, "KL/std": 40.2339973449707, "epoch": 0.9130890052356021, "fcm_dpo/beta": 0.021259302273392677, "fcm_dpo/delta": 0.1383521556854248, "fcm_dpo/margin": 22.028366088867188, "fcm_dpo/q_t": 0.40718600153923035, "grad_norm": 53.56985855102539, "learning_rate": 1.1731874863145142e-08, "logits/chosen": 1.462636113166809, "logits/rejected": 1.5538408756256104, "logps/chosen": -323.319091796875, "logps/ref_chosen": -312.9300537109375, "logps/ref_rejected": -296.2721862792969, "logps/rejected": -328.6895751953125, "loss": 4.6678, "margin_dpo/margin_mean": 22.028366088867188, "margin_dpo/margin_std": 43.784568786621094, "step": 436 }, { "KL/chosen_KL_mean": -6.537776947021484, "KL/mean": -24.114627838134766, "KL/rejected_KL_mean": -41.691471099853516, "KL/std": 41.110382080078125, "epoch": 0.9151832460732985, "fcm_dpo/beta": 0.020916461944580078, "fcm_dpo/delta": -0.1481207013130188, "fcm_dpo/margin": 35.153690338134766, "fcm_dpo/q_t": 0.35055267810821533, "grad_norm": 39.32798385620117, "learning_rate": 1.118401890024001e-08, "logits/chosen": 1.6977591514587402, "logits/rejected": 1.745481014251709, "logps/chosen": -274.8516845703125, "logps/ref_chosen": -268.31390380859375, "logps/ref_rejected": -268.7603759765625, "logps/rejected": -310.4518737792969, "loss": 3.9117, "margin_dpo/margin_mean": 35.153690338134766, "margin_dpo/margin_std": 44.421180725097656, "step": 437 }, { "KL/chosen_KL_mean": -13.689395904541016, "KL/mean": -22.959537506103516, "KL/rejected_KL_mean": -32.22968292236328, "KL/std": 38.005943298339844, "epoch": 0.9172774869109948, "fcm_dpo/beta": 0.02057368867099285, "fcm_dpo/delta": 0.08883843570947647, "fcm_dpo/margin": 18.540285110473633, "fcm_dpo/q_t": 0.42163756489753723, "grad_norm": 90.26850891113281, "learning_rate": 1.06489699136324e-08, "logits/chosen": 1.33683443069458, "logits/rejected": 1.407370686531067, "logps/chosen": -268.9212646484375, "logps/ref_chosen": -255.23190307617188, "logps/ref_rejected": -241.1186981201172, "logps/rejected": -273.348388671875, "loss": 4.9188, "margin_dpo/margin_mean": 18.540285110473633, "margin_dpo/margin_std": 43.72941970825195, "step": 438 }, { "KL/chosen_KL_mean": -10.832399368286133, "KL/mean": -25.905786514282227, "KL/rejected_KL_mean": -40.979164123535156, "KL/std": 37.94921112060547, "epoch": 0.9193717277486911, "fcm_dpo/beta": 0.02108529582619667, "fcm_dpo/delta": -0.03847494348883629, "fcm_dpo/margin": 30.146764755249023, "fcm_dpo/q_t": 0.37095460295677185, "grad_norm": 40.88623046875, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 1.2603108882904053, "logits/rejected": 1.2645988464355469, "logps/chosen": -261.812744140625, "logps/ref_chosen": -250.9803466796875, "logps/ref_rejected": -239.9314422607422, "logps/rejected": -280.9106140136719, "loss": 4.1643, "margin_dpo/margin_mean": 30.146760940551758, "margin_dpo/margin_std": 43.66949462890625, "step": 439 }, { "KL/chosen_KL_mean": -13.518768310546875, "KL/mean": -25.87021827697754, "KL/rejected_KL_mean": -38.22166442871094, "KL/std": 38.02260208129883, "epoch": 0.9214659685863874, "fcm_dpo/beta": 0.02056608349084854, "fcm_dpo/delta": -0.03422413766384125, "fcm_dpo/margin": 24.702899932861328, "fcm_dpo/q_t": 0.39521169662475586, "grad_norm": 50.304046630859375, "learning_rate": 9.617406953185136e-09, "logits/chosen": 1.4519972801208496, "logits/rejected": 1.371519684791565, "logps/chosen": -314.52154541015625, "logps/ref_chosen": -301.0027770996094, "logps/ref_rejected": -263.628662109375, "logps/rejected": -301.850341796875, "loss": 4.5864, "margin_dpo/margin_mean": 24.70290184020996, "margin_dpo/margin_std": 44.9271240234375, "step": 440 }, { "KL/chosen_KL_mean": -6.745321273803711, "KL/mean": -24.43891143798828, "KL/rejected_KL_mean": -42.132503509521484, "KL/std": 39.826961517333984, "epoch": 0.9235602094240838, "fcm_dpo/beta": 0.019525915384292603, "fcm_dpo/delta": -0.09813890606164932, "fcm_dpo/margin": 35.387176513671875, "fcm_dpo/q_t": 0.35289281606674194, "grad_norm": 41.78523254394531, "learning_rate": 9.12094829893642e-09, "logits/chosen": 1.5125982761383057, "logits/rejected": 1.5976024866104126, "logps/chosen": -307.5940856933594, "logps/ref_chosen": -300.8487854003906, "logps/ref_rejected": -296.7118835449219, "logps/rejected": -338.8443908691406, "loss": 3.8317, "margin_dpo/margin_mean": 35.38717269897461, "margin_dpo/margin_std": 38.58313751220703, "step": 441 }, { "KL/chosen_KL_mean": -12.945383071899414, "KL/mean": -26.59016990661621, "KL/rejected_KL_mean": -40.234962463378906, "KL/std": 37.29020309448242, "epoch": 0.9256544502617801, "fcm_dpo/beta": 0.01888934150338173, "fcm_dpo/delta": 0.09072023630142212, "fcm_dpo/margin": 27.289583206176758, "fcm_dpo/q_t": 0.3936287760734558, "grad_norm": 54.41708755493164, "learning_rate": 8.637407257200496e-09, "logits/chosen": 1.6090409755706787, "logits/rejected": 1.670328140258789, "logps/chosen": -284.59381103515625, "logps/ref_chosen": -271.6484069824219, "logps/ref_rejected": -238.21197509765625, "logps/rejected": -278.4469299316406, "loss": 4.5528, "margin_dpo/margin_mean": 27.289583206176758, "margin_dpo/margin_std": 49.01494598388672, "step": 442 }, { "KL/chosen_KL_mean": -9.426856994628906, "KL/mean": -23.713916778564453, "KL/rejected_KL_mean": -38.0009765625, "KL/std": 37.76384735107422, "epoch": 0.9277486910994764, "fcm_dpo/beta": 0.020260203629732132, "fcm_dpo/delta": 0.01686916872859001, "fcm_dpo/margin": 28.57412338256836, "fcm_dpo/q_t": 0.37880489230155945, "grad_norm": 45.84555435180664, "learning_rate": 8.166809758815895e-09, "logits/chosen": 1.3293415307998657, "logits/rejected": 1.3950724601745605, "logps/chosen": -275.723388671875, "logps/ref_chosen": -266.2965087890625, "logps/ref_rejected": -263.7326965332031, "logps/rejected": -301.733642578125, "loss": 4.2638, "margin_dpo/margin_mean": 28.574121475219727, "margin_dpo/margin_std": 41.3916015625, "step": 443 }, { "KL/chosen_KL_mean": -10.304304122924805, "KL/mean": -24.973583221435547, "KL/rejected_KL_mean": -39.64287185668945, "KL/std": 39.80085754394531, "epoch": 0.9298429319371728, "fcm_dpo/beta": 0.020134340971708298, "fcm_dpo/delta": 0.008157186210155487, "fcm_dpo/margin": 29.338579177856445, "fcm_dpo/q_t": 0.3796614408493042, "grad_norm": 50.080379486083984, "learning_rate": 7.709181040498253e-09, "logits/chosen": 1.194259762763977, "logits/rejected": 1.1803237199783325, "logps/chosen": -302.6650390625, "logps/ref_chosen": -292.36077880859375, "logps/ref_rejected": -264.35662841796875, "logps/rejected": -303.99951171875, "loss": 4.316, "margin_dpo/margin_mean": 29.338579177856445, "margin_dpo/margin_std": 46.49433135986328, "step": 444 }, { "KL/chosen_KL_mean": -9.584835052490234, "KL/mean": -22.41143035888672, "KL/rejected_KL_mean": -35.23802185058594, "KL/std": 38.90290832519531, "epoch": 0.9319371727748691, "fcm_dpo/beta": 0.020845849066972733, "fcm_dpo/delta": -0.007416628301143646, "fcm_dpo/margin": 25.65318489074707, "fcm_dpo/q_t": 0.39004504680633545, "grad_norm": 45.288604736328125, "learning_rate": 7.2645456434869965e-09, "logits/chosen": 1.257015585899353, "logits/rejected": 1.344290852546692, "logps/chosen": -256.30462646484375, "logps/ref_chosen": -246.71978759765625, "logps/ref_rejected": -261.339599609375, "logps/rejected": -296.57763671875, "loss": 4.5136, "margin_dpo/margin_mean": 25.653186798095703, "margin_dpo/margin_std": 42.89739227294922, "step": 445 }, { "KL/chosen_KL_mean": -5.0773468017578125, "KL/mean": -21.905426025390625, "KL/rejected_KL_mean": -38.733516693115234, "KL/std": 38.742462158203125, "epoch": 0.9340314136125655, "fcm_dpo/beta": 0.01960984244942665, "fcm_dpo/delta": -0.06857600808143616, "fcm_dpo/margin": 33.65616226196289, "fcm_dpo/q_t": 0.36124396324157715, "grad_norm": 61.7397575378418, "learning_rate": 6.832927412229017e-09, "logits/chosen": 1.2315505743026733, "logits/rejected": 1.2126425504684448, "logps/chosen": -303.6360778808594, "logps/ref_chosen": -298.5587158203125, "logps/ref_rejected": -264.5877685546875, "logps/rejected": -303.3212890625, "loss": 4.0676, "margin_dpo/margin_mean": 33.656158447265625, "margin_dpo/margin_std": 44.581233978271484, "step": 446 }, { "KL/chosen_KL_mean": -3.0303268432617188, "KL/mean": -20.40411949157715, "KL/rejected_KL_mean": -37.77790832519531, "KL/std": 36.89284133911133, "epoch": 0.9361256544502617, "fcm_dpo/beta": 0.018508557230234146, "fcm_dpo/delta": -0.0480596087872982, "fcm_dpo/margin": 34.747581481933594, "fcm_dpo/q_t": 0.36139318346977234, "grad_norm": 36.14704895019531, "learning_rate": 6.414349493100129e-09, "logits/chosen": 1.2298775911331177, "logits/rejected": 1.1924989223480225, "logps/chosen": -252.883544921875, "logps/ref_chosen": -249.8532257080078, "logps/ref_rejected": -233.2117156982422, "logps/rejected": -270.9896240234375, "loss": 3.8808, "margin_dpo/margin_mean": 34.747581481933594, "margin_dpo/margin_std": 39.805580139160156, "step": 447 }, { "KL/chosen_KL_mean": -6.709333419799805, "KL/mean": -21.88568687438965, "KL/rejected_KL_mean": -37.062042236328125, "KL/std": 38.11823272705078, "epoch": 0.9382198952879581, "fcm_dpo/beta": 0.018123872578144073, "fcm_dpo/delta": -0.01662895828485489, "fcm_dpo/margin": 30.352706909179688, "fcm_dpo/q_t": 0.3823128640651703, "grad_norm": 52.687721252441406, "learning_rate": 6.0088343331638756e-09, "logits/chosen": 1.4540094137191772, "logits/rejected": 1.6012420654296875, "logps/chosen": -265.5309753417969, "logps/ref_chosen": -258.8216247558594, "logps/ref_rejected": -260.4953308105469, "logps/rejected": -297.557373046875, "loss": 4.1937, "margin_dpo/margin_mean": 30.352706909179688, "margin_dpo/margin_std": 41.40434265136719, "step": 448 }, { "KL/chosen_KL_mean": -5.103841781616211, "KL/mean": -21.434587478637695, "KL/rejected_KL_mean": -37.76533508300781, "KL/std": 37.36093521118164, "epoch": 0.9403141361256544, "fcm_dpo/beta": 0.018367934972047806, "fcm_dpo/delta": -0.001986050046980381, "fcm_dpo/margin": 32.661495208740234, "fcm_dpo/q_t": 0.3707224428653717, "grad_norm": 43.69072723388672, "learning_rate": 5.616403678967624e-09, "logits/chosen": 1.5900778770446777, "logits/rejected": 1.4752076864242554, "logps/chosen": -329.3203430175781, "logps/ref_chosen": -324.2165222167969, "logps/ref_rejected": -237.05453491210938, "logps/rejected": -274.81988525390625, "loss": 4.0803, "margin_dpo/margin_mean": 32.661495208740234, "margin_dpo/margin_std": 42.267757415771484, "step": 449 }, { "KL/chosen_KL_mean": -12.929838180541992, "KL/mean": -27.109384536743164, "KL/rejected_KL_mean": -41.28893280029297, "KL/std": 39.57197570800781, "epoch": 0.9424083769633508, "fcm_dpo/beta": 0.01822546496987343, "fcm_dpo/delta": 0.0388304740190506, "fcm_dpo/margin": 28.359092712402344, "fcm_dpo/q_t": 0.38950634002685547, "grad_norm": 49.931846618652344, "learning_rate": 5.2370785753763356e-09, "logits/chosen": 1.437721610069275, "logits/rejected": 1.4123148918151855, "logps/chosen": -291.3310852050781, "logps/ref_chosen": -278.4012451171875, "logps/ref_rejected": -247.72671508789062, "logps/rejected": -289.0156555175781, "loss": 4.2995, "margin_dpo/margin_mean": 28.35909080505371, "margin_dpo/margin_std": 42.51638412475586, "step": 450 }, { "KL/chosen_KL_mean": -7.553045272827148, "KL/mean": -23.329837799072266, "KL/rejected_KL_mean": -39.10662841796875, "KL/std": 38.305355072021484, "epoch": 0.9445026178010472, "fcm_dpo/beta": 0.018385058268904686, "fcm_dpo/delta": -0.04042164981365204, "fcm_dpo/margin": 31.5535888671875, "fcm_dpo/q_t": 0.37636423110961914, "grad_norm": 44.744327545166016, "learning_rate": 4.8708793644441086e-09, "logits/chosen": 1.3844701051712036, "logits/rejected": 1.392881989479065, "logps/chosen": -299.6476745605469, "logps/ref_chosen": -292.0946350097656, "logps/ref_rejected": -252.24237060546875, "logps/rejected": -291.3489990234375, "loss": 4.1547, "margin_dpo/margin_mean": 31.5535888671875, "margin_dpo/margin_std": 41.81111526489258, "step": 451 }, { "KL/chosen_KL_mean": -3.678445816040039, "KL/mean": -19.315462112426758, "KL/rejected_KL_mean": -34.952491760253906, "KL/std": 39.858558654785156, "epoch": 0.9465968586387434, "fcm_dpo/beta": 0.01781679131090641, "fcm_dpo/delta": 0.04480419307947159, "fcm_dpo/margin": 31.274038314819336, "fcm_dpo/q_t": 0.38180726766586304, "grad_norm": 33.700416564941406, "learning_rate": 4.517825684323323e-09, "logits/chosen": 1.3815429210662842, "logits/rejected": 1.4320133924484253, "logps/chosen": -294.28924560546875, "logps/ref_chosen": -290.61077880859375, "logps/ref_rejected": -279.8963928222656, "logps/rejected": -314.8488464355469, "loss": 4.1625, "margin_dpo/margin_mean": 31.2740421295166, "margin_dpo/margin_std": 41.90071487426758, "step": 452 }, { "KL/chosen_KL_mean": -8.826772689819336, "KL/mean": -24.943347930908203, "KL/rejected_KL_mean": -41.05990982055664, "KL/std": 39.05281066894531, "epoch": 0.9486910994764398, "fcm_dpo/beta": 0.018621232360601425, "fcm_dpo/delta": -0.0006048642098903656, "fcm_dpo/margin": 32.233131408691406, "fcm_dpo/q_t": 0.37227222323417664, "grad_norm": 40.410343170166016, "learning_rate": 4.1779364682113794e-09, "logits/chosen": 1.4764719009399414, "logits/rejected": 1.4323137998580933, "logps/chosen": -325.14202880859375, "logps/ref_chosen": -316.3152770996094, "logps/ref_rejected": -300.5086975097656, "logps/rejected": -341.5686340332031, "loss": 4.062, "margin_dpo/margin_mean": 32.233131408691406, "margin_dpo/margin_std": 42.46658706665039, "step": 453 }, { "KL/chosen_KL_mean": -7.95329475402832, "KL/mean": -23.164857864379883, "KL/rejected_KL_mean": -38.376426696777344, "KL/std": 40.72514343261719, "epoch": 0.9507853403141361, "fcm_dpo/beta": 0.01902606338262558, "fcm_dpo/delta": 0.0204494409263134, "fcm_dpo/margin": 30.423128128051758, "fcm_dpo/q_t": 0.3785085678100586, "grad_norm": 40.92826843261719, "learning_rate": 3.851229943335393e-09, "logits/chosen": 1.430807113647461, "logits/rejected": 1.5160000324249268, "logps/chosen": -298.49859619140625, "logps/ref_chosen": -290.5452575683594, "logps/ref_rejected": -297.2102355957031, "logps/rejected": -335.586669921875, "loss": 4.1744, "margin_dpo/margin_mean": 30.423131942749023, "margin_dpo/margin_std": 41.89031982421875, "step": 454 }, { "KL/chosen_KL_mean": -5.455175399780273, "KL/mean": -18.343996047973633, "KL/rejected_KL_mean": -31.23281478881836, "KL/std": 37.38507080078125, "epoch": 0.9528795811518325, "fcm_dpo/beta": 0.01976756751537323, "fcm_dpo/delta": 0.09486553072929382, "fcm_dpo/margin": 25.777637481689453, "fcm_dpo/q_t": 0.39325541257858276, "grad_norm": 47.402862548828125, "learning_rate": 3.5377236299748147e-09, "logits/chosen": 1.4440865516662598, "logits/rejected": 1.5377838611602783, "logps/chosen": -268.8755187988281, "logps/ref_chosen": -263.4203186035156, "logps/ref_rejected": -270.30291748046875, "logps/rejected": -301.53570556640625, "loss": 4.4293, "margin_dpo/margin_mean": 25.777637481689453, "margin_dpo/margin_std": 42.960609436035156, "step": 455 }, { "KL/chosen_KL_mean": -11.12274169921875, "KL/mean": -29.427356719970703, "KL/rejected_KL_mean": -47.73197937011719, "KL/std": 42.497840881347656, "epoch": 0.9549738219895288, "fcm_dpo/beta": 0.01928105764091015, "fcm_dpo/delta": -0.11647738516330719, "fcm_dpo/margin": 36.60923385620117, "fcm_dpo/q_t": 0.358774334192276, "grad_norm": 39.07716751098633, "learning_rate": 3.2374343405217884e-09, "logits/chosen": 1.2928086519241333, "logits/rejected": 1.4272217750549316, "logps/chosen": -323.1601257324219, "logps/ref_chosen": -312.0373840332031, "logps/ref_rejected": -290.50946044921875, "logps/rejected": -338.2414245605469, "loss": 4.0107, "margin_dpo/margin_mean": 36.609230041503906, "margin_dpo/margin_std": 49.75794219970703, "step": 456 }, { "KL/chosen_KL_mean": -9.828344345092773, "KL/mean": -26.873695373535156, "KL/rejected_KL_mean": -43.919044494628906, "KL/std": 41.28899002075195, "epoch": 0.9570680628272251, "fcm_dpo/beta": 0.01832345686852932, "fcm_dpo/delta": -0.02747979573905468, "fcm_dpo/margin": 34.09069061279297, "fcm_dpo/q_t": 0.3670627176761627, "grad_norm": 38.195186614990234, "learning_rate": 2.9503781785795713e-09, "logits/chosen": 1.3719782829284668, "logits/rejected": 1.3869181871414185, "logps/chosen": -307.0003356933594, "logps/ref_chosen": -297.1719970703125, "logps/ref_rejected": -269.15557861328125, "logps/rejected": -313.07464599609375, "loss": 4.0677, "margin_dpo/margin_mean": 34.0906867980957, "margin_dpo/margin_std": 45.770652770996094, "step": 457 }, { "KL/chosen_KL_mean": -7.7780609130859375, "KL/mean": -23.371183395385742, "KL/rejected_KL_mean": -38.96431350708008, "KL/std": 41.964454650878906, "epoch": 0.9591623036649215, "fcm_dpo/beta": 0.018945634365081787, "fcm_dpo/delta": 0.0071364520117640495, "fcm_dpo/margin": 31.186248779296875, "fcm_dpo/q_t": 0.37805095314979553, "grad_norm": 43.36493682861328, "learning_rate": 2.6765705380989432e-09, "logits/chosen": 1.3832483291625977, "logits/rejected": 1.2892534732818604, "logps/chosen": -295.1429748535156, "logps/ref_chosen": -287.3648986816406, "logps/ref_rejected": -271.1472473144531, "logps/rejected": -310.111572265625, "loss": 4.2142, "margin_dpo/margin_mean": 31.186248779296875, "margin_dpo/margin_std": 44.78030776977539, "step": 458 }, { "KL/chosen_KL_mean": -9.517560958862305, "KL/mean": -21.69217300415039, "KL/rejected_KL_mean": -33.866790771484375, "KL/std": 38.41093063354492, "epoch": 0.9612565445026178, "fcm_dpo/beta": 0.018657265231013298, "fcm_dpo/delta": 0.007451209239661694, "fcm_dpo/margin": 24.349231719970703, "fcm_dpo/q_t": 0.4049905836582184, "grad_norm": 43.45826721191406, "learning_rate": 2.416026102552732e-09, "logits/chosen": 1.2569316625595093, "logits/rejected": 1.3241119384765625, "logps/chosen": -298.1236572265625, "logps/ref_chosen": -288.6061096191406, "logps/ref_rejected": -285.6366271972656, "logps/rejected": -319.50341796875, "loss": 4.6272, "margin_dpo/margin_mean": 24.349231719970703, "margin_dpo/margin_std": 44.906341552734375, "step": 459 }, { "KL/chosen_KL_mean": -10.26303482055664, "KL/mean": -22.68872833251953, "KL/rejected_KL_mean": -35.114418029785156, "KL/std": 38.66323471069336, "epoch": 0.9633507853403142, "fcm_dpo/beta": 0.01909796893596649, "fcm_dpo/delta": 0.060689061880111694, "fcm_dpo/margin": 24.85137176513672, "fcm_dpo/q_t": 0.40231454372406006, "grad_norm": 62.3796272277832, "learning_rate": 2.168758844148272e-09, "logits/chosen": 1.3401086330413818, "logits/rejected": 1.2732634544372559, "logps/chosen": -320.5149841308594, "logps/ref_chosen": -310.251953125, "logps/ref_rejected": -252.962158203125, "logps/rejected": -288.0765686035156, "loss": 4.661, "margin_dpo/margin_mean": 24.85137176513672, "margin_dpo/margin_std": 47.31897735595703, "step": 460 }, { "KL/chosen_KL_mean": -8.58036994934082, "KL/mean": -23.64044189453125, "KL/rejected_KL_mean": -38.70050811767578, "KL/std": 39.988468170166016, "epoch": 0.9654450261780104, "fcm_dpo/beta": 0.01960652321577072, "fcm_dpo/delta": 0.00908164493739605, "fcm_dpo/margin": 30.120145797729492, "fcm_dpo/q_t": 0.3788868486881256, "grad_norm": 45.759761810302734, "learning_rate": 1.9347820230782295e-09, "logits/chosen": 1.1388990879058838, "logits/rejected": 1.248889446258545, "logps/chosen": -245.3607940673828, "logps/ref_chosen": -236.7804412841797, "logps/ref_rejected": -229.67550659179688, "logps/rejected": -268.37603759765625, "loss": 4.2962, "margin_dpo/margin_mean": 30.120145797729492, "margin_dpo/margin_std": 47.037715911865234, "step": 461 }, { "KL/chosen_KL_mean": -5.170505523681641, "KL/mean": -23.70345115661621, "KL/rejected_KL_mean": -42.236385345458984, "KL/std": 39.78120803833008, "epoch": 0.9675392670157068, "fcm_dpo/beta": 0.018582535907626152, "fcm_dpo/delta": -0.09749210625886917, "fcm_dpo/margin": 37.065879821777344, "fcm_dpo/q_t": 0.3565976619720459, "grad_norm": 50.27894973754883, "learning_rate": 1.7141081868094209e-09, "logits/chosen": 1.4427176713943481, "logits/rejected": 1.3286992311477661, "logps/chosen": -340.4740295410156, "logps/ref_chosen": -335.30352783203125, "logps/ref_rejected": -250.6796875, "logps/rejected": -292.9160461425781, "loss": 3.9905, "margin_dpo/margin_mean": 37.065879821777344, "margin_dpo/margin_std": 49.16842269897461, "step": 462 }, { "KL/chosen_KL_mean": -10.044492721557617, "KL/mean": -24.297210693359375, "KL/rejected_KL_mean": -38.5499267578125, "KL/std": 40.091426849365234, "epoch": 0.9696335078534032, "fcm_dpo/beta": 0.018351979553699493, "fcm_dpo/delta": 0.03311692178249359, "fcm_dpo/margin": 28.505441665649414, "fcm_dpo/q_t": 0.38852405548095703, "grad_norm": 37.71248245239258, "learning_rate": 1.5067491694100153e-09, "logits/chosen": 1.7018460035324097, "logits/rejected": 1.5499235391616821, "logps/chosen": -298.2647399902344, "logps/ref_chosen": -288.22021484375, "logps/ref_rejected": -227.96673583984375, "logps/rejected": -266.5166320800781, "loss": 4.3149, "margin_dpo/margin_mean": 28.505441665649414, "margin_dpo/margin_std": 42.738094329833984, "step": 463 }, { "KL/chosen_KL_mean": -10.695844650268555, "KL/mean": -25.198429107666016, "KL/rejected_KL_mean": -39.70100784301758, "KL/std": 39.06782150268555, "epoch": 0.9717277486910995, "fcm_dpo/beta": 0.01894655078649521, "fcm_dpo/delta": 0.05104460194706917, "fcm_dpo/margin": 29.005159378051758, "fcm_dpo/q_t": 0.3809618353843689, "grad_norm": 35.92738723754883, "learning_rate": 1.3127160909147672e-09, "logits/chosen": 1.6909222602844238, "logits/rejected": 1.7193284034729004, "logps/chosen": -271.0328063964844, "logps/ref_chosen": -260.3369140625, "logps/ref_rejected": -254.2083740234375, "logps/rejected": -293.9093933105469, "loss": 4.2979, "margin_dpo/margin_mean": 29.005159378051758, "margin_dpo/margin_std": 42.85988998413086, "step": 464 }, { "KL/chosen_KL_mean": -2.0730247497558594, "KL/mean": -20.170202255249023, "KL/rejected_KL_mean": -38.26738357543945, "KL/std": 39.66697692871094, "epoch": 0.9738219895287958, "fcm_dpo/beta": 0.01791943982243538, "fcm_dpo/delta": -0.13475660979747772, "fcm_dpo/margin": 36.19435119628906, "fcm_dpo/q_t": 0.3630122244358063, "grad_norm": 38.98749542236328, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 1.6583516597747803, "logits/rejected": 1.7033283710479736, "logps/chosen": -285.14251708984375, "logps/ref_chosen": -283.0694885253906, "logps/ref_rejected": -289.1370544433594, "logps/rejected": -327.40447998046875, "loss": 3.9985, "margin_dpo/margin_mean": 36.19435119628906, "margin_dpo/margin_std": 43.46207046508789, "step": 465 }, { "KL/chosen_KL_mean": -3.785524368286133, "KL/mean": -22.948883056640625, "KL/rejected_KL_mean": -42.112239837646484, "KL/std": 39.4052848815918, "epoch": 0.9759162303664921, "fcm_dpo/beta": 0.01683930680155754, "fcm_dpo/delta": -0.05012017488479614, "fcm_dpo/margin": 38.32670593261719, "fcm_dpo/q_t": 0.3580254316329956, "grad_norm": 42.31191635131836, "learning_rate": 9.64668657069706e-10, "logits/chosen": 1.0171492099761963, "logits/rejected": 1.056290626525879, "logps/chosen": -260.6705627441406, "logps/ref_chosen": -256.8850402832031, "logps/ref_rejected": -247.0802459716797, "logps/rejected": -289.1924743652344, "loss": 3.7963, "margin_dpo/margin_mean": 38.32670593261719, "margin_dpo/margin_std": 40.48317337036133, "step": 466 }, { "KL/chosen_KL_mean": -10.045860290527344, "KL/mean": -24.08479118347168, "KL/rejected_KL_mean": -38.12372589111328, "KL/std": 39.32081985473633, "epoch": 0.9780104712041885, "fcm_dpo/beta": 0.016987893730401993, "fcm_dpo/delta": 0.006081026047468185, "fcm_dpo/margin": 28.077857971191406, "fcm_dpo/q_t": 0.39910078048706055, "grad_norm": 38.65256118774414, "learning_rate": 8.106729664475176e-10, "logits/chosen": 1.1129744052886963, "logits/rejected": 1.2602795362472534, "logps/chosen": -267.5699462890625, "logps/ref_chosen": -257.5240783691406, "logps/ref_rejected": -274.82122802734375, "logps/rejected": -312.9449462890625, "loss": 4.4977, "margin_dpo/margin_mean": 28.077857971191406, "margin_dpo/margin_std": 46.522178649902344, "step": 467 }, { "KL/chosen_KL_mean": -5.589506149291992, "KL/mean": -18.8349666595459, "KL/rejected_KL_mean": -32.0804328918457, "KL/std": 38.555419921875, "epoch": 0.9801047120418848, "fcm_dpo/beta": 0.017922434955835342, "fcm_dpo/delta": 0.1293361335992813, "fcm_dpo/margin": 26.490917205810547, "fcm_dpo/q_t": 0.3997229337692261, "grad_norm": 36.69874572753906, "learning_rate": 6.700405431837585e-10, "logits/chosen": 1.524809718132019, "logits/rejected": 1.4071067571640015, "logps/chosen": -317.8492431640625, "logps/ref_chosen": -312.2597351074219, "logps/ref_rejected": -259.74383544921875, "logps/rejected": -291.8242492675781, "loss": 4.4111, "margin_dpo/margin_mean": 26.490917205810547, "margin_dpo/margin_std": 41.01814651489258, "step": 468 }, { "KL/chosen_KL_mean": -4.054059982299805, "KL/mean": -21.49460220336914, "KL/rejected_KL_mean": -38.935142517089844, "KL/std": 40.927608489990234, "epoch": 0.9821989528795811, "fcm_dpo/beta": 0.018519805744290352, "fcm_dpo/delta": -0.05003425478935242, "fcm_dpo/margin": 34.88108444213867, "fcm_dpo/q_t": 0.363657146692276, "grad_norm": 39.32550811767578, "learning_rate": 5.427789289685347e-10, "logits/chosen": 1.2567800283432007, "logits/rejected": 1.1510202884674072, "logps/chosen": -321.0937805175781, "logps/ref_chosen": -317.03973388671875, "logps/ref_rejected": -260.12628173828125, "logps/rejected": -299.0614013671875, "loss": 4.0756, "margin_dpo/margin_mean": 34.88108444213867, "margin_dpo/margin_std": 47.22452926635742, "step": 469 }, { "KL/chosen_KL_mean": -2.6450576782226562, "KL/mean": -20.932540893554688, "KL/rejected_KL_mean": -39.22002410888672, "KL/std": 40.9278564453125, "epoch": 0.9842931937172775, "fcm_dpo/beta": 0.01805165596306324, "fcm_dpo/delta": -0.06591328233480453, "fcm_dpo/margin": 36.574974060058594, "fcm_dpo/q_t": 0.36200690269470215, "grad_norm": 36.61351776123047, "learning_rate": 4.288949484559934e-10, "logits/chosen": 1.4258358478546143, "logits/rejected": 1.345468282699585, "logps/chosen": -307.03900146484375, "logps/ref_chosen": -304.3939514160156, "logps/ref_rejected": -257.6587219238281, "logps/rejected": -296.8786926269531, "loss": 4.0182, "margin_dpo/margin_mean": 36.574974060058594, "margin_dpo/margin_std": 46.32111358642578, "step": 470 }, { "KL/chosen_KL_mean": -10.424781799316406, "KL/mean": -23.33281707763672, "KL/rejected_KL_mean": -36.240848541259766, "KL/std": 40.707862854003906, "epoch": 0.9863874345549738, "fcm_dpo/beta": 0.01781153865158558, "fcm_dpo/delta": 0.09338882565498352, "fcm_dpo/margin": 25.816076278686523, "fcm_dpo/q_t": 0.4024926722049713, "grad_norm": 45.583438873291016, "learning_rate": 3.2839470889836627e-10, "logits/chosen": 1.3496334552764893, "logits/rejected": 1.4129079580307007, "logps/chosen": -295.3686218261719, "logps/ref_chosen": -284.94384765625, "logps/ref_rejected": -297.4357604980469, "logps/rejected": -333.6766357421875, "loss": 4.463, "margin_dpo/margin_mean": 25.81607437133789, "margin_dpo/margin_std": 42.50419235229492, "step": 471 }, { "KL/chosen_KL_mean": -4.572975158691406, "KL/mean": -21.51951026916504, "KL/rejected_KL_mean": -38.466064453125, "KL/std": 38.62905502319336, "epoch": 0.9884816753926702, "fcm_dpo/beta": 0.018027406185865402, "fcm_dpo/delta": -0.012763120234012604, "fcm_dpo/margin": 33.89308547973633, "fcm_dpo/q_t": 0.37206608057022095, "grad_norm": 34.75439453125, "learning_rate": 2.412835998185092e-10, "logits/chosen": 1.3935006856918335, "logits/rejected": 1.5130993127822876, "logps/chosen": -236.92724609375, "logps/ref_chosen": -232.35427856445312, "logps/ref_rejected": -245.6057891845703, "logps/rejected": -284.07183837890625, "loss": 4.0583, "margin_dpo/margin_mean": 33.89308547973633, "margin_dpo/margin_std": 44.567867279052734, "step": 472 }, { "KL/chosen_KL_mean": -7.315509796142578, "KL/mean": -24.49239730834961, "KL/rejected_KL_mean": -41.669288635253906, "KL/std": 38.954444885253906, "epoch": 0.9905759162303664, "fcm_dpo/beta": 0.018295086920261383, "fcm_dpo/delta": -0.031036358326673508, "fcm_dpo/margin": 34.353790283203125, "fcm_dpo/q_t": 0.36757490038871765, "grad_norm": 38.30513381958008, "learning_rate": 1.6756629272085544e-10, "logits/chosen": 1.2387901544570923, "logits/rejected": 1.2626218795776367, "logps/chosen": -285.3223876953125, "logps/ref_chosen": -278.0068664550781, "logps/ref_rejected": -257.19256591796875, "logps/rejected": -298.86187744140625, "loss": 4.0698, "margin_dpo/margin_mean": 34.353790283203125, "margin_dpo/margin_std": 45.340179443359375, "step": 473 }, { "KL/chosen_KL_mean": -11.093404769897461, "KL/mean": -25.184301376342773, "KL/rejected_KL_mean": -39.27520751953125, "KL/std": 38.69279479980469, "epoch": 0.9926701570680628, "fcm_dpo/beta": 0.017912233248353004, "fcm_dpo/delta": 0.10232597589492798, "fcm_dpo/margin": 28.181808471679688, "fcm_dpo/q_t": 0.390829861164093, "grad_norm": 54.18507766723633, "learning_rate": 1.072467408408384e-10, "logits/chosen": 1.256346344947815, "logits/rejected": 1.2883105278015137, "logps/chosen": -291.85089111328125, "logps/ref_chosen": -280.75750732421875, "logps/ref_rejected": -264.34295654296875, "logps/rejected": -303.6181335449219, "loss": 4.2791, "margin_dpo/margin_mean": 28.18181037902832, "margin_dpo/margin_std": 37.79776382446289, "step": 474 }, { "KL/chosen_KL_mean": -5.9772186279296875, "KL/mean": -19.791828155517578, "KL/rejected_KL_mean": -33.60643005371094, "KL/std": 37.01370620727539, "epoch": 0.9947643979057592, "fcm_dpo/beta": 0.01921505481004715, "fcm_dpo/delta": -0.009704768657684326, "fcm_dpo/margin": 27.629199981689453, "fcm_dpo/q_t": 0.38869670033454895, "grad_norm": 39.49391174316406, "learning_rate": 6.032817893297793e-11, "logits/chosen": 1.2344403266906738, "logits/rejected": 1.3401638269424438, "logps/chosen": -247.4485626220703, "logps/ref_chosen": -241.4713592529297, "logps/ref_rejected": -244.0782928466797, "logps/rejected": -277.6847229003906, "loss": 4.3437, "margin_dpo/margin_mean": 27.629199981689453, "margin_dpo/margin_std": 42.206539154052734, "step": 475 }, { "KL/chosen_KL_mean": -1.0645999908447266, "KL/mean": -17.27362060546875, "KL/rejected_KL_mean": -33.482635498046875, "KL/std": 37.50159454345703, "epoch": 0.9968586387434555, "fcm_dpo/beta": 0.01907847449183464, "fcm_dpo/delta": -0.02217945083975792, "fcm_dpo/margin": 32.41803741455078, "fcm_dpo/q_t": 0.3712438941001892, "grad_norm": 37.655029296875, "learning_rate": 2.6813123097352287e-11, "logits/chosen": 1.4651304483413696, "logits/rejected": 1.4587892293930054, "logps/chosen": -319.83892822265625, "logps/ref_chosen": -318.7742919921875, "logps/ref_rejected": -293.66595458984375, "logps/rejected": -327.1485595703125, "loss": 4.0899, "margin_dpo/margin_mean": 32.41803741455078, "margin_dpo/margin_std": 43.326416015625, "step": 476 }, { "KL/chosen_KL_mean": -8.864704132080078, "KL/mean": -25.756881713867188, "KL/rejected_KL_mean": -42.649051666259766, "KL/std": 40.76933288574219, "epoch": 0.9989528795811519, "fcm_dpo/beta": 0.018941551446914673, "fcm_dpo/delta": -0.04303491488099098, "fcm_dpo/margin": 33.78434753417969, "fcm_dpo/q_t": 0.3677205443382263, "grad_norm": 38.56641387939453, "learning_rate": 6.7033706447061635e-12, "logits/chosen": 1.3105425834655762, "logits/rejected": 1.332196593284607, "logps/chosen": -274.8004150390625, "logps/ref_chosen": -265.9356994628906, "logps/ref_rejected": -243.6986083984375, "logps/rejected": -286.34765625, "loss": 4.1499, "margin_dpo/margin_mean": 33.78434753417969, "margin_dpo/margin_std": 47.822933197021484, "step": 477 }, { "epoch": 0.9989528795811519, "step": 477, "total_flos": 0.0, "train_loss": 4.596999753196284, "train_runtime": 7387.5434, "train_samples_per_second": 8.275, "train_steps_per_second": 0.065 } ], "logging_steps": 1, "max_steps": 477, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 200, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }