{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.999244142101285, "eval_steps": 200, "global_step": 661, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.02867889404296875, "KL/mean": 0.029354453086853027, "KL/rejected_KL_mean": 0.030029296875, "KL/std": 0.2071000635623932, "epoch": 0.0015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013532638549804688, "fcm_dpo/q_t": 0.5000336766242981, "grad_norm": 28.2196044921875, "learning_rate": 0.0, "logits/chosen": 0.13337239623069763, "logits/rejected": 0.12492949515581131, "logps/chosen": -64.5841293334961, "logps/ref_chosen": -64.61280822753906, "logps/ref_rejected": -64.17195129394531, "logps/rejected": -64.14192199707031, "loss": 1.3866, "margin_dpo/margin_mean": -0.0013527870178222656, "margin_dpo/margin_std": 0.2561596930027008, "step": 1 }, { "KL/chosen_KL_mean": -0.00289154052734375, "KL/mean": -0.021616414189338684, "KL/rejected_KL_mean": -0.04033660888671875, "KL/std": 0.19624735414981842, "epoch": 0.0030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.037450045347213745, "fcm_dpo/q_t": 0.4990639090538025, "grad_norm": 27.82891273498535, "learning_rate": 7.462686567164179e-09, "logits/chosen": 0.09414851665496826, "logits/rejected": 0.07363267242908478, "logps/chosen": -56.101890563964844, "logps/ref_chosen": -56.0989990234375, "logps/ref_rejected": -66.59971618652344, "logps/rejected": -66.64006042480469, "loss": 1.3828, "margin_dpo/margin_mean": 0.03744968771934509, "margin_dpo/margin_std": 0.27811938524246216, "step": 2 }, { "KL/chosen_KL_mean": 0.0621337890625, "KL/mean": 0.06413677334785461, "KL/rejected_KL_mean": 0.06613540649414062, "KL/std": 0.22972196340560913, "epoch": 0.0045351473922902496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0040013790130615234, "fcm_dpo/q_t": 0.5000998377799988, "grad_norm": 31.116546630859375, "learning_rate": 1.4925373134328357e-08, "logits/chosen": 0.09941669553518295, "logits/rejected": 0.06143157184123993, "logps/chosen": -65.39512634277344, "logps/ref_chosen": -65.45726013183594, "logps/ref_rejected": -90.82853698730469, "logps/rejected": -90.76239776611328, "loss": 1.3869, "margin_dpo/margin_mean": -0.00400128960609436, "margin_dpo/margin_std": 0.290522038936615, "step": 3 }, { "KL/chosen_KL_mean": 0.000263214111328125, "KL/mean": -0.023683160543441772, "KL/rejected_KL_mean": -0.047637939453125, "KL/std": 0.24327343702316284, "epoch": 0.006046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04789847135543823, "fcm_dpo/q_t": 0.49880456924438477, "grad_norm": 34.16933059692383, "learning_rate": 2.2388059701492534e-08, "logits/chosen": 0.11254644393920898, "logits/rejected": 0.0962742418050766, "logps/chosen": -76.85992431640625, "logps/ref_chosen": -76.86018371582031, "logps/ref_rejected": -79.91523742675781, "logps/rejected": -79.96287536621094, "loss": 1.3818, "margin_dpo/margin_mean": 0.04789838194847107, "margin_dpo/margin_std": 0.3381476402282715, "step": 4 }, { "KL/chosen_KL_mean": -0.027805328369140625, "KL/mean": -0.015260711312294006, "KL/rejected_KL_mean": -0.002712249755859375, "KL/std": 0.18707606196403503, "epoch": 0.007558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.025092214345932007, "fcm_dpo/q_t": 0.5006270408630371, "grad_norm": 29.659147262573242, "learning_rate": 2.9850746268656714e-08, "logits/chosen": 0.07833173871040344, "logits/rejected": 0.03964802622795105, "logps/chosen": -62.9991455078125, "logps/ref_chosen": -62.97134017944336, "logps/ref_rejected": -79.9192123413086, "logps/rejected": -79.92192077636719, "loss": 1.3889, "margin_dpo/margin_mean": -0.02509164810180664, "margin_dpo/margin_std": 0.23738428950309753, "step": 5 }, { "KL/chosen_KL_mean": -0.0323944091796875, "KL/mean": 0.007522225379943848, "KL/rejected_KL_mean": 0.047443389892578125, "KL/std": 0.25012582540512085, "epoch": 0.009070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07983356714248657, "fcm_dpo/q_t": 0.501994788646698, "grad_norm": 29.858251571655273, "learning_rate": 3.731343283582089e-08, "logits/chosen": 0.14125987887382507, "logits/rejected": 0.10181364417076111, "logps/chosen": -51.339759826660156, "logps/ref_chosen": -51.30736541748047, "logps/ref_rejected": -82.77239227294922, "logps/rejected": -82.7249526977539, "loss": 1.3945, "margin_dpo/margin_mean": -0.07983347773551941, "margin_dpo/margin_std": 0.3165475130081177, "step": 6 }, { "KL/chosen_KL_mean": 0.05200386047363281, "KL/mean": 0.038868606090545654, "KL/rejected_KL_mean": 0.025732040405273438, "KL/std": 0.21536785364151, "epoch": 0.010582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02626955509185791, "fcm_dpo/q_t": 0.4993433356285095, "grad_norm": 27.112102508544922, "learning_rate": 4.477611940298507e-08, "logits/chosen": 0.019042763859033585, "logits/rejected": -0.02486884593963623, "logps/chosen": -51.40740966796875, "logps/ref_chosen": -51.45941162109375, "logps/ref_rejected": -66.3828125, "logps/rejected": -66.3570785522461, "loss": 1.3839, "margin_dpo/margin_mean": 0.026269584894180298, "margin_dpo/margin_std": 0.3134131133556366, "step": 7 }, { "KL/chosen_KL_mean": -0.024341583251953125, "KL/mean": -0.011224046349525452, "KL/rejected_KL_mean": 0.001895904541015625, "KL/std": 0.2582741677761078, "epoch": 0.012093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.026233285665512085, "fcm_dpo/q_t": 0.5006557703018188, "grad_norm": 28.65567970275879, "learning_rate": 5.223880597014925e-08, "logits/chosen": 0.07061031460762024, "logits/rejected": 0.04848004877567291, "logps/chosen": -62.221885681152344, "logps/ref_chosen": -62.197547912597656, "logps/ref_rejected": -74.66180419921875, "logps/rejected": -74.659912109375, "loss": 1.3892, "margin_dpo/margin_mean": -0.026233792304992676, "margin_dpo/margin_std": 0.357799768447876, "step": 8 }, { "KL/chosen_KL_mean": -0.012987136840820312, "KL/mean": -0.02591407299041748, "KL/rejected_KL_mean": -0.038845062255859375, "KL/std": 0.2304016649723053, "epoch": 0.013605442176870748, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025857150554656982, "fcm_dpo/q_t": 0.4993537664413452, "grad_norm": 31.47567367553711, "learning_rate": 5.970149253731343e-08, "logits/chosen": 0.15661674737930298, "logits/rejected": 0.09843897819519043, "logps/chosen": -55.64270782470703, "logps/ref_chosen": -55.629722595214844, "logps/ref_rejected": -86.21221923828125, "logps/rejected": -86.25106811523438, "loss": 1.3839, "margin_dpo/margin_mean": 0.02585729956626892, "margin_dpo/margin_std": 0.3030700385570526, "step": 9 }, { "KL/chosen_KL_mean": -0.0495147705078125, "KL/mean": -0.027920737862586975, "KL/rejected_KL_mean": -0.006328582763671875, "KL/std": 0.25644826889038086, "epoch": 0.015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.043188005685806274, "fcm_dpo/q_t": 0.5010795593261719, "grad_norm": 29.739234924316406, "learning_rate": 6.71641791044776e-08, "logits/chosen": 0.12492163479328156, "logits/rejected": 0.09430065751075745, "logps/chosen": -62.740116119384766, "logps/ref_chosen": -62.69060134887695, "logps/ref_rejected": -90.610107421875, "logps/rejected": -90.61643981933594, "loss": 1.391, "margin_dpo/margin_mean": -0.043188244104385376, "margin_dpo/margin_std": 0.3699379861354828, "step": 10 }, { "KL/chosen_KL_mean": 0.019266128540039062, "KL/mean": -0.0006564557552337646, "KL/rejected_KL_mean": -0.0205841064453125, "KL/std": 0.20153909921646118, "epoch": 0.016628873771730914, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03984951972961426, "fcm_dpo/q_t": 0.49900442361831665, "grad_norm": 29.289079666137695, "learning_rate": 7.462686567164178e-08, "logits/chosen": 0.1184384673833847, "logits/rejected": 0.11146491020917892, "logps/chosen": -65.74784851074219, "logps/ref_chosen": -65.76712036132812, "logps/ref_rejected": -72.4764633178711, "logps/rejected": -72.4970474243164, "loss": 1.3826, "margin_dpo/margin_mean": 0.039849698543548584, "margin_dpo/margin_std": 0.301962286233902, "step": 11 }, { "KL/chosen_KL_mean": -0.03316497802734375, "KL/mean": -0.036665767431259155, "KL/rejected_KL_mean": -0.04016876220703125, "KL/std": 0.2160276174545288, "epoch": 0.018140589569160998, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007006347179412842, "fcm_dpo/q_t": 0.4998248219490051, "grad_norm": 28.296424865722656, "learning_rate": 8.208955223880596e-08, "logits/chosen": 0.00660199299454689, "logits/rejected": -0.009146112017333508, "logps/chosen": -60.73805618286133, "logps/ref_chosen": -60.704891204833984, "logps/ref_rejected": -69.41564178466797, "logps/rejected": -69.455810546875, "loss": 1.3858, "margin_dpo/margin_mean": 0.007006734609603882, "margin_dpo/margin_std": 0.2850860357284546, "step": 12 }, { "KL/chosen_KL_mean": 0.041828155517578125, "KL/mean": 0.04208873212337494, "KL/rejected_KL_mean": 0.0423583984375, "KL/std": 0.19688871502876282, "epoch": 0.019652305366591082, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.000529557466506958, "fcm_dpo/q_t": 0.5000130534172058, "grad_norm": 29.095415115356445, "learning_rate": 8.955223880597014e-08, "logits/chosen": 0.10334185510873795, "logits/rejected": 0.04153688997030258, "logps/chosen": -49.867427825927734, "logps/ref_chosen": -49.90925598144531, "logps/ref_rejected": -92.37818145751953, "logps/rejected": -92.33582305908203, "loss": 1.3866, "margin_dpo/margin_mean": -0.0005303919315338135, "margin_dpo/margin_std": 0.302842915058136, "step": 13 }, { "KL/chosen_KL_mean": 0.022100448608398438, "KL/mean": 0.00214308500289917, "KL/rejected_KL_mean": -0.017810821533203125, "KL/std": 0.17682762444019318, "epoch": 0.021164021164021163, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03991013765335083, "fcm_dpo/q_t": 0.4990024268627167, "grad_norm": 29.589576721191406, "learning_rate": 9.701492537313432e-08, "logits/chosen": 0.0741623118519783, "logits/rejected": 0.056773897260427475, "logps/chosen": -60.59669494628906, "logps/ref_chosen": -60.61879348754883, "logps/ref_rejected": -71.79306030273438, "logps/rejected": -71.81086730957031, "loss": 1.3825, "margin_dpo/margin_mean": 0.039910584688186646, "margin_dpo/margin_std": 0.25130391120910645, "step": 14 }, { "KL/chosen_KL_mean": -0.03235435485839844, "KL/mean": -0.033513978123664856, "KL/rejected_KL_mean": -0.03466796875, "KL/std": 0.23530861735343933, "epoch": 0.022675736961451247, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0023103058338165283, "fcm_dpo/q_t": 0.4999410808086395, "grad_norm": 33.306461334228516, "learning_rate": 1.044776119402985e-07, "logits/chosen": 0.07059284299612045, "logits/rejected": 0.027335654944181442, "logps/chosen": -63.50189208984375, "logps/ref_chosen": -63.46953582763672, "logps/ref_rejected": -88.88951110839844, "logps/rejected": -88.92417907714844, "loss": 1.3864, "margin_dpo/margin_mean": 0.0023106038570404053, "margin_dpo/margin_std": 0.35110729932785034, "step": 15 }, { "KL/chosen_KL_mean": 0.0051555633544921875, "KL/mean": -0.00991896539926529, "KL/rejected_KL_mean": -0.024997711181640625, "KL/std": 0.20466622710227966, "epoch": 0.02418745275888133, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03015093505382538, "fcm_dpo/q_t": 0.49924665689468384, "grad_norm": 26.78266716003418, "learning_rate": 1.1194029850746268e-07, "logits/chosen": 0.11953230202198029, "logits/rejected": 0.08214319497346878, "logps/chosen": -46.52714538574219, "logps/ref_chosen": -46.53229904174805, "logps/ref_rejected": -74.27533721923828, "logps/rejected": -74.30033874511719, "loss": 1.3835, "margin_dpo/margin_mean": 0.030150994658470154, "margin_dpo/margin_std": 0.3233643174171448, "step": 16 }, { "KL/chosen_KL_mean": -0.03330421447753906, "KL/mean": -0.026638299226760864, "KL/rejected_KL_mean": -0.0199737548828125, "KL/std": 0.21004769206047058, "epoch": 0.025699168556311415, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.013328969478607178, "fcm_dpo/q_t": 0.5003334283828735, "grad_norm": 32.7893180847168, "learning_rate": 1.1940298507462686e-07, "logits/chosen": 0.05357828736305237, "logits/rejected": 0.03485024347901344, "logps/chosen": -64.11113739013672, "logps/ref_chosen": -64.07783508300781, "logps/ref_rejected": -86.40876770019531, "logps/rejected": -86.42874145507812, "loss": 1.3878, "margin_dpo/margin_mean": -0.013329803943634033, "margin_dpo/margin_std": 0.27140507102012634, "step": 17 }, { "KL/chosen_KL_mean": -0.001186370849609375, "KL/mean": -0.005001351237297058, "KL/rejected_KL_mean": -0.008813858032226562, "KL/std": 0.21888390183448792, "epoch": 0.027210884353741496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007630079984664917, "fcm_dpo/q_t": 0.4998093545436859, "grad_norm": 27.819944381713867, "learning_rate": 1.2686567164179106e-07, "logits/chosen": 0.0680946335196495, "logits/rejected": 0.024525661021471024, "logps/chosen": -44.87552261352539, "logps/ref_chosen": -44.87433624267578, "logps/ref_rejected": -70.97604370117188, "logps/rejected": -70.98486328125, "loss": 1.3858, "margin_dpo/margin_mean": 0.007630020380020142, "margin_dpo/margin_std": 0.3008061647415161, "step": 18 }, { "KL/chosen_KL_mean": 0.01468658447265625, "KL/mean": -0.019955851137638092, "KL/rejected_KL_mean": -0.054599761962890625, "KL/std": 0.20606964826583862, "epoch": 0.02872260015117158, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06928969919681549, "fcm_dpo/q_t": 0.4982680082321167, "grad_norm": 30.73091697692871, "learning_rate": 1.343283582089552e-07, "logits/chosen": 0.0740823969244957, "logits/rejected": 0.06053391098976135, "logps/chosen": -68.14512634277344, "logps/ref_chosen": -68.1598129272461, "logps/ref_rejected": -81.17138671875, "logps/rejected": -81.22599029541016, "loss": 1.3795, "margin_dpo/margin_mean": 0.06928981840610504, "margin_dpo/margin_std": 0.22947733104228973, "step": 19 }, { "KL/chosen_KL_mean": 0.0164337158203125, "KL/mean": 0.01836688071489334, "KL/rejected_KL_mean": 0.020305633544921875, "KL/std": 0.2009357511997223, "epoch": 0.030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003872022032737732, "fcm_dpo/q_t": 0.5000965595245361, "grad_norm": 29.24262046813965, "learning_rate": 1.4179104477611938e-07, "logits/chosen": 0.14226043224334717, "logits/rejected": 0.1189441829919815, "logps/chosen": -53.66212844848633, "logps/ref_chosen": -53.67856216430664, "logps/ref_rejected": -74.16911315917969, "logps/rejected": -74.14881134033203, "loss": 1.3869, "margin_dpo/margin_mean": -0.0038716942071914673, "margin_dpo/margin_std": 0.2613745629787445, "step": 20 }, { "KL/chosen_KL_mean": -0.013164520263671875, "KL/mean": -0.004536345601081848, "KL/rejected_KL_mean": 0.00409698486328125, "KL/std": 0.20376834273338318, "epoch": 0.031746031746031744, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.017256349325180054, "fcm_dpo/q_t": 0.5004312992095947, "grad_norm": 29.48101806640625, "learning_rate": 1.4925373134328355e-07, "logits/chosen": 0.1120043694972992, "logits/rejected": 0.08625894784927368, "logps/chosen": -64.7147216796875, "logps/ref_chosen": -64.70155334472656, "logps/ref_rejected": -81.02095031738281, "logps/rejected": -81.016845703125, "loss": 1.3882, "margin_dpo/margin_mean": -0.01725691556930542, "margin_dpo/margin_std": 0.2752753496170044, "step": 21 }, { "KL/chosen_KL_mean": 0.0014190673828125, "KL/mean": -0.024805426597595215, "KL/rejected_KL_mean": -0.051029205322265625, "KL/std": 0.2457861751317978, "epoch": 0.03325774754346183, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05245363712310791, "fcm_dpo/q_t": 0.49868905544281006, "grad_norm": 28.892513275146484, "learning_rate": 1.5671641791044775e-07, "logits/chosen": -0.0012458395212888718, "logits/rejected": -0.02185724303126335, "logps/chosen": -58.03457260131836, "logps/ref_chosen": -58.03599166870117, "logps/ref_rejected": -80.72721862792969, "logps/rejected": -80.77824401855469, "loss": 1.3814, "margin_dpo/margin_mean": 0.052453696727752686, "margin_dpo/margin_std": 0.36326271295547485, "step": 22 }, { "KL/chosen_KL_mean": -0.005031585693359375, "KL/mean": -0.004315197467803955, "KL/rejected_KL_mean": -0.003604888916015625, "KL/std": 0.22622814774513245, "epoch": 0.03476946334089191, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0014243721961975098, "fcm_dpo/q_t": 0.5000349879264832, "grad_norm": 32.9621696472168, "learning_rate": 1.6417910447761193e-07, "logits/chosen": 0.14867977797985077, "logits/rejected": 0.12283647060394287, "logps/chosen": -66.36111450195312, "logps/ref_chosen": -66.35608673095703, "logps/ref_rejected": -93.02769470214844, "logps/rejected": -93.03129577636719, "loss": 1.3867, "margin_dpo/margin_mean": -0.0014249980449676514, "margin_dpo/margin_std": 0.3283410668373108, "step": 23 }, { "KL/chosen_KL_mean": -0.023939132690429688, "KL/mean": -0.010058224201202393, "KL/rejected_KL_mean": 0.0038280487060546875, "KL/std": 0.2233589142560959, "epoch": 0.036281179138321996, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.027758657932281494, "fcm_dpo/q_t": 0.5006934404373169, "grad_norm": 26.37616539001465, "learning_rate": 1.716417910447761e-07, "logits/chosen": 0.11849376559257507, "logits/rejected": 0.08617529273033142, "logps/chosen": -54.48517608642578, "logps/ref_chosen": -54.461238861083984, "logps/ref_rejected": -68.33817291259766, "logps/rejected": -68.3343505859375, "loss": 1.3893, "margin_dpo/margin_mean": -0.027759432792663574, "margin_dpo/margin_std": 0.3043414354324341, "step": 24 }, { "KL/chosen_KL_mean": -0.07821273803710938, "KL/mean": -0.042458370327949524, "KL/rejected_KL_mean": -0.00670623779296875, "KL/std": 0.2250480055809021, "epoch": 0.03779289493575208, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0715077817440033, "fcm_dpo/q_t": 0.5017871856689453, "grad_norm": 29.881423950195312, "learning_rate": 1.7910447761194027e-07, "logits/chosen": 0.10595692694187164, "logits/rejected": 0.05439333990216255, "logps/chosen": -60.08241653442383, "logps/ref_chosen": -60.00420379638672, "logps/ref_rejected": -90.47376251220703, "logps/rejected": -90.48046875, "loss": 1.3937, "margin_dpo/margin_mean": -0.07150781154632568, "margin_dpo/margin_std": 0.2951575517654419, "step": 25 }, { "KL/chosen_KL_mean": -0.025112152099609375, "KL/mean": -0.030633173882961273, "KL/rejected_KL_mean": -0.036151885986328125, "KL/std": 0.2159086912870407, "epoch": 0.039304610733182165, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.011040762066841125, "fcm_dpo/q_t": 0.4997239112854004, "grad_norm": 29.78719711303711, "learning_rate": 1.8656716417910447e-07, "logits/chosen": 0.10197645425796509, "logits/rejected": 0.08369824290275574, "logps/chosen": -56.844268798828125, "logps/ref_chosen": -56.81915283203125, "logps/ref_rejected": -77.84333038330078, "logps/rejected": -77.87948608398438, "loss": 1.3855, "margin_dpo/margin_mean": 0.011040925979614258, "margin_dpo/margin_std": 0.3176705837249756, "step": 26 }, { "KL/chosen_KL_mean": -0.040157318115234375, "KL/mean": -0.01530282199382782, "KL/rejected_KL_mean": 0.009548187255859375, "KL/std": 0.24493414163589478, "epoch": 0.04081632653061224, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04971262812614441, "fcm_dpo/q_t": 0.5012425184249878, "grad_norm": 29.07723617553711, "learning_rate": 1.9402985074626865e-07, "logits/chosen": 0.0994294285774231, "logits/rejected": 0.07478873431682587, "logps/chosen": -62.91718292236328, "logps/ref_chosen": -62.87702560424805, "logps/ref_rejected": -71.34437561035156, "logps/rejected": -71.33482360839844, "loss": 1.3915, "margin_dpo/margin_mean": -0.04971283674240112, "margin_dpo/margin_std": 0.2961461544036865, "step": 27 }, { "KL/chosen_KL_mean": -0.025875091552734375, "KL/mean": -0.01864849030971527, "KL/rejected_KL_mean": -0.011423110961914062, "KL/std": 0.2145068347454071, "epoch": 0.042328042328042326, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014449745416641235, "fcm_dpo/q_t": 0.500361442565918, "grad_norm": 27.64870834350586, "learning_rate": 2.0149253731343282e-07, "logits/chosen": 0.061025142669677734, "logits/rejected": 0.05223534256219864, "logps/chosen": -59.8592529296875, "logps/ref_chosen": -59.8333740234375, "logps/ref_rejected": -70.39804077148438, "logps/rejected": -70.40946197509766, "loss": 1.388, "margin_dpo/margin_mean": -0.014449775218963623, "margin_dpo/margin_std": 0.28998634219169617, "step": 28 }, { "KL/chosen_KL_mean": -0.011623382568359375, "KL/mean": -0.024745166301727295, "KL/rejected_KL_mean": -0.03786468505859375, "KL/std": 0.22506870329380035, "epoch": 0.04383975812547241, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02624613046646118, "fcm_dpo/q_t": 0.49934351444244385, "grad_norm": 32.635955810546875, "learning_rate": 2.08955223880597e-07, "logits/chosen": 0.1302889883518219, "logits/rejected": 0.11286543309688568, "logps/chosen": -74.13182067871094, "logps/ref_chosen": -74.12020111083984, "logps/ref_rejected": -83.33099365234375, "logps/rejected": -83.36885070800781, "loss": 1.3839, "margin_dpo/margin_mean": 0.026246458292007446, "margin_dpo/margin_std": 0.3253002464771271, "step": 29 }, { "KL/chosen_KL_mean": -0.0457916259765625, "KL/mean": -0.03270387649536133, "KL/rejected_KL_mean": -0.01961517333984375, "KL/std": 0.2302623689174652, "epoch": 0.045351473922902494, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.026176929473876953, "fcm_dpo/q_t": 0.5006539225578308, "grad_norm": 30.555559158325195, "learning_rate": 2.1641791044776117e-07, "logits/chosen": 0.12713855504989624, "logits/rejected": 0.07250553369522095, "logps/chosen": -50.797080993652344, "logps/ref_chosen": -50.75128936767578, "logps/ref_rejected": -89.29063415527344, "logps/rejected": -89.31024932861328, "loss": 1.3891, "margin_dpo/margin_mean": -0.026176661252975464, "margin_dpo/margin_std": 0.2921898066997528, "step": 30 }, { "KL/chosen_KL_mean": -0.028470993041992188, "KL/mean": -0.032488495111465454, "KL/rejected_KL_mean": -0.03650665283203125, "KL/std": 0.22643673419952393, "epoch": 0.04686318972033258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00803285837173462, "fcm_dpo/q_t": 0.49979931116104126, "grad_norm": 34.28264617919922, "learning_rate": 2.2388059701492537e-07, "logits/chosen": 0.09353435784578323, "logits/rejected": 0.04833420366048813, "logps/chosen": -65.36522674560547, "logps/ref_chosen": -65.33675384521484, "logps/ref_rejected": -100.76666259765625, "logps/rejected": -100.80316925048828, "loss": 1.3857, "margin_dpo/margin_mean": 0.008032828569412231, "margin_dpo/margin_std": 0.31350964307785034, "step": 31 }, { "KL/chosen_KL_mean": 0.023860931396484375, "KL/mean": -0.0067330896854400635, "KL/rejected_KL_mean": -0.037326812744140625, "KL/std": 0.2645628750324249, "epoch": 0.04837490551776266, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.061192214488983154, "fcm_dpo/q_t": 0.49847108125686646, "grad_norm": 29.951520919799805, "learning_rate": 2.3134328358208954e-07, "logits/chosen": 0.06752718985080719, "logits/rejected": 0.059862636029720306, "logps/chosen": -67.15946960449219, "logps/ref_chosen": -67.18333435058594, "logps/ref_rejected": -82.80763244628906, "logps/rejected": -82.84496307373047, "loss": 1.3805, "margin_dpo/margin_mean": 0.06119275093078613, "margin_dpo/margin_std": 0.3593711256980896, "step": 32 }, { "KL/chosen_KL_mean": -0.0693359375, "KL/mean": -0.0568530336022377, "KL/rejected_KL_mean": -0.04436492919921875, "KL/std": 0.23845943808555603, "epoch": 0.049886621315192746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0249737948179245, "fcm_dpo/q_t": 0.5006238222122192, "grad_norm": 31.129165649414062, "learning_rate": 2.388059701492537e-07, "logits/chosen": 0.026731964200735092, "logits/rejected": 0.0009631272405385971, "logps/chosen": -64.10881805419922, "logps/ref_chosen": -64.03948211669922, "logps/ref_rejected": -75.68357849121094, "logps/rejected": -75.72795104980469, "loss": 1.3891, "margin_dpo/margin_mean": -0.024973228573799133, "margin_dpo/margin_std": 0.34394294023513794, "step": 33 }, { "KL/chosen_KL_mean": -0.00998687744140625, "KL/mean": -0.04686702787876129, "KL/rejected_KL_mean": -0.08374404907226562, "KL/std": 0.2107549011707306, "epoch": 0.05139833711262283, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07376006245613098, "fcm_dpo/q_t": 0.4981565773487091, "grad_norm": 28.369434356689453, "learning_rate": 2.4626865671641786e-07, "logits/chosen": 0.11266010254621506, "logits/rejected": 0.08217991888523102, "logps/chosen": -53.67428207397461, "logps/ref_chosen": -53.6642951965332, "logps/ref_rejected": -65.77989959716797, "logps/rejected": -65.8636474609375, "loss": 1.3792, "margin_dpo/margin_mean": 0.07376009225845337, "margin_dpo/margin_std": 0.28919702768325806, "step": 34 }, { "KL/chosen_KL_mean": -0.05603218078613281, "KL/mean": -0.08405140042304993, "KL/rejected_KL_mean": -0.11207199096679688, "KL/std": 0.21035067737102509, "epoch": 0.05291005291005291, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.056039512157440186, "fcm_dpo/q_t": 0.4985993802547455, "grad_norm": 27.643247604370117, "learning_rate": 2.537313432835821e-07, "logits/chosen": 0.04645001143217087, "logits/rejected": 0.024315200746059418, "logps/chosen": -61.07289505004883, "logps/ref_chosen": -61.01686096191406, "logps/ref_rejected": -72.78598022460938, "logps/rejected": -72.89805603027344, "loss": 1.3809, "margin_dpo/margin_mean": 0.05603942275047302, "margin_dpo/margin_std": 0.2715159058570862, "step": 35 }, { "KL/chosen_KL_mean": -0.06983184814453125, "KL/mean": -0.08555865287780762, "KL/rejected_KL_mean": -0.10128402709960938, "KL/std": 0.23037990927696228, "epoch": 0.05442176870748299, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03145325183868408, "fcm_dpo/q_t": 0.49921417236328125, "grad_norm": 28.695158004760742, "learning_rate": 2.611940298507462e-07, "logits/chosen": 0.09576475620269775, "logits/rejected": 0.043359238654375076, "logps/chosen": -50.60719299316406, "logps/ref_chosen": -50.53736114501953, "logps/ref_rejected": -78.11678314208984, "logps/rejected": -78.21806335449219, "loss": 1.3834, "margin_dpo/margin_mean": 0.03145378828048706, "margin_dpo/margin_std": 0.34016674757003784, "step": 36 }, { "KL/chosen_KL_mean": -0.032245635986328125, "KL/mean": -0.10205593705177307, "KL/rejected_KL_mean": -0.17187118530273438, "KL/std": 0.2938510775566101, "epoch": 0.055933484504913075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13962048292160034, "fcm_dpo/q_t": 0.4965134263038635, "grad_norm": 37.48423767089844, "learning_rate": 2.686567164179104e-07, "logits/chosen": 0.10579690337181091, "logits/rejected": 0.025799859315156937, "logps/chosen": -59.5861930847168, "logps/ref_chosen": -59.55394744873047, "logps/ref_rejected": -108.27702331542969, "logps/rejected": -108.44889831542969, "loss": 1.3729, "margin_dpo/margin_mean": 0.13962051272392273, "margin_dpo/margin_std": 0.4410157799720764, "step": 37 }, { "KL/chosen_KL_mean": -0.0857086181640625, "KL/mean": -0.11043363809585571, "KL/rejected_KL_mean": -0.13515663146972656, "KL/std": 0.25614839792251587, "epoch": 0.05744520030234316, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04944509267807007, "fcm_dpo/q_t": 0.4987648129463196, "grad_norm": 29.67615509033203, "learning_rate": 2.761194029850746e-07, "logits/chosen": 0.08986853808164597, "logits/rejected": 0.0756976380944252, "logps/chosen": -65.87406921386719, "logps/ref_chosen": -65.78836059570312, "logps/ref_rejected": -76.1619873046875, "logps/rejected": -76.2971420288086, "loss": 1.3817, "margin_dpo/margin_mean": 0.04944491386413574, "margin_dpo/margin_std": 0.3537583351135254, "step": 38 }, { "KL/chosen_KL_mean": -0.10344505310058594, "KL/mean": -0.12965010106563568, "KL/rejected_KL_mean": -0.155853271484375, "KL/std": 0.25784462690353394, "epoch": 0.05895691609977324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05241057276725769, "fcm_dpo/q_t": 0.4986896514892578, "grad_norm": 29.232746124267578, "learning_rate": 2.8358208955223876e-07, "logits/chosen": 0.1692015826702118, "logits/rejected": 0.14214356243610382, "logps/chosen": -57.28025436401367, "logps/ref_chosen": -57.17681121826172, "logps/ref_rejected": -79.486328125, "logps/rejected": -79.64218139648438, "loss": 1.3814, "margin_dpo/margin_mean": 0.052410632371902466, "margin_dpo/margin_std": 0.3558116555213928, "step": 39 }, { "KL/chosen_KL_mean": -0.11075210571289062, "KL/mean": -0.1190851703286171, "KL/rejected_KL_mean": -0.12742233276367188, "KL/std": 0.24136286973953247, "epoch": 0.06046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016666367650032043, "fcm_dpo/q_t": 0.49958330392837524, "grad_norm": 31.398401260375977, "learning_rate": 2.9104477611940296e-07, "logits/chosen": 0.1105370819568634, "logits/rejected": 0.06097438558936119, "logps/chosen": -61.44491958618164, "logps/ref_chosen": -61.33416748046875, "logps/ref_rejected": -79.10697174072266, "logps/rejected": -79.23439025878906, "loss": 1.3849, "margin_dpo/margin_mean": 0.016666486859321594, "margin_dpo/margin_std": 0.327433705329895, "step": 40 }, { "KL/chosen_KL_mean": -0.11747360229492188, "KL/mean": -0.16613994538784027, "KL/rejected_KL_mean": -0.21480178833007812, "KL/std": 0.2806848883628845, "epoch": 0.06198034769463341, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09732410311698914, "fcm_dpo/q_t": 0.49756765365600586, "grad_norm": 30.464834213256836, "learning_rate": 2.985074626865671e-07, "logits/chosen": 0.06427644938230515, "logits/rejected": 0.04391469433903694, "logps/chosen": -67.66419982910156, "logps/ref_chosen": -67.5467300415039, "logps/ref_rejected": -83.87788391113281, "logps/rejected": -84.09268951416016, "loss": 1.377, "margin_dpo/margin_mean": 0.09732472896575928, "margin_dpo/margin_std": 0.3751354217529297, "step": 41 }, { "KL/chosen_KL_mean": -0.10732078552246094, "KL/mean": -0.1636602282524109, "KL/rejected_KL_mean": -0.22000503540039062, "KL/std": 0.3067892789840698, "epoch": 0.06349206349206349, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11268658936023712, "fcm_dpo/q_t": 0.4971844553947449, "grad_norm": 29.092308044433594, "learning_rate": 3.059701492537313e-07, "logits/chosen": 0.05194007605314255, "logits/rejected": 0.0304647795855999, "logps/chosen": -61.37217712402344, "logps/ref_chosen": -61.26485824584961, "logps/ref_rejected": -76.3629150390625, "logps/rejected": -76.58291625976562, "loss": 1.3755, "margin_dpo/margin_mean": 0.1126861423254013, "margin_dpo/margin_std": 0.4176199436187744, "step": 42 }, { "KL/chosen_KL_mean": -0.09345245361328125, "KL/mean": -0.111243337392807, "KL/rejected_KL_mean": -0.12903594970703125, "KL/std": 0.28760355710983276, "epoch": 0.06500377928949358, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.035582125186920166, "fcm_dpo/q_t": 0.49911123514175415, "grad_norm": 34.529136657714844, "learning_rate": 3.134328358208955e-07, "logits/chosen": 0.08271262049674988, "logits/rejected": 0.07184702157974243, "logps/chosen": -71.90248107910156, "logps/ref_chosen": -71.80902862548828, "logps/ref_rejected": -81.12464141845703, "logps/rejected": -81.25367736816406, "loss": 1.3832, "margin_dpo/margin_mean": 0.035581886768341064, "margin_dpo/margin_std": 0.3950834274291992, "step": 43 }, { "KL/chosen_KL_mean": -0.16973114013671875, "KL/mean": -0.1733190417289734, "KL/rejected_KL_mean": -0.176910400390625, "KL/std": 0.3061409592628479, "epoch": 0.06651549508692366, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.007180750370025635, "fcm_dpo/q_t": 0.4998205602169037, "grad_norm": 32.519447326660156, "learning_rate": 3.2089552238805965e-07, "logits/chosen": 0.04154222086071968, "logits/rejected": 0.011357433162629604, "logps/chosen": -66.72016143798828, "logps/ref_chosen": -66.55043029785156, "logps/ref_rejected": -85.06198120117188, "logps/rejected": -85.2388916015625, "loss": 1.3861, "margin_dpo/margin_mean": 0.0071804821491241455, "margin_dpo/margin_std": 0.45991164445877075, "step": 44 }, { "KL/chosen_KL_mean": -0.13983726501464844, "KL/mean": -0.17917409539222717, "KL/rejected_KL_mean": -0.21851348876953125, "KL/std": 0.2951315641403198, "epoch": 0.06802721088435375, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0786718875169754, "fcm_dpo/q_t": 0.49803340435028076, "grad_norm": 31.8658504486084, "learning_rate": 3.2835820895522385e-07, "logits/chosen": 0.13182783126831055, "logits/rejected": 0.07822854816913605, "logps/chosen": -62.383689880371094, "logps/ref_chosen": -62.24385452270508, "logps/ref_rejected": -92.96665954589844, "logps/rejected": -93.1851806640625, "loss": 1.3789, "margin_dpo/margin_mean": 0.07867246866226196, "margin_dpo/margin_std": 0.4071173369884491, "step": 45 }, { "KL/chosen_KL_mean": -0.11602401733398438, "KL/mean": -0.19016796350479126, "KL/rejected_KL_mean": -0.2643165588378906, "KL/std": 0.31932127475738525, "epoch": 0.06953892668178382, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14828699827194214, "fcm_dpo/q_t": 0.4962952733039856, "grad_norm": 31.104398727416992, "learning_rate": 3.3582089552238805e-07, "logits/chosen": 0.13636288046836853, "logits/rejected": 0.08986735343933105, "logps/chosen": -61.61492919921875, "logps/ref_chosen": -61.498905181884766, "logps/ref_rejected": -78.91172790527344, "logps/rejected": -79.17604064941406, "loss": 1.372, "margin_dpo/margin_mean": 0.14828673005104065, "margin_dpo/margin_std": 0.4339235723018646, "step": 46 }, { "KL/chosen_KL_mean": -0.14241790771484375, "KL/mean": -0.20898690819740295, "KL/rejected_KL_mean": -0.2755584716796875, "KL/std": 0.2778022289276123, "epoch": 0.0710506424792139, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.13313710689544678, "fcm_dpo/q_t": 0.49667197465896606, "grad_norm": 28.65785789489746, "learning_rate": 3.432835820895522e-07, "logits/chosen": 0.05303880572319031, "logits/rejected": 0.009075134992599487, "logps/chosen": -51.72076416015625, "logps/ref_chosen": -51.578346252441406, "logps/ref_rejected": -68.2215576171875, "logps/rejected": -68.49711608886719, "loss": 1.3734, "margin_dpo/margin_mean": 0.13313719630241394, "margin_dpo/margin_std": 0.38025137782096863, "step": 47 }, { "KL/chosen_KL_mean": -0.2553062438964844, "KL/mean": -0.2546156048774719, "KL/rejected_KL_mean": -0.2539253234863281, "KL/std": 0.34144163131713867, "epoch": 0.07256235827664399, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013832151889801025, "fcm_dpo/q_t": 0.5000360608100891, "grad_norm": 26.89451789855957, "learning_rate": 3.507462686567164e-07, "logits/chosen": 0.16787520051002502, "logits/rejected": 0.1375715285539627, "logps/chosen": -52.04895782470703, "logps/ref_chosen": -51.79365158081055, "logps/ref_rejected": -64.22503662109375, "logps/rejected": -64.47896575927734, "loss": 1.3871, "margin_dpo/margin_mean": -0.0013829171657562256, "margin_dpo/margin_std": 0.4911916255950928, "step": 48 }, { "KL/chosen_KL_mean": -0.20473098754882812, "KL/mean": -0.23833119869232178, "KL/rejected_KL_mean": -0.2719306945800781, "KL/std": 0.333074688911438, "epoch": 0.07407407407407407, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06719930469989777, "fcm_dpo/q_t": 0.4983193874359131, "grad_norm": 27.35091781616211, "learning_rate": 3.5820895522388055e-07, "logits/chosen": 0.03351443260908127, "logits/rejected": 0.011932183057069778, "logps/chosen": -58.339332580566406, "logps/ref_chosen": -58.13460159301758, "logps/ref_rejected": -64.63206481933594, "logps/rejected": -64.90399932861328, "loss": 1.3801, "margin_dpo/margin_mean": 0.06719934940338135, "margin_dpo/margin_std": 0.4344923794269562, "step": 49 }, { "KL/chosen_KL_mean": -0.25493812561035156, "KL/mean": -0.3313453197479248, "KL/rejected_KL_mean": -0.40775299072265625, "KL/std": 0.3598587214946747, "epoch": 0.07558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15281718969345093, "fcm_dpo/q_t": 0.4961821436882019, "grad_norm": 27.900310516357422, "learning_rate": 3.6567164179104475e-07, "logits/chosen": 0.11298177391290665, "logits/rejected": 0.08316892385482788, "logps/chosen": -53.11137771606445, "logps/ref_chosen": -52.85643768310547, "logps/ref_rejected": -72.17460632324219, "logps/rejected": -72.58235168457031, "loss": 1.3715, "margin_dpo/margin_mean": 0.15281730890274048, "margin_dpo/margin_std": 0.40361297130584717, "step": 50 }, { "KL/chosen_KL_mean": -0.3104381561279297, "KL/mean": -0.41523122787475586, "KL/rejected_KL_mean": -0.5200233459472656, "KL/std": 0.4115421772003174, "epoch": 0.07709750566893424, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20958328247070312, "fcm_dpo/q_t": 0.49476635456085205, "grad_norm": 30.999216079711914, "learning_rate": 3.7313432835820895e-07, "logits/chosen": 0.07924918085336685, "logits/rejected": 0.05170644819736481, "logps/chosen": -63.966880798339844, "logps/ref_chosen": -63.65644073486328, "logps/ref_rejected": -86.13229370117188, "logps/rejected": -86.6523208618164, "loss": 1.3661, "margin_dpo/margin_mean": 0.209583580493927, "margin_dpo/margin_std": 0.49226850271224976, "step": 51 }, { "KL/chosen_KL_mean": -0.32818603515625, "KL/mean": -0.4305916428565979, "KL/rejected_KL_mean": -0.5329971313476562, "KL/std": 0.40797537565231323, "epoch": 0.07860922146636433, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2048064023256302, "fcm_dpo/q_t": 0.4948863089084625, "grad_norm": 32.73704528808594, "learning_rate": 3.805970149253731e-07, "logits/chosen": 0.07293161749839783, "logits/rejected": 0.024682261049747467, "logps/chosen": -68.16839599609375, "logps/ref_chosen": -67.8402099609375, "logps/ref_rejected": -96.97090911865234, "logps/rejected": -97.50390625, "loss": 1.3666, "margin_dpo/margin_mean": 0.2048058956861496, "margin_dpo/margin_std": 0.5083526372909546, "step": 52 }, { "KL/chosen_KL_mean": -0.34824371337890625, "KL/mean": -0.43194323778152466, "KL/rejected_KL_mean": -0.5156440734863281, "KL/std": 0.3921298384666443, "epoch": 0.0801209372637944, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1674003303050995, "fcm_dpo/q_t": 0.4958154559135437, "grad_norm": 27.480405807495117, "learning_rate": 3.880597014925373e-07, "logits/chosen": 0.07155004888772964, "logits/rejected": 0.06118874251842499, "logps/chosen": -57.226383209228516, "logps/ref_chosen": -56.87813949584961, "logps/ref_rejected": -60.75569152832031, "logps/rejected": -61.271339416503906, "loss": 1.3702, "margin_dpo/margin_mean": 0.1674005091190338, "margin_dpo/margin_std": 0.4693652391433716, "step": 53 }, { "KL/chosen_KL_mean": -0.3873310089111328, "KL/mean": -0.46201127767562866, "KL/rejected_KL_mean": -0.5366916656494141, "KL/std": 0.39615002274513245, "epoch": 0.08163265306122448, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14936020970344543, "fcm_dpo/q_t": 0.4962688088417053, "grad_norm": 26.520944595336914, "learning_rate": 3.9552238805970144e-07, "logits/chosen": 0.03022252582013607, "logits/rejected": 0.015316802076995373, "logps/chosen": -47.65425109863281, "logps/ref_chosen": -47.26692199707031, "logps/ref_rejected": -62.19426727294922, "logps/rejected": -62.73095703125, "loss": 1.3721, "margin_dpo/margin_mean": 0.14936044812202454, "margin_dpo/margin_std": 0.5132778286933899, "step": 54 }, { "KL/chosen_KL_mean": -0.3409919738769531, "KL/mean": -0.4967942535877228, "KL/rejected_KL_mean": -0.6526031494140625, "KL/std": 0.48860490322113037, "epoch": 0.08314436885865457, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3116055130958557, "fcm_dpo/q_t": 0.49222302436828613, "grad_norm": 31.283021926879883, "learning_rate": 4.0298507462686564e-07, "logits/chosen": 0.04874131828546524, "logits/rejected": -0.029031695798039436, "logps/chosen": -50.66718292236328, "logps/ref_chosen": -50.32619094848633, "logps/ref_rejected": -92.44389343261719, "logps/rejected": -93.09649658203125, "loss": 1.3565, "margin_dpo/margin_mean": 0.31160497665405273, "margin_dpo/margin_std": 0.6381343603134155, "step": 55 }, { "KL/chosen_KL_mean": -0.34792327880859375, "KL/mean": -0.44482094049453735, "KL/rejected_KL_mean": -0.5417137145996094, "KL/std": 0.4808979034423828, "epoch": 0.08465608465608465, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19379210472106934, "fcm_dpo/q_t": 0.49515196681022644, "grad_norm": 27.554279327392578, "learning_rate": 4.1044776119402984e-07, "logits/chosen": 0.13023659586906433, "logits/rejected": 0.10778436064720154, "logps/chosen": -57.11489486694336, "logps/ref_chosen": -56.766971588134766, "logps/ref_rejected": -66.30504608154297, "logps/rejected": -66.84675598144531, "loss": 1.3682, "margin_dpo/margin_mean": 0.19379198551177979, "margin_dpo/margin_std": 0.671954333782196, "step": 56 }, { "KL/chosen_KL_mean": -0.5123443603515625, "KL/mean": -0.6745944023132324, "KL/rejected_KL_mean": -0.8368415832519531, "KL/std": 0.6111703515052795, "epoch": 0.08616780045351474, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3244953751564026, "fcm_dpo/q_t": 0.49190065264701843, "grad_norm": 30.53541374206543, "learning_rate": 4.17910447761194e-07, "logits/chosen": 0.12122000753879547, "logits/rejected": 0.05556311830878258, "logps/chosen": -58.28009033203125, "logps/ref_chosen": -57.76774597167969, "logps/ref_rejected": -82.75698852539062, "logps/rejected": -83.59382629394531, "loss": 1.3553, "margin_dpo/margin_mean": 0.3244949281215668, "margin_dpo/margin_std": 0.6604194641113281, "step": 57 }, { "KL/chosen_KL_mean": -0.5639533996582031, "KL/mean": -0.6952039003372192, "KL/rejected_KL_mean": -0.8264541625976562, "KL/std": 0.6729289889335632, "epoch": 0.08767951625094482, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26249879598617554, "fcm_dpo/q_t": 0.4935028851032257, "grad_norm": 30.190555572509766, "learning_rate": 4.253731343283582e-07, "logits/chosen": 0.05941343307495117, "logits/rejected": 0.04385446757078171, "logps/chosen": -73.32804107666016, "logps/ref_chosen": -72.76408386230469, "logps/ref_rejected": -84.49275207519531, "logps/rejected": -85.31920623779297, "loss": 1.363, "margin_dpo/margin_mean": 0.2624986469745636, "margin_dpo/margin_std": 0.9827470183372498, "step": 58 }, { "KL/chosen_KL_mean": -0.5376720428466797, "KL/mean": -0.6911669969558716, "KL/rejected_KL_mean": -0.8446674346923828, "KL/std": 0.6432867050170898, "epoch": 0.08919123204837491, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3069899380207062, "fcm_dpo/q_t": 0.4923567771911621, "grad_norm": 26.56601905822754, "learning_rate": 4.3283582089552234e-07, "logits/chosen": 0.12795722484588623, "logits/rejected": 0.06226852163672447, "logps/chosen": -50.35844802856445, "logps/ref_chosen": -49.820777893066406, "logps/ref_rejected": -77.14368438720703, "logps/rejected": -77.98834991455078, "loss": 1.3577, "margin_dpo/margin_mean": 0.3069910705089569, "margin_dpo/margin_std": 0.8052564859390259, "step": 59 }, { "KL/chosen_KL_mean": -0.6147499084472656, "KL/mean": -0.6438243389129639, "KL/rejected_KL_mean": -0.6729011535644531, "KL/std": 0.6342208981513977, "epoch": 0.09070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05814960598945618, "fcm_dpo/q_t": 0.498546302318573, "grad_norm": 29.609012603759766, "learning_rate": 4.4029850746268654e-07, "logits/chosen": 0.13567467033863068, "logits/rejected": 0.13422152400016785, "logps/chosen": -63.83952331542969, "logps/ref_chosen": -63.22477340698242, "logps/ref_rejected": -61.360477447509766, "logps/rejected": -62.03337860107422, "loss": 1.3822, "margin_dpo/margin_mean": 0.05814942717552185, "margin_dpo/margin_std": 0.809841513633728, "step": 60 }, { "KL/chosen_KL_mean": -0.7202339172363281, "KL/mean": -0.7759703397750854, "KL/rejected_KL_mean": -0.8317070007324219, "KL/std": 0.7063132524490356, "epoch": 0.09221466364323508, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11147007346153259, "fcm_dpo/q_t": 0.49721381068229675, "grad_norm": 27.97510528564453, "learning_rate": 4.4776119402985074e-07, "logits/chosen": 0.14454835653305054, "logits/rejected": 0.11166666448116302, "logps/chosen": -49.737030029296875, "logps/ref_chosen": -49.01679992675781, "logps/ref_rejected": -74.90817260742188, "logps/rejected": -75.73987579345703, "loss": 1.3776, "margin_dpo/margin_mean": 0.11146968603134155, "margin_dpo/margin_std": 0.9815603494644165, "step": 61 }, { "KL/chosen_KL_mean": -0.7713375091552734, "KL/mean": -0.8586764931678772, "KL/rejected_KL_mean": -0.9460220336914062, "KL/std": 0.6308267712593079, "epoch": 0.09372637944066516, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1746814250946045, "fcm_dpo/q_t": 0.4956405758857727, "grad_norm": 28.683074951171875, "learning_rate": 4.552238805970149e-07, "logits/chosen": 0.09312943369150162, "logits/rejected": 0.0548982173204422, "logps/chosen": -63.52320861816406, "logps/ref_chosen": -62.751869201660156, "logps/ref_rejected": -78.93360900878906, "logps/rejected": -79.87963104248047, "loss": 1.3708, "margin_dpo/margin_mean": 0.1746809184551239, "margin_dpo/margin_std": 0.8726956248283386, "step": 62 }, { "KL/chosen_KL_mean": -0.5630893707275391, "KL/mean": -0.8304413557052612, "KL/rejected_KL_mean": -1.0978012084960938, "KL/std": 0.6817377209663391, "epoch": 0.09523809523809523, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5347055196762085, "fcm_dpo/q_t": 0.48666688799858093, "grad_norm": 31.4731502532959, "learning_rate": 4.626865671641791e-07, "logits/chosen": 0.1990186870098114, "logits/rejected": 0.1737050712108612, "logps/chosen": -61.078338623046875, "logps/ref_chosen": -60.51525115966797, "logps/ref_rejected": -85.11021423339844, "logps/rejected": -86.20801544189453, "loss": 1.3355, "margin_dpo/margin_mean": 0.534705638885498, "margin_dpo/margin_std": 0.8916192054748535, "step": 63 }, { "KL/chosen_KL_mean": -0.8345146179199219, "KL/mean": -0.8636895418167114, "KL/rejected_KL_mean": -0.8928680419921875, "KL/std": 0.7093801498413086, "epoch": 0.09674981103552532, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.058351099491119385, "fcm_dpo/q_t": 0.4985370635986328, "grad_norm": 26.187711715698242, "learning_rate": 4.701492537313433e-07, "logits/chosen": 0.07552579045295715, "logits/rejected": 0.05123044550418854, "logps/chosen": -52.041358947753906, "logps/ref_chosen": -51.20684814453125, "logps/ref_rejected": -66.93081665039062, "logps/rejected": -67.82368469238281, "loss": 1.3824, "margin_dpo/margin_mean": 0.058351099491119385, "margin_dpo/margin_std": 0.8890206813812256, "step": 64 }, { "KL/chosen_KL_mean": -0.8652439117431641, "KL/mean": -1.0923452377319336, "KL/rejected_KL_mean": -1.3194541931152344, "KL/std": 0.8883547782897949, "epoch": 0.0982615268329554, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4542124569416046, "fcm_dpo/q_t": 0.48869743943214417, "grad_norm": 30.891233444213867, "learning_rate": 4.776119402985074e-07, "logits/chosen": 0.1984306275844574, "logits/rejected": 0.16880089044570923, "logps/chosen": -68.15393829345703, "logps/ref_chosen": -67.2886962890625, "logps/ref_rejected": -74.44281005859375, "logps/rejected": -75.76226043701172, "loss": 1.3448, "margin_dpo/margin_mean": 0.4542120397090912, "margin_dpo/margin_std": 1.1676384210586548, "step": 65 }, { "KL/chosen_KL_mean": -0.9264984130859375, "KL/mean": -1.099461555480957, "KL/rejected_KL_mean": -1.2724266052246094, "KL/std": 0.8448681831359863, "epoch": 0.09977324263038549, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.34592562913894653, "fcm_dpo/q_t": 0.4913817346096039, "grad_norm": 29.38481330871582, "learning_rate": 4.850746268656717e-07, "logits/chosen": 0.11555850505828857, "logits/rejected": 0.09126327931880951, "logps/chosen": -71.66990661621094, "logps/ref_chosen": -70.743408203125, "logps/ref_rejected": -77.26499938964844, "logps/rejected": -78.53742980957031, "loss": 1.3552, "margin_dpo/margin_mean": 0.34592553973197937, "margin_dpo/margin_std": 1.115427017211914, "step": 66 }, { "KL/chosen_KL_mean": -0.7729225158691406, "KL/mean": -0.9254127144813538, "KL/rejected_KL_mean": -1.0778999328613281, "KL/std": 0.7637161612510681, "epoch": 0.10128495842781557, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.30497562885284424, "fcm_dpo/q_t": 0.49239760637283325, "grad_norm": 29.26283073425293, "learning_rate": 4.925373134328357e-07, "logits/chosen": 0.10749173164367676, "logits/rejected": 0.05099410563707352, "logps/chosen": -61.375526428222656, "logps/ref_chosen": -60.60260009765625, "logps/ref_rejected": -75.22235870361328, "logps/rejected": -76.30026245117188, "loss": 1.3582, "margin_dpo/margin_mean": 0.3049759268760681, "margin_dpo/margin_std": 0.9200187921524048, "step": 67 }, { "KL/chosen_KL_mean": -1.2506217956542969, "KL/mean": -1.421552062034607, "KL/rejected_KL_mean": -1.5924835205078125, "KL/std": 0.9510532021522522, "epoch": 0.10279667422524566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.34186407923698425, "fcm_dpo/q_t": 0.49147123098373413, "grad_norm": 31.39417266845703, "learning_rate": 5e-07, "logits/chosen": 0.06202474609017372, "logits/rejected": 0.03223402798175812, "logps/chosen": -78.77899169921875, "logps/ref_chosen": -77.52836608886719, "logps/ref_rejected": -93.17778015136719, "logps/rejected": -94.770263671875, "loss": 1.3563, "margin_dpo/margin_mean": 0.34186458587646484, "margin_dpo/margin_std": 1.2264301776885986, "step": 68 }, { "KL/chosen_KL_mean": -1.184225082397461, "KL/mean": -1.4764790534973145, "KL/rejected_KL_mean": -1.7687301635742188, "KL/std": 1.024730920791626, "epoch": 0.10430839002267574, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5845098495483398, "fcm_dpo/q_t": 0.4854779541492462, "grad_norm": 30.998098373413086, "learning_rate": 4.999965034812934e-07, "logits/chosen": 0.08792998641729355, "logits/rejected": 0.045593686401844025, "logps/chosen": -67.12727355957031, "logps/ref_chosen": -65.94305419921875, "logps/ref_rejected": -89.7735595703125, "logps/rejected": -91.54228973388672, "loss": 1.3326, "margin_dpo/margin_mean": 0.5845092535018921, "margin_dpo/margin_std": 1.2174975872039795, "step": 69 }, { "KL/chosen_KL_mean": -1.2389240264892578, "KL/mean": -1.3959846496582031, "KL/rejected_KL_mean": -1.5530376434326172, "KL/std": 0.9079196453094482, "epoch": 0.10582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.31411120295524597, "fcm_dpo/q_t": 0.49217674136161804, "grad_norm": 28.801374435424805, "learning_rate": 4.999860140229787e-07, "logits/chosen": 0.16327008605003357, "logits/rejected": 0.1396973729133606, "logps/chosen": -63.19683837890625, "logps/ref_chosen": -61.95791244506836, "logps/ref_rejected": -75.80945587158203, "logps/rejected": -77.36249542236328, "loss": 1.3588, "margin_dpo/margin_mean": 0.3141113817691803, "margin_dpo/margin_std": 1.214660406112671, "step": 70 }, { "KL/chosen_KL_mean": -1.4482479095458984, "KL/mean": -1.5614473819732666, "KL/rejected_KL_mean": -1.6746501922607422, "KL/std": 1.0249958038330078, "epoch": 0.1073318216175359, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22640283405780792, "fcm_dpo/q_t": 0.49432241916656494, "grad_norm": 28.454469680786133, "learning_rate": 4.999685319184688e-07, "logits/chosen": 0.11360330879688263, "logits/rejected": 0.09736478328704834, "logps/chosen": -64.79582214355469, "logps/ref_chosen": -63.34757995605469, "logps/ref_rejected": -67.49658203125, "logps/rejected": -69.17123413085938, "loss": 1.3693, "margin_dpo/margin_mean": 0.22640304267406464, "margin_dpo/margin_std": 1.474951982498169, "step": 71 }, { "KL/chosen_KL_mean": -1.1897659301757812, "KL/mean": -1.553931713104248, "KL/rejected_KL_mean": -1.9180984497070312, "KL/std": 1.0742740631103516, "epoch": 0.10884353741496598, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7283300161361694, "fcm_dpo/q_t": 0.48187729716300964, "grad_norm": 30.26369857788086, "learning_rate": 4.999440576567755e-07, "logits/chosen": 0.1204986423254013, "logits/rejected": 0.05717724561691284, "logps/chosen": -57.04906463623047, "logps/ref_chosen": -55.85929870605469, "logps/ref_rejected": -68.45423889160156, "logps/rejected": -70.37234497070312, "loss": 1.3191, "margin_dpo/margin_mean": 0.7283297181129456, "margin_dpo/margin_std": 1.2839291095733643, "step": 72 }, { "KL/chosen_KL_mean": -1.8362503051757812, "KL/mean": -1.8976032733917236, "KL/rejected_KL_mean": -1.9589519500732422, "KL/std": 1.2357733249664307, "epoch": 0.11035525321239607, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12270593643188477, "fcm_dpo/q_t": 0.4970957636833191, "grad_norm": 31.9867000579834, "learning_rate": 4.999125919224965e-07, "logits/chosen": 0.11336830258369446, "logits/rejected": 0.09915591776371002, "logps/chosen": -70.97505187988281, "logps/ref_chosen": -69.13880920410156, "logps/ref_rejected": -79.04586791992188, "logps/rejected": -81.00482177734375, "loss": 1.3809, "margin_dpo/margin_mean": 0.12270653247833252, "margin_dpo/margin_std": 1.5762428045272827, "step": 73 }, { "KL/chosen_KL_mean": -1.2705440521240234, "KL/mean": -1.6108319759368896, "KL/rejected_KL_mean": -1.9511184692382812, "KL/std": 1.0245332717895508, "epoch": 0.11186696900982615, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6805762648582458, "fcm_dpo/q_t": 0.48308998346328735, "grad_norm": 27.936552047729492, "learning_rate": 4.998741355957963e-07, "logits/chosen": 0.1540667563676834, "logits/rejected": 0.09951502084732056, "logps/chosen": -51.19428253173828, "logps/ref_chosen": -49.923736572265625, "logps/ref_rejected": -81.73213958740234, "logps/rejected": -83.68325805664062, "loss": 1.3243, "margin_dpo/margin_mean": 0.6805763244628906, "margin_dpo/margin_std": 1.3530315160751343, "step": 74 }, { "KL/chosen_KL_mean": -1.3861274719238281, "KL/mean": -1.7660695314407349, "KL/rejected_KL_mean": -2.146007537841797, "KL/std": 1.1102235317230225, "epoch": 0.11337868480725624, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7598824501037598, "fcm_dpo/q_t": 0.4811212420463562, "grad_norm": 26.1428165435791, "learning_rate": 4.998286897523808e-07, "logits/chosen": 0.12207367271184921, "logits/rejected": 0.0893176943063736, "logps/chosen": -47.45487976074219, "logps/ref_chosen": -46.06875228881836, "logps/ref_rejected": -66.1181411743164, "logps/rejected": -68.26415252685547, "loss": 1.3163, "margin_dpo/margin_mean": 0.7598823308944702, "margin_dpo/margin_std": 1.3477892875671387, "step": 75 }, { "KL/chosen_KL_mean": -1.5627460479736328, "KL/mean": -1.734074354171753, "KL/rejected_KL_mean": -1.9054031372070312, "KL/std": 1.2022194862365723, "epoch": 0.11489040060468632, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3426612615585327, "fcm_dpo/q_t": 0.49147289991378784, "grad_norm": 29.220705032348633, "learning_rate": 4.997762556634679e-07, "logits/chosen": 0.12868475914001465, "logits/rejected": 0.08463678508996964, "logps/chosen": -55.62549591064453, "logps/ref_chosen": -54.06275177001953, "logps/ref_rejected": -74.87464141845703, "logps/rejected": -76.78004455566406, "loss": 1.3586, "margin_dpo/margin_mean": 0.3426612913608551, "margin_dpo/margin_std": 1.5809069871902466, "step": 76 }, { "KL/chosen_KL_mean": -1.6425457000732422, "KL/mean": -2.0247294902801514, "KL/rejected_KL_mean": -2.406909942626953, "KL/std": 1.2334535121917725, "epoch": 0.1164021164021164, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7643673419952393, "fcm_dpo/q_t": 0.4810134768486023, "grad_norm": 29.239410400390625, "learning_rate": 4.99716834795752e-07, "logits/chosen": 0.17115390300750732, "logits/rejected": 0.12893466651439667, "logps/chosen": -54.71863555908203, "logps/ref_chosen": -53.07609176635742, "logps/ref_rejected": -74.45601654052734, "logps/rejected": -76.86293029785156, "loss": 1.3169, "margin_dpo/margin_mean": 0.7643671631813049, "margin_dpo/margin_std": 1.4656544923782349, "step": 77 }, { "KL/chosen_KL_mean": -1.8553199768066406, "KL/mean": -2.048530340194702, "KL/rejected_KL_mean": -2.2417449951171875, "KL/std": 1.2667438983917236, "epoch": 0.11791383219954649, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.38642293214797974, "fcm_dpo/q_t": 0.4903751313686371, "grad_norm": 29.433731079101562, "learning_rate": 4.996504288113623e-07, "logits/chosen": 0.11265414953231812, "logits/rejected": 0.09231537580490112, "logps/chosen": -69.58074188232422, "logps/ref_chosen": -67.72541809082031, "logps/ref_rejected": -79.03926849365234, "logps/rejected": -81.28101348876953, "loss": 1.3531, "margin_dpo/margin_mean": 0.3864227533340454, "margin_dpo/margin_std": 1.4329341650009155, "step": 78 }, { "KL/chosen_KL_mean": -1.7077217102050781, "KL/mean": -2.3071465492248535, "KL/rejected_KL_mean": -2.9065704345703125, "KL/std": 1.5384122133255005, "epoch": 0.11942554799697656, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1988508701324463, "fcm_dpo/q_t": 0.47053754329681396, "grad_norm": 30.798582077026367, "learning_rate": 4.995770395678171e-07, "logits/chosen": 0.18950387835502625, "logits/rejected": 0.12797753512859344, "logps/chosen": -53.86836624145508, "logps/ref_chosen": -52.16064453125, "logps/ref_rejected": -83.31062316894531, "logps/rejected": -86.21719360351562, "loss": 1.2795, "margin_dpo/margin_mean": 1.198850393295288, "margin_dpo/margin_std": 1.959941029548645, "step": 79 }, { "KL/chosen_KL_mean": -2.024747848510742, "KL/mean": -2.3166556358337402, "KL/rejected_KL_mean": -2.6085662841796875, "KL/std": 1.369431495666504, "epoch": 0.12093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5838208198547363, "fcm_dpo/q_t": 0.48558056354522705, "grad_norm": 28.381168365478516, "learning_rate": 4.994966691179711e-07, "logits/chosen": 0.14689680933952332, "logits/rejected": 0.08690305054187775, "logps/chosen": -63.43531036376953, "logps/ref_chosen": -61.410560607910156, "logps/ref_rejected": -78.66004943847656, "logps/rejected": -81.26861572265625, "loss": 1.3383, "margin_dpo/margin_mean": 0.5838209390640259, "margin_dpo/margin_std": 1.8922240734100342, "step": 80 }, { "KL/chosen_KL_mean": -2.1597976684570312, "KL/mean": -2.5934157371520996, "KL/rejected_KL_mean": -3.0270309448242188, "KL/std": 1.434091329574585, "epoch": 0.12244897959183673, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8672326803207397, "fcm_dpo/q_t": 0.47862696647644043, "grad_norm": 29.007713317871094, "learning_rate": 4.994093197099587e-07, "logits/chosen": 0.13177847862243652, "logits/rejected": 0.09794065356254578, "logps/chosen": -65.96417236328125, "logps/ref_chosen": -63.80437088012695, "logps/ref_rejected": -79.3484115600586, "logps/rejected": -82.37544250488281, "loss": 1.3097, "margin_dpo/margin_mean": 0.8672324419021606, "margin_dpo/margin_std": 1.7289981842041016, "step": 81 }, { "KL/chosen_KL_mean": -1.7621707916259766, "KL/mean": -2.3001327514648438, "KL/rejected_KL_mean": -2.8380966186523438, "KL/std": 1.359740972518921, "epoch": 0.12396069538926682, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.075925588607788, "fcm_dpo/q_t": 0.4732835590839386, "grad_norm": 26.572818756103516, "learning_rate": 4.993149937871306e-07, "logits/chosen": 0.12116063386201859, "logits/rejected": 0.057523056864738464, "logps/chosen": -50.58006286621094, "logps/ref_chosen": -48.817893981933594, "logps/ref_rejected": -70.31497955322266, "logps/rejected": -73.153076171875, "loss": 1.2876, "margin_dpo/margin_mean": 1.0759259462356567, "margin_dpo/margin_std": 1.5349102020263672, "step": 82 }, { "KL/chosen_KL_mean": -2.0419349670410156, "KL/mean": -2.62204647064209, "KL/rejected_KL_mean": -3.202159881591797, "KL/std": 1.4831633567810059, "epoch": 0.1254724111866969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1602277755737305, "fcm_dpo/q_t": 0.47140026092529297, "grad_norm": 29.156784057617188, "learning_rate": 4.992136939879856e-07, "logits/chosen": 0.18060043454170227, "logits/rejected": 0.13148349523544312, "logps/chosen": -59.192710876464844, "logps/ref_chosen": -57.15077209472656, "logps/ref_rejected": -75.1710205078125, "logps/rejected": -78.37318420410156, "loss": 1.2821, "margin_dpo/margin_mean": 1.1602275371551514, "margin_dpo/margin_std": 1.8395249843597412, "step": 83 }, { "KL/chosen_KL_mean": -2.5159435272216797, "KL/mean": -2.952871322631836, "KL/rejected_KL_mean": -3.389801025390625, "KL/std": 1.8310933113098145, "epoch": 0.12698412698412698, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8738585710525513, "fcm_dpo/q_t": 0.4783720076084137, "grad_norm": 30.70692253112793, "learning_rate": 4.991054231460969e-07, "logits/chosen": 0.16691604256629944, "logits/rejected": 0.1250467747449875, "logps/chosen": -67.29324340820312, "logps/ref_chosen": -64.77729797363281, "logps/ref_rejected": -84.71949768066406, "logps/rejected": -88.10929870605469, "loss": 1.3101, "margin_dpo/margin_mean": 0.8738582134246826, "margin_dpo/margin_std": 1.9217841625213623, "step": 84 }, { "KL/chosen_KL_mean": -2.2303695678710938, "KL/mean": -2.8911349773406982, "KL/rejected_KL_mean": -3.5518951416015625, "KL/std": 1.786381721496582, "epoch": 0.12849584278155707, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3215255737304688, "fcm_dpo/q_t": 0.46737951040267944, "grad_norm": 26.60832405090332, "learning_rate": 4.989901842900325e-07, "logits/chosen": 0.14956089854240417, "logits/rejected": 0.10656014084815979, "logps/chosen": -52.48206329345703, "logps/ref_chosen": -50.25169372558594, "logps/ref_rejected": -66.55439758300781, "logps/rejected": -70.10629272460938, "loss": 1.2688, "margin_dpo/margin_mean": 1.3215255737304688, "margin_dpo/margin_std": 1.9796284437179565, "step": 85 }, { "KL/chosen_KL_mean": -2.857900619506836, "KL/mean": -3.3198752403259277, "KL/rejected_KL_mean": -3.781848907470703, "KL/std": 1.9224777221679688, "epoch": 0.13000755857898716, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9239518642425537, "fcm_dpo/q_t": 0.47717535495758057, "grad_norm": 27.272216796875, "learning_rate": 4.988679806432711e-07, "logits/chosen": 0.16544556617736816, "logits/rejected": 0.147491455078125, "logps/chosen": -63.58708190917969, "logps/ref_chosen": -60.72917938232422, "logps/ref_rejected": -72.30961608886719, "logps/rejected": -76.09146118164062, "loss": 1.3065, "margin_dpo/margin_mean": 0.9239521026611328, "margin_dpo/margin_std": 2.0516607761383057, "step": 86 }, { "KL/chosen_KL_mean": -3.028636932373047, "KL/mean": -3.547091007232666, "KL/rejected_KL_mean": -4.065544128417969, "KL/std": 2.0648305416107178, "epoch": 0.13151927437641722, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0369104146957397, "fcm_dpo/q_t": 0.4744616746902466, "grad_norm": 30.57183837890625, "learning_rate": 4.987388156241114e-07, "logits/chosen": 0.14023475348949432, "logits/rejected": 0.08374907076358795, "logps/chosen": -68.78660583496094, "logps/ref_chosen": -65.75796508789062, "logps/ref_rejected": -84.81159973144531, "logps/rejected": -88.87713623046875, "loss": 1.3007, "margin_dpo/margin_mean": 1.0369102954864502, "margin_dpo/margin_std": 2.476595640182495, "step": 87 }, { "KL/chosen_KL_mean": -3.0699501037597656, "KL/mean": -3.4808826446533203, "KL/rejected_KL_mean": -3.8918113708496094, "KL/std": 2.3378701210021973, "epoch": 0.1330309901738473, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8218619227409363, "fcm_dpo/q_t": 0.4799662232398987, "grad_norm": 30.38210105895996, "learning_rate": 4.986026928455767e-07, "logits/chosen": 0.23296837508678436, "logits/rejected": 0.2060338854789734, "logps/chosen": -65.89398193359375, "logps/ref_chosen": -62.82402801513672, "logps/ref_rejected": -74.9607162475586, "logps/rejected": -78.85252380371094, "loss": 1.3288, "margin_dpo/margin_mean": 0.821861982345581, "margin_dpo/margin_std": 2.9647645950317383, "step": 88 }, { "KL/chosen_KL_mean": -2.5609130859375, "KL/mean": -3.2582736015319824, "KL/rejected_KL_mean": -3.9556312561035156, "KL/std": 2.156515598297119, "epoch": 0.1345427059712774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3947181701660156, "fcm_dpo/q_t": 0.4660217761993408, "grad_norm": 29.258838653564453, "learning_rate": 4.984596161153135e-07, "logits/chosen": 0.237514466047287, "logits/rejected": 0.15626181662082672, "logps/chosen": -43.752349853515625, "logps/ref_chosen": -41.191436767578125, "logps/ref_rejected": -85.44769287109375, "logps/rejected": -89.40332794189453, "loss": 1.2685, "margin_dpo/margin_mean": 1.3947186470031738, "margin_dpo/margin_std": 2.536036968231201, "step": 89 }, { "KL/chosen_KL_mean": -3.1376190185546875, "KL/mean": -3.7834763526916504, "KL/rejected_KL_mean": -4.4293365478515625, "KL/std": 2.46217942237854, "epoch": 0.1360544217687075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2917141914367676, "fcm_dpo/q_t": 0.4683051109313965, "grad_norm": 29.12344741821289, "learning_rate": 4.983095894354857e-07, "logits/chosen": 0.168723464012146, "logits/rejected": 0.11255622655153275, "logps/chosen": -59.721527099609375, "logps/ref_chosen": -56.58390808105469, "logps/ref_rejected": -86.86978149414062, "logps/rejected": -91.29911804199219, "loss": 1.2769, "margin_dpo/margin_mean": 1.291713833808899, "margin_dpo/margin_std": 2.4582650661468506, "step": 90 }, { "KL/chosen_KL_mean": -3.149148941040039, "KL/mean": -3.722820281982422, "KL/rejected_KL_mean": -4.296489715576172, "KL/std": 2.303582191467285, "epoch": 0.13756613756613756, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1473394632339478, "fcm_dpo/q_t": 0.47223398089408875, "grad_norm": 25.720582962036133, "learning_rate": 4.98152617002662e-07, "logits/chosen": 0.1862848550081253, "logits/rejected": 0.14085903763771057, "logps/chosen": -55.531494140625, "logps/ref_chosen": -52.38234329223633, "logps/ref_rejected": -72.17642211914062, "logps/rejected": -76.47291564941406, "loss": 1.2941, "margin_dpo/margin_mean": 1.147339105606079, "margin_dpo/margin_std": 2.7885167598724365, "step": 91 }, { "KL/chosen_KL_mean": -3.230426788330078, "KL/mean": -4.01956844329834, "KL/rejected_KL_mean": -4.808712005615234, "KL/std": 2.6578006744384766, "epoch": 0.13907785336356765, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.578284740447998, "fcm_dpo/q_t": 0.4619179964065552, "grad_norm": 26.956829071044922, "learning_rate": 4.979887032076988e-07, "logits/chosen": 0.22094112634658813, "logits/rejected": 0.1804642379283905, "logps/chosen": -56.23912811279297, "logps/ref_chosen": -53.00870132446289, "logps/ref_rejected": -79.77812957763672, "logps/rejected": -84.58683776855469, "loss": 1.2588, "margin_dpo/margin_mean": 1.578284502029419, "margin_dpo/margin_std": 3.120558738708496, "step": 92 }, { "KL/chosen_KL_mean": -3.615804672241211, "KL/mean": -4.103872299194336, "KL/rejected_KL_mean": -4.591936111450195, "KL/std": 2.629575729370117, "epoch": 0.14058956916099774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9761362075805664, "fcm_dpo/q_t": 0.4767376184463501, "grad_norm": 24.554616928100586, "learning_rate": 4.978178526356172e-07, "logits/chosen": 0.20379352569580078, "logits/rejected": 0.17453384399414062, "logps/chosen": -48.522857666015625, "logps/ref_chosen": -44.90705108642578, "logps/ref_rejected": -58.7879524230957, "logps/rejected": -63.37989044189453, "loss": 1.3171, "margin_dpo/margin_mean": 0.9761360883712769, "margin_dpo/margin_std": 3.2695860862731934, "step": 93 }, { "KL/chosen_KL_mean": -3.2867393493652344, "KL/mean": -4.221029758453369, "KL/rejected_KL_mean": -5.1553192138671875, "KL/std": 3.2257328033447266, "epoch": 0.1421012849584278, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8685789108276367, "fcm_dpo/q_t": 0.45501282811164856, "grad_norm": 27.455610275268555, "learning_rate": 4.976400700654751e-07, "logits/chosen": 0.2643049359321594, "logits/rejected": 0.22353461384773254, "logps/chosen": -63.224510192871094, "logps/ref_chosen": -59.93777084350586, "logps/ref_rejected": -79.3138427734375, "logps/rejected": -84.46916198730469, "loss": 1.2467, "margin_dpo/margin_mean": 1.868578553199768, "margin_dpo/margin_std": 3.965810537338257, "step": 94 }, { "KL/chosen_KL_mean": -4.280330657958984, "KL/mean": -5.080137252807617, "KL/rejected_KL_mean": -5.87994384765625, "KL/std": 2.722566604614258, "epoch": 0.1436130007558579, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5996131896972656, "fcm_dpo/q_t": 0.4613925516605377, "grad_norm": 29.14002799987793, "learning_rate": 4.974553604702332e-07, "logits/chosen": 0.15258441865444183, "logits/rejected": 0.0883299708366394, "logps/chosen": -64.44882202148438, "logps/ref_chosen": -60.168487548828125, "logps/ref_rejected": -90.73665618896484, "logps/rejected": -96.6166000366211, "loss": 1.2618, "margin_dpo/margin_mean": 1.5996129512786865, "margin_dpo/margin_std": 3.4419288635253906, "step": 95 }, { "KL/chosen_KL_mean": -4.186677932739258, "KL/mean": -5.113858222961426, "KL/rejected_KL_mean": -6.041038513183594, "KL/std": 3.425692081451416, "epoch": 0.14512471655328799, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8543610572814941, "fcm_dpo/q_t": 0.4570516049861908, "grad_norm": 27.39577293395996, "learning_rate": 4.972637290166157e-07, "logits/chosen": 0.18779154121875763, "logits/rejected": 0.14556936919689178, "logps/chosen": -64.85545349121094, "logps/ref_chosen": -60.66877746582031, "logps/ref_rejected": -88.30673217773438, "logps/rejected": -94.34777069091797, "loss": 1.2594, "margin_dpo/margin_mean": 1.854360818862915, "margin_dpo/margin_std": 4.501690864562988, "step": 96 }, { "KL/chosen_KL_mean": -4.962148666381836, "KL/mean": -5.490826606750488, "KL/rejected_KL_mean": -6.019508361816406, "KL/std": 3.6154727935791016, "epoch": 0.14663643235071808, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0573649406433105, "fcm_dpo/q_t": 0.47493094205856323, "grad_norm": 36.246150970458984, "learning_rate": 4.970651810649666e-07, "logits/chosen": 0.12958383560180664, "logits/rejected": 0.08639142662286758, "logps/chosen": -70.00627136230469, "logps/ref_chosen": -65.04412078857422, "logps/ref_rejected": -78.42092895507812, "logps/rejected": -84.44043731689453, "loss": 1.3422, "margin_dpo/margin_mean": 1.0573647022247314, "margin_dpo/margin_std": 4.932512283325195, "step": 97 }, { "KL/chosen_KL_mean": -4.633974075317383, "KL/mean": -4.971208095550537, "KL/rejected_KL_mean": -5.308444976806641, "KL/std": 3.280994415283203, "epoch": 0.14814814814814814, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6744681596755981, "fcm_dpo/q_t": 0.4831414818763733, "grad_norm": 30.31377410888672, "learning_rate": 4.968597221690985e-07, "logits/chosen": 0.23023685812950134, "logits/rejected": 0.2029840350151062, "logps/chosen": -60.13720703125, "logps/ref_chosen": -55.503231048583984, "logps/ref_rejected": -72.81553649902344, "logps/rejected": -78.12398529052734, "loss": 1.3469, "margin_dpo/margin_mean": 0.6744686365127563, "margin_dpo/margin_std": 3.2570295333862305, "step": 98 }, { "KL/chosen_KL_mean": -4.691219329833984, "KL/mean": -5.205721855163574, "KL/rejected_KL_mean": -5.7202301025390625, "KL/std": 3.546834945678711, "epoch": 0.14965986394557823, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0290096998214722, "fcm_dpo/q_t": 0.4757050573825836, "grad_norm": 32.59518814086914, "learning_rate": 4.966473580761389e-07, "logits/chosen": 0.2541065216064453, "logits/rejected": 0.21417078375816345, "logps/chosen": -63.2668571472168, "logps/ref_chosen": -58.57563781738281, "logps/ref_rejected": -78.693603515625, "logps/rejected": -84.41383361816406, "loss": 1.3378, "margin_dpo/margin_mean": 1.0290095806121826, "margin_dpo/margin_std": 4.595114707946777, "step": 99 }, { "KL/chosen_KL_mean": -5.293548583984375, "KL/mean": -6.042011260986328, "KL/rejected_KL_mean": -6.790477752685547, "KL/std": 4.170048236846924, "epoch": 0.15117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.496924638748169, "fcm_dpo/q_t": 0.46660345792770386, "grad_norm": 32.89773941040039, "learning_rate": 4.964280947263676e-07, "logits/chosen": 0.20524325966835022, "logits/rejected": 0.19783976674079895, "logps/chosen": -84.87698364257812, "logps/ref_chosen": -79.58343505859375, "logps/ref_rejected": -92.152587890625, "logps/rejected": -98.94306182861328, "loss": 1.3287, "margin_dpo/margin_mean": 1.4969251155853271, "margin_dpo/margin_std": 5.857483386993408, "step": 100 }, { "KL/chosen_KL_mean": -4.28514289855957, "KL/mean": -5.326085090637207, "KL/rejected_KL_mean": -6.367027282714844, "KL/std": 3.696342945098877, "epoch": 0.15268329554043839, "fcm_dpo/beta": 0.22752070426940918, "fcm_dpo/delta": 0.07919151335954666, "fcm_dpo/margin": 2.0818839073181152, "fcm_dpo/q_t": 0.4497947096824646, "grad_norm": 81.98783111572266, "learning_rate": 4.96201938253052e-07, "logits/chosen": 0.21574577689170837, "logits/rejected": 0.17738720774650574, "logps/chosen": -56.61792755126953, "logps/ref_chosen": -52.332786560058594, "logps/ref_rejected": -69.55589294433594, "logps/rejected": -75.92292785644531, "loss": 1.2433, "margin_dpo/margin_mean": 2.0818839073181152, "margin_dpo/margin_std": 4.205405235290527, "step": 101 }, { "KL/chosen_KL_mean": -5.278135299682617, "KL/mean": -5.710550308227539, "KL/rejected_KL_mean": -6.142971038818359, "KL/std": 3.6688332557678223, "epoch": 0.15419501133786848, "fcm_dpo/beta": 0.35483452677726746, "fcm_dpo/delta": -7.2879483923316e-05, "fcm_dpo/margin": 0.8648367524147034, "fcm_dpo/q_t": 0.47156286239624023, "grad_norm": 162.13876342773438, "learning_rate": 4.959688949822748e-07, "logits/chosen": 0.13888651132583618, "logits/rejected": 0.10068652778863907, "logps/chosen": -70.02162170410156, "logps/ref_chosen": -64.74348449707031, "logps/ref_rejected": -69.06132507324219, "logps/rejected": -75.20429992675781, "loss": 1.7198, "margin_dpo/margin_mean": 0.8648366928100586, "margin_dpo/margin_std": 5.406614303588867, "step": 102 }, { "KL/chosen_KL_mean": -4.759641647338867, "KL/mean": -5.5327959060668945, "KL/rejected_KL_mean": -6.305950164794922, "KL/std": 3.811723232269287, "epoch": 0.15570672713529857, "fcm_dpo/beta": 0.2257329523563385, "fcm_dpo/delta": -0.23946459591388702, "fcm_dpo/margin": 1.5463120937347412, "fcm_dpo/q_t": 0.3969665467739105, "grad_norm": 121.77664947509766, "learning_rate": 4.957289714327572e-07, "logits/chosen": 0.2512431740760803, "logits/rejected": 0.2177349030971527, "logps/chosen": -68.59628295898438, "logps/ref_chosen": -63.83664321899414, "logps/ref_rejected": -79.32362365722656, "logps/rejected": -85.62957763671875, "loss": 1.4546, "margin_dpo/margin_mean": 1.546311855316162, "margin_dpo/margin_std": 3.6003336906433105, "step": 103 }, { "KL/chosen_KL_mean": -4.559057235717773, "KL/mean": -5.69294548034668, "KL/rejected_KL_mean": -6.826839447021484, "KL/std": 4.804152011871338, "epoch": 0.15721844293272866, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.2677783966064453, "fcm_dpo/q_t": 0.49564671516418457, "grad_norm": 3.038849353790283, "learning_rate": 4.954821743156767e-07, "logits/chosen": 0.23869748413562775, "logits/rejected": 0.15296530723571777, "logps/chosen": -65.55826568603516, "logps/ref_chosen": -60.99920654296875, "logps/ref_rejected": -98.84645080566406, "logps/rejected": -105.67329406738281, "loss": 1.3695, "margin_dpo/margin_mean": 2.267778158187866, "margin_dpo/margin_std": 6.126588821411133, "step": 104 }, { "KL/chosen_KL_mean": -5.151271820068359, "KL/mean": -6.049022197723389, "KL/rejected_KL_mean": -6.94677734375, "KL/std": 3.8394556045532227, "epoch": 0.15873015873015872, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7955060005187988, "fcm_dpo/q_t": 0.4965510368347168, "grad_norm": 2.853419303894043, "learning_rate": 4.952285105344791e-07, "logits/chosen": 0.17378970980644226, "logits/rejected": 0.12029208987951279, "logps/chosen": -76.1015396118164, "logps/ref_chosen": -70.95027160644531, "logps/ref_rejected": -87.88340759277344, "logps/rejected": -94.83018493652344, "loss": 1.3729, "margin_dpo/margin_mean": 1.7955060005187988, "margin_dpo/margin_std": 4.681982040405273, "step": 105 }, { "KL/chosen_KL_mean": -4.813375473022461, "KL/mean": -5.589751243591309, "KL/rejected_KL_mean": -6.366130828857422, "KL/std": 3.7389588356018066, "epoch": 0.1602418745275888, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5527547597885132, "fcm_dpo/q_t": 0.4970164895057678, "grad_norm": 2.309171438217163, "learning_rate": 4.949679871846857e-07, "logits/chosen": 0.2168569266796112, "logits/rejected": 0.20323088765144348, "logps/chosen": -67.272705078125, "logps/ref_chosen": -62.45933151245117, "logps/ref_rejected": -67.00595092773438, "logps/rejected": -73.37207794189453, "loss": 1.3748, "margin_dpo/margin_mean": 1.5527546405792236, "margin_dpo/margin_std": 5.025470733642578, "step": 106 }, { "KL/chosen_KL_mean": -6.303882598876953, "KL/mean": -6.7031097412109375, "KL/rejected_KL_mean": -7.102336883544922, "KL/std": 4.748565196990967, "epoch": 0.1617535903250189, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7984510064125061, "fcm_dpo/q_t": 0.49846479296684265, "grad_norm": 2.6205790042877197, "learning_rate": 4.947006115536947e-07, "logits/chosen": 0.19034229218959808, "logits/rejected": 0.16588297486305237, "logps/chosen": -82.141845703125, "logps/ref_chosen": -75.83796691894531, "logps/ref_rejected": -87.74038696289062, "logps/rejected": -94.84272003173828, "loss": 1.3807, "margin_dpo/margin_mean": 0.7984509468078613, "margin_dpo/margin_std": 5.902289390563965, "step": 107 }, { "KL/chosen_KL_mean": -5.396709442138672, "KL/mean": -6.335811138153076, "KL/rejected_KL_mean": -7.274906158447266, "KL/std": 4.074840545654297, "epoch": 0.16326530612244897, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8781964778900146, "fcm_dpo/q_t": 0.49639254808425903, "grad_norm": 2.582226037979126, "learning_rate": 4.944263911205772e-07, "logits/chosen": 0.19318030774593353, "logits/rejected": 0.16133251786231995, "logps/chosen": -73.78994750976562, "logps/ref_chosen": -68.39323425292969, "logps/ref_rejected": -83.24267578125, "logps/rejected": -90.517578125, "loss": 1.3723, "margin_dpo/margin_mean": 1.8781968355178833, "margin_dpo/margin_std": 5.379508972167969, "step": 108 }, { "KL/chosen_KL_mean": -4.847797393798828, "KL/mean": -6.345134735107422, "KL/rejected_KL_mean": -7.842475891113281, "KL/std": 5.034552574157715, "epoch": 0.16477702191987906, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.994675636291504, "fcm_dpo/q_t": 0.49424993991851807, "grad_norm": 2.9282801151275635, "learning_rate": 4.941453335558681e-07, "logits/chosen": 0.20009146630764008, "logits/rejected": 0.14455196261405945, "logps/chosen": -60.37527847290039, "logps/ref_chosen": -55.52748107910156, "logps/ref_rejected": -83.55218505859375, "logps/rejected": -91.39466857910156, "loss": 1.3638, "margin_dpo/margin_mean": 2.994675636291504, "margin_dpo/margin_std": 5.442989349365234, "step": 109 }, { "KL/chosen_KL_mean": -6.40728759765625, "KL/mean": -6.496090888977051, "KL/rejected_KL_mean": -6.584896087646484, "KL/std": 4.201725006103516, "epoch": 0.16628873771730915, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17760717868804932, "fcm_dpo/q_t": 0.49965718388557434, "grad_norm": 2.743898630142212, "learning_rate": 4.938574467213517e-07, "logits/chosen": 0.16291868686676025, "logits/rejected": 0.17184513807296753, "logps/chosen": -87.56603240966797, "logps/ref_chosen": -81.15874481201172, "logps/ref_rejected": -72.56021118164062, "logps/rejected": -79.14511108398438, "loss": 1.3854, "margin_dpo/margin_mean": 0.1776072382926941, "margin_dpo/margin_std": 5.59630012512207, "step": 110 }, { "KL/chosen_KL_mean": -5.86181640625, "KL/mean": -6.897708892822266, "KL/rejected_KL_mean": -7.933605194091797, "KL/std": 4.601965427398682, "epoch": 0.16780045351473924, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0717906951904297, "fcm_dpo/q_t": 0.4960215389728546, "grad_norm": 2.4579412937164307, "learning_rate": 4.935627386698418e-07, "logits/chosen": 0.2789857089519501, "logits/rejected": 0.24208299815654755, "logps/chosen": -58.220802307128906, "logps/ref_chosen": -52.358985900878906, "logps/ref_rejected": -77.06150817871094, "logps/rejected": -84.99510955810547, "loss": 1.3709, "margin_dpo/margin_mean": 2.0717906951904297, "margin_dpo/margin_std": 5.603476047515869, "step": 111 }, { "KL/chosen_KL_mean": -5.752780914306641, "KL/mean": -7.0450944900512695, "KL/rejected_KL_mean": -8.337406158447266, "KL/std": 4.755733966827393, "epoch": 0.1693121693121693, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.5846307277679443, "fcm_dpo/q_t": 0.49503466486930847, "grad_norm": 3.0572729110717773, "learning_rate": 4.932612176449559e-07, "logits/chosen": 0.1905369758605957, "logits/rejected": 0.12637881934642792, "logps/chosen": -68.77284240722656, "logps/ref_chosen": -63.02006530761719, "logps/ref_rejected": -111.36941528320312, "logps/rejected": -119.70681762695312, "loss": 1.3669, "margin_dpo/margin_mean": 2.584629535675049, "margin_dpo/margin_std": 5.197779655456543, "step": 112 }, { "KL/chosen_KL_mean": -5.822654724121094, "KL/mean": -6.4955949783325195, "KL/rejected_KL_mean": -7.1685333251953125, "KL/std": 3.995758533477783, "epoch": 0.1708238851095994, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3458778858184814, "fcm_dpo/q_t": 0.49741414189338684, "grad_norm": 2.6555185317993164, "learning_rate": 4.929528920808854e-07, "logits/chosen": 0.2019399255514145, "logits/rejected": 0.16312995553016663, "logps/chosen": -61.63031768798828, "logps/ref_chosen": -55.80766296386719, "logps/ref_rejected": -69.84014129638672, "logps/rejected": -77.00867462158203, "loss": 1.3764, "margin_dpo/margin_mean": 1.3458775281906128, "margin_dpo/margin_std": 5.301340103149414, "step": 113 }, { "KL/chosen_KL_mean": -5.752845764160156, "KL/mean": -6.942575454711914, "KL/rejected_KL_mean": -8.132308959960938, "KL/std": 4.493397235870361, "epoch": 0.17233560090702948, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.3794596195220947, "fcm_dpo/q_t": 0.495430052280426, "grad_norm": 2.5707952976226807, "learning_rate": 4.92637770602159e-07, "logits/chosen": 0.26456063985824585, "logits/rejected": 0.20437636971473694, "logps/chosen": -72.08561706542969, "logps/ref_chosen": -66.33277130126953, "logps/ref_rejected": -71.61489868164062, "logps/rejected": -79.74720764160156, "loss": 1.3685, "margin_dpo/margin_mean": 2.3794591426849365, "margin_dpo/margin_std": 5.449493885040283, "step": 114 }, { "KL/chosen_KL_mean": -6.265972137451172, "KL/mean": -7.2780961990356445, "KL/rejected_KL_mean": -8.290224075317383, "KL/std": 4.36235237121582, "epoch": 0.17384731670445955, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0242466926574707, "fcm_dpo/q_t": 0.49611204862594604, "grad_norm": 2.4445128440856934, "learning_rate": 4.923158620234019e-07, "logits/chosen": 0.2416526824235916, "logits/rejected": 0.17938242852687836, "logps/chosen": -62.015010833740234, "logps/ref_chosen": -55.74903869628906, "logps/ref_rejected": -79.59849548339844, "logps/rejected": -87.88871765136719, "loss": 1.3712, "margin_dpo/margin_mean": 2.0242466926574707, "margin_dpo/margin_std": 5.242649078369141, "step": 115 }, { "KL/chosen_KL_mean": -6.046346664428711, "KL/mean": -7.712983131408691, "KL/rejected_KL_mean": -9.379619598388672, "KL/std": 4.501278877258301, "epoch": 0.17535903250188964, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3332784175872803, "fcm_dpo/q_t": 0.49359679222106934, "grad_norm": 2.852839708328247, "learning_rate": 4.91987175349089e-07, "logits/chosen": 0.2487809658050537, "logits/rejected": 0.1809816211462021, "logps/chosen": -55.41151428222656, "logps/ref_chosen": -49.36516571044922, "logps/ref_rejected": -72.84671020507812, "logps/rejected": -82.22632598876953, "loss": 1.3612, "margin_dpo/margin_mean": 3.3332786560058594, "margin_dpo/margin_std": 5.188892364501953, "step": 116 }, { "KL/chosen_KL_mean": -6.292646408081055, "KL/mean": -7.328824043273926, "KL/rejected_KL_mean": -8.365001678466797, "KL/std": 4.449042320251465, "epoch": 0.17687074829931973, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0723490715026855, "fcm_dpo/q_t": 0.496019184589386, "grad_norm": 2.2913970947265625, "learning_rate": 4.916517197732933e-07, "logits/chosen": 0.26744014024734497, "logits/rejected": 0.23043423891067505, "logps/chosen": -64.0035400390625, "logps/ref_chosen": -57.710899353027344, "logps/ref_rejected": -69.77253723144531, "logps/rejected": -78.13754272460938, "loss": 1.371, "margin_dpo/margin_mean": 2.0723495483398438, "margin_dpo/margin_std": 5.764100551605225, "step": 117 }, { "KL/chosen_KL_mean": -6.316926956176758, "KL/mean": -7.789030075073242, "KL/rejected_KL_mean": -9.261127471923828, "KL/std": 5.045524597167969, "epoch": 0.17838246409674982, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.9442031383514404, "fcm_dpo/q_t": 0.4943440854549408, "grad_norm": 2.639897346496582, "learning_rate": 4.913095046794281e-07, "logits/chosen": 0.33026766777038574, "logits/rejected": 0.28865814208984375, "logps/chosen": -58.79682540893555, "logps/ref_chosen": -52.479896545410156, "logps/ref_rejected": -81.359130859375, "logps/rejected": -90.62025451660156, "loss": 1.3642, "margin_dpo/margin_mean": 2.9442026615142822, "margin_dpo/margin_std": 5.081968307495117, "step": 118 }, { "KL/chosen_KL_mean": -8.139110565185547, "KL/mean": -9.161100387573242, "KL/rejected_KL_mean": -10.183094024658203, "KL/std": 6.225261688232422, "epoch": 0.17989417989417988, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0439817905426025, "fcm_dpo/q_t": 0.4960786998271942, "grad_norm": 2.5201234817504883, "learning_rate": 4.909605396399855e-07, "logits/chosen": 0.2730048894882202, "logits/rejected": 0.2327132672071457, "logps/chosen": -69.49678039550781, "logps/ref_chosen": -61.35767364501953, "logps/ref_rejected": -75.71510314941406, "logps/rejected": -85.898193359375, "loss": 1.3716, "margin_dpo/margin_mean": 2.0439815521240234, "margin_dpo/margin_std": 7.831289768218994, "step": 119 }, { "KL/chosen_KL_mean": -6.7714385986328125, "KL/mean": -8.515937805175781, "KL/rejected_KL_mean": -10.260440826416016, "KL/std": 5.0966644287109375, "epoch": 0.18140589569160998, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.4890010356903076, "fcm_dpo/q_t": 0.49329859018325806, "grad_norm": 2.945573091506958, "learning_rate": 4.906048344162676e-07, "logits/chosen": 0.23838064074516296, "logits/rejected": 0.17867352068424225, "logps/chosen": -66.67900848388672, "logps/ref_chosen": -59.907569885253906, "logps/ref_rejected": -79.6910629272461, "logps/rejected": -89.95150756835938, "loss": 1.3601, "margin_dpo/margin_mean": 3.4890012741088867, "margin_dpo/margin_std": 5.204916000366211, "step": 120 }, { "KL/chosen_KL_mean": -7.601043701171875, "KL/mean": -9.675811767578125, "KL/rejected_KL_mean": -11.75058364868164, "KL/std": 6.724109649658203, "epoch": 0.18291761148904007, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.149538516998291, "fcm_dpo/q_t": 0.49204063415527344, "grad_norm": 3.3024706840515137, "learning_rate": 4.902423989581143e-07, "logits/chosen": 0.35299158096313477, "logits/rejected": 0.26683202385902405, "logps/chosen": -63.267086029052734, "logps/ref_chosen": -55.66604232788086, "logps/ref_rejected": -101.56233978271484, "logps/rejected": -113.31292724609375, "loss": 1.3556, "margin_dpo/margin_mean": 4.149538516998291, "margin_dpo/margin_std": 7.892666816711426, "step": 121 }, { "KL/chosen_KL_mean": -8.613723754882812, "KL/mean": -10.285037994384766, "KL/rejected_KL_mean": -11.956352233886719, "KL/std": 7.303105354309082, "epoch": 0.18442932728647016, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3426246643066406, "fcm_dpo/q_t": 0.49359560012817383, "grad_norm": 2.7032148838043213, "learning_rate": 4.898732434036243e-07, "logits/chosen": 0.26826468110084534, "logits/rejected": 0.2332470566034317, "logps/chosen": -71.9480972290039, "logps/ref_chosen": -63.334373474121094, "logps/ref_rejected": -73.67523193359375, "logps/rejected": -85.631591796875, "loss": 1.362, "margin_dpo/margin_mean": 3.3426246643066406, "margin_dpo/margin_std": 9.229589462280273, "step": 122 }, { "KL/chosen_KL_mean": -8.551752090454102, "KL/mean": -10.23425579071045, "KL/rejected_KL_mean": -11.916763305664062, "KL/std": 6.5701446533203125, "epoch": 0.18594104308390022, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 3.3650121688842773, "fcm_dpo/q_t": 0.4935457706451416, "grad_norm": 2.6148831844329834, "learning_rate": 4.894973780788722e-07, "logits/chosen": 0.3093302845954895, "logits/rejected": 0.2672804594039917, "logps/chosen": -65.45049285888672, "logps/ref_chosen": -56.89874267578125, "logps/ref_rejected": -78.97028350830078, "logps/rejected": -90.88704681396484, "loss": 1.3614, "margin_dpo/margin_mean": 3.3650124073028564, "margin_dpo/margin_std": 7.255990505218506, "step": 123 }, { "KL/chosen_KL_mean": -9.072261810302734, "KL/mean": -11.368237495422363, "KL/rejected_KL_mean": -13.664215087890625, "KL/std": 7.325522422790527, "epoch": 0.1874527588813303, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.591949939727783, "fcm_dpo/q_t": 0.491183876991272, "grad_norm": 2.9130699634552, "learning_rate": 4.89114813497619e-07, "logits/chosen": 0.3153066039085388, "logits/rejected": 0.25329259037971497, "logps/chosen": -66.18834686279297, "logps/ref_chosen": -57.116085052490234, "logps/ref_rejected": -87.93074035644531, "logps/rejected": -101.59495544433594, "loss": 1.3524, "margin_dpo/margin_mean": 4.591949462890625, "margin_dpo/margin_std": 8.780609130859375, "step": 124 }, { "KL/chosen_KL_mean": -10.442159652709961, "KL/mean": -12.644015312194824, "KL/rejected_KL_mean": -14.845863342285156, "KL/std": 8.506006240844727, "epoch": 0.1889644746787604, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.4036970138549805, "fcm_dpo/q_t": 0.4915594458580017, "grad_norm": 3.0990495681762695, "learning_rate": 4.887255603610184e-07, "logits/chosen": 0.34969115257263184, "logits/rejected": 0.28866562247276306, "logps/chosen": -76.1483383178711, "logps/ref_chosen": -65.7061767578125, "logps/ref_rejected": -91.72711944580078, "logps/rejected": -106.57298278808594, "loss": 1.3542, "margin_dpo/margin_mean": 4.4036970138549805, "margin_dpo/margin_std": 9.770685195922852, "step": 125 }, { "KL/chosen_KL_mean": -10.139768600463867, "KL/mean": -10.864148139953613, "KL/rejected_KL_mean": -11.588525772094727, "KL/std": 6.056389808654785, "epoch": 0.19047619047619047, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4487552642822266, "fcm_dpo/q_t": 0.4972168207168579, "grad_norm": 2.5714292526245117, "learning_rate": 4.883296295573176e-07, "logits/chosen": 0.16045671701431274, "logits/rejected": 0.15517394244670868, "logps/chosen": -78.31585693359375, "logps/ref_chosen": -68.17608642578125, "logps/ref_rejected": -65.1175537109375, "logps/rejected": -76.7060775756836, "loss": 1.3758, "margin_dpo/margin_mean": 1.4487552642822266, "margin_dpo/margin_std": 6.551729679107666, "step": 126 }, { "KL/chosen_KL_mean": -10.118179321289062, "KL/mean": -12.15770149230957, "KL/rejected_KL_mean": -14.197227478027344, "KL/std": 5.957196235656738, "epoch": 0.19198790627362056, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.07904577255249, "fcm_dpo/q_t": 0.49216848611831665, "grad_norm": 2.511080265045166, "learning_rate": 4.87927032161552e-07, "logits/chosen": 0.27716439962387085, "logits/rejected": 0.2475191354751587, "logps/chosen": -71.9984130859375, "logps/ref_chosen": -61.88023376464844, "logps/ref_rejected": -68.46012878417969, "logps/rejected": -82.6573486328125, "loss": 1.356, "margin_dpo/margin_mean": 4.07904577255249, "margin_dpo/margin_std": 7.116820812225342, "step": 127 }, { "KL/chosen_KL_mean": -11.079221725463867, "KL/mean": -14.211316108703613, "KL/rejected_KL_mean": -17.343414306640625, "KL/std": 9.948829650878906, "epoch": 0.19349962207105065, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.264186859130859, "fcm_dpo/q_t": 0.4880082905292511, "grad_norm": 3.778376817703247, "learning_rate": 4.875177794352363e-07, "logits/chosen": 0.3209880590438843, "logits/rejected": 0.2622468173503876, "logps/chosen": -77.7882080078125, "logps/ref_chosen": -66.708984375, "logps/ref_rejected": -94.97969055175781, "logps/rejected": -112.32310485839844, "loss": 1.3411, "margin_dpo/margin_mean": 6.264187812805176, "margin_dpo/margin_std": 12.51683235168457, "step": 128 }, { "KL/chosen_KL_mean": -13.11799430847168, "KL/mean": -14.13327407836914, "KL/rejected_KL_mean": -15.148551940917969, "KL/std": 9.112518310546875, "epoch": 0.19501133786848074, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.0305609703063965, "fcm_dpo/q_t": 0.49610602855682373, "grad_norm": 2.6070330142974854, "learning_rate": 4.871018828260491e-07, "logits/chosen": 0.30411797761917114, "logits/rejected": 0.2988065481185913, "logps/chosen": -78.45681762695312, "logps/ref_chosen": -65.33882904052734, "logps/ref_rejected": -68.06109619140625, "logps/rejected": -83.20964050292969, "loss": 1.3727, "margin_dpo/margin_mean": 2.0305609703063965, "margin_dpo/margin_std": 11.41251277923584, "step": 129 }, { "KL/chosen_KL_mean": -12.702459335327148, "KL/mean": -15.399238586425781, "KL/rejected_KL_mean": -18.09601593017578, "KL/std": 9.932580947875977, "epoch": 0.1965230536659108, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.393557548522949, "fcm_dpo/q_t": 0.4896560311317444, "grad_norm": 3.132603883743286, "learning_rate": 4.866793539675126e-07, "logits/chosen": 0.26491034030914307, "logits/rejected": 0.21366286277770996, "logps/chosen": -71.36320495605469, "logps/ref_chosen": -58.660743713378906, "logps/ref_rejected": -79.24510192871094, "logps/rejected": -97.34111785888672, "loss": 1.3468, "margin_dpo/margin_mean": 5.393556594848633, "margin_dpo/margin_std": 10.183626174926758, "step": 130 }, { "KL/chosen_KL_mean": -11.906011581420898, "KL/mean": -15.45162582397461, "KL/rejected_KL_mean": -18.997241973876953, "KL/std": 10.712788581848145, "epoch": 0.1980347694633409, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.091231346130371, "fcm_dpo/q_t": 0.4864232838153839, "grad_norm": 3.4537715911865234, "learning_rate": 4.86250204678667e-07, "logits/chosen": 0.28533366322517395, "logits/rejected": 0.2199201136827469, "logps/chosen": -64.42054748535156, "logps/ref_chosen": -52.51453399658203, "logps/ref_rejected": -85.18299865722656, "logps/rejected": -104.18023681640625, "loss": 1.3349, "margin_dpo/margin_mean": 7.091231346130371, "margin_dpo/margin_std": 12.101551055908203, "step": 131 }, { "KL/chosen_KL_mean": -13.843952178955078, "KL/mean": -16.509641647338867, "KL/rejected_KL_mean": -19.175325393676758, "KL/std": 11.187821388244629, "epoch": 0.19954648526077098, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.331372261047363, "fcm_dpo/q_t": 0.4897724986076355, "grad_norm": 2.8171606063842773, "learning_rate": 4.858144469637408e-07, "logits/chosen": 0.3724517822265625, "logits/rejected": 0.341764897108078, "logps/chosen": -79.52908325195312, "logps/ref_chosen": -65.68513488769531, "logps/ref_rejected": -69.54120635986328, "logps/rejected": -88.71653747558594, "loss": 1.3473, "margin_dpo/margin_mean": 5.331372261047363, "margin_dpo/margin_std": 10.194868087768555, "step": 132 }, { "KL/chosen_KL_mean": -15.300018310546875, "KL/mean": -16.778060913085938, "KL/rejected_KL_mean": -18.25609588623047, "KL/std": 9.998517990112305, "epoch": 0.20105820105820105, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 2.956071376800537, "fcm_dpo/q_t": 0.4943294823169708, "grad_norm": 2.656231164932251, "learning_rate": 4.853720930118138e-07, "logits/chosen": 0.2910245954990387, "logits/rejected": 0.28055471181869507, "logps/chosen": -78.89813232421875, "logps/ref_chosen": -63.598114013671875, "logps/ref_rejected": -73.72798156738281, "logps/rejected": -91.98407745361328, "loss": 1.3658, "margin_dpo/margin_mean": 2.956071376800537, "margin_dpo/margin_std": 11.712414741516113, "step": 133 }, { "KL/chosen_KL_mean": -14.085258483886719, "KL/mean": -17.977962493896484, "KL/rejected_KL_mean": -21.87066650390625, "KL/std": 12.072915077209473, "epoch": 0.20256991685563114, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.785408973693848, "fcm_dpo/q_t": 0.4851023554801941, "grad_norm": 3.1521494388580322, "learning_rate": 4.849231551964771e-07, "logits/chosen": 0.41877007484436035, "logits/rejected": 0.36156511306762695, "logps/chosen": -67.87982940673828, "logps/ref_chosen": -53.79457092285156, "logps/ref_rejected": -74.16741943359375, "logps/rejected": -96.0380859375, "loss": 1.3303, "margin_dpo/margin_mean": 7.785409927368164, "margin_dpo/margin_std": 14.063751220703125, "step": 134 }, { "KL/chosen_KL_mean": -14.210046768188477, "KL/mean": -16.44684410095215, "KL/rejected_KL_mean": -18.683643341064453, "KL/std": 10.793325424194336, "epoch": 0.20408163265306123, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.473599910736084, "fcm_dpo/q_t": 0.4914175271987915, "grad_norm": 2.542938232421875, "learning_rate": 4.844676460754862e-07, "logits/chosen": 0.3599017262458801, "logits/rejected": 0.32789885997772217, "logps/chosen": -63.651123046875, "logps/ref_chosen": -49.441078186035156, "logps/ref_rejected": -65.96878051757812, "logps/rejected": -84.65242004394531, "loss": 1.354, "margin_dpo/margin_mean": 4.473599433898926, "margin_dpo/margin_std": 11.142325401306152, "step": 135 }, { "KL/chosen_KL_mean": -18.304481506347656, "KL/mean": -21.979602813720703, "KL/rejected_KL_mean": -25.654720306396484, "KL/std": 14.195917129516602, "epoch": 0.20559334845049132, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.350245475769043, "fcm_dpo/q_t": 0.4859556555747986, "grad_norm": 3.6188459396362305, "learning_rate": 4.840055783904106e-07, "logits/chosen": 0.35441240668296814, "logits/rejected": 0.27984628081321716, "logps/chosen": -85.0637435913086, "logps/ref_chosen": -66.75926208496094, "logps/ref_rejected": -94.61787414550781, "logps/rejected": -120.27259826660156, "loss": 1.3349, "margin_dpo/margin_mean": 7.350246429443359, "margin_dpo/margin_std": 16.540786743164062, "step": 136 }, { "KL/chosen_KL_mean": -16.800689697265625, "KL/mean": -20.018104553222656, "KL/rejected_KL_mean": -23.235515594482422, "KL/std": 12.882331848144531, "epoch": 0.20710506424792138, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.434828281402588, "fcm_dpo/q_t": 0.48770397901535034, "grad_norm": 2.851135492324829, "learning_rate": 4.835369650662767e-07, "logits/chosen": 0.39496833086013794, "logits/rejected": 0.3645436465740204, "logps/chosen": -73.58448791503906, "logps/ref_chosen": -56.78379821777344, "logps/ref_rejected": -69.89952087402344, "logps/rejected": -93.13504028320312, "loss": 1.3409, "margin_dpo/margin_mean": 6.434828758239746, "margin_dpo/margin_std": 14.978425979614258, "step": 137 }, { "KL/chosen_KL_mean": -20.657804489135742, "KL/mean": -23.541412353515625, "KL/rejected_KL_mean": -26.425025939941406, "KL/std": 14.655818939208984, "epoch": 0.20861678004535147, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.767213821411133, "fcm_dpo/q_t": 0.48898112773895264, "grad_norm": 2.8203554153442383, "learning_rate": 4.830618192112065e-07, "logits/chosen": 0.3970043361186981, "logits/rejected": 0.36176303029060364, "logps/chosen": -79.42381286621094, "logps/ref_chosen": -58.766014099121094, "logps/ref_rejected": -68.12371826171875, "logps/rejected": -94.54874420166016, "loss": 1.347, "margin_dpo/margin_mean": 5.767213821411133, "margin_dpo/margin_std": 17.624059677124023, "step": 138 }, { "KL/chosen_KL_mean": -22.82883071899414, "KL/mean": -25.602848052978516, "KL/rejected_KL_mean": -28.376876831054688, "KL/std": 12.521333694458008, "epoch": 0.21012849584278157, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 5.548043727874756, "fcm_dpo/q_t": 0.48936471343040466, "grad_norm": 3.1577208042144775, "learning_rate": 4.825801541160509e-07, "logits/chosen": 0.3622744083404541, "logits/rejected": 0.3314181864261627, "logps/chosen": -94.0544204711914, "logps/ref_chosen": -71.2255859375, "logps/ref_rejected": -82.1834716796875, "logps/rejected": -110.56034851074219, "loss": 1.3473, "margin_dpo/margin_mean": 5.548043727874756, "margin_dpo/margin_std": 14.130794525146484, "step": 139 }, { "KL/chosen_KL_mean": -21.173185348510742, "KL/mean": -25.737133026123047, "KL/rejected_KL_mean": -30.30107879638672, "KL/std": 16.705089569091797, "epoch": 0.21164021164021163, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.127891540527344, "fcm_dpo/q_t": 0.4825606942176819, "grad_norm": 3.3629658222198486, "learning_rate": 4.820919832540181e-07, "logits/chosen": 0.35938477516174316, "logits/rejected": 0.3098704516887665, "logps/chosen": -84.45085144042969, "logps/ref_chosen": -63.27766418457031, "logps/ref_rejected": -83.30647277832031, "logps/rejected": -113.60755157470703, "loss": 1.324, "margin_dpo/margin_mean": 9.127891540527344, "margin_dpo/margin_std": 21.27337646484375, "step": 140 }, { "KL/chosen_KL_mean": -23.774330139160156, "KL/mean": -28.25469970703125, "KL/rejected_KL_mean": -32.73507308959961, "KL/std": 16.478158950805664, "epoch": 0.21315192743764172, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 8.960737228393555, "fcm_dpo/q_t": 0.48290276527404785, "grad_norm": 3.510462999343872, "learning_rate": 4.815973202802966e-07, "logits/chosen": 0.43517443537712097, "logits/rejected": 0.38708174228668213, "logps/chosen": -85.54109954833984, "logps/ref_chosen": -61.76676940917969, "logps/ref_rejected": -88.60601806640625, "logps/rejected": -121.34109497070312, "loss": 1.3247, "margin_dpo/margin_mean": 8.960738182067871, "margin_dpo/margin_std": 19.960668563842773, "step": 141 }, { "KL/chosen_KL_mean": -23.207942962646484, "KL/mean": -26.78273582458496, "KL/rejected_KL_mean": -30.357532501220703, "KL/std": 14.630638122558594, "epoch": 0.2146636432350718, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.149584770202637, "fcm_dpo/q_t": 0.4863271713256836, "grad_norm": 3.2684221267700195, "learning_rate": 4.810961790316729e-07, "logits/chosen": 0.425723671913147, "logits/rejected": 0.40205299854278564, "logps/chosen": -88.48271942138672, "logps/ref_chosen": -65.2747802734375, "logps/ref_rejected": -81.1378173828125, "logps/rejected": -111.49534606933594, "loss": 1.3353, "margin_dpo/margin_mean": 7.149584770202637, "margin_dpo/margin_std": 14.595829010009766, "step": 142 }, { "KL/chosen_KL_mean": -25.313261032104492, "KL/mean": -27.440980911254883, "KL/rejected_KL_mean": -29.56869888305664, "KL/std": 18.696537017822266, "epoch": 0.2161753590325019, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 4.255438327789307, "fcm_dpo/q_t": 0.49176695942878723, "grad_norm": 3.4273107051849365, "learning_rate": 4.805885735261454e-07, "logits/chosen": 0.4723702371120453, "logits/rejected": 0.4537476897239685, "logps/chosen": -87.93109130859375, "logps/ref_chosen": -62.617828369140625, "logps/ref_rejected": -70.39239501953125, "logps/rejected": -99.96109008789062, "loss": 1.3612, "margin_dpo/margin_mean": 4.255438327789307, "margin_dpo/margin_std": 22.319290161132812, "step": 143 }, { "KL/chosen_KL_mean": -26.418071746826172, "KL/mean": -30.407001495361328, "KL/rejected_KL_mean": -34.39592361450195, "KL/std": 20.060935974121094, "epoch": 0.21768707482993196, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.977849006652832, "fcm_dpo/q_t": 0.4849676489830017, "grad_norm": 3.241164207458496, "learning_rate": 4.800745179625307e-07, "logits/chosen": 0.4205203056335449, "logits/rejected": 0.38700443506240845, "logps/chosen": -87.22076416015625, "logps/ref_chosen": -60.80268859863281, "logps/ref_rejected": -79.07284545898438, "logps/rejected": -113.46876525878906, "loss": 1.3353, "margin_dpo/margin_mean": 7.977849006652832, "margin_dpo/margin_std": 25.233558654785156, "step": 144 }, { "KL/chosen_KL_mean": -27.356273651123047, "KL/mean": -31.281274795532227, "KL/rejected_KL_mean": -35.206275939941406, "KL/std": 23.064613342285156, "epoch": 0.21919879062736206, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.850009918212891, "fcm_dpo/q_t": 0.48525863885879517, "grad_norm": 3.983025074005127, "learning_rate": 4.795540267200686e-07, "logits/chosen": 0.4053303003311157, "logits/rejected": 0.4174395799636841, "logps/chosen": -101.96773529052734, "logps/ref_chosen": -74.61146545410156, "logps/ref_rejected": -83.24461364746094, "logps/rejected": -118.45088958740234, "loss": 1.3407, "margin_dpo/margin_mean": 7.850009441375732, "margin_dpo/margin_std": 30.250947952270508, "step": 145 }, { "KL/chosen_KL_mean": -26.02267837524414, "KL/mean": -29.3727970123291, "KL/rejected_KL_mean": -32.72291946411133, "KL/std": 19.601165771484375, "epoch": 0.22071050642479215, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.700242519378662, "fcm_dpo/q_t": 0.4872305989265442, "grad_norm": 3.2747786045074463, "learning_rate": 4.790271143580173e-07, "logits/chosen": 0.4032752215862274, "logits/rejected": 0.39034542441368103, "logps/chosen": -83.86366271972656, "logps/ref_chosen": -57.84098434448242, "logps/ref_rejected": -67.47422790527344, "logps/rejected": -100.1971435546875, "loss": 1.3418, "margin_dpo/margin_mean": 6.70024299621582, "margin_dpo/margin_std": 20.67666244506836, "step": 146 }, { "KL/chosen_KL_mean": -29.016862869262695, "KL/mean": -32.25214385986328, "KL/rejected_KL_mean": -35.48743438720703, "KL/std": 21.1727237701416, "epoch": 0.2222222222222222, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.470568656921387, "fcm_dpo/q_t": 0.4874805808067322, "grad_norm": 3.599782943725586, "learning_rate": 4.784937956152489e-07, "logits/chosen": 0.414762020111084, "logits/rejected": 0.3716977834701538, "logps/chosen": -95.83033752441406, "logps/ref_chosen": -66.81346893310547, "logps/ref_rejected": -81.1796875, "logps/rejected": -116.66712951660156, "loss": 1.3455, "margin_dpo/margin_mean": 6.470569610595703, "margin_dpo/margin_std": 23.536331176757812, "step": 147 }, { "KL/chosen_KL_mean": -24.12586212158203, "KL/mean": -30.170164108276367, "KL/rejected_KL_mean": -36.21445846557617, "KL/std": 20.31801414489746, "epoch": 0.2237339380196523, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.088604927062988, "fcm_dpo/q_t": 0.4769914448261261, "grad_norm": 3.680696487426758, "learning_rate": 4.779540854098347e-07, "logits/chosen": 0.5683670043945312, "logits/rejected": 0.49711206555366516, "logps/chosen": -72.81361389160156, "logps/ref_chosen": -48.6877555847168, "logps/ref_rejected": -67.50503540039062, "logps/rejected": -103.71949768066406, "loss": 1.3031, "margin_dpo/margin_mean": 12.088605880737305, "margin_dpo/margin_std": 22.556541442871094, "step": 148 }, { "KL/chosen_KL_mean": -29.621259689331055, "KL/mean": -35.046180725097656, "KL/rejected_KL_mean": -40.471099853515625, "KL/std": 22.03968620300293, "epoch": 0.2252456538170824, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 10.849838256835938, "fcm_dpo/q_t": 0.47930651903152466, "grad_norm": 3.6473169326782227, "learning_rate": 4.774079988386296e-07, "logits/chosen": 0.46499621868133545, "logits/rejected": 0.4208984971046448, "logps/chosen": -84.7650375366211, "logps/ref_chosen": -55.143775939941406, "logps/ref_rejected": -64.79888916015625, "logps/rejected": -105.26998901367188, "loss": 1.3114, "margin_dpo/margin_mean": 10.849838256835938, "margin_dpo/margin_std": 21.40958595275879, "step": 149 }, { "KL/chosen_KL_mean": -29.565271377563477, "KL/mean": -36.95281219482422, "KL/rejected_KL_mean": -44.34034729003906, "KL/std": 23.726795196533203, "epoch": 0.22675736961451248, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.775071144104004, "fcm_dpo/q_t": 0.47204089164733887, "grad_norm": 3.51343035697937, "learning_rate": 4.768555511768486e-07, "logits/chosen": 0.4633652865886688, "logits/rejected": 0.4186779260635376, "logps/chosen": -97.03601837158203, "logps/ref_chosen": -67.47074890136719, "logps/ref_rejected": -89.21170806884766, "logps/rejected": -133.55206298828125, "loss": 1.2891, "margin_dpo/margin_mean": 14.775071144104004, "margin_dpo/margin_std": 29.063199996948242, "step": 150 }, { "KL/chosen_KL_mean": -21.63004493713379, "KL/mean": -31.206939697265625, "KL/rejected_KL_mean": -40.78384017944336, "KL/std": 26.35299301147461, "epoch": 0.22826908541194255, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.153793334960938, "fcm_dpo/q_t": 0.46416527032852173, "grad_norm": 4.2144775390625, "learning_rate": 4.762967578776406e-07, "logits/chosen": 0.4840549826622009, "logits/rejected": 0.4279939532279968, "logps/chosen": -74.08958435058594, "logps/ref_chosen": -52.45954132080078, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -119.84684753417969, "loss": 1.2569, "margin_dpo/margin_mean": 19.153793334960938, "margin_dpo/margin_std": 28.657949447631836, "step": 151 }, { "KL/chosen_KL_mean": -28.358936309814453, "KL/mean": -34.87946701049805, "KL/rejected_KL_mean": -41.39999771118164, "KL/std": 23.711170196533203, "epoch": 0.22978080120937264, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 13.041062355041504, "fcm_dpo/q_t": 0.47520697116851807, "grad_norm": 3.737586498260498, "learning_rate": 4.757316345716553e-07, "logits/chosen": 0.5636520981788635, "logits/rejected": 0.5086890459060669, "logps/chosen": -84.91276550292969, "logps/ref_chosen": -56.5538330078125, "logps/ref_rejected": -76.55074310302734, "logps/rejected": -117.95074462890625, "loss": 1.3005, "margin_dpo/margin_mean": 13.041061401367188, "margin_dpo/margin_std": 28.46988868713379, "step": 152 }, { "KL/chosen_KL_mean": -30.26366424560547, "KL/mean": -36.74250030517578, "KL/rejected_KL_mean": -43.22133255004883, "KL/std": 24.817947387695312, "epoch": 0.23129251700680273, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.957663536071777, "fcm_dpo/q_t": 0.4754427969455719, "grad_norm": 3.9588053226470947, "learning_rate": 4.751601970666064e-07, "logits/chosen": 0.48824167251586914, "logits/rejected": 0.4558285176753998, "logps/chosen": -98.27055358886719, "logps/ref_chosen": -68.00689697265625, "logps/ref_rejected": -74.83482360839844, "logps/rejected": -118.05615234375, "loss": 1.2993, "margin_dpo/margin_mean": 12.957662582397461, "margin_dpo/margin_std": 25.99228286743164, "step": 153 }, { "KL/chosen_KL_mean": -29.722576141357422, "KL/mean": -34.363807678222656, "KL/rejected_KL_mean": -39.00503921508789, "KL/std": 29.986373901367188, "epoch": 0.2328042328042328, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 9.282459259033203, "fcm_dpo/q_t": 0.4823741316795349, "grad_norm": 3.928870677947998, "learning_rate": 4.745824613468292e-07, "logits/chosen": 0.5707046389579773, "logits/rejected": 0.5677356719970703, "logps/chosen": -88.94511413574219, "logps/ref_chosen": -59.222537994384766, "logps/ref_rejected": -64.19131469726562, "logps/rejected": -103.19635772705078, "loss": 1.3314, "margin_dpo/margin_mean": 9.282459259033203, "margin_dpo/margin_std": 31.800521850585938, "step": 154 }, { "KL/chosen_KL_mean": -33.82355499267578, "KL/mean": -41.13679122924805, "KL/rejected_KL_mean": -48.45003128051758, "KL/std": 29.099594116210938, "epoch": 0.23431594860166288, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 14.62647819519043, "fcm_dpo/q_t": 0.4723411202430725, "grad_norm": 4.301799297332764, "learning_rate": 4.7399844357283393e-07, "logits/chosen": 0.580405592918396, "logits/rejected": 0.5614101886749268, "logps/chosen": -102.27825164794922, "logps/ref_chosen": -68.45469665527344, "logps/ref_rejected": -77.91763305664062, "logps/rejected": -126.36766815185547, "loss": 1.296, "margin_dpo/margin_mean": 14.626476287841797, "margin_dpo/margin_std": 35.91053009033203, "step": 155 }, { "KL/chosen_KL_mean": -34.29534149169922, "KL/mean": -44.44483184814453, "KL/rejected_KL_mean": -54.594322204589844, "KL/std": 32.78551483154297, "epoch": 0.23582766439909297, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.29897689819336, "fcm_dpo/q_t": 0.4618450999259949, "grad_norm": 4.438568115234375, "learning_rate": 4.7340816008085305e-07, "logits/chosen": 0.5352226495742798, "logits/rejected": 0.49391427636146545, "logps/chosen": -101.56494140625, "logps/ref_chosen": -67.26959991455078, "logps/ref_rejected": -86.95914459228516, "logps/rejected": -141.553466796875, "loss": 1.2513, "margin_dpo/margin_mean": 20.29897689819336, "margin_dpo/margin_std": 31.937244415283203, "step": 156 }, { "KL/chosen_KL_mean": -28.4747371673584, "KL/mean": -34.87397003173828, "KL/rejected_KL_mean": -41.2732048034668, "KL/std": 27.206077575683594, "epoch": 0.23733938019652306, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 12.798463821411133, "fcm_dpo/q_t": 0.4756951630115509, "grad_norm": 4.1154913902282715, "learning_rate": 4.728116273823847e-07, "logits/chosen": 0.5524776577949524, "logits/rejected": 0.5371452569961548, "logps/chosen": -83.24761199951172, "logps/ref_chosen": -54.77287292480469, "logps/ref_rejected": -63.87866973876953, "logps/rejected": -105.15187072753906, "loss": 1.3008, "margin_dpo/margin_mean": 12.798463821411133, "margin_dpo/margin_std": 25.809242248535156, "step": 157 }, { "KL/chosen_KL_mean": -34.29819107055664, "KL/mean": -42.93422317504883, "KL/rejected_KL_mean": -51.570255279541016, "KL/std": 35.774105072021484, "epoch": 0.23885109599395313, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.272056579589844, "fcm_dpo/q_t": 0.46861717104911804, "grad_norm": 4.501484394073486, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 0.5918360948562622, "logits/rejected": 0.5534389019012451, "logps/chosen": -99.22090911865234, "logps/ref_chosen": -64.92271423339844, "logps/ref_rejected": -82.23789978027344, "logps/rejected": -133.8081512451172, "loss": 1.286, "margin_dpo/margin_mean": 17.272056579589844, "margin_dpo/margin_std": 43.68254089355469, "step": 158 }, { "KL/chosen_KL_mean": -33.22077178955078, "KL/mean": -41.89005661010742, "KL/rejected_KL_mean": -50.559349060058594, "KL/std": 29.38928985595703, "epoch": 0.24036281179138322, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 17.338571548461914, "fcm_dpo/q_t": 0.46729815006256104, "grad_norm": 4.624864101409912, "learning_rate": 4.715998812855304e-07, "logits/chosen": 0.5776375532150269, "logits/rejected": 0.5369961857795715, "logps/chosen": -90.26776123046875, "logps/ref_chosen": -57.046993255615234, "logps/ref_rejected": -73.32441711425781, "logps/rejected": -123.8837661743164, "loss": 1.2746, "margin_dpo/margin_mean": 17.338571548461914, "margin_dpo/margin_std": 34.329891204833984, "step": 159 }, { "KL/chosen_KL_mean": -34.37767791748047, "KL/mean": -43.98714065551758, "KL/rejected_KL_mean": -53.59660339355469, "KL/std": 33.58818054199219, "epoch": 0.2418745275888133, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.21892547607422, "fcm_dpo/q_t": 0.46424031257629395, "grad_norm": 5.148886203765869, "learning_rate": 4.7098470178228755e-07, "logits/chosen": 0.5072331428527832, "logits/rejected": 0.4576966464519501, "logps/chosen": -84.1845932006836, "logps/ref_chosen": -49.806915283203125, "logps/ref_rejected": -68.3370132446289, "logps/rejected": -121.9336166381836, "loss": 1.265, "margin_dpo/margin_mean": 19.21892547607422, "margin_dpo/margin_std": 37.81275177001953, "step": 160 }, { "KL/chosen_KL_mean": -35.280982971191406, "KL/mean": -45.04498291015625, "KL/rejected_KL_mean": -54.808990478515625, "KL/std": 29.070030212402344, "epoch": 0.24338624338624337, "fcm_dpo/beta": 0.007687800098210573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.527999877929688, "fcm_dpo/q_t": 0.46312597393989563, "grad_norm": 5.976346015930176, "learning_rate": 4.703633408618955e-07, "logits/chosen": 0.5991806983947754, "logits/rejected": 0.5647536516189575, "logps/chosen": -87.7814712524414, "logps/ref_chosen": -52.50048828125, "logps/ref_rejected": -66.04540252685547, "logps/rejected": -120.8543930053711, "loss": 1.2569, "margin_dpo/margin_mean": 19.52800178527832, "margin_dpo/margin_std": 31.474956512451172, "step": 161 }, { "KL/chosen_KL_mean": -45.48345947265625, "KL/mean": -58.5546989440918, "KL/rejected_KL_mean": -71.62594604492188, "KL/std": 45.28847885131836, "epoch": 0.24489795918367346, "fcm_dpo/beta": 0.019494613632559776, "fcm_dpo/delta": 0.08775182068347931, "fcm_dpo/margin": 26.142498016357422, "fcm_dpo/q_t": 0.45138120651245117, "grad_norm": 21.5263614654541, "learning_rate": 4.697358159051549e-07, "logits/chosen": 0.6398231983184814, "logits/rejected": 0.5949274301528931, "logps/chosen": -114.95265197753906, "logps/ref_chosen": -69.46919250488281, "logps/ref_rejected": -92.00952911376953, "logps/rejected": -163.63546752929688, "loss": 1.1634, "margin_dpo/margin_mean": 26.142494201660156, "margin_dpo/margin_std": 50.92291259765625, "step": 162 }, { "KL/chosen_KL_mean": -32.73987579345703, "KL/mean": -46.96985626220703, "KL/rejected_KL_mean": -61.1998405456543, "KL/std": 35.424747467041016, "epoch": 0.24640967498110355, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": -0.12022696435451508, "fcm_dpo/margin": 28.459957122802734, "fcm_dpo/q_t": 0.4269000291824341, "grad_norm": 3.908928155899048, "learning_rate": 4.691021444652876e-07, "logits/chosen": 0.6149941682815552, "logits/rejected": 0.5667312145233154, "logps/chosen": -83.35371398925781, "logps/ref_chosen": -50.613834381103516, "logps/ref_rejected": -74.62033081054688, "logps/rejected": -135.82015991210938, "loss": 1.2688, "margin_dpo/margin_mean": 28.459957122802734, "margin_dpo/margin_std": 38.75524139404297, "step": 163 }, { "KL/chosen_KL_mean": -39.49297332763672, "KL/mean": -53.75981140136719, "KL/rejected_KL_mean": -68.02665710449219, "KL/std": 41.714744567871094, "epoch": 0.24792139077853365, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.533679962158203, "fcm_dpo/q_t": 0.46781498193740845, "grad_norm": 4.191322326660156, "learning_rate": 4.6846234426744624e-07, "logits/chosen": 0.5729007720947266, "logits/rejected": 0.5150625705718994, "logps/chosen": -94.34107971191406, "logps/ref_chosen": -54.848114013671875, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -147.08966064453125, "loss": 1.2704, "margin_dpo/margin_mean": 28.53368377685547, "margin_dpo/margin_std": 42.69341278076172, "step": 164 }, { "KL/chosen_KL_mean": -45.441650390625, "KL/mean": -58.43135070800781, "KL/rejected_KL_mean": -71.42105102539062, "KL/std": 41.49935531616211, "epoch": 0.2494331065759637, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.979393005371094, "fcm_dpo/q_t": 0.4706171154975891, "grad_norm": 4.249202251434326, "learning_rate": 4.678164332082175e-07, "logits/chosen": 0.6801586151123047, "logits/rejected": 0.6297128200531006, "logps/chosen": -96.5308609008789, "logps/ref_chosen": -51.089210510253906, "logps/ref_rejected": -71.23370361328125, "logps/rejected": -142.65475463867188, "loss": 1.2815, "margin_dpo/margin_mean": 25.979393005371094, "margin_dpo/margin_std": 44.35696029663086, "step": 165 }, { "KL/chosen_KL_mean": -49.10633850097656, "KL/mean": -61.71703338623047, "KL/rejected_KL_mean": -74.3277359008789, "KL/std": 45.09733581542969, "epoch": 0.2509448223733938, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.221389770507812, "fcm_dpo/q_t": 0.47154539823532104, "grad_norm": 4.738693714141846, "learning_rate": 4.6716442935512214e-07, "logits/chosen": 0.6274275779724121, "logits/rejected": 0.5359787940979004, "logps/chosen": -112.29714965820312, "logps/ref_chosen": -63.19081115722656, "logps/ref_rejected": -93.8402099609375, "logps/rejected": -168.16793823242188, "loss": 1.2842, "margin_dpo/margin_mean": 25.221389770507812, "margin_dpo/margin_std": 43.412750244140625, "step": 166 }, { "KL/chosen_KL_mean": -42.655426025390625, "KL/mean": -56.4607048034668, "KL/rejected_KL_mean": -70.26599884033203, "KL/std": 39.476051330566406, "epoch": 0.25245653817082386, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.610572814941406, "fcm_dpo/q_t": 0.4687613248825073, "grad_norm": 5.55941915512085, "learning_rate": 4.6650635094610966e-07, "logits/chosen": 0.6002979278564453, "logits/rejected": 0.5730512142181396, "logps/chosen": -101.57969665527344, "logps/ref_chosen": -58.92427062988281, "logps/ref_rejected": -72.97377014160156, "logps/rejected": -143.23977661132812, "loss": 1.2729, "margin_dpo/margin_mean": 27.610572814941406, "margin_dpo/margin_std": 41.076995849609375, "step": 167 }, { "KL/chosen_KL_mean": -52.67481231689453, "KL/mean": -66.61177062988281, "KL/rejected_KL_mean": -80.54873657226562, "KL/std": 47.3160400390625, "epoch": 0.25396825396825395, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.873924255371094, "fcm_dpo/q_t": 0.4689233899116516, "grad_norm": 4.620728492736816, "learning_rate": 4.6584221638904767e-07, "logits/chosen": 0.6193612813949585, "logits/rejected": 0.5847088098526001, "logps/chosen": -118.3261947631836, "logps/ref_chosen": -65.65138244628906, "logps/ref_rejected": -79.71418762207031, "logps/rejected": -160.26292419433594, "loss": 1.2779, "margin_dpo/margin_mean": 27.873924255371094, "margin_dpo/margin_std": 53.34053039550781, "step": 168 }, { "KL/chosen_KL_mean": -43.973472595214844, "KL/mean": -56.64049530029297, "KL/rejected_KL_mean": -69.30752563476562, "KL/std": 43.73821258544922, "epoch": 0.25547996976568405, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.33405113220215, "fcm_dpo/q_t": 0.47141051292419434, "grad_norm": 4.314192295074463, "learning_rate": 4.651720442612075e-07, "logits/chosen": 0.7225455641746521, "logits/rejected": 0.6896216869354248, "logps/chosen": -105.39933776855469, "logps/ref_chosen": -61.425865173339844, "logps/ref_rejected": -76.09590148925781, "logps/rejected": -145.40342712402344, "loss": 1.2896, "margin_dpo/margin_mean": 25.33405113220215, "margin_dpo/margin_std": 54.63517379760742, "step": 169 }, { "KL/chosen_KL_mean": -39.26036834716797, "KL/mean": -53.16771697998047, "KL/rejected_KL_mean": -67.07505798339844, "KL/std": 41.21588134765625, "epoch": 0.25699168556311414, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.814695358276367, "fcm_dpo/q_t": 0.46871745586395264, "grad_norm": 5.0231099128723145, "learning_rate": 4.6449585330874425e-07, "logits/chosen": 0.689607560634613, "logits/rejected": 0.6932332515716553, "logps/chosen": -95.91355895996094, "logps/ref_chosen": -56.65319061279297, "logps/ref_rejected": -63.45965576171875, "logps/rejected": -130.5347137451172, "loss": 1.2806, "margin_dpo/margin_mean": 27.814697265625, "margin_dpo/margin_std": 54.60728454589844, "step": 170 }, { "KL/chosen_KL_mean": -52.41896438598633, "KL/mean": -69.64653778076172, "KL/rejected_KL_mean": -86.87411499023438, "KL/std": 57.269317626953125, "epoch": 0.2585034013605442, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.45514678955078, "fcm_dpo/q_t": 0.46149614453315735, "grad_norm": 4.3705902099609375, "learning_rate": 4.6381366244617224e-07, "logits/chosen": 0.767911434173584, "logits/rejected": 0.7232061624526978, "logps/chosen": -116.15372467041016, "logps/ref_chosen": -63.73476028442383, "logps/ref_rejected": -78.50328063964844, "logps/rejected": -165.37738037109375, "loss": 1.2571, "margin_dpo/margin_mean": 34.45514678955078, "margin_dpo/margin_std": 64.5282974243164, "step": 171 }, { "KL/chosen_KL_mean": -55.21881866455078, "KL/mean": -71.57858276367188, "KL/rejected_KL_mean": -87.93833923339844, "KL/std": 53.955963134765625, "epoch": 0.2600151171579743, "fcm_dpo/beta": 0.004572372883558273, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.719520568847656, "fcm_dpo/q_t": 0.4635423421859741, "grad_norm": 6.561391353607178, "learning_rate": 4.631254907558365e-07, "logits/chosen": 0.7488380670547485, "logits/rejected": 0.686285674571991, "logps/chosen": -107.42057800292969, "logps/ref_chosen": -52.201759338378906, "logps/ref_rejected": -82.85285949707031, "logps/rejected": -170.79119873046875, "loss": 1.259, "margin_dpo/margin_mean": 32.719520568847656, "margin_dpo/margin_std": 55.974552154541016, "step": 172 }, { "KL/chosen_KL_mean": -52.73722839355469, "KL/mean": -72.90831756591797, "KL/rejected_KL_mean": -93.07940673828125, "KL/std": 59.073570251464844, "epoch": 0.2615268329554044, "fcm_dpo/beta": 0.014145536348223686, "fcm_dpo/delta": 0.06264997273683548, "fcm_dpo/margin": 40.342193603515625, "fcm_dpo/q_t": 0.4352329969406128, "grad_norm": 13.129067420959473, "learning_rate": 4.624313574873786e-07, "logits/chosen": 0.7639884352684021, "logits/rejected": 0.6851507425308228, "logps/chosen": -108.17195129394531, "logps/ref_chosen": -55.434722900390625, "logps/ref_rejected": -77.81967163085938, "logps/rejected": -170.89907836914062, "loss": 1.1037, "margin_dpo/margin_mean": 40.342193603515625, "margin_dpo/margin_std": 72.60908508300781, "step": 173 }, { "KL/chosen_KL_mean": -61.133522033691406, "KL/mean": -83.47229766845703, "KL/rejected_KL_mean": -105.81106567382812, "KL/std": 60.41809844970703, "epoch": 0.26303854875283444, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": -0.1022532507777214, "fcm_dpo/margin": 44.67753601074219, "fcm_dpo/q_t": 0.41912031173706055, "grad_norm": 3.89009428024292, "learning_rate": 4.61731282057198e-07, "logits/chosen": 0.7750641107559204, "logits/rejected": 0.7090386748313904, "logps/chosen": -118.30548095703125, "logps/ref_chosen": -57.17195129394531, "logps/ref_rejected": -85.47578430175781, "logps/rejected": -191.28684997558594, "loss": 1.2858, "margin_dpo/margin_mean": 44.67753601074219, "margin_dpo/margin_std": 57.747344970703125, "step": 174 }, { "KL/chosen_KL_mean": -57.48314666748047, "KL/mean": -79.11009216308594, "KL/rejected_KL_mean": -100.73703002929688, "KL/std": 61.092506408691406, "epoch": 0.26455026455026454, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.253883361816406, "fcm_dpo/q_t": 0.47405892610549927, "grad_norm": 3.5935990810394287, "learning_rate": 4.6102528404790965e-07, "logits/chosen": 0.799199104309082, "logits/rejected": 0.7752283811569214, "logps/chosen": -125.14877319335938, "logps/ref_chosen": -67.6656265258789, "logps/ref_rejected": -84.36766815185547, "logps/rejected": -185.10470581054688, "loss": 1.2931, "margin_dpo/margin_mean": 43.253883361816406, "margin_dpo/margin_std": 77.93309020996094, "step": 175 }, { "KL/chosen_KL_mean": -75.63528442382812, "KL/mean": -89.86036682128906, "KL/rejected_KL_mean": -104.08544158935547, "KL/std": 67.660888671875, "epoch": 0.2660619803476946, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.450157165527344, "fcm_dpo/q_t": 0.4830489456653595, "grad_norm": 4.446000576019287, "learning_rate": 4.603133832077953e-07, "logits/chosen": 0.7807189226150513, "logits/rejected": 0.762683093547821, "logps/chosen": -153.49404907226562, "logps/ref_chosen": -77.8587646484375, "logps/ref_rejected": -81.08732604980469, "logps/rejected": -185.17276000976562, "loss": 1.3297, "margin_dpo/margin_mean": 28.450157165527344, "margin_dpo/margin_std": 85.36820220947266, "step": 176 }, { "KL/chosen_KL_mean": -56.47824478149414, "KL/mean": -86.89440155029297, "KL/rejected_KL_mean": -117.310546875, "KL/std": 66.56551361083984, "epoch": 0.2675736961451247, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.83230209350586, "fcm_dpo/q_t": 0.4634092152118683, "grad_norm": 4.204408645629883, "learning_rate": 4.5959559945025183e-07, "logits/chosen": 0.8780996799468994, "logits/rejected": 0.7931272983551025, "logps/chosen": -111.69863891601562, "logps/ref_chosen": -55.22039794921875, "logps/ref_rejected": -92.54973602294922, "logps/rejected": -209.86029052734375, "loss": 1.2499, "margin_dpo/margin_mean": 60.832298278808594, "margin_dpo/margin_std": 62.22540283203125, "step": 177 }, { "KL/chosen_KL_mean": -71.03286743164062, "KL/mean": -90.96412658691406, "KL/rejected_KL_mean": -110.89537048339844, "KL/std": 68.37006378173828, "epoch": 0.2690854119425548, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.86250305175781, "fcm_dpo/q_t": 0.47601377964019775, "grad_norm": 5.275820732116699, "learning_rate": 4.588719528532341e-07, "logits/chosen": 0.7893873453140259, "logits/rejected": 0.7460411787033081, "logps/chosen": -131.84335327148438, "logps/ref_chosen": -60.81049346923828, "logps/ref_rejected": -81.12973022460938, "logps/rejected": -192.0251007080078, "loss": 1.2979, "margin_dpo/margin_mean": 39.86250686645508, "margin_dpo/margin_std": 62.40123748779297, "step": 178 }, { "KL/chosen_KL_mean": -76.19467163085938, "KL/mean": -97.18215942382812, "KL/rejected_KL_mean": -118.16964721679688, "KL/std": 71.81806945800781, "epoch": 0.2705971277399849, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.9749755859375, "fcm_dpo/q_t": 0.474868506193161, "grad_norm": 3.9109606742858887, "learning_rate": 4.581424636586928e-07, "logits/chosen": 0.8630691170692444, "logits/rejected": 0.8520238995552063, "logps/chosen": -141.86639404296875, "logps/ref_chosen": -65.67171478271484, "logps/ref_rejected": -75.32586669921875, "logps/rejected": -193.49551391601562, "loss": 1.2974, "margin_dpo/margin_mean": 41.9749755859375, "margin_dpo/margin_std": 83.55865478515625, "step": 179 }, { "KL/chosen_KL_mean": -63.65786361694336, "KL/mean": -79.41153717041016, "KL/rejected_KL_mean": -95.16521453857422, "KL/std": 63.367218017578125, "epoch": 0.272108843537415, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 31.507352828979492, "fcm_dpo/q_t": 0.4810509979724884, "grad_norm": 3.097667932510376, "learning_rate": 4.5740715227200897e-07, "logits/chosen": 0.7191129922866821, "logits/rejected": 0.7070032358169556, "logps/chosen": -120.34066772460938, "logps/ref_chosen": -56.68280792236328, "logps/ref_rejected": -64.94414520263672, "logps/rejected": -160.10935974121094, "loss": 1.3188, "margin_dpo/margin_mean": 31.507354736328125, "margin_dpo/margin_std": 71.05530548095703, "step": 180 }, { "KL/chosen_KL_mean": -61.297054290771484, "KL/mean": -88.07896423339844, "KL/rejected_KL_mean": -114.86087036132812, "KL/std": 77.27511596679688, "epoch": 0.273620559334845, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.56382751464844, "fcm_dpo/q_t": 0.4678885340690613, "grad_norm": 4.882580280303955, "learning_rate": 4.566660392614228e-07, "logits/chosen": 0.89243084192276, "logits/rejected": 0.8556175827980042, "logps/chosen": -122.07310485839844, "logps/ref_chosen": -60.77604675292969, "logps/ref_rejected": -83.98361206054688, "logps/rejected": -198.844482421875, "loss": 1.2695, "margin_dpo/margin_mean": 53.56382369995117, "margin_dpo/margin_std": 77.41310119628906, "step": 181 }, { "KL/chosen_KL_mean": -78.02749633789062, "KL/mean": -106.99266815185547, "KL/rejected_KL_mean": -135.95785522460938, "KL/std": 78.79426574707031, "epoch": 0.2751322751322751, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.93036651611328, "fcm_dpo/q_t": 0.4655247628688812, "grad_norm": 5.966950416564941, "learning_rate": 4.5591914535745817e-07, "logits/chosen": 0.8725330829620361, "logits/rejected": 0.7979344129562378, "logps/chosen": -138.28128051757812, "logps/ref_chosen": -60.2537841796875, "logps/ref_rejected": -89.7706298828125, "logps/rejected": -225.72848510742188, "loss": 1.2631, "margin_dpo/margin_mean": 57.93036651611328, "margin_dpo/margin_std": 90.61637115478516, "step": 182 }, { "KL/chosen_KL_mean": -96.92032623291016, "KL/mean": -109.78304290771484, "KL/rejected_KL_mean": -122.645751953125, "KL/std": 79.40963745117188, "epoch": 0.2766439909297052, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.725440979003906, "fcm_dpo/q_t": 0.48464155197143555, "grad_norm": 4.779506683349609, "learning_rate": 4.551664914523433e-07, "logits/chosen": 0.8302876949310303, "logits/rejected": 0.8160231113433838, "logps/chosen": -158.68174743652344, "logps/ref_chosen": -61.76142120361328, "logps/ref_rejected": -72.54627990722656, "logps/rejected": -195.19204711914062, "loss": 1.3378, "margin_dpo/margin_mean": 25.725440979003906, "margin_dpo/margin_std": 93.7728271484375, "step": 183 }, { "KL/chosen_KL_mean": -62.56498718261719, "KL/mean": -87.00849151611328, "KL/rejected_KL_mean": -111.45199584960938, "KL/std": 66.37240600585938, "epoch": 0.2781557067271353, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.887001037597656, "fcm_dpo/q_t": 0.4707995057106018, "grad_norm": 6.9039998054504395, "learning_rate": 4.544080985994258e-07, "logits/chosen": 0.9358631372451782, "logits/rejected": 0.8840564489364624, "logps/chosen": -109.40570831298828, "logps/ref_chosen": -46.840721130371094, "logps/ref_rejected": -69.3609390258789, "logps/rejected": -180.81292724609375, "loss": 1.2804, "margin_dpo/margin_mean": 48.887001037597656, "margin_dpo/margin_std": 79.2603759765625, "step": 184 }, { "KL/chosen_KL_mean": -80.97999572753906, "KL/mean": -106.63510131835938, "KL/rejected_KL_mean": -132.29022216796875, "KL/std": 85.46723937988281, "epoch": 0.2796674225245654, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 51.310218811035156, "fcm_dpo/q_t": 0.46929919719696045, "grad_norm": 6.995913028717041, "learning_rate": 4.5364398801258394e-07, "logits/chosen": 0.9254462122917175, "logits/rejected": 0.8940056562423706, "logps/chosen": -133.3011474609375, "logps/ref_chosen": -52.32114028930664, "logps/ref_rejected": -68.3885726928711, "logps/rejected": -200.6787872314453, "loss": 1.2813, "margin_dpo/margin_mean": 51.31022262573242, "margin_dpo/margin_std": 100.49153137207031, "step": 185 }, { "KL/chosen_KL_mean": -84.80559539794922, "KL/mean": -113.83717346191406, "KL/rejected_KL_mean": -142.86874389648438, "KL/std": 107.6434555053711, "epoch": 0.2811791383219955, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.063148498535156, "fcm_dpo/q_t": 0.46672147512435913, "grad_norm": 6.379158973693848, "learning_rate": 4.5287418106563354e-07, "logits/chosen": 0.8682913184165955, "logits/rejected": 0.8399237394332886, "logps/chosen": -152.22572326660156, "logps/ref_chosen": -67.42012786865234, "logps/ref_rejected": -82.50968933105469, "logps/rejected": -225.37843322753906, "loss": 1.2795, "margin_dpo/margin_mean": 58.063148498535156, "margin_dpo/margin_std": 143.8511962890625, "step": 186 }, { "KL/chosen_KL_mean": -106.72709655761719, "KL/mean": -132.92059326171875, "KL/rejected_KL_mean": -159.11410522460938, "KL/std": 99.10030364990234, "epoch": 0.28269085411942557, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.38701248168945, "fcm_dpo/q_t": 0.46877333521842957, "grad_norm": 5.65391206741333, "learning_rate": 4.520986992917297e-07, "logits/chosen": 0.8852934241294861, "logits/rejected": 0.8451642990112305, "logps/chosen": -182.25259399414062, "logps/ref_chosen": -75.52549743652344, "logps/ref_rejected": -94.76289367675781, "logps/rejected": -253.8769989013672, "loss": 1.2815, "margin_dpo/margin_mean": 52.38701629638672, "margin_dpo/margin_std": 109.4050521850586, "step": 187 }, { "KL/chosen_KL_mean": -94.13689422607422, "KL/mean": -118.1536865234375, "KL/rejected_KL_mean": -142.17047119140625, "KL/std": 93.5236587524414, "epoch": 0.2842025699168556, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.0335807800293, "fcm_dpo/q_t": 0.4715660512447357, "grad_norm": 5.645846366882324, "learning_rate": 4.5131756438276466e-07, "logits/chosen": 0.9208471775054932, "logits/rejected": 0.8986366987228394, "logps/chosen": -165.66021728515625, "logps/ref_chosen": -71.52333068847656, "logps/ref_rejected": -78.29949951171875, "logps/rejected": -220.469970703125, "loss": 1.2893, "margin_dpo/margin_mean": 48.03358459472656, "margin_dpo/margin_std": 105.69105529785156, "step": 188 }, { "KL/chosen_KL_mean": -113.9881591796875, "KL/mean": -126.44110107421875, "KL/rejected_KL_mean": -138.89404296875, "KL/std": 106.01944732666016, "epoch": 0.2857142857142857, "fcm_dpo/beta": 0.0024263300001621246, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.9058895111084, "fcm_dpo/q_t": 0.48495352268218994, "grad_norm": 7.330818176269531, "learning_rate": 4.5053079818876096e-07, "logits/chosen": 0.984262228012085, "logits/rejected": 1.0013400316238403, "logps/chosen": -186.1644287109375, "logps/ref_chosen": -72.17626953125, "logps/ref_rejected": -75.26313781738281, "logps/rejected": -214.15716552734375, "loss": 1.3461, "margin_dpo/margin_mean": 24.90589141845703, "margin_dpo/margin_std": 109.00585174560547, "step": 189 }, { "KL/chosen_KL_mean": -87.60731506347656, "KL/mean": -137.9809112548828, "KL/rejected_KL_mean": -188.35450744628906, "KL/std": 113.34170532226562, "epoch": 0.2872260015117158, "fcm_dpo/beta": 0.00413139583542943, "fcm_dpo/delta": -0.01652386784553528, "fcm_dpo/margin": 100.7471923828125, "fcm_dpo/q_t": 0.4048711657524109, "grad_norm": 11.910893440246582, "learning_rate": 4.4973842271726024e-07, "logits/chosen": 1.0319868326187134, "logits/rejected": 0.919386625289917, "logps/chosen": -142.23158264160156, "logps/ref_chosen": -54.624271392822266, "logps/ref_rejected": -101.47068786621094, "logps/rejected": -289.8251953125, "loss": 1.0797, "margin_dpo/margin_mean": 100.74720001220703, "margin_dpo/margin_std": 116.15534973144531, "step": 190 }, { "KL/chosen_KL_mean": -128.3540802001953, "KL/mean": -150.7650146484375, "KL/rejected_KL_mean": -173.17596435546875, "KL/std": 121.36798095703125, "epoch": 0.2887377173091459, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.8218994140625, "fcm_dpo/q_t": 0.4793294072151184, "grad_norm": 7.475344181060791, "learning_rate": 4.48940460132708e-07, "logits/chosen": 0.9936637878417969, "logits/rejected": 0.9684906005859375, "logps/chosen": -201.28659057617188, "logps/ref_chosen": -72.93251037597656, "logps/ref_rejected": -89.95103454589844, "logps/rejected": -263.12701416015625, "loss": 1.3288, "margin_dpo/margin_mean": 44.821895599365234, "margin_dpo/margin_std": 163.15121459960938, "step": 191 }, { "KL/chosen_KL_mean": -115.16537475585938, "KL/mean": -129.2557830810547, "KL/rejected_KL_mean": -143.34620666503906, "KL/std": 98.83819580078125, "epoch": 0.29024943310657597, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.18083953857422, "fcm_dpo/q_t": 0.48696666955947876, "grad_norm": 3.313401222229004, "learning_rate": 4.481369327558329e-07, "logits/chosen": 1.02927827835083, "logits/rejected": 1.007131576538086, "logps/chosen": -169.16648864746094, "logps/ref_chosen": -54.001121520996094, "logps/ref_rejected": -63.531551361083984, "logps/rejected": -206.8777618408203, "loss": 1.3415, "margin_dpo/margin_mean": 28.180843353271484, "margin_dpo/margin_std": 89.10528564453125, "step": 192 }, { "KL/chosen_KL_mean": -104.46321105957031, "KL/mean": -128.8253173828125, "KL/rejected_KL_mean": -153.18743896484375, "KL/std": 88.49467468261719, "epoch": 0.29176114890400606, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.724239349365234, "fcm_dpo/q_t": 0.47747933864593506, "grad_norm": 3.7360472679138184, "learning_rate": 4.47327863063023e-07, "logits/chosen": 1.0135523080825806, "logits/rejected": 1.0168956518173218, "logps/chosen": -161.21249389648438, "logps/ref_chosen": -56.74927520751953, "logps/ref_rejected": -58.80629348754883, "logps/rejected": -211.99374389648438, "loss": 1.3057, "margin_dpo/margin_mean": 48.7242431640625, "margin_dpo/margin_std": 94.30784606933594, "step": 193 }, { "KL/chosen_KL_mean": -115.06211853027344, "KL/mean": -141.86691284179688, "KL/rejected_KL_mean": -168.67172241210938, "KL/std": 117.1404800415039, "epoch": 0.29327286470143615, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.60961151123047, "fcm_dpo/q_t": 0.47572097182273865, "grad_norm": 6.185281276702881, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 1.052534818649292, "logits/rejected": 1.035210371017456, "logps/chosen": -171.7115478515625, "logps/ref_chosen": -56.64944076538086, "logps/ref_rejected": -69.98954772949219, "logps/rejected": -238.66127014160156, "loss": 1.3049, "margin_dpo/margin_mean": 53.60961151123047, "margin_dpo/margin_std": 132.62509155273438, "step": 194 }, { "KL/chosen_KL_mean": -144.16168212890625, "KL/mean": -165.70416259765625, "KL/rejected_KL_mean": -187.2466278076172, "KL/std": 127.5001220703125, "epoch": 0.2947845804988662, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.0849494934082, "fcm_dpo/q_t": 0.4802978038787842, "grad_norm": 5.549439430236816, "learning_rate": 4.4569318740967043e-07, "logits/chosen": 0.9337140321731567, "logits/rejected": 0.9414522051811218, "logps/chosen": -214.5714569091797, "logps/ref_chosen": -70.40977478027344, "logps/ref_rejected": -74.39448547363281, "logps/rejected": -261.64111328125, "loss": 1.326, "margin_dpo/margin_mean": 43.08494567871094, "margin_dpo/margin_std": 144.31869506835938, "step": 195 }, { "KL/chosen_KL_mean": -121.95760345458984, "KL/mean": -155.3830108642578, "KL/rejected_KL_mean": -188.80841064453125, "KL/std": 146.87850952148438, "epoch": 0.2962962962962963, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.8508071899414, "fcm_dpo/q_t": 0.4697552025318146, "grad_norm": 5.599655628204346, "learning_rate": 4.448676271745197e-07, "logits/chosen": 1.0035507678985596, "logits/rejected": 0.9772269129753113, "logps/chosen": -181.1851806640625, "logps/ref_chosen": -59.227577209472656, "logps/ref_rejected": -83.54757690429688, "logps/rejected": -272.3559875488281, "loss": 1.2815, "margin_dpo/margin_mean": 66.8508071899414, "margin_dpo/margin_std": 129.74188232421875, "step": 196 }, { "KL/chosen_KL_mean": -116.84405517578125, "KL/mean": -151.54763793945312, "KL/rejected_KL_mean": -186.251220703125, "KL/std": 130.40406799316406, "epoch": 0.29780801209372637, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.40716552734375, "fcm_dpo/q_t": 0.4688175320625305, "grad_norm": 4.895405292510986, "learning_rate": 4.440366160729392e-07, "logits/chosen": 1.0863635540008545, "logits/rejected": 1.0468790531158447, "logps/chosen": -168.37318420410156, "logps/ref_chosen": -51.52912902832031, "logps/ref_rejected": -73.70631408691406, "logps/rejected": -259.95751953125, "loss": 1.2843, "margin_dpo/margin_mean": 69.40716552734375, "margin_dpo/margin_std": 164.8914794921875, "step": 197 }, { "KL/chosen_KL_mean": -125.76316833496094, "KL/mean": -156.66259765625, "KL/rejected_KL_mean": -187.56204223632812, "KL/std": 126.10319519042969, "epoch": 0.29931972789115646, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.798851013183594, "fcm_dpo/q_t": 0.4717727303504944, "grad_norm": 4.650595664978027, "learning_rate": 4.432001773500957e-07, "logits/chosen": 1.1201140880584717, "logits/rejected": 1.0980479717254639, "logps/chosen": -185.54583740234375, "logps/ref_chosen": -59.78268051147461, "logps/ref_rejected": -72.24533081054688, "logps/rejected": -259.807373046875, "loss": 1.2896, "margin_dpo/margin_mean": 61.79884719848633, "margin_dpo/margin_std": 131.0926513671875, "step": 198 }, { "KL/chosen_KL_mean": -128.4625244140625, "KL/mean": -160.48162841796875, "KL/rejected_KL_mean": -192.500732421875, "KL/std": 118.23385620117188, "epoch": 0.30083144368858655, "fcm_dpo/beta": 0.001862645847722888, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.0382308959961, "fcm_dpo/q_t": 0.47144871950149536, "grad_norm": 5.723056316375732, "learning_rate": 4.4235833440297856e-07, "logits/chosen": 1.049532175064087, "logits/rejected": 0.9984226226806641, "logps/chosen": -184.84930419921875, "logps/ref_chosen": -56.38677215576172, "logps/ref_rejected": -74.56779479980469, "logps/rejected": -267.06854248046875, "loss": 1.2908, "margin_dpo/margin_mean": 64.0382308959961, "margin_dpo/margin_std": 147.04953002929688, "step": 199 }, { "KL/chosen_KL_mean": -128.42446899414062, "KL/mean": -175.69747924804688, "KL/rejected_KL_mean": -222.97048950195312, "KL/std": 161.18472290039062, "epoch": 0.30234315948601664, "fcm_dpo/beta": 0.006587632466107607, "fcm_dpo/delta": 0.0778110921382904, "fcm_dpo/margin": 94.5460205078125, "fcm_dpo/q_t": 0.4322563111782074, "grad_norm": 19.645002365112305, "learning_rate": 4.415111107797445e-07, "logits/chosen": 1.102787971496582, "logits/rejected": 1.0492348670959473, "logps/chosen": -186.248779296875, "logps/ref_chosen": -57.82432556152344, "logps/ref_rejected": -89.28246307373047, "logps/rejected": -312.2529296875, "loss": 1.1653, "margin_dpo/margin_mean": 94.5460205078125, "margin_dpo/margin_std": 190.74359130859375, "step": 200 }, { "KL/chosen_KL_mean": -153.81317138671875, "KL/mean": -185.98416137695312, "KL/rejected_KL_mean": -218.15512084960938, "KL/std": 170.94137573242188, "epoch": 0.30385487528344673, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": -0.08959634602069855, "fcm_dpo/margin": 64.3419418334961, "fcm_dpo/q_t": 0.4359002113342285, "grad_norm": 4.09077787399292, "learning_rate": 4.4065853017905953e-07, "logits/chosen": 1.068925380706787, "logits/rejected": 1.0387152433395386, "logps/chosen": -212.8129425048828, "logps/ref_chosen": -58.999759674072266, "logps/ref_rejected": -84.67575073242188, "logps/rejected": -302.83087158203125, "loss": 1.3112, "margin_dpo/margin_mean": 64.34194946289062, "margin_dpo/margin_std": 189.60845947265625, "step": 201 }, { "KL/chosen_KL_mean": -113.20198822021484, "KL/mean": -160.37185668945312, "KL/rejected_KL_mean": -207.541748046875, "KL/std": 120.42765808105469, "epoch": 0.30536659108087677, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.33975982666016, "fcm_dpo/q_t": 0.46428173780441284, "grad_norm": 5.2453508377075195, "learning_rate": 4.3980061644943575e-07, "logits/chosen": 1.0624161958694458, "logits/rejected": 1.0226997137069702, "logps/chosen": -160.86264038085938, "logps/ref_chosen": -47.660648345947266, "logps/ref_rejected": -73.63249969482422, "logps/rejected": -281.17425537109375, "loss": 1.2583, "margin_dpo/margin_mean": 94.33975982666016, "margin_dpo/margin_std": 137.9876708984375, "step": 202 }, { "KL/chosen_KL_mean": -150.9142303466797, "KL/mean": -200.44468688964844, "KL/rejected_KL_mean": -249.97515869140625, "KL/std": 143.50985717773438, "epoch": 0.30687830687830686, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.0609130859375, "fcm_dpo/q_t": 0.4625791609287262, "grad_norm": 6.239743232727051, "learning_rate": 4.3893739358856455e-07, "logits/chosen": 1.1303417682647705, "logits/rejected": 1.0701119899749756, "logps/chosen": -213.23977661132812, "logps/ref_chosen": -62.32553482055664, "logps/ref_rejected": -99.37226104736328, "logps/rejected": -349.347412109375, "loss": 1.2536, "margin_dpo/margin_mean": 99.0609130859375, "margin_dpo/margin_std": 154.81906127929688, "step": 203 }, { "KL/chosen_KL_mean": -129.41946411132812, "KL/mean": -163.46804809570312, "KL/rejected_KL_mean": -197.5166015625, "KL/std": 131.462158203125, "epoch": 0.30839002267573695, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.09713745117188, "fcm_dpo/q_t": 0.4740549921989441, "grad_norm": 4.741433143615723, "learning_rate": 4.380688857426449e-07, "logits/chosen": 1.1037497520446777, "logits/rejected": 1.0652885437011719, "logps/chosen": -180.0487823486328, "logps/ref_chosen": -50.62931823730469, "logps/ref_rejected": -66.60475158691406, "logps/rejected": -264.121337890625, "loss": 1.3004, "margin_dpo/margin_mean": 68.09713745117188, "margin_dpo/margin_std": 162.0704803466797, "step": 204 }, { "KL/chosen_KL_mean": -178.677978515625, "KL/mean": -224.63375854492188, "KL/rejected_KL_mean": -270.58953857421875, "KL/std": 211.23817443847656, "epoch": 0.30990173847316704, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.91157531738281, "fcm_dpo/q_t": 0.4660814702510834, "grad_norm": 4.968754291534424, "learning_rate": 4.3719511720570814e-07, "logits/chosen": 1.1142351627349854, "logits/rejected": 1.0800881385803223, "logps/chosen": -249.03414916992188, "logps/ref_chosen": -70.3561782836914, "logps/ref_rejected": -93.39848327636719, "logps/rejected": -363.988037109375, "loss": 1.2784, "margin_dpo/margin_mean": 91.91158294677734, "margin_dpo/margin_std": 221.230224609375, "step": 205 }, { "KL/chosen_KL_mean": -190.87460327148438, "KL/mean": -210.524169921875, "KL/rejected_KL_mean": -230.17376708984375, "KL/std": 179.7605743408203, "epoch": 0.31141345427059713, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.29918670654297, "fcm_dpo/q_t": 0.48608922958374023, "grad_norm": 6.895484447479248, "learning_rate": 4.363161124189387e-07, "logits/chosen": 1.153550624847412, "logits/rejected": 1.1396384239196777, "logps/chosen": -258.52008056640625, "logps/ref_chosen": -67.64547729492188, "logps/ref_rejected": -79.89584350585938, "logps/rejected": -310.0696105957031, "loss": 1.3557, "margin_dpo/margin_mean": 39.2991828918457, "margin_dpo/margin_std": 221.7179412841797, "step": 206 }, { "KL/chosen_KL_mean": -192.2279052734375, "KL/mean": -239.26425170898438, "KL/rejected_KL_mean": -286.30059814453125, "KL/std": 181.48193359375, "epoch": 0.3129251700680272, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.07268524169922, "fcm_dpo/q_t": 0.465423583984375, "grad_norm": 4.847929954528809, "learning_rate": 4.3543189596998986e-07, "logits/chosen": 1.124064326286316, "logits/rejected": 1.0906260013580322, "logps/chosen": -259.8921203613281, "logps/ref_chosen": -67.66419219970703, "logps/ref_rejected": -85.10249328613281, "logps/rejected": -371.403076171875, "loss": 1.2714, "margin_dpo/margin_mean": 94.07269287109375, "margin_dpo/margin_std": 202.92820739746094, "step": 207 }, { "KL/chosen_KL_mean": -176.33616638183594, "KL/mean": -196.31283569335938, "KL/rejected_KL_mean": -216.28952026367188, "KL/std": 157.74716186523438, "epoch": 0.3144368858654573, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.953346252441406, "fcm_dpo/q_t": 0.48543232679367065, "grad_norm": 5.455617427825928, "learning_rate": 4.3454249259229664e-07, "logits/chosen": 1.1541500091552734, "logits/rejected": 1.136106014251709, "logps/chosen": -234.06787109375, "logps/ref_chosen": -57.731712341308594, "logps/ref_rejected": -74.19276428222656, "logps/rejected": -290.4822998046875, "loss": 1.3437, "margin_dpo/margin_mean": 39.95335388183594, "margin_dpo/margin_std": 167.1756134033203, "step": 208 }, { "KL/chosen_KL_mean": -220.13638305664062, "KL/mean": -263.32781982421875, "KL/rejected_KL_mean": -306.5192565917969, "KL/std": 255.47915649414062, "epoch": 0.31594860166288735, "fcm_dpo/beta": 0.0015425452729687095, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.38284301757812, "fcm_dpo/q_t": 0.46814075112342834, "grad_norm": 6.359311580657959, "learning_rate": 4.336479271643833e-07, "logits/chosen": 1.1173896789550781, "logits/rejected": 1.0917067527770996, "logps/chosen": -288.68646240234375, "logps/ref_chosen": -68.55007934570312, "logps/ref_rejected": -87.90541076660156, "logps/rejected": -394.4246826171875, "loss": 1.3044, "margin_dpo/margin_mean": 86.38285064697266, "margin_dpo/margin_std": 283.98956298828125, "step": 209 }, { "KL/chosen_KL_mean": -179.193603515625, "KL/mean": -246.36642456054688, "KL/rejected_KL_mean": -313.53924560546875, "KL/std": 254.68429565429688, "epoch": 0.31746031746031744, "fcm_dpo/beta": 0.0019031364936381578, "fcm_dpo/delta": 0.023973697796463966, "fcm_dpo/margin": 134.34561157226562, "fcm_dpo/q_t": 0.45283159613609314, "grad_norm": 9.294327735900879, "learning_rate": 4.327482247091679e-07, "logits/chosen": 1.2122018337249756, "logits/rejected": 1.1607775688171387, "logps/chosen": -236.46188354492188, "logps/ref_chosen": -57.268272399902344, "logps/ref_rejected": -85.72807312011719, "logps/rejected": -399.2673034667969, "loss": 1.198, "margin_dpo/margin_mean": 134.34561157226562, "margin_dpo/margin_std": 250.78106689453125, "step": 210 }, { "KL/chosen_KL_mean": -167.25848388671875, "KL/mean": -236.9961395263672, "KL/rejected_KL_mean": -306.7337951660156, "KL/std": 225.39352416992188, "epoch": 0.31897203325774753, "fcm_dpo/beta": 0.004614786710590124, "fcm_dpo/delta": -0.5421129465103149, "fcm_dpo/margin": 139.4752960205078, "fcm_dpo/q_t": 0.3901931047439575, "grad_norm": 23.795278549194336, "learning_rate": 4.3184341039326217e-07, "logits/chosen": 1.2343952655792236, "logits/rejected": 1.1764979362487793, "logps/chosen": -220.89920043945312, "logps/ref_chosen": -53.640708923339844, "logps/ref_rejected": -93.0387954711914, "logps/rejected": -399.7725830078125, "loss": 1.3759, "margin_dpo/margin_mean": 139.4752960205078, "margin_dpo/margin_std": 275.88897705078125, "step": 211 }, { "KL/chosen_KL_mean": -185.3031463623047, "KL/mean": -240.04290771484375, "KL/rejected_KL_mean": -294.78265380859375, "KL/std": 222.0595703125, "epoch": 0.3204837490551776, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.47950744628906, "fcm_dpo/q_t": 0.49998939037323, "grad_norm": 0.001861194847151637, "learning_rate": 4.309335095262675e-07, "logits/chosen": 1.2033497095108032, "logits/rejected": 1.167579174041748, "logps/chosen": -242.66989135742188, "logps/ref_chosen": -57.36674499511719, "logps/ref_rejected": -79.89643096923828, "logps/rejected": -374.6790771484375, "loss": 1.3863, "margin_dpo/margin_mean": 109.47950744628906, "margin_dpo/margin_std": 288.2134704589844, "step": 212 }, { "KL/chosen_KL_mean": -211.92974853515625, "KL/mean": -252.25604248046875, "KL/rejected_KL_mean": -292.5823974609375, "KL/std": 271.2462463378906, "epoch": 0.3219954648526077, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.65261840820312, "fcm_dpo/q_t": 0.4999921917915344, "grad_norm": 0.0020023249089717865, "learning_rate": 4.3001854756006724e-07, "logits/chosen": 1.1829383373260498, "logits/rejected": 1.178153157234192, "logps/chosen": -277.15087890625, "logps/ref_chosen": -65.22111511230469, "logps/ref_rejected": -80.1810302734375, "logps/rejected": -372.7633972167969, "loss": 1.3863, "margin_dpo/margin_mean": 80.65261840820312, "margin_dpo/margin_std": 359.29901123046875, "step": 213 }, { "KL/chosen_KL_mean": -217.9564208984375, "KL/mean": -248.85366821289062, "KL/rejected_KL_mean": -279.75091552734375, "KL/std": 257.87176513671875, "epoch": 0.3235071806500378, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.79451370239258, "fcm_dpo/q_t": 0.49999403953552246, "grad_norm": 0.0014851855812594295, "learning_rate": 4.290985500881143e-07, "logits/chosen": 1.1230930089950562, "logits/rejected": 1.1280714273452759, "logps/chosen": -279.2487487792969, "logps/ref_chosen": -61.292327880859375, "logps/ref_rejected": -67.69841003417969, "logps/rejected": -347.4493408203125, "loss": 1.3863, "margin_dpo/margin_mean": 61.794517517089844, "margin_dpo/margin_std": 338.4102783203125, "step": 214 }, { "KL/chosen_KL_mean": -213.97157287597656, "KL/mean": -296.07861328125, "KL/rejected_KL_mean": -378.18560791015625, "KL/std": 297.4847106933594, "epoch": 0.3250188964474679, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.21408081054688, "fcm_dpo/q_t": 0.49998411536216736, "grad_norm": 0.003093123435974121, "learning_rate": 4.281735428447157e-07, "logits/chosen": 1.1404362916946411, "logits/rejected": 1.096791386604309, "logps/chosen": -277.8406982421875, "logps/ref_chosen": -63.869136810302734, "logps/ref_rejected": -98.7657241821289, "logps/rejected": -476.95135498046875, "loss": 1.3862, "margin_dpo/margin_mean": 164.21408081054688, "margin_dpo/margin_std": 372.910400390625, "step": 215 }, { "KL/chosen_KL_mean": -207.86676025390625, "KL/mean": -284.94354248046875, "KL/rejected_KL_mean": -362.020263671875, "KL/std": 262.0420837402344, "epoch": 0.32653061224489793, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.15354919433594, "fcm_dpo/q_t": 0.49998509883880615, "grad_norm": 0.003247037995606661, "learning_rate": 4.2724355170431247e-07, "logits/chosen": 1.1896278858184814, "logits/rejected": 1.1537840366363525, "logps/chosen": -275.69171142578125, "logps/ref_chosen": -67.824951171875, "logps/ref_rejected": -96.40231323242188, "logps/rejected": -458.422607421875, "loss": 1.3862, "margin_dpo/margin_mean": 154.15353393554688, "margin_dpo/margin_std": 316.47894287109375, "step": 216 }, { "KL/chosen_KL_mean": -208.4380645751953, "KL/mean": -280.11431884765625, "KL/rejected_KL_mean": -351.79052734375, "KL/std": 250.4637908935547, "epoch": 0.328042328042328, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 143.35247802734375, "fcm_dpo/q_t": 0.49998611211776733, "grad_norm": 0.002652729395776987, "learning_rate": 4.26308602680756e-07, "logits/chosen": 1.1905633211135864, "logits/rejected": 1.1464626789093018, "logps/chosen": -268.94305419921875, "logps/ref_chosen": -60.5049934387207, "logps/ref_rejected": -84.26618194580078, "logps/rejected": -436.0567321777344, "loss": 1.3862, "margin_dpo/margin_mean": 143.35247802734375, "margin_dpo/margin_std": 322.68109130859375, "step": 217 }, { "KL/chosen_KL_mean": -261.1753845214844, "KL/mean": -280.609130859375, "KL/rejected_KL_mean": -300.0428466796875, "KL/std": 274.44744873046875, "epoch": 0.3295540438397581, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.86746597290039, "fcm_dpo/q_t": 0.49999624490737915, "grad_norm": 0.0014461593236774206, "learning_rate": 4.253687219265803e-07, "logits/chosen": 1.2026422023773193, "logits/rejected": 1.20313560962677, "logps/chosen": -331.76971435546875, "logps/ref_chosen": -70.59431457519531, "logps/ref_rejected": -73.89038848876953, "logps/rejected": -373.9332275390625, "loss": 1.3863, "margin_dpo/margin_mean": 38.867462158203125, "margin_dpo/margin_std": 299.7601318359375, "step": 218 }, { "KL/chosen_KL_mean": -212.3046112060547, "KL/mean": -256.71380615234375, "KL/rejected_KL_mean": -301.1230163574219, "KL/std": 195.33700561523438, "epoch": 0.3310657596371882, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.81842041015625, "fcm_dpo/q_t": 0.49999141693115234, "grad_norm": 0.0015157065354287624, "learning_rate": 4.2442393573227043e-07, "logits/chosen": 1.2033343315124512, "logits/rejected": 1.186316967010498, "logps/chosen": -272.7955322265625, "logps/ref_chosen": -60.490943908691406, "logps/ref_rejected": -75.85001373291016, "logps/rejected": -376.9730224609375, "loss": 1.3863, "margin_dpo/margin_mean": 88.81842041015625, "margin_dpo/margin_std": 246.12535095214844, "step": 219 }, { "KL/chosen_KL_mean": -152.8551025390625, "KL/mean": -207.81396484375, "KL/rejected_KL_mean": -262.7728271484375, "KL/std": 192.8766326904297, "epoch": 0.3325774754346183, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.91773986816406, "fcm_dpo/q_t": 0.4999893307685852, "grad_norm": 0.00206238916143775, "learning_rate": 4.234742705255272e-07, "logits/chosen": 1.2526534795761108, "logits/rejected": 1.2216923236846924, "logps/chosen": -197.86849975585938, "logps/ref_chosen": -45.013397216796875, "logps/ref_rejected": -70.49369812011719, "logps/rejected": -333.26654052734375, "loss": 1.3863, "margin_dpo/margin_mean": 109.91773986816406, "margin_dpo/margin_std": 220.33956909179688, "step": 220 }, { "KL/chosen_KL_mean": -189.33047485351562, "KL/mean": -266.7629089355469, "KL/rejected_KL_mean": -344.19537353515625, "KL/std": 269.7894287109375, "epoch": 0.3340891912320484, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.86489868164062, "fcm_dpo/q_t": 0.4999850392341614, "grad_norm": 0.002508159726858139, "learning_rate": 4.22519752870528e-07, "logits/chosen": 1.1693766117095947, "logits/rejected": 1.1354807615280151, "logps/chosen": -248.4263153076172, "logps/ref_chosen": -59.09584045410156, "logps/ref_rejected": -88.64388275146484, "logps/rejected": -432.8392333984375, "loss": 1.3862, "margin_dpo/margin_mean": 154.86488342285156, "margin_dpo/margin_std": 335.72869873046875, "step": 221 }, { "KL/chosen_KL_mean": -219.88815307617188, "KL/mean": -301.087890625, "KL/rejected_KL_mean": -382.28759765625, "KL/std": 298.183349609375, "epoch": 0.3356009070294785, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 162.39944458007812, "fcm_dpo/q_t": 0.4999842941761017, "grad_norm": 0.003092326922342181, "learning_rate": 4.2156040946718343e-07, "logits/chosen": 1.2328643798828125, "logits/rejected": 1.1719518899917603, "logps/chosen": -275.8858337402344, "logps/ref_chosen": -55.9976921081543, "logps/ref_rejected": -111.94727325439453, "logps/rejected": -494.2348937988281, "loss": 1.3862, "margin_dpo/margin_mean": 162.39944458007812, "margin_dpo/margin_std": 360.7342529296875, "step": 222 }, { "KL/chosen_KL_mean": -218.1814422607422, "KL/mean": -282.19677734375, "KL/rejected_KL_mean": -346.21209716796875, "KL/std": 276.27166748046875, "epoch": 0.3371126228269085, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.03065490722656, "fcm_dpo/q_t": 0.4999876022338867, "grad_norm": 0.0024099198635667562, "learning_rate": 4.2059626715039065e-07, "logits/chosen": 1.2278566360473633, "logits/rejected": 1.2019100189208984, "logps/chosen": -278.0728759765625, "logps/ref_chosen": -59.891422271728516, "logps/ref_rejected": -86.28954315185547, "logps/rejected": -432.50164794921875, "loss": 1.3862, "margin_dpo/margin_mean": 128.03065490722656, "margin_dpo/margin_std": 309.4208679199219, "step": 223 }, { "KL/chosen_KL_mean": -233.63507080078125, "KL/mean": -257.69244384765625, "KL/rejected_KL_mean": -281.74981689453125, "KL/std": 263.045166015625, "epoch": 0.3386243386243386, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.114749908447266, "fcm_dpo/q_t": 0.4999953508377075, "grad_norm": 0.0018048178171738982, "learning_rate": 4.1962735288928304e-07, "logits/chosen": 1.2750532627105713, "logits/rejected": 1.2633410692214966, "logps/chosen": -297.6797180175781, "logps/ref_chosen": -64.04463195800781, "logps/ref_rejected": -75.05450439453125, "logps/rejected": -356.8043212890625, "loss": 1.3863, "margin_dpo/margin_mean": 48.11475372314453, "margin_dpo/margin_std": 270.31732177734375, "step": 224 }, { "KL/chosen_KL_mean": -257.35723876953125, "KL/mean": -354.2248840332031, "KL/rejected_KL_mean": -451.092529296875, "KL/std": 388.58221435546875, "epoch": 0.3401360544217687, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 193.7353057861328, "fcm_dpo/q_t": 0.49998122453689575, "grad_norm": 0.003719078376889229, "learning_rate": 4.186536937864752e-07, "logits/chosen": 1.2433876991271973, "logits/rejected": 1.2075812816619873, "logps/chosen": -323.45306396484375, "logps/ref_chosen": -66.0958251953125, "logps/ref_rejected": -97.68675231933594, "logps/rejected": -548.779296875, "loss": 1.3862, "margin_dpo/margin_mean": 193.73529052734375, "margin_dpo/margin_std": 417.85693359375, "step": 225 }, { "KL/chosen_KL_mean": -184.22708129882812, "KL/mean": -231.06289672851562, "KL/rejected_KL_mean": -277.8987121582031, "KL/std": 214.7318115234375, "epoch": 0.3416477702191988, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.67162322998047, "fcm_dpo/q_t": 0.49999094009399414, "grad_norm": 0.0018737937789410353, "learning_rate": 4.176753170773052e-07, "logits/chosen": 1.2286086082458496, "logits/rejected": 1.2162175178527832, "logps/chosen": -235.64395141601562, "logps/ref_chosen": -51.4168701171875, "logps/ref_rejected": -66.30068969726562, "logps/rejected": -344.19940185546875, "loss": 1.3863, "margin_dpo/margin_mean": 93.67161560058594, "margin_dpo/margin_std": 288.7680358886719, "step": 226 }, { "KL/chosen_KL_mean": -203.26730346679688, "KL/mean": -247.32821655273438, "KL/rejected_KL_mean": -291.38916015625, "KL/std": 236.57278442382812, "epoch": 0.3431594860166289, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.12184143066406, "fcm_dpo/q_t": 0.4999914765357971, "grad_norm": 0.0017340828198939562, "learning_rate": 4.166922501290729e-07, "logits/chosen": 1.2588739395141602, "logits/rejected": 1.2475314140319824, "logps/chosen": -261.257080078125, "logps/ref_chosen": -57.989776611328125, "logps/ref_rejected": -75.05464172363281, "logps/rejected": -366.44378662109375, "loss": 1.3863, "margin_dpo/margin_mean": 88.12184143066406, "margin_dpo/margin_std": 271.02618408203125, "step": 227 }, { "KL/chosen_KL_mean": -190.28192138671875, "KL/mean": -252.29940795898438, "KL/rejected_KL_mean": -314.31695556640625, "KL/std": 251.63345336914062, "epoch": 0.34467120181405897, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.0350112915039, "fcm_dpo/q_t": 0.49998798966407776, "grad_norm": 0.0025014770217239857, "learning_rate": 4.1570452044027405e-07, "logits/chosen": 1.2646236419677734, "logits/rejected": 1.2317761182785034, "logps/chosen": -245.84127807617188, "logps/ref_chosen": -55.55936813354492, "logps/ref_rejected": -77.02364349365234, "logps/rejected": -391.340576171875, "loss": 1.3862, "margin_dpo/margin_mean": 124.0350112915039, "margin_dpo/margin_std": 312.650390625, "step": 228 }, { "KL/chosen_KL_mean": -173.30197143554688, "KL/mean": -235.5060272216797, "KL/rejected_KL_mean": -297.7100830078125, "KL/std": 271.6685485839844, "epoch": 0.34618291761148906, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.40812683105469, "fcm_dpo/q_t": 0.49998795986175537, "grad_norm": 0.0023492288310080767, "learning_rate": 4.147121556398312e-07, "logits/chosen": 1.2890279293060303, "logits/rejected": 1.2542376518249512, "logps/chosen": -224.09661865234375, "logps/ref_chosen": -50.79466247558594, "logps/ref_rejected": -78.4474105834961, "logps/rejected": -376.157470703125, "loss": 1.3862, "margin_dpo/margin_mean": 124.40812683105469, "margin_dpo/margin_std": 323.8258972167969, "step": 229 }, { "KL/chosen_KL_mean": -214.48695373535156, "KL/mean": -247.63502502441406, "KL/rejected_KL_mean": -280.7830810546875, "KL/std": 275.4522705078125, "epoch": 0.3476946334089191, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.2961196899414, "fcm_dpo/q_t": 0.49999356269836426, "grad_norm": 0.0015789554454386234, "learning_rate": 4.137151834863213e-07, "logits/chosen": 1.2542227506637573, "logits/rejected": 1.2686011791229248, "logps/chosen": -271.2161865234375, "logps/ref_chosen": -56.729225158691406, "logps/ref_rejected": -62.99180603027344, "logps/rejected": -343.77490234375, "loss": 1.3863, "margin_dpo/margin_mean": 66.2961196899414, "margin_dpo/margin_std": 330.39410400390625, "step": 230 }, { "KL/chosen_KL_mean": -251.24246215820312, "KL/mean": -291.47467041015625, "KL/rejected_KL_mean": -331.7068786621094, "KL/std": 262.79315185546875, "epoch": 0.3492063492063492, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.46441650390625, "fcm_dpo/q_t": 0.4999921917915344, "grad_norm": 0.0015881173312664032, "learning_rate": 4.1271363186719835e-07, "logits/chosen": 1.2051804065704346, "logits/rejected": 1.2067768573760986, "logps/chosen": -323.83953857421875, "logps/ref_chosen": -72.59709930419922, "logps/ref_rejected": -86.2322998046875, "logps/rejected": -417.9391784667969, "loss": 1.3863, "margin_dpo/margin_mean": 80.46441650390625, "margin_dpo/margin_std": 341.13519287109375, "step": 231 }, { "KL/chosen_KL_mean": -247.03854370117188, "KL/mean": -286.765869140625, "KL/rejected_KL_mean": -326.4931945800781, "KL/std": 314.61639404296875, "epoch": 0.3507180650037793, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.45465087890625, "fcm_dpo/q_t": 0.499992311000824, "grad_norm": 0.0016828159568831325, "learning_rate": 4.1170752879801436e-07, "logits/chosen": 1.212486982345581, "logits/rejected": 1.1974080801010132, "logps/chosen": -315.1571044921875, "logps/ref_chosen": -68.1185302734375, "logps/ref_rejected": -83.79415893554688, "logps/rejected": -410.287353515625, "loss": 1.3863, "margin_dpo/margin_mean": 79.45465087890625, "margin_dpo/margin_std": 385.42352294921875, "step": 232 }, { "KL/chosen_KL_mean": -209.6459503173828, "KL/mean": -239.39019775390625, "KL/rejected_KL_mean": -269.1343994140625, "KL/std": 206.01983642578125, "epoch": 0.35222978080120937, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 59.48847198486328, "fcm_dpo/q_t": 0.4999942183494568, "grad_norm": 0.0019632186740636826, "learning_rate": 4.106969024216348e-07, "logits/chosen": 1.2710832357406616, "logits/rejected": 1.2419123649597168, "logps/chosen": -264.7160949707031, "logps/ref_chosen": -55.070152282714844, "logps/ref_rejected": -66.61845397949219, "logps/rejected": -335.75286865234375, "loss": 1.3863, "margin_dpo/margin_mean": 59.48847198486328, "margin_dpo/margin_std": 254.86038208007812, "step": 233 }, { "KL/chosen_KL_mean": -177.97811889648438, "KL/mean": -201.326416015625, "KL/rejected_KL_mean": -224.67471313476562, "KL/std": 178.39566040039062, "epoch": 0.35374149659863946, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 46.69660568237305, "fcm_dpo/q_t": 0.49999547004699707, "grad_norm": 0.0013678089017048478, "learning_rate": 4.09681781007452e-07, "logits/chosen": 1.230893611907959, "logits/rejected": 1.2400652170181274, "logps/chosen": -233.9040069580078, "logps/ref_chosen": -55.92589569091797, "logps/ref_rejected": -51.11608123779297, "logps/rejected": -275.7908020019531, "loss": 1.3863, "margin_dpo/margin_mean": 46.69660568237305, "margin_dpo/margin_std": 229.29837036132812, "step": 234 }, { "KL/chosen_KL_mean": -215.87887573242188, "KL/mean": -269.4683532714844, "KL/rejected_KL_mean": -323.057861328125, "KL/std": 284.41656494140625, "epoch": 0.35525321239606955, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.17898559570312, "fcm_dpo/q_t": 0.4999896287918091, "grad_norm": 0.0015956625575199723, "learning_rate": 4.08662192950594e-07, "logits/chosen": 1.2453619241714478, "logits/rejected": 1.2564499378204346, "logps/chosen": -280.4186096191406, "logps/ref_chosen": -64.53972625732422, "logps/ref_rejected": -77.69151306152344, "logps/rejected": -400.7493896484375, "loss": 1.3863, "margin_dpo/margin_mean": 107.17898559570312, "margin_dpo/margin_std": 307.4293212890625, "step": 235 }, { "KL/chosen_KL_mean": -268.6509704589844, "KL/mean": -299.5308837890625, "KL/rejected_KL_mean": -330.4107971191406, "KL/std": 262.4644775390625, "epoch": 0.35676492819349964, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.75979995727539, "fcm_dpo/q_t": 0.4999940097332001, "grad_norm": 0.0013398700393736362, "learning_rate": 4.076381667711306e-07, "logits/chosen": 1.2493544816970825, "logits/rejected": 1.2446727752685547, "logps/chosen": -339.80572509765625, "logps/ref_chosen": -71.15473937988281, "logps/ref_rejected": -84.88541412353516, "logps/rejected": -415.29620361328125, "loss": 1.3863, "margin_dpo/margin_mean": 61.759796142578125, "margin_dpo/margin_std": 344.2894287109375, "step": 236 }, { "KL/chosen_KL_mean": -239.0124969482422, "KL/mean": -271.1570739746094, "KL/rejected_KL_mean": -303.30169677734375, "KL/std": 226.06419372558594, "epoch": 0.35827664399092973, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.28919219970703, "fcm_dpo/q_t": 0.49999377131462097, "grad_norm": 0.0016555647598579526, "learning_rate": 4.066097311132753e-07, "logits/chosen": 1.203998327255249, "logits/rejected": 1.211648941040039, "logps/chosen": -315.1545104980469, "logps/ref_chosen": -76.14201354980469, "logps/ref_rejected": -80.88479614257812, "logps/rejected": -384.1864929199219, "loss": 1.3863, "margin_dpo/margin_mean": 64.2891845703125, "margin_dpo/margin_std": 271.71954345703125, "step": 237 }, { "KL/chosen_KL_mean": -209.5856475830078, "KL/mean": -240.08590698242188, "KL/rejected_KL_mean": -270.586181640625, "KL/std": 247.57437133789062, "epoch": 0.35978835978835977, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.00053405761719, "fcm_dpo/q_t": 0.49999409914016724, "grad_norm": 0.0015964685007929802, "learning_rate": 4.0557691474458414e-07, "logits/chosen": 1.198110580444336, "logits/rejected": 1.1851861476898193, "logps/chosen": -278.4704895019531, "logps/ref_chosen": -68.88484954833984, "logps/ref_rejected": -75.8946304321289, "logps/rejected": -346.48077392578125, "loss": 1.3863, "margin_dpo/margin_mean": 61.000526428222656, "margin_dpo/margin_std": 281.09130859375, "step": 238 }, { "KL/chosen_KL_mean": -206.03028869628906, "KL/mean": -302.91278076171875, "KL/rejected_KL_mean": -399.7953186035156, "KL/std": 299.86492919921875, "epoch": 0.36130007558578986, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 193.76502990722656, "fcm_dpo/q_t": 0.49998122453689575, "grad_norm": 0.0037673686165362597, "learning_rate": 4.045397465551513e-07, "logits/chosen": 1.2611656188964844, "logits/rejected": 1.1864782571792603, "logps/chosen": -262.8021240234375, "logps/ref_chosen": -56.771827697753906, "logps/ref_rejected": -116.23050689697266, "logps/rejected": -516.0258178710938, "loss": 1.3862, "margin_dpo/margin_mean": 193.76504516601562, "margin_dpo/margin_std": 343.26318359375, "step": 239 }, { "KL/chosen_KL_mean": -220.20172119140625, "KL/mean": -302.418212890625, "KL/rejected_KL_mean": -384.63470458984375, "KL/std": 321.66656494140625, "epoch": 0.36281179138321995, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.43296813964844, "fcm_dpo/q_t": 0.49998408555984497, "grad_norm": 0.00302553316578269, "learning_rate": 4.0349825555680045e-07, "logits/chosen": 1.2720229625701904, "logits/rejected": 1.2404829263687134, "logps/chosen": -273.55584716796875, "logps/ref_chosen": -53.35411071777344, "logps/ref_rejected": -80.12019348144531, "logps/rejected": -464.7548828125, "loss": 1.3862, "margin_dpo/margin_mean": 164.43295288085938, "margin_dpo/margin_std": 391.212890625, "step": 240 }, { "KL/chosen_KL_mean": -226.899658203125, "KL/mean": -280.0133056640625, "KL/rejected_KL_mean": -333.1269226074219, "KL/std": 283.84722900390625, "epoch": 0.36432350718065004, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.22727966308594, "fcm_dpo/q_t": 0.49998971819877625, "grad_norm": 0.0025036621373146772, "learning_rate": 4.0245247088227377e-07, "logits/chosen": 1.2458418607711792, "logits/rejected": 1.2394804954528809, "logps/chosen": -298.7950744628906, "logps/ref_chosen": -71.89541625976562, "logps/ref_rejected": -83.03492736816406, "logps/rejected": -416.161865234375, "loss": 1.3863, "margin_dpo/margin_mean": 106.22727966308594, "margin_dpo/margin_std": 347.1982421875, "step": 241 }, { "KL/chosen_KL_mean": -213.5513916015625, "KL/mean": -256.9989318847656, "KL/rejected_KL_mean": -300.44647216796875, "KL/std": 220.97384643554688, "epoch": 0.36583522297808013, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.89505004882812, "fcm_dpo/q_t": 0.49999159574508667, "grad_norm": 0.0013653092319145799, "learning_rate": 4.0140242178441665e-07, "logits/chosen": 1.2530593872070312, "logits/rejected": 1.252440094947815, "logps/chosen": -271.47882080078125, "logps/ref_chosen": -57.927433013916016, "logps/ref_rejected": -67.838623046875, "logps/rejected": -368.28509521484375, "loss": 1.3863, "margin_dpo/margin_mean": 86.89505767822266, "margin_dpo/margin_std": 233.89271545410156, "step": 242 }, { "KL/chosen_KL_mean": -261.82867431640625, "KL/mean": -280.63238525390625, "KL/rejected_KL_mean": -299.43609619140625, "KL/std": 240.62506103515625, "epoch": 0.3673469387755102, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 37.60741424560547, "fcm_dpo/q_t": 0.4999963641166687, "grad_norm": 0.0014760139165446162, "learning_rate": 4.003481376353596e-07, "logits/chosen": 1.2306547164916992, "logits/rejected": 1.242539405822754, "logps/chosen": -336.1053466796875, "logps/ref_chosen": -74.27667236328125, "logps/ref_rejected": -73.24340057373047, "logps/rejected": -372.67950439453125, "loss": 1.3863, "margin_dpo/margin_mean": 37.60741424560547, "margin_dpo/margin_std": 283.18731689453125, "step": 243 }, { "KL/chosen_KL_mean": -188.20855712890625, "KL/mean": -253.76040649414062, "KL/rejected_KL_mean": -319.312255859375, "KL/std": 258.85791015625, "epoch": 0.3688586545729403, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.1037139892578, "fcm_dpo/q_t": 0.49998730421066284, "grad_norm": 0.0026807633694261312, "learning_rate": 3.9928964792569654e-07, "logits/chosen": 1.2482179403305054, "logits/rejected": 1.2198402881622314, "logps/chosen": -241.57244873046875, "logps/ref_chosen": -53.36390686035156, "logps/ref_rejected": -71.10276794433594, "logps/rejected": -390.4150390625, "loss": 1.3862, "margin_dpo/margin_mean": 131.1037139892578, "margin_dpo/margin_std": 323.0280456542969, "step": 244 }, { "KL/chosen_KL_mean": -225.77105712890625, "KL/mean": -294.74444580078125, "KL/rejected_KL_mean": -363.7178649902344, "KL/std": 293.7945556640625, "epoch": 0.37037037037037035, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.94680786132812, "fcm_dpo/q_t": 0.4999866485595703, "grad_norm": 0.0021245733369141817, "learning_rate": 3.982269822636601e-07, "logits/chosen": 1.2210156917572021, "logits/rejected": 1.2317429780960083, "logps/chosen": -296.9661560058594, "logps/ref_chosen": -71.19510650634766, "logps/ref_rejected": -80.76235961914062, "logps/rejected": -444.480224609375, "loss": 1.3862, "margin_dpo/margin_mean": 137.94680786132812, "margin_dpo/margin_std": 270.6390075683594, "step": 245 }, { "KL/chosen_KL_mean": -266.89068603515625, "KL/mean": -320.71197509765625, "KL/rejected_KL_mean": -374.5332336425781, "KL/std": 293.4222412109375, "epoch": 0.37188208616780044, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.6425552368164, "fcm_dpo/q_t": 0.4999895691871643, "grad_norm": 0.0021127404179424047, "learning_rate": 3.971601703742932e-07, "logits/chosen": 1.265270471572876, "logits/rejected": 1.237593650817871, "logps/chosen": -338.5117492675781, "logps/ref_chosen": -71.62104797363281, "logps/ref_rejected": -94.03392028808594, "logps/rejected": -468.567138671875, "loss": 1.3863, "margin_dpo/margin_mean": 107.64256286621094, "margin_dpo/margin_std": 309.1661376953125, "step": 246 }, { "KL/chosen_KL_mean": -302.8369140625, "KL/mean": -326.8985595703125, "KL/rejected_KL_mean": -350.9602355957031, "KL/std": 334.1150817871094, "epoch": 0.37339380196523053, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 48.12330627441406, "fcm_dpo/q_t": 0.4999953508377075, "grad_norm": 0.002026443835347891, "learning_rate": 3.960892420986177e-07, "logits/chosen": 1.2411327362060547, "logits/rejected": 1.2402052879333496, "logps/chosen": -382.85943603515625, "logps/ref_chosen": -80.02254486083984, "logps/ref_rejected": -89.22705841064453, "logps/rejected": -440.1872863769531, "loss": 1.3863, "margin_dpo/margin_mean": 48.12330627441406, "margin_dpo/margin_std": 404.8681640625, "step": 247 }, { "KL/chosen_KL_mean": -273.5013427734375, "KL/mean": -274.2215881347656, "KL/rejected_KL_mean": -274.94183349609375, "KL/std": 300.5850830078125, "epoch": 0.3749055177626606, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.44049072265625, "fcm_dpo/q_t": 0.49999985098838806, "grad_norm": 0.0020725768990814686, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 1.2465314865112305, "logits/rejected": 1.2722697257995605, "logps/chosen": -338.8793029785156, "logps/ref_chosen": -65.37796020507812, "logps/ref_rejected": -61.365787506103516, "logps/rejected": -336.3076171875, "loss": 1.3863, "margin_dpo/margin_mean": 1.44049072265625, "margin_dpo/margin_std": 416.7322998046875, "step": 248 }, { "KL/chosen_KL_mean": -266.708984375, "KL/mean": -261.5863037109375, "KL/rejected_KL_mean": -256.463623046875, "KL/std": 257.3307800292969, "epoch": 0.3764172335600907, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -10.245379447937012, "fcm_dpo/q_t": 0.5000010132789612, "grad_norm": 0.0021689594723284245, "learning_rate": 3.9393515632731094e-07, "logits/chosen": 1.2218635082244873, "logits/rejected": 1.247019648551941, "logps/chosen": -341.3104553222656, "logps/ref_chosen": -74.60145568847656, "logps/ref_rejected": -63.79338455200195, "logps/rejected": -320.25701904296875, "loss": 1.3863, "margin_dpo/margin_mean": -10.245381355285645, "margin_dpo/margin_std": 330.6340026855469, "step": 249 }, { "KL/chosen_KL_mean": -243.35177612304688, "KL/mean": -302.5293884277344, "KL/rejected_KL_mean": -361.70703125, "KL/std": 364.4193115234375, "epoch": 0.3779289493575208, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.3552474975586, "fcm_dpo/q_t": 0.49998852610588074, "grad_norm": 0.0022773679811507463, "learning_rate": 3.9285205908608934e-07, "logits/chosen": 1.2806836366653442, "logits/rejected": 1.28066885471344, "logps/chosen": -305.28997802734375, "logps/ref_chosen": -61.938209533691406, "logps/ref_rejected": -72.21602630615234, "logps/rejected": -433.9230651855469, "loss": 1.3862, "margin_dpo/margin_mean": 118.3552474975586, "margin_dpo/margin_std": 424.7938537597656, "step": 250 }, { "KL/chosen_KL_mean": -238.873779296875, "KL/mean": -284.80145263671875, "KL/rejected_KL_mean": -330.7291259765625, "KL/std": 258.41424560546875, "epoch": 0.3794406651549509, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.85538482666016, "fcm_dpo/q_t": 0.49999111890792847, "grad_norm": 0.002100482815876603, "learning_rate": 3.9176496596569265e-07, "logits/chosen": 1.240132451057434, "logits/rejected": 1.2237201929092407, "logps/chosen": -305.730712890625, "logps/ref_chosen": -66.85694885253906, "logps/ref_rejected": -84.83396911621094, "logps/rejected": -415.5631103515625, "loss": 1.3863, "margin_dpo/margin_mean": 91.85537719726562, "margin_dpo/margin_std": 292.9080810546875, "step": 251 }, { "KL/chosen_KL_mean": -204.33717346191406, "KL/mean": -248.474853515625, "KL/rejected_KL_mean": -292.612548828125, "KL/std": 251.9514923095703, "epoch": 0.38095238095238093, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.27537536621094, "fcm_dpo/q_t": 0.4999914765357971, "grad_norm": 0.0017355838790535927, "learning_rate": 3.9067390737445254e-07, "logits/chosen": 1.2478840351104736, "logits/rejected": 1.2181034088134766, "logps/chosen": -260.56109619140625, "logps/ref_chosen": -56.22393035888672, "logps/ref_rejected": -77.1136245727539, "logps/rejected": -369.7261657714844, "loss": 1.3863, "margin_dpo/margin_mean": 88.27537536621094, "margin_dpo/margin_std": 345.6888122558594, "step": 252 }, { "KL/chosen_KL_mean": -171.81988525390625, "KL/mean": -219.31829833984375, "KL/rejected_KL_mean": -266.81671142578125, "KL/std": 209.09054565429688, "epoch": 0.382464096749811, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.99684143066406, "fcm_dpo/q_t": 0.4999908208847046, "grad_norm": 0.0018323827534914017, "learning_rate": 3.8957891383162304e-07, "logits/chosen": 1.2669950723648071, "logits/rejected": 1.26656174659729, "logps/chosen": -224.02987670898438, "logps/ref_chosen": -52.21001434326172, "logps/ref_rejected": -58.75764846801758, "logps/rejected": -325.5743408203125, "loss": 1.3863, "margin_dpo/margin_mean": 94.99683380126953, "margin_dpo/margin_std": 259.6316223144531, "step": 253 }, { "KL/chosen_KL_mean": -270.77874755859375, "KL/mean": -316.91168212890625, "KL/rejected_KL_mean": -363.0445556640625, "KL/std": 343.09942626953125, "epoch": 0.3839758125472411, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.26583862304688, "fcm_dpo/q_t": 0.4999910593032837, "grad_norm": 0.0022480462212115526, "learning_rate": 3.884800159665276e-07, "logits/chosen": 1.238560438156128, "logits/rejected": 1.2194675207138062, "logps/chosen": -336.4150695800781, "logps/ref_chosen": -65.63632202148438, "logps/ref_rejected": -82.34425354003906, "logps/rejected": -445.3888244628906, "loss": 1.3863, "margin_dpo/margin_mean": 92.26583862304688, "margin_dpo/margin_std": 337.8890380859375, "step": 254 }, { "KL/chosen_KL_mean": -255.44696044921875, "KL/mean": -305.6675720214844, "KL/rejected_KL_mean": -355.88818359375, "KL/std": 318.76397705078125, "epoch": 0.3854875283446712, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.44125366210938, "fcm_dpo/q_t": 0.4999902844429016, "grad_norm": 0.0018314458429813385, "learning_rate": 3.873772445177015e-07, "logits/chosen": 1.2495830059051514, "logits/rejected": 1.2430639266967773, "logps/chosen": -323.3580627441406, "logps/ref_chosen": -67.91108703613281, "logps/ref_rejected": -83.89114379882812, "logps/rejected": -439.77935791015625, "loss": 1.3863, "margin_dpo/margin_mean": 100.44124603271484, "margin_dpo/margin_std": 398.81390380859375, "step": 255 }, { "KL/chosen_KL_mean": -259.4943542480469, "KL/mean": -310.67401123046875, "KL/rejected_KL_mean": -361.85369873046875, "KL/std": 321.3037414550781, "epoch": 0.3869992441421013, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.35935974121094, "fcm_dpo/q_t": 0.4999900758266449, "grad_norm": 0.0024800151586532593, "learning_rate": 3.862706303320329e-07, "logits/chosen": 1.2634713649749756, "logits/rejected": 1.2241191864013672, "logps/chosen": -322.99432373046875, "logps/ref_chosen": -63.49998474121094, "logps/ref_rejected": -90.77104187011719, "logps/rejected": -452.624755859375, "loss": 1.3863, "margin_dpo/margin_mean": 102.35935974121094, "margin_dpo/margin_std": 402.4932556152344, "step": 256 }, { "KL/chosen_KL_mean": -262.64410400390625, "KL/mean": -331.921875, "KL/rejected_KL_mean": -401.1995849609375, "KL/std": 297.68072509765625, "epoch": 0.3885109599395314, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.5554656982422, "fcm_dpo/q_t": 0.49998658895492554, "grad_norm": 0.0026133295614272356, "learning_rate": 3.851602043638994e-07, "logits/chosen": 1.2339143753051758, "logits/rejected": 1.192386507987976, "logps/chosen": -333.2447509765625, "logps/ref_chosen": -70.60064697265625, "logps/ref_rejected": -108.58313751220703, "logps/rejected": -509.78271484375, "loss": 1.3862, "margin_dpo/margin_mean": 138.5554656982422, "margin_dpo/margin_std": 377.41680908203125, "step": 257 }, { "KL/chosen_KL_mean": -226.23171997070312, "KL/mean": -291.40777587890625, "KL/rejected_KL_mean": -356.5838623046875, "KL/std": 314.12420654296875, "epoch": 0.3900226757369615, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.35211181640625, "fcm_dpo/q_t": 0.49998739361763, "grad_norm": 0.002391485497355461, "learning_rate": 3.840459976743023e-07, "logits/chosen": 1.226086139678955, "logits/rejected": 1.204946517944336, "logps/chosen": -285.48590087890625, "logps/ref_chosen": -59.25416564941406, "logps/ref_rejected": -85.58709716796875, "logps/rejected": -442.1709289550781, "loss": 1.3862, "margin_dpo/margin_mean": 130.35211181640625, "margin_dpo/margin_std": 371.4399108886719, "step": 258 }, { "KL/chosen_KL_mean": -232.07115173339844, "KL/mean": -321.46673583984375, "KL/rejected_KL_mean": -410.86236572265625, "KL/std": 316.74798583984375, "epoch": 0.3915343915343915, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 178.79119873046875, "fcm_dpo/q_t": 0.49998271465301514, "grad_norm": 0.003150097792968154, "learning_rate": 3.8292804142999796e-07, "logits/chosen": 1.2068517208099365, "logits/rejected": 1.1704658269882202, "logps/chosen": -297.50604248046875, "logps/ref_chosen": -65.43487548828125, "logps/ref_rejected": -95.41731262207031, "logps/rejected": -506.2796630859375, "loss": 1.3862, "margin_dpo/margin_mean": 178.79119873046875, "margin_dpo/margin_std": 385.9300537109375, "step": 259 }, { "KL/chosen_KL_mean": -184.13815307617188, "KL/mean": -257.88519287109375, "KL/rejected_KL_mean": -331.63226318359375, "KL/std": 256.04803466796875, "epoch": 0.3930461073318216, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 147.49411010742188, "fcm_dpo/q_t": 0.4999857246875763, "grad_norm": 0.002410466317087412, "learning_rate": 3.818063669026256e-07, "logits/chosen": 1.246681809425354, "logits/rejected": 1.2082734107971191, "logps/chosen": -233.2277374267578, "logps/ref_chosen": -49.08958435058594, "logps/ref_rejected": -79.01708221435547, "logps/rejected": -410.64935302734375, "loss": 1.3862, "margin_dpo/margin_mean": 147.49411010742188, "margin_dpo/margin_std": 310.56329345703125, "step": 260 }, { "KL/chosen_KL_mean": -234.46389770507812, "KL/mean": -250.5965576171875, "KL/rejected_KL_mean": -266.7292175292969, "KL/std": 255.46902465820312, "epoch": 0.3945578231292517, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 32.26532745361328, "fcm_dpo/q_t": 0.4999969005584717, "grad_norm": 0.001933022402226925, "learning_rate": 3.806810054678331e-07, "logits/chosen": 1.184006929397583, "logits/rejected": 1.216488242149353, "logps/chosen": -305.3363037109375, "logps/ref_chosen": -70.87239074707031, "logps/ref_rejected": -65.01522064208984, "logps/rejected": -331.74444580078125, "loss": 1.3863, "margin_dpo/margin_mean": 32.265323638916016, "margin_dpo/margin_std": 333.6747741699219, "step": 261 }, { "KL/chosen_KL_mean": -257.26776123046875, "KL/mean": -322.60302734375, "KL/rejected_KL_mean": -387.93829345703125, "KL/std": 322.8201904296875, "epoch": 0.3960695389266818, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.67054748535156, "fcm_dpo/q_t": 0.4999873638153076, "grad_norm": 0.0025448398664593697, "learning_rate": 3.7955198860439887e-07, "logits/chosen": 1.293022871017456, "logits/rejected": 1.268181324005127, "logps/chosen": -325.13836669921875, "logps/ref_chosen": -67.8706283569336, "logps/ref_rejected": -88.7205810546875, "logps/rejected": -476.65887451171875, "loss": 1.3862, "margin_dpo/margin_mean": 130.67056274414062, "margin_dpo/margin_std": 383.709716796875, "step": 262 }, { "KL/chosen_KL_mean": -204.814208984375, "KL/mean": -286.87152099609375, "KL/rejected_KL_mean": -368.9288024902344, "KL/std": 260.5648193359375, "epoch": 0.3975812547241119, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.11460876464844, "fcm_dpo/q_t": 0.49998411536216736, "grad_norm": 0.0031319339759647846, "learning_rate": 3.784193478933516e-07, "logits/chosen": 1.2445664405822754, "logits/rejected": 1.1973962783813477, "logps/chosen": -260.0087890625, "logps/ref_chosen": -55.194583892822266, "logps/ref_rejected": -80.54048156738281, "logps/rejected": -449.46929931640625, "loss": 1.3862, "margin_dpo/margin_mean": 164.11460876464844, "margin_dpo/margin_std": 305.0133056640625, "step": 263 }, { "KL/chosen_KL_mean": -304.02825927734375, "KL/mean": -360.13616943359375, "KL/rejected_KL_mean": -416.244140625, "KL/std": 354.64105224609375, "epoch": 0.39909297052154197, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.21583557128906, "fcm_dpo/q_t": 0.4999891221523285, "grad_norm": 0.0019485560478642583, "learning_rate": 3.7728311501708674e-07, "logits/chosen": 1.213137149810791, "logits/rejected": 1.212262511253357, "logps/chosen": -387.198974609375, "logps/ref_chosen": -83.17068481445312, "logps/ref_rejected": -88.33625793457031, "logps/rejected": -504.58038330078125, "loss": 1.3863, "margin_dpo/margin_mean": 112.21582794189453, "margin_dpo/margin_std": 419.0724792480469, "step": 264 }, { "KL/chosen_KL_mean": -178.7972412109375, "KL/mean": -248.697021484375, "KL/rejected_KL_mean": -318.5968017578125, "KL/std": 211.929931640625, "epoch": 0.40060468631897206, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.79954528808594, "fcm_dpo/q_t": 0.499986469745636, "grad_norm": 0.0025794145185500383, "learning_rate": 3.7614332175848027e-07, "logits/chosen": 1.2378535270690918, "logits/rejected": 1.2246911525726318, "logps/chosen": -230.4600830078125, "logps/ref_chosen": -51.66284942626953, "logps/ref_rejected": -67.1720962524414, "logps/rejected": -385.7688903808594, "loss": 1.3862, "margin_dpo/margin_mean": 139.79954528808594, "margin_dpo/margin_std": 223.987548828125, "step": 265 }, { "KL/chosen_KL_mean": -226.0594024658203, "KL/mean": -293.44244384765625, "KL/rejected_KL_mean": -360.8254699707031, "KL/std": 271.87078857421875, "epoch": 0.4021164021164021, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.7660675048828, "fcm_dpo/q_t": 0.4999869465827942, "grad_norm": 0.0027317116037011147, "learning_rate": 3.75e-07, "logits/chosen": 1.2706271409988403, "logits/rejected": 1.2448146343231201, "logps/chosen": -283.5098876953125, "logps/ref_chosen": -57.45049285888672, "logps/ref_rejected": -77.60826110839844, "logps/rejected": -438.4337158203125, "loss": 1.3862, "margin_dpo/margin_mean": 134.76608276367188, "margin_dpo/margin_std": 255.4769744873047, "step": 266 }, { "KL/chosen_KL_mean": -187.67311096191406, "KL/mean": -220.25643920898438, "KL/rejected_KL_mean": -252.83978271484375, "KL/std": 226.619140625, "epoch": 0.4036281179138322, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 65.16666412353516, "fcm_dpo/q_t": 0.4999936819076538, "grad_norm": 0.0015339229721575975, "learning_rate": 3.738531817228131e-07, "logits/chosen": 1.2775146961212158, "logits/rejected": 1.2753092050552368, "logps/chosen": -242.70846557617188, "logps/ref_chosen": -55.03535079956055, "logps/ref_rejected": -66.0953369140625, "logps/rejected": -318.93511962890625, "loss": 1.3863, "margin_dpo/margin_mean": 65.16666412353516, "margin_dpo/margin_std": 247.80099487304688, "step": 267 }, { "KL/chosen_KL_mean": -225.61331176757812, "KL/mean": -269.3935546875, "KL/rejected_KL_mean": -313.173828125, "KL/std": 249.95361328125, "epoch": 0.4051398337112623, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.56051635742188, "fcm_dpo/q_t": 0.4999915361404419, "grad_norm": 0.0014259773306548595, "learning_rate": 3.7270289900589204e-07, "logits/chosen": 1.2123204469680786, "logits/rejected": 1.2190499305725098, "logps/chosen": -290.68505859375, "logps/ref_chosen": -65.07174682617188, "logps/ref_rejected": -71.42485809326172, "logps/rejected": -384.59869384765625, "loss": 1.3863, "margin_dpo/margin_mean": 87.56051635742188, "margin_dpo/margin_std": 257.95928955078125, "step": 268 }, { "KL/chosen_KL_mean": -245.9951171875, "KL/mean": -291.13848876953125, "KL/rejected_KL_mean": -336.2818603515625, "KL/std": 273.3773498535156, "epoch": 0.40665154950869237, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.28675079345703, "fcm_dpo/q_t": 0.499991238117218, "grad_norm": 0.002284642308950424, "learning_rate": 3.7154918402511714e-07, "logits/chosen": 1.2732834815979004, "logits/rejected": 1.2535003423690796, "logps/chosen": -313.13134765625, "logps/ref_chosen": -67.1362075805664, "logps/ref_rejected": -82.55778503417969, "logps/rejected": -418.83966064453125, "loss": 1.3863, "margin_dpo/margin_mean": 90.28675842285156, "margin_dpo/margin_std": 311.7049560546875, "step": 269 }, { "KL/chosen_KL_mean": -231.08285522460938, "KL/mean": -305.00152587890625, "KL/rejected_KL_mean": -378.920166015625, "KL/std": 334.8675537109375, "epoch": 0.40816326530612246, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 147.83731079101562, "fcm_dpo/q_t": 0.4999856948852539, "grad_norm": 0.0029855347238481045, "learning_rate": 3.7039206905237656e-07, "logits/chosen": 1.265000343322754, "logits/rejected": 1.234115481376648, "logps/chosen": -297.77154541015625, "logps/ref_chosen": -66.6886978149414, "logps/ref_rejected": -85.16129302978516, "logps/rejected": -464.0814208984375, "loss": 1.3862, "margin_dpo/margin_mean": 147.83731079101562, "margin_dpo/margin_std": 414.43560791015625, "step": 270 }, { "KL/chosen_KL_mean": -261.5472106933594, "KL/mean": -319.3465576171875, "KL/rejected_KL_mean": -377.14593505859375, "KL/std": 378.90277099609375, "epoch": 0.40967498110355255, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.59872436523438, "fcm_dpo/q_t": 0.4999887943267822, "grad_norm": 0.002353003015741706, "learning_rate": 3.692315864546635e-07, "logits/chosen": 1.3018203973770142, "logits/rejected": 1.2758502960205078, "logps/chosen": -333.95477294921875, "logps/ref_chosen": -72.40754699707031, "logps/ref_rejected": -92.06311798095703, "logps/rejected": -469.20904541015625, "loss": 1.3862, "margin_dpo/margin_mean": 115.5987319946289, "margin_dpo/margin_std": 363.4158935546875, "step": 271 }, { "KL/chosen_KL_mean": -240.21646118164062, "KL/mean": -269.6424255371094, "KL/rejected_KL_mean": -299.06842041015625, "KL/std": 277.5693359375, "epoch": 0.41118669690098264, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.8519287109375, "fcm_dpo/q_t": 0.49999427795410156, "grad_norm": 0.001313323387876153, "learning_rate": 3.6806776869317067e-07, "logits/chosen": 1.288849949836731, "logits/rejected": 1.30269455909729, "logps/chosen": -306.81787109375, "logps/ref_chosen": -66.60140228271484, "logps/ref_rejected": -67.74340057373047, "logps/rejected": -366.8117980957031, "loss": 1.3863, "margin_dpo/margin_mean": 58.8519287109375, "margin_dpo/margin_std": 330.32196044921875, "step": 272 }, { "KL/chosen_KL_mean": -218.98214721679688, "KL/mean": -287.33990478515625, "KL/rejected_KL_mean": -355.6976318359375, "KL/std": 279.4718017578125, "epoch": 0.4126984126984127, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 136.71548461914062, "fcm_dpo/q_t": 0.49998676776885986, "grad_norm": 0.0026562055572867393, "learning_rate": 3.669006483223828e-07, "logits/chosen": 1.2323360443115234, "logits/rejected": 1.204160451889038, "logps/chosen": -276.3370361328125, "logps/ref_chosen": -57.35487747192383, "logps/ref_rejected": -84.17168426513672, "logps/rejected": -439.86932373046875, "loss": 1.3862, "margin_dpo/margin_mean": 136.71546936035156, "margin_dpo/margin_std": 341.8489074707031, "step": 273 }, { "KL/chosen_KL_mean": -227.13885498046875, "KL/mean": -261.77813720703125, "KL/rejected_KL_mean": -296.41741943359375, "KL/std": 222.0265350341797, "epoch": 0.41421012849584277, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.27857971191406, "fcm_dpo/q_t": 0.4999932646751404, "grad_norm": 0.001474517397582531, "learning_rate": 3.657302579891656e-07, "logits/chosen": 1.2174702882766724, "logits/rejected": 1.2115811109542847, "logps/chosen": -286.78033447265625, "logps/ref_chosen": -59.64149475097656, "logps/ref_rejected": -68.29348754882812, "logps/rejected": -364.71087646484375, "loss": 1.3863, "margin_dpo/margin_mean": 69.27857208251953, "margin_dpo/margin_std": 245.24844360351562, "step": 274 }, { "KL/chosen_KL_mean": -202.14822387695312, "KL/mean": -266.6926574707031, "KL/rejected_KL_mean": -331.2370910644531, "KL/std": 231.38563537597656, "epoch": 0.41572184429327286, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.0888671875, "fcm_dpo/q_t": 0.49998751282691956, "grad_norm": 0.0028466566000133753, "learning_rate": 3.645566304318526e-07, "logits/chosen": 1.286808967590332, "logits/rejected": 1.2461142539978027, "logps/chosen": -255.4148712158203, "logps/ref_chosen": -53.26664352416992, "logps/ref_rejected": -73.84062194824219, "logps/rejected": -405.07769775390625, "loss": 1.3862, "margin_dpo/margin_mean": 129.0888671875, "margin_dpo/margin_std": 244.29052734375, "step": 275 }, { "KL/chosen_KL_mean": -184.27493286132812, "KL/mean": -225.28565979003906, "KL/rejected_KL_mean": -266.2964172363281, "KL/std": 173.41802978515625, "epoch": 0.41723356009070295, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.021484375, "fcm_dpo/q_t": 0.4999920725822449, "grad_norm": 0.0015512220561504364, "learning_rate": 3.633797984793294e-07, "logits/chosen": 1.2420344352722168, "logits/rejected": 1.2362403869628906, "logps/chosen": -237.29571533203125, "logps/ref_chosen": -53.02079772949219, "logps/ref_rejected": -61.56678771972656, "logps/rejected": -327.86322021484375, "loss": 1.3863, "margin_dpo/margin_mean": 82.021484375, "margin_dpo/margin_std": 235.60247802734375, "step": 276 }, { "KL/chosen_KL_mean": -272.57080078125, "KL/mean": -275.81597900390625, "KL/rejected_KL_mean": -279.0611572265625, "KL/std": 280.1846618652344, "epoch": 0.41874527588813304, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 6.490352630615234, "fcm_dpo/q_t": 0.49999934434890747, "grad_norm": 0.0013179422821849585, "learning_rate": 3.6219979505011555e-07, "logits/chosen": 1.2742948532104492, "logits/rejected": 1.291219711303711, "logps/chosen": -344.0037841796875, "logps/ref_chosen": -71.43299102783203, "logps/ref_rejected": -67.65852355957031, "logps/rejected": -346.71966552734375, "loss": 1.3863, "margin_dpo/margin_mean": 6.490348815917969, "margin_dpo/margin_std": 291.0250244140625, "step": 277 }, { "KL/chosen_KL_mean": -271.00958251953125, "KL/mean": -318.64300537109375, "KL/rejected_KL_mean": -366.27642822265625, "KL/std": 313.33929443359375, "epoch": 0.42025699168556313, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.26683807373047, "fcm_dpo/q_t": 0.4999907612800598, "grad_norm": 0.0023758604656904936, "learning_rate": 3.6101665315144353e-07, "logits/chosen": 1.2502532005310059, "logits/rejected": 1.2217738628387451, "logps/chosen": -338.120361328125, "logps/ref_chosen": -67.11076354980469, "logps/ref_rejected": -88.74851989746094, "logps/rejected": -455.02496337890625, "loss": 1.3863, "margin_dpo/margin_mean": 95.26683044433594, "margin_dpo/margin_std": 256.470947265625, "step": 278 }, { "KL/chosen_KL_mean": -200.28704833984375, "KL/mean": -251.19207763671875, "KL/rejected_KL_mean": -302.09710693359375, "KL/std": 222.035400390625, "epoch": 0.4217687074829932, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.81007385253906, "fcm_dpo/q_t": 0.49999016523361206, "grad_norm": 0.002032017568126321, "learning_rate": 3.5983040587833563e-07, "logits/chosen": 1.242897868156433, "logits/rejected": 1.225640058517456, "logps/chosen": -254.78451538085938, "logps/ref_chosen": -54.49748611450195, "logps/ref_rejected": -70.42373657226562, "logps/rejected": -372.5208435058594, "loss": 1.3863, "margin_dpo/margin_mean": 101.8100814819336, "margin_dpo/margin_std": 249.4945068359375, "step": 279 }, { "KL/chosen_KL_mean": -205.01568603515625, "KL/mean": -279.7425842285156, "KL/rejected_KL_mean": -354.469482421875, "KL/std": 266.3010559082031, "epoch": 0.42328042328042326, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 149.45379638671875, "fcm_dpo/q_t": 0.4999855160713196, "grad_norm": 0.002008727751672268, "learning_rate": 3.586410864126781e-07, "logits/chosen": 1.2439789772033691, "logits/rejected": 1.2364301681518555, "logps/chosen": -265.4485168457031, "logps/ref_chosen": -60.43281173706055, "logps/ref_rejected": -78.39051818847656, "logps/rejected": -432.8599853515625, "loss": 1.3862, "margin_dpo/margin_mean": 149.45379638671875, "margin_dpo/margin_std": 288.43994140625, "step": 280 }, { "KL/chosen_KL_mean": -224.62643432617188, "KL/mean": -239.99485778808594, "KL/rejected_KL_mean": -255.36328125, "KL/std": 230.7904052734375, "epoch": 0.42479213907785335, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.736845016479492, "fcm_dpo/q_t": 0.49999701976776123, "grad_norm": 0.0013363017933443189, "learning_rate": 3.574487280222929e-07, "logits/chosen": 1.2372968196868896, "logits/rejected": 1.2535717487335205, "logps/chosen": -284.90850830078125, "logps/ref_chosen": -60.2820930480957, "logps/ref_rejected": -62.04009246826172, "logps/rejected": -317.40338134765625, "loss": 1.3863, "margin_dpo/margin_mean": 30.736841201782227, "margin_dpo/margin_std": 231.83021545410156, "step": 281 }, { "KL/chosen_KL_mean": -230.42965698242188, "KL/mean": -267.6971130371094, "KL/rejected_KL_mean": -304.96453857421875, "KL/std": 253.47097778320312, "epoch": 0.42630385487528344, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.53489685058594, "fcm_dpo/q_t": 0.4999927878379822, "grad_norm": 0.0015713232569396496, "learning_rate": 3.562533640600075e-07, "logits/chosen": 1.2360237836837769, "logits/rejected": 1.222773790359497, "logps/chosen": -291.0535888671875, "logps/ref_chosen": -60.623924255371094, "logps/ref_rejected": -68.67400360107422, "logps/rejected": -373.6385498046875, "loss": 1.3863, "margin_dpo/margin_mean": 74.5348892211914, "margin_dpo/margin_std": 305.5966796875, "step": 282 }, { "KL/chosen_KL_mean": -207.01063537597656, "KL/mean": -313.7869873046875, "KL/rejected_KL_mean": -420.56329345703125, "KL/std": 381.99566650390625, "epoch": 0.42781557067271353, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 213.55267333984375, "fcm_dpo/q_t": 0.4999793469905853, "grad_norm": 0.004121567588299513, "learning_rate": 3.550550279627215e-07, "logits/chosen": 1.2334487438201904, "logits/rejected": 1.195540189743042, "logps/chosen": -274.65838623046875, "logps/ref_chosen": -67.64775085449219, "logps/ref_rejected": -99.96835327148438, "logps/rejected": -520.5316772460938, "loss": 1.3862, "margin_dpo/margin_mean": 213.5526580810547, "margin_dpo/margin_std": 493.8293151855469, "step": 283 }, { "KL/chosen_KL_mean": -213.08843994140625, "KL/mean": -276.4241027832031, "KL/rejected_KL_mean": -339.759765625, "KL/std": 258.0605773925781, "epoch": 0.4293272864701436, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.67131805419922, "fcm_dpo/q_t": 0.49998775124549866, "grad_norm": 0.0023277702275663614, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 1.2803890705108643, "logits/rejected": 1.249000906944275, "logps/chosen": -270.0558776855469, "logps/ref_chosen": -56.96742630004883, "logps/ref_rejected": -86.36236572265625, "logps/rejected": -426.12213134765625, "loss": 1.3862, "margin_dpo/margin_mean": 126.67131805419922, "margin_dpo/margin_std": 305.05230712890625, "step": 284 }, { "KL/chosen_KL_mean": -243.08445739746094, "KL/mean": -282.5973815917969, "KL/rejected_KL_mean": -322.11029052734375, "KL/std": 253.41421508789062, "epoch": 0.4308390022675737, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 79.02581024169922, "fcm_dpo/q_t": 0.49999234080314636, "grad_norm": 0.0040868110954761505, "learning_rate": 3.5264957352549375e-07, "logits/chosen": 1.2414788007736206, "logits/rejected": 1.229378342628479, "logps/chosen": -314.7405700683594, "logps/ref_chosen": -71.65611267089844, "logps/ref_rejected": -81.63829803466797, "logps/rejected": -403.74859619140625, "loss": 1.3863, "margin_dpo/margin_mean": 79.02581787109375, "margin_dpo/margin_std": 294.8158264160156, "step": 285 }, { "KL/chosen_KL_mean": -242.94229125976562, "KL/mean": -322.85455322265625, "KL/rejected_KL_mean": -402.766845703125, "KL/std": 337.2320251464844, "epoch": 0.4323507180650038, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.82455444335938, "fcm_dpo/q_t": 0.4999845325946808, "grad_norm": 0.0029409886337816715, "learning_rate": 3.514425224712835e-07, "logits/chosen": 1.2100770473480225, "logits/rejected": 1.176485300064087, "logps/chosen": -304.0218200683594, "logps/ref_chosen": -61.07952117919922, "logps/ref_rejected": -91.28128051757812, "logps/rejected": -494.0481262207031, "loss": 1.3862, "margin_dpo/margin_mean": 159.82455444335938, "margin_dpo/margin_std": 381.1865539550781, "step": 286 }, { "KL/chosen_KL_mean": -163.37152099609375, "KL/mean": -214.27389526367188, "KL/rejected_KL_mean": -265.1762390136719, "KL/std": 199.1695556640625, "epoch": 0.43386243386243384, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.80474090576172, "fcm_dpo/q_t": 0.49999016523361206, "grad_norm": 0.0017581077991053462, "learning_rate": 3.502326338516534e-07, "logits/chosen": 1.243018388748169, "logits/rejected": 1.237166166305542, "logps/chosen": -209.40731811523438, "logps/ref_chosen": -46.035789489746094, "logps/ref_rejected": -59.95293426513672, "logps/rejected": -325.1291809082031, "loss": 1.3863, "margin_dpo/margin_mean": 101.80474090576172, "margin_dpo/margin_std": 249.35662841796875, "step": 287 }, { "KL/chosen_KL_mean": -247.86630249023438, "KL/mean": -297.19573974609375, "KL/rejected_KL_mean": -346.5251159667969, "KL/std": 270.19976806640625, "epoch": 0.43537414965986393, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.65884399414062, "fcm_dpo/q_t": 0.49999046325683594, "grad_norm": 0.002023276872932911, "learning_rate": 3.490199415097892e-07, "logits/chosen": 1.1788934469223022, "logits/rejected": 1.1556057929992676, "logps/chosen": -313.25714111328125, "logps/ref_chosen": -65.3908462524414, "logps/ref_rejected": -88.53607940673828, "logps/rejected": -435.06121826171875, "loss": 1.3863, "margin_dpo/margin_mean": 98.6588363647461, "margin_dpo/margin_std": 299.93707275390625, "step": 288 }, { "KL/chosen_KL_mean": -218.3133087158203, "KL/mean": -268.06744384765625, "KL/rejected_KL_mean": -317.82159423828125, "KL/std": 313.28009033203125, "epoch": 0.436885865457294, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.50828552246094, "fcm_dpo/q_t": 0.4999903440475464, "grad_norm": 0.00193495349958539, "learning_rate": 3.4780447936730247e-07, "logits/chosen": 1.2431552410125732, "logits/rejected": 1.243152141571045, "logps/chosen": -272.90692138671875, "logps/ref_chosen": -54.5936279296875, "logps/ref_rejected": -67.20855712890625, "logps/rejected": -385.0301513671875, "loss": 1.3863, "margin_dpo/margin_mean": 99.50830078125, "margin_dpo/margin_std": 335.50543212890625, "step": 289 }, { "KL/chosen_KL_mean": -221.37855529785156, "KL/mean": -276.5778503417969, "KL/rejected_KL_mean": -331.77716064453125, "KL/std": 264.361083984375, "epoch": 0.4383975812547241, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.39859771728516, "fcm_dpo/q_t": 0.4999893307685852, "grad_norm": 0.001686166855506599, "learning_rate": 3.465862814232821e-07, "logits/chosen": 1.2381784915924072, "logits/rejected": 1.1985375881195068, "logps/chosen": -282.76312255859375, "logps/ref_chosen": -61.38457489013672, "logps/ref_rejected": -91.92778015136719, "logps/rejected": -423.7049255371094, "loss": 1.3863, "margin_dpo/margin_mean": 110.39860534667969, "margin_dpo/margin_std": 287.36102294921875, "step": 290 }, { "KL/chosen_KL_mean": -188.7603759765625, "KL/mean": -280.0110168457031, "KL/rejected_KL_mean": -371.2616882324219, "KL/std": 305.65924072265625, "epoch": 0.4399092970521542, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 182.5012969970703, "fcm_dpo/q_t": 0.4999823570251465, "grad_norm": 0.0030123190954327583, "learning_rate": 3.4536538175334343e-07, "logits/chosen": 1.3322393894195557, "logits/rejected": 1.3048069477081299, "logps/chosen": -239.6234130859375, "logps/ref_chosen": -50.863037109375, "logps/ref_rejected": -82.20868682861328, "logps/rejected": -453.4703674316406, "loss": 1.3862, "margin_dpo/margin_mean": 182.5012969970703, "margin_dpo/margin_std": 367.24835205078125, "step": 291 }, { "KL/chosen_KL_mean": -225.71701049804688, "KL/mean": -268.5120849609375, "KL/rejected_KL_mean": -311.30718994140625, "KL/std": 296.009033203125, "epoch": 0.4414210128495843, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.59013366699219, "fcm_dpo/q_t": 0.4999917149543762, "grad_norm": 0.0019401878817006946, "learning_rate": 3.4414181450867465e-07, "logits/chosen": 1.244645118713379, "logits/rejected": 1.2302597761154175, "logps/chosen": -290.06591796875, "logps/ref_chosen": -64.34888458251953, "logps/ref_rejected": -72.86434173583984, "logps/rejected": -384.1715087890625, "loss": 1.3863, "margin_dpo/margin_mean": 85.59012603759766, "margin_dpo/margin_std": 375.63006591796875, "step": 292 }, { "KL/chosen_KL_mean": -197.04302978515625, "KL/mean": -276.2276306152344, "KL/rejected_KL_mean": -355.4122314453125, "KL/std": 286.4289855957031, "epoch": 0.4429327286470144, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 158.36923217773438, "fcm_dpo/q_t": 0.4999846816062927, "grad_norm": 0.002683456754311919, "learning_rate": 3.4291561391508185e-07, "logits/chosen": 1.3139989376068115, "logits/rejected": 1.2797266244888306, "logps/chosen": -251.91250610351562, "logps/ref_chosen": -54.869468688964844, "logps/ref_rejected": -81.858642578125, "logps/rejected": -437.2708740234375, "loss": 1.3862, "margin_dpo/margin_mean": 158.36923217773438, "margin_dpo/margin_std": 348.2574157714844, "step": 293 }, { "KL/chosen_KL_mean": -204.25213623046875, "KL/mean": -246.90603637695312, "KL/rejected_KL_mean": -289.5599365234375, "KL/std": 281.533447265625, "epoch": 0.4444444444444444, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 85.30781555175781, "fcm_dpo/q_t": 0.4999917447566986, "grad_norm": 0.0014272573171183467, "learning_rate": 3.4168681427203153e-07, "logits/chosen": 1.244971752166748, "logits/rejected": 1.2297112941741943, "logps/chosen": -260.92303466796875, "logps/ref_chosen": -56.670902252197266, "logps/ref_rejected": -70.32819366455078, "logps/rejected": -359.88812255859375, "loss": 1.3863, "margin_dpo/margin_mean": 85.30781555175781, "margin_dpo/margin_std": 254.677978515625, "step": 294 }, { "KL/chosen_KL_mean": -193.20266723632812, "KL/mean": -270.8068542480469, "KL/rejected_KL_mean": -348.4110107421875, "KL/std": 270.56781005859375, "epoch": 0.4459561602418745, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.20838928222656, "fcm_dpo/q_t": 0.4999849796295166, "grad_norm": 0.0031207390129566193, "learning_rate": 3.4045544995169125e-07, "logits/chosen": 1.2604093551635742, "logits/rejected": 1.2129838466644287, "logps/chosen": -243.60354614257812, "logps/ref_chosen": -50.40088653564453, "logps/ref_rejected": -83.43521881103516, "logps/rejected": -431.84625244140625, "loss": 1.3862, "margin_dpo/margin_mean": 155.20838928222656, "margin_dpo/margin_std": 278.3273010253906, "step": 295 }, { "KL/chosen_KL_mean": -232.06796264648438, "KL/mean": -298.0061340332031, "KL/rejected_KL_mean": -363.9443359375, "KL/std": 285.2636413574219, "epoch": 0.4474678760393046, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.87635803222656, "fcm_dpo/q_t": 0.49998724460601807, "grad_norm": 0.0020598177798092365, "learning_rate": 3.392215553979679e-07, "logits/chosen": 1.1887896060943604, "logits/rejected": 1.1723339557647705, "logps/chosen": -301.21832275390625, "logps/ref_chosen": -69.15034484863281, "logps/ref_rejected": -89.60166931152344, "logps/rejected": -453.5459899902344, "loss": 1.3862, "margin_dpo/margin_mean": 131.87635803222656, "margin_dpo/margin_std": 339.91888427734375, "step": 296 }, { "KL/chosen_KL_mean": -209.5154266357422, "KL/mean": -267.8277587890625, "KL/rejected_KL_mean": -326.14007568359375, "KL/std": 264.28759765625, "epoch": 0.4489795918367347, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.6246566772461, "fcm_dpo/q_t": 0.49998870491981506, "grad_norm": 0.0021352798212319613, "learning_rate": 3.3798516512554485e-07, "logits/chosen": 1.2609082460403442, "logits/rejected": 1.251166820526123, "logps/chosen": -267.53173828125, "logps/ref_chosen": -58.01630401611328, "logps/ref_rejected": -69.95780944824219, "logps/rejected": -396.0978698730469, "loss": 1.3862, "margin_dpo/margin_mean": 116.6246566772461, "margin_dpo/margin_std": 218.6939239501953, "step": 297 }, { "KL/chosen_KL_mean": -233.57615661621094, "KL/mean": -274.2930908203125, "KL/rejected_KL_mean": -315.010009765625, "KL/std": 299.4732360839844, "epoch": 0.4504913076341648, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.43384552001953, "fcm_dpo/q_t": 0.49999210238456726, "grad_norm": 0.0023514467757195234, "learning_rate": 3.367463137189156e-07, "logits/chosen": 1.2886168956756592, "logits/rejected": 1.2632899284362793, "logps/chosen": -289.7454833984375, "logps/ref_chosen": -56.1693115234375, "logps/ref_rejected": -68.55052185058594, "logps/rejected": -383.5605163574219, "loss": 1.3863, "margin_dpo/margin_mean": 81.43385314941406, "margin_dpo/margin_std": 265.78594970703125, "step": 298 }, { "KL/chosen_KL_mean": -219.94754028320312, "KL/mean": -259.0394287109375, "KL/rejected_KL_mean": -298.13128662109375, "KL/std": 257.43145751953125, "epoch": 0.4520030234315949, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.1837387084961, "fcm_dpo/q_t": 0.4999924302101135, "grad_norm": 0.0015664704842492938, "learning_rate": 3.355050358314172e-07, "logits/chosen": 1.1981974840164185, "logits/rejected": 1.1902220249176025, "logps/chosen": -282.2653503417969, "logps/ref_chosen": -62.31780242919922, "logps/ref_rejected": -72.60028839111328, "logps/rejected": -370.7315673828125, "loss": 1.3863, "margin_dpo/margin_mean": 78.18374633789062, "margin_dpo/margin_std": 312.6253967285156, "step": 299 }, { "KL/chosen_KL_mean": -220.5366973876953, "KL/mean": -281.3580322265625, "KL/rejected_KL_mean": -342.1794128417969, "KL/std": 277.378173828125, "epoch": 0.45351473922902497, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.64270782470703, "fcm_dpo/q_t": 0.4999881982803345, "grad_norm": 0.0024519728031009436, "learning_rate": 3.3426136618426043e-07, "logits/chosen": 1.2705652713775635, "logits/rejected": 1.2518296241760254, "logps/chosen": -280.91827392578125, "logps/ref_chosen": -60.38157653808594, "logps/ref_rejected": -75.45442199707031, "logps/rejected": -417.63385009765625, "loss": 1.3862, "margin_dpo/margin_mean": 121.6427001953125, "margin_dpo/margin_std": 316.0571594238281, "step": 300 }, { "KL/chosen_KL_mean": -221.11685180664062, "KL/mean": -255.0898895263672, "KL/rejected_KL_mean": -289.06292724609375, "KL/std": 244.7019805908203, "epoch": 0.455026455026455, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.9460678100586, "fcm_dpo/q_t": 0.4999934434890747, "grad_norm": 0.0015637945616617799, "learning_rate": 3.3301533956555885e-07, "logits/chosen": 1.2348217964172363, "logits/rejected": 1.2190877199172974, "logps/chosen": -273.9677429199219, "logps/ref_chosen": -52.85089111328125, "logps/ref_rejected": -69.97584533691406, "logps/rejected": -359.03875732421875, "loss": 1.3863, "margin_dpo/margin_mean": 67.94606018066406, "margin_dpo/margin_std": 319.27197265625, "step": 301 }, { "KL/chosen_KL_mean": -285.2535400390625, "KL/mean": -327.0502624511719, "KL/rejected_KL_mean": -368.8470153808594, "KL/std": 318.28948974609375, "epoch": 0.4565381708238851, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.593505859375, "fcm_dpo/q_t": 0.49999189376831055, "grad_norm": 0.002295486629009247, "learning_rate": 3.317669908293554e-07, "logits/chosen": 1.2259061336517334, "logits/rejected": 1.203969955444336, "logps/chosen": -352.22003173828125, "logps/ref_chosen": -66.96650695800781, "logps/ref_rejected": -88.09510803222656, "logps/rejected": -456.942138671875, "loss": 1.3863, "margin_dpo/margin_mean": 83.593505859375, "margin_dpo/margin_std": 367.9168395996094, "step": 302 }, { "KL/chosen_KL_mean": -222.12728881835938, "KL/mean": -289.8243103027344, "KL/rejected_KL_mean": -357.5213317871094, "KL/std": 335.4458923339844, "epoch": 0.4580498866213152, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.39398193359375, "fcm_dpo/q_t": 0.4999868869781494, "grad_norm": 0.0023559981491416693, "learning_rate": 3.3051635489464793e-07, "logits/chosen": 1.1870148181915283, "logits/rejected": 1.1635546684265137, "logps/chosen": -284.24884033203125, "logps/ref_chosen": -62.12152862548828, "logps/ref_rejected": -90.31204223632812, "logps/rejected": -447.8333740234375, "loss": 1.3862, "margin_dpo/margin_mean": 135.3939971923828, "margin_dpo/margin_std": 403.8096923828125, "step": 303 }, { "KL/chosen_KL_mean": -235.18692016601562, "KL/mean": -295.52618408203125, "KL/rejected_KL_mean": -355.86541748046875, "KL/std": 304.51690673828125, "epoch": 0.4595616024187453, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.678466796875, "fcm_dpo/q_t": 0.499988317489624, "grad_norm": 0.0019933602306991816, "learning_rate": 3.292634667444117e-07, "logits/chosen": 1.2784829139709473, "logits/rejected": 1.2663378715515137, "logps/chosen": -295.88201904296875, "logps/ref_chosen": -60.695091247558594, "logps/ref_rejected": -78.2525405883789, "logps/rejected": -434.1179504394531, "loss": 1.3862, "margin_dpo/margin_mean": 120.67847442626953, "margin_dpo/margin_std": 319.9189453125, "step": 304 }, { "KL/chosen_KL_mean": -272.2154541015625, "KL/mean": -285.18829345703125, "KL/rejected_KL_mean": -298.16107177734375, "KL/std": 275.7799072265625, "epoch": 0.46107331821617537, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 25.945667266845703, "fcm_dpo/q_t": 0.49999749660491943, "grad_norm": 0.0017613671952858567, "learning_rate": 3.280083614246217e-07, "logits/chosen": 1.2120630741119385, "logits/rejected": 1.2468100786209106, "logps/chosen": -344.9145812988281, "logps/ref_chosen": -72.69914245605469, "logps/ref_rejected": -65.65670776367188, "logps/rejected": -363.81781005859375, "loss": 1.3863, "margin_dpo/margin_mean": 25.945667266845703, "margin_dpo/margin_std": 326.3166198730469, "step": 305 }, { "KL/chosen_KL_mean": -198.40487670898438, "KL/mean": -271.77508544921875, "KL/rejected_KL_mean": -345.14532470703125, "KL/std": 292.58148193359375, "epoch": 0.46258503401360546, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 146.74044799804688, "fcm_dpo/q_t": 0.49998578429222107, "grad_norm": 0.002900703577324748, "learning_rate": 3.267510740432719e-07, "logits/chosen": 1.2891820669174194, "logits/rejected": 1.249337077140808, "logps/chosen": -252.37539672851562, "logps/ref_chosen": -53.97052764892578, "logps/ref_rejected": -71.02423095703125, "logps/rejected": -416.1695556640625, "loss": 1.3862, "margin_dpo/margin_mean": 146.74044799804688, "margin_dpo/margin_std": 371.2115478515625, "step": 306 }, { "KL/chosen_KL_mean": -208.95254516601562, "KL/mean": -243.92665100097656, "KL/rejected_KL_mean": -278.9007568359375, "KL/std": 252.12908935546875, "epoch": 0.46409674981103555, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.94822692871094, "fcm_dpo/q_t": 0.4999932646751404, "grad_norm": 0.0016844801139086485, "learning_rate": 3.2549163976939285e-07, "logits/chosen": 1.2810804843902588, "logits/rejected": 1.2607386112213135, "logps/chosen": -266.36566162109375, "logps/ref_chosen": -57.413108825683594, "logps/ref_rejected": -68.68010711669922, "logps/rejected": -347.58087158203125, "loss": 1.3863, "margin_dpo/margin_mean": 69.94822692871094, "margin_dpo/margin_std": 359.750732421875, "step": 307 }, { "KL/chosen_KL_mean": -280.36407470703125, "KL/mean": -300.9876708984375, "KL/rejected_KL_mean": -321.61126708984375, "KL/std": 320.69512939453125, "epoch": 0.4656084656084656, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.247154235839844, "fcm_dpo/q_t": 0.49999600648880005, "grad_norm": 0.00144182366784662, "learning_rate": 3.2423009383206874e-07, "logits/chosen": 1.2935481071472168, "logits/rejected": 1.287482738494873, "logps/chosen": -346.962890625, "logps/ref_chosen": -66.59879302978516, "logps/ref_rejected": -74.337158203125, "logps/rejected": -395.9483947753906, "loss": 1.3863, "margin_dpo/margin_mean": 41.247154235839844, "margin_dpo/margin_std": 429.9683837890625, "step": 308 }, { "KL/chosen_KL_mean": -248.10739135742188, "KL/mean": -301.85650634765625, "KL/rejected_KL_mean": -355.60565185546875, "KL/std": 268.0228271484375, "epoch": 0.4671201814058957, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.498291015625, "fcm_dpo/q_t": 0.4999895989894867, "grad_norm": 0.002324699657037854, "learning_rate": 3.229664715194511e-07, "logits/chosen": 1.25961434841156, "logits/rejected": 1.245445728302002, "logps/chosen": -313.50213623046875, "logps/ref_chosen": -65.39474487304688, "logps/ref_rejected": -75.70930480957031, "logps/rejected": -431.31494140625, "loss": 1.3863, "margin_dpo/margin_mean": 107.49828338623047, "margin_dpo/margin_std": 310.15728759765625, "step": 309 }, { "KL/chosen_KL_mean": -295.44293212890625, "KL/mean": -333.5084228515625, "KL/rejected_KL_mean": -371.5738830566406, "KL/std": 341.388671875, "epoch": 0.46863189720332576, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.13092041015625, "fcm_dpo/q_t": 0.49999260902404785, "grad_norm": 0.0016792939277365804, "learning_rate": 3.2170080817777257e-07, "logits/chosen": 1.264609456062317, "logits/rejected": 1.2663490772247314, "logps/chosen": -370.1112060546875, "logps/ref_chosen": -74.66827392578125, "logps/ref_rejected": -80.5689697265625, "logps/rejected": -452.142822265625, "loss": 1.3863, "margin_dpo/margin_mean": 76.13092803955078, "margin_dpo/margin_std": 361.7975769042969, "step": 310 }, { "KL/chosen_KL_mean": -204.85330200195312, "KL/mean": -294.4068603515625, "KL/rejected_KL_mean": -383.9604187011719, "KL/std": 327.627685546875, "epoch": 0.47014361300075586, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 179.10711669921875, "fcm_dpo/q_t": 0.49998265504837036, "grad_norm": 0.0032888497225940228, "learning_rate": 3.204331392103574e-07, "logits/chosen": 1.2235231399536133, "logits/rejected": 1.1665198802947998, "logps/chosen": -264.5913391113281, "logps/ref_chosen": -59.738033294677734, "logps/ref_rejected": -93.60757446289062, "logps/rejected": -477.5679931640625, "loss": 1.3862, "margin_dpo/margin_mean": 179.10711669921875, "margin_dpo/margin_std": 429.142822265625, "step": 311 }, { "KL/chosen_KL_mean": -197.45437622070312, "KL/mean": -267.0279541015625, "KL/rejected_KL_mean": -336.6015319824219, "KL/std": 230.93252563476562, "epoch": 0.47165532879818595, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.14715576171875, "fcm_dpo/q_t": 0.49998652935028076, "grad_norm": 0.0022931136190891266, "learning_rate": 3.1916350007663176e-07, "logits/chosen": 1.2868523597717285, "logits/rejected": 1.2605751752853394, "logps/chosen": -251.27081298828125, "logps/ref_chosen": -53.816436767578125, "logps/ref_rejected": -68.6575698852539, "logps/rejected": -405.25909423828125, "loss": 1.3862, "margin_dpo/margin_mean": 139.1471710205078, "margin_dpo/margin_std": 282.67626953125, "step": 312 }, { "KL/chosen_KL_mean": -232.93560791015625, "KL/mean": -266.92388916015625, "KL/rejected_KL_mean": -300.91217041015625, "KL/std": 304.7332763671875, "epoch": 0.47316704459561604, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.97655487060547, "fcm_dpo/q_t": 0.4999934136867523, "grad_norm": 0.0016308584017679095, "learning_rate": 3.178919262911314e-07, "logits/chosen": 1.3045995235443115, "logits/rejected": 1.302783489227295, "logps/chosen": -292.8929748535156, "logps/ref_chosen": -59.957359313964844, "logps/ref_rejected": -69.31729888916016, "logps/rejected": -370.2294921875, "loss": 1.3863, "margin_dpo/margin_mean": 67.9765625, "margin_dpo/margin_std": 373.98345947265625, "step": 313 }, { "KL/chosen_KL_mean": -240.1637420654297, "KL/mean": -261.7298583984375, "KL/rejected_KL_mean": -283.29595947265625, "KL/std": 226.59564208984375, "epoch": 0.47467876039304613, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.13225555419922, "fcm_dpo/q_t": 0.4999958276748657, "grad_norm": 0.0014463362749665976, "learning_rate": 3.166184534225087e-07, "logits/chosen": 1.2623672485351562, "logits/rejected": 1.2793068885803223, "logps/chosen": -310.431884765625, "logps/ref_chosen": -70.26815795898438, "logps/ref_rejected": -69.23971557617188, "logps/rejected": -352.53570556640625, "loss": 1.3863, "margin_dpo/margin_mean": 43.13225555419922, "margin_dpo/margin_std": 263.3436279296875, "step": 314 }, { "KL/chosen_KL_mean": -246.52500915527344, "KL/mean": -288.8182373046875, "KL/rejected_KL_mean": -331.1114196777344, "KL/std": 295.01458740234375, "epoch": 0.47619047619047616, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 84.58641052246094, "fcm_dpo/q_t": 0.49999183416366577, "grad_norm": 0.0019202815601602197, "learning_rate": 3.1534311709253723e-07, "logits/chosen": 1.2083520889282227, "logits/rejected": 1.2015604972839355, "logps/chosen": -314.3197021484375, "logps/ref_chosen": -67.79469299316406, "logps/ref_rejected": -74.55148315429688, "logps/rejected": -405.66290283203125, "loss": 1.3863, "margin_dpo/margin_mean": 84.58641052246094, "margin_dpo/margin_std": 364.86529541015625, "step": 315 }, { "KL/chosen_KL_mean": -200.84915161132812, "KL/mean": -298.4619445800781, "KL/rejected_KL_mean": -396.07470703125, "KL/std": 297.4620666503906, "epoch": 0.47770219198790626, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 195.2255859375, "fcm_dpo/q_t": 0.4999811053276062, "grad_norm": 0.0035327484365552664, "learning_rate": 3.1406595297511564e-07, "logits/chosen": 1.211460828781128, "logits/rejected": 1.1569902896881104, "logps/chosen": -256.13763427734375, "logps/ref_chosen": -55.288482666015625, "logps/ref_rejected": -96.15723419189453, "logps/rejected": -492.2319641113281, "loss": 1.3862, "margin_dpo/margin_mean": 195.22557067871094, "margin_dpo/margin_std": 394.2180480957031, "step": 316 }, { "KL/chosen_KL_mean": -203.72149658203125, "KL/mean": -248.2875518798828, "KL/rejected_KL_mean": -292.8536376953125, "KL/std": 228.49501037597656, "epoch": 0.47921390778533635, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.13215637207031, "fcm_dpo/q_t": 0.49999138712882996, "grad_norm": 0.0016370030352845788, "learning_rate": 3.1278699679526975e-07, "logits/chosen": 1.2447960376739502, "logits/rejected": 1.2230143547058105, "logps/chosen": -258.3028564453125, "logps/ref_chosen": -54.58137512207031, "logps/ref_rejected": -72.77232360839844, "logps/rejected": -365.6259460449219, "loss": 1.3863, "margin_dpo/margin_mean": 89.13215637207031, "margin_dpo/margin_std": 290.63079833984375, "step": 317 }, { "KL/chosen_KL_mean": -209.48098754882812, "KL/mean": -272.79022216796875, "KL/rejected_KL_mean": -336.09942626953125, "KL/std": 281.35205078125, "epoch": 0.48072562358276644, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.61843872070312, "fcm_dpo/q_t": 0.49998778104782104, "grad_norm": 0.0026248276699334383, "learning_rate": 3.1150628432815336e-07, "logits/chosen": 1.296776294708252, "logits/rejected": 1.268967866897583, "logps/chosen": -262.3692321777344, "logps/ref_chosen": -52.88822937011719, "logps/ref_rejected": -80.63988494873047, "logps/rejected": -416.73931884765625, "loss": 1.3862, "margin_dpo/margin_mean": 126.6184310913086, "margin_dpo/margin_std": 386.39801025390625, "step": 318 }, { "KL/chosen_KL_mean": -246.42294311523438, "KL/mean": -298.90435791015625, "KL/rejected_KL_mean": -351.38580322265625, "KL/std": 296.37994384765625, "epoch": 0.48223733938019653, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.962890625, "fcm_dpo/q_t": 0.4999898672103882, "grad_norm": 0.0018176967278122902, "learning_rate": 3.1022385139804707e-07, "logits/chosen": 1.2094240188598633, "logits/rejected": 1.211363434791565, "logps/chosen": -310.7862548828125, "logps/ref_chosen": -64.36333465576172, "logps/ref_rejected": -79.47296142578125, "logps/rejected": -430.8587646484375, "loss": 1.3863, "margin_dpo/margin_mean": 104.96288299560547, "margin_dpo/margin_std": 335.21063232421875, "step": 319 }, { "KL/chosen_KL_mean": -198.46250915527344, "KL/mean": -244.89617919921875, "KL/rejected_KL_mean": -291.32989501953125, "KL/std": 290.3658142089844, "epoch": 0.4837490551776266, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.86738586425781, "fcm_dpo/q_t": 0.4999910295009613, "grad_norm": 0.0021061012521386147, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 1.236513614654541, "logits/rejected": 1.227114200592041, "logps/chosen": -248.02125549316406, "logps/ref_chosen": -49.558746337890625, "logps/ref_rejected": -71.23444366455078, "logps/rejected": -362.5643310546875, "loss": 1.3863, "margin_dpo/margin_mean": 92.86739349365234, "margin_dpo/margin_std": 356.69232177734375, "step": 320 }, { "KL/chosen_KL_mean": -178.01779174804688, "KL/mean": -218.20223999023438, "KL/rejected_KL_mean": -258.3866882324219, "KL/std": 188.52374267578125, "epoch": 0.4852607709750567, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 80.36886596679688, "fcm_dpo/q_t": 0.4999922215938568, "grad_norm": 0.0019223628332838416, "learning_rate": 3.0765396768561004e-07, "logits/chosen": 1.2715182304382324, "logits/rejected": 1.2804362773895264, "logps/chosen": -230.1030731201172, "logps/ref_chosen": -52.08526611328125, "logps/ref_rejected": -55.58674621582031, "logps/rejected": -313.9734191894531, "loss": 1.3863, "margin_dpo/margin_mean": 80.36886596679688, "margin_dpo/margin_std": 233.33819580078125, "step": 321 }, { "KL/chosen_KL_mean": -205.60140991210938, "KL/mean": -281.4060974121094, "KL/rejected_KL_mean": -357.21075439453125, "KL/std": 279.41717529296875, "epoch": 0.48677248677248675, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 151.609375, "fcm_dpo/q_t": 0.49998533725738525, "grad_norm": 0.0024833802599459887, "learning_rate": 3.063665887884511e-07, "logits/chosen": 1.2712690830230713, "logits/rejected": 1.2434958219528198, "logps/chosen": -253.00552368164062, "logps/ref_chosen": -47.404109954833984, "logps/ref_rejected": -73.4260025024414, "logps/rejected": -430.63677978515625, "loss": 1.3862, "margin_dpo/margin_mean": 151.609375, "margin_dpo/margin_std": 249.66696166992188, "step": 322 }, { "KL/chosen_KL_mean": -240.9448699951172, "KL/mean": -307.22540283203125, "KL/rejected_KL_mean": -373.50592041015625, "KL/std": 300.21624755859375, "epoch": 0.48828420256991684, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.5610809326172, "fcm_dpo/q_t": 0.4999871551990509, "grad_norm": 0.0025218951050192118, "learning_rate": 3.0507763319663517e-07, "logits/chosen": 1.174757480621338, "logits/rejected": 1.1535224914550781, "logps/chosen": -310.951171875, "logps/ref_chosen": -70.00630187988281, "logps/ref_rejected": -86.96690368652344, "logps/rejected": -460.47283935546875, "loss": 1.3862, "margin_dpo/margin_mean": 132.5610809326172, "margin_dpo/margin_std": 364.93914794921875, "step": 323 }, { "KL/chosen_KL_mean": -213.75881958007812, "KL/mean": -267.80682373046875, "KL/rejected_KL_mean": -321.8548583984375, "KL/std": 297.6637268066406, "epoch": 0.4897959183673469, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.09601593017578, "fcm_dpo/q_t": 0.4999895393848419, "grad_norm": 0.00212281895801425, "learning_rate": 3.0378713696502097e-07, "logits/chosen": 1.2725311517715454, "logits/rejected": 1.250030755996704, "logps/chosen": -269.64764404296875, "logps/ref_chosen": -55.88882064819336, "logps/ref_rejected": -75.23088073730469, "logps/rejected": -397.0857238769531, "loss": 1.3863, "margin_dpo/margin_mean": 108.09602355957031, "margin_dpo/margin_std": 297.931640625, "step": 324 }, { "KL/chosen_KL_mean": -228.80210876464844, "KL/mean": -300.92083740234375, "KL/rejected_KL_mean": -373.03955078125, "KL/std": 327.6212158203125, "epoch": 0.491307634164777, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 144.23745727539062, "fcm_dpo/q_t": 0.49998605251312256, "grad_norm": 0.002541890600696206, "learning_rate": 3.0249513619156206e-07, "logits/chosen": 1.2248497009277344, "logits/rejected": 1.2180355787277222, "logps/chosen": -292.9491271972656, "logps/ref_chosen": -64.14701843261719, "logps/ref_rejected": -79.91143798828125, "logps/rejected": -452.95098876953125, "loss": 1.3862, "margin_dpo/margin_mean": 144.23745727539062, "margin_dpo/margin_std": 414.3074645996094, "step": 325 }, { "KL/chosen_KL_mean": -288.5466003417969, "KL/mean": -298.1192626953125, "KL/rejected_KL_mean": -307.69195556640625, "KL/std": 281.703125, "epoch": 0.4928193499622071, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 19.14533233642578, "fcm_dpo/q_t": 0.49999815225601196, "grad_norm": 0.0015972210094332695, "learning_rate": 3.012016670162977e-07, "logits/chosen": 1.2351655960083008, "logits/rejected": 1.2426598072052002, "logps/chosen": -364.07794189453125, "logps/ref_chosen": -75.53131103515625, "logps/ref_rejected": -76.5898666381836, "logps/rejected": -384.28179931640625, "loss": 1.3863, "margin_dpo/margin_mean": 19.14533233642578, "margin_dpo/margin_std": 339.36480712890625, "step": 326 }, { "KL/chosen_KL_mean": -287.528564453125, "KL/mean": -324.689453125, "KL/rejected_KL_mean": -361.85040283203125, "KL/std": 321.2103271484375, "epoch": 0.4943310657596372, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.32182312011719, "fcm_dpo/q_t": 0.49999281764030457, "grad_norm": 0.0020750907715409994, "learning_rate": 2.99906765620341e-07, "logits/chosen": 1.1873157024383545, "logits/rejected": 1.1866263151168823, "logps/chosen": -356.86572265625, "logps/ref_chosen": -69.33717346191406, "logps/ref_rejected": -73.37751770019531, "logps/rejected": -435.2279052734375, "loss": 1.3863, "margin_dpo/margin_mean": 74.32182312011719, "margin_dpo/margin_std": 364.2970275878906, "step": 327 }, { "KL/chosen_KL_mean": -234.11676025390625, "KL/mean": -285.86553955078125, "KL/rejected_KL_mean": -337.6143798828125, "KL/std": 277.8924865722656, "epoch": 0.4958427815570673, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.49760437011719, "fcm_dpo/q_t": 0.49998998641967773, "grad_norm": 0.0018194402800872922, "learning_rate": 2.9861046822486766e-07, "logits/chosen": 1.1889562606811523, "logits/rejected": 1.171844244003296, "logps/chosen": -295.822998046875, "logps/ref_chosen": -61.70623016357422, "logps/ref_rejected": -83.73808288574219, "logps/rejected": -421.3524475097656, "loss": 1.3863, "margin_dpo/margin_mean": 103.49760437011719, "margin_dpo/margin_std": 310.90997314453125, "step": 328 }, { "KL/chosen_KL_mean": -241.38299560546875, "KL/mean": -305.01055908203125, "KL/rejected_KL_mean": -368.63812255859375, "KL/std": 312.40496826171875, "epoch": 0.4973544973544973, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.255126953125, "fcm_dpo/q_t": 0.4999876618385315, "grad_norm": 0.002292838878929615, "learning_rate": 2.9731281109010253e-07, "logits/chosen": 1.2591251134872437, "logits/rejected": 1.2390198707580566, "logps/chosen": -305.88140869140625, "logps/ref_chosen": -64.4984130859375, "logps/ref_rejected": -83.6591796875, "logps/rejected": -452.29730224609375, "loss": 1.3862, "margin_dpo/margin_mean": 127.255126953125, "margin_dpo/margin_std": 396.40557861328125, "step": 329 }, { "KL/chosen_KL_mean": -189.281494140625, "KL/mean": -244.40090942382812, "KL/rejected_KL_mean": -299.52032470703125, "KL/std": 252.27133178710938, "epoch": 0.4988662131519274, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.23881530761719, "fcm_dpo/q_t": 0.4999893307685852, "grad_norm": 0.0021174049470573664, "learning_rate": 2.9601383051430505e-07, "logits/chosen": 1.2592849731445312, "logits/rejected": 1.2317944765090942, "logps/chosen": -244.08615112304688, "logps/ref_chosen": -54.80464172363281, "logps/ref_rejected": -75.3194351196289, "logps/rejected": -374.83978271484375, "loss": 1.3863, "margin_dpo/margin_mean": 110.23881530761719, "margin_dpo/margin_std": 354.689697265625, "step": 330 }, { "KL/chosen_KL_mean": -210.95858764648438, "KL/mean": -264.14520263671875, "KL/rejected_KL_mean": -317.331787109375, "KL/std": 248.91436767578125, "epoch": 0.5003779289493575, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.37322235107422, "fcm_dpo/q_t": 0.49998968839645386, "grad_norm": 0.0013487640535458922, "learning_rate": 2.947135628327544e-07, "logits/chosen": 1.2729997634887695, "logits/rejected": 1.273748755455017, "logps/chosen": -270.201171875, "logps/ref_chosen": -59.242584228515625, "logps/ref_rejected": -69.87483215332031, "logps/rejected": -387.2066345214844, "loss": 1.3863, "margin_dpo/margin_mean": 106.37321472167969, "margin_dpo/margin_std": 286.5001220703125, "step": 331 }, { "KL/chosen_KL_mean": -226.46791076660156, "KL/mean": -281.8634033203125, "KL/rejected_KL_mean": -337.2588806152344, "KL/std": 250.1396484375, "epoch": 0.5018896447467877, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.7909927368164, "fcm_dpo/q_t": 0.4999893009662628, "grad_norm": 0.0019972885493189096, "learning_rate": 2.934120444167326e-07, "logits/chosen": 1.2130635976791382, "logits/rejected": 1.2021818161010742, "logps/chosen": -293.5776672363281, "logps/ref_chosen": -67.10975646972656, "logps/ref_rejected": -77.11839294433594, "logps/rejected": -414.37725830078125, "loss": 1.3863, "margin_dpo/margin_mean": 110.7909927368164, "margin_dpo/margin_std": 291.0926208496094, "step": 332 }, { "KL/chosen_KL_mean": -233.87643432617188, "KL/mean": -303.2012939453125, "KL/rejected_KL_mean": -372.526123046875, "KL/std": 304.5091247558594, "epoch": 0.5034013605442177, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.64968872070312, "fcm_dpo/q_t": 0.49998658895492554, "grad_norm": 0.0025588450953364372, "learning_rate": 2.921093116725076e-07, "logits/chosen": 1.2254364490509033, "logits/rejected": 1.1889737844467163, "logps/chosen": -292.257568359375, "logps/ref_chosen": -58.381134033203125, "logps/ref_rejected": -85.02839660644531, "logps/rejected": -457.55450439453125, "loss": 1.3862, "margin_dpo/margin_mean": 138.64968872070312, "margin_dpo/margin_std": 301.95068359375, "step": 333 }, { "KL/chosen_KL_mean": -223.56558227539062, "KL/mean": -282.3580322265625, "KL/rejected_KL_mean": -341.15045166015625, "KL/std": 278.36016845703125, "epoch": 0.5049130763416477, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.58486938476562, "fcm_dpo/q_t": 0.4999886155128479, "grad_norm": 0.0021759173832833767, "learning_rate": 2.9080540104031484e-07, "logits/chosen": 1.2243032455444336, "logits/rejected": 1.1974499225616455, "logps/chosen": -290.45758056640625, "logps/ref_chosen": -66.89199829101562, "logps/ref_rejected": -91.83695220947266, "logps/rejected": -432.9873962402344, "loss": 1.3862, "margin_dpo/margin_mean": 117.58486938476562, "margin_dpo/margin_std": 335.0116882324219, "step": 334 }, { "KL/chosen_KL_mean": -209.07431030273438, "KL/mean": -258.6519775390625, "KL/rejected_KL_mean": -308.2296142578125, "KL/std": 264.89276123046875, "epoch": 0.5064247921390779, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.15528869628906, "fcm_dpo/q_t": 0.49999040365219116, "grad_norm": 0.0017754482105374336, "learning_rate": 2.895003489933375e-07, "logits/chosen": 1.212792992591858, "logits/rejected": 1.2027299404144287, "logps/chosen": -270.5887756347656, "logps/ref_chosen": -61.51445770263672, "logps/ref_rejected": -75.68916320800781, "logps/rejected": -383.9187927246094, "loss": 1.3863, "margin_dpo/margin_mean": 99.15528869628906, "margin_dpo/margin_std": 291.6207275390625, "step": 335 }, { "KL/chosen_KL_mean": -274.1320495605469, "KL/mean": -322.80322265625, "KL/rejected_KL_mean": -371.474365234375, "KL/std": 263.66119384765625, "epoch": 0.5079365079365079, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.34229278564453, "fcm_dpo/q_t": 0.4999905824661255, "grad_norm": 0.002034288365393877, "learning_rate": 2.8819419203668675e-07, "logits/chosen": 1.2427122592926025, "logits/rejected": 1.2363805770874023, "logps/chosen": -342.98211669921875, "logps/ref_chosen": -68.85006713867188, "logps/ref_rejected": -92.99603271484375, "logps/rejected": -464.4703674316406, "loss": 1.3863, "margin_dpo/margin_mean": 97.34230041503906, "margin_dpo/margin_std": 293.6593017578125, "step": 336 }, { "KL/chosen_KL_mean": -289.3245849609375, "KL/mean": -334.5281066894531, "KL/rejected_KL_mean": -379.7316589355469, "KL/std": 313.8486328125, "epoch": 0.509448223733938, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.40706634521484, "fcm_dpo/q_t": 0.4999912679195404, "grad_norm": 0.0020725426729768515, "learning_rate": 2.8688696670638053e-07, "logits/chosen": 1.1902989149093628, "logits/rejected": 1.1826756000518799, "logps/chosen": -362.51239013671875, "logps/ref_chosen": -73.18783569335938, "logps/ref_rejected": -86.89118957519531, "logps/rejected": -466.62286376953125, "loss": 1.3863, "margin_dpo/margin_mean": 90.40706634521484, "margin_dpo/margin_std": 352.4085693359375, "step": 337 }, { "KL/chosen_KL_mean": -244.63876342773438, "KL/mean": -279.78643798828125, "KL/rejected_KL_mean": -314.93414306640625, "KL/std": 228.41464233398438, "epoch": 0.5109599395313681, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.29537963867188, "fcm_dpo/q_t": 0.4999932050704956, "grad_norm": 0.0021237812470644712, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 1.2304810285568237, "logits/rejected": 1.2233942747116089, "logps/chosen": -308.578369140625, "logps/ref_chosen": -63.939613342285156, "logps/ref_rejected": -75.34243774414062, "logps/rejected": -390.27655029296875, "loss": 1.3863, "margin_dpo/margin_mean": 70.29537963867188, "margin_dpo/margin_std": 193.20977783203125, "step": 338 }, { "KL/chosen_KL_mean": -184.398681640625, "KL/mean": -237.96548461914062, "KL/rejected_KL_mean": -291.5323181152344, "KL/std": 243.76315307617188, "epoch": 0.5124716553287982, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.13363647460938, "fcm_dpo/q_t": 0.4999896287918091, "grad_norm": 0.002144779311493039, "learning_rate": 2.842694572172736e-07, "logits/chosen": 1.333993673324585, "logits/rejected": 1.2993600368499756, "logps/chosen": -229.94781494140625, "logps/ref_chosen": -45.54913330078125, "logps/ref_rejected": -67.0482177734375, "logps/rejected": -358.5805358886719, "loss": 1.3863, "margin_dpo/margin_mean": 107.13363647460938, "margin_dpo/margin_std": 302.3402099609375, "step": 339 }, { "KL/chosen_KL_mean": -211.99609375, "KL/mean": -246.68463134765625, "KL/rejected_KL_mean": -281.3731994628906, "KL/std": 209.9356689453125, "epoch": 0.5139833711262283, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.37712860107422, "fcm_dpo/q_t": 0.49999329447746277, "grad_norm": 0.0015614746371284127, "learning_rate": 2.8295924627584004e-07, "logits/chosen": 1.2425800561904907, "logits/rejected": 1.241082787513733, "logps/chosen": -266.0017395019531, "logps/ref_chosen": -54.00564956665039, "logps/ref_rejected": -61.314430236816406, "logps/rejected": -342.6876220703125, "loss": 1.3863, "margin_dpo/margin_mean": 69.37713623046875, "margin_dpo/margin_std": 277.55560302734375, "step": 340 }, { "KL/chosen_KL_mean": -240.11708068847656, "KL/mean": -302.804931640625, "KL/rejected_KL_mean": -365.4927062988281, "KL/std": 279.3319091796875, "epoch": 0.5154950869236583, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.37564086914062, "fcm_dpo/q_t": 0.4999878406524658, "grad_norm": 0.002194821136072278, "learning_rate": 2.816481133934373e-07, "logits/chosen": 1.2894377708435059, "logits/rejected": 1.27662992477417, "logps/chosen": -303.5121765136719, "logps/ref_chosen": -63.39509582519531, "logps/ref_rejected": -76.20973205566406, "logps/rejected": -441.70245361328125, "loss": 1.3862, "margin_dpo/margin_mean": 125.37564086914062, "margin_dpo/margin_std": 277.63555908203125, "step": 341 }, { "KL/chosen_KL_mean": -215.72718811035156, "KL/mean": -266.249755859375, "KL/rejected_KL_mean": -316.7723388671875, "KL/std": 273.38848876953125, "epoch": 0.5170068027210885, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.045166015625, "fcm_dpo/q_t": 0.49999022483825684, "grad_norm": 0.0017220752779394388, "learning_rate": 2.8033609524527046e-07, "logits/chosen": 1.278684139251709, "logits/rejected": 1.2664589881896973, "logps/chosen": -268.7749938964844, "logps/ref_chosen": -53.047813415527344, "logps/ref_rejected": -68.2854232788086, "logps/rejected": -385.0577697753906, "loss": 1.3863, "margin_dpo/margin_mean": 101.045166015625, "margin_dpo/margin_std": 286.5677795410156, "step": 342 }, { "KL/chosen_KL_mean": -263.2760009765625, "KL/mean": -300.4712829589844, "KL/rejected_KL_mean": -337.6665344238281, "KL/std": 306.28350830078125, "epoch": 0.5185185185185185, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 74.39053344726562, "fcm_dpo/q_t": 0.4999927878379822, "grad_norm": 0.0017567041795700788, "learning_rate": 2.7902322853130753e-07, "logits/chosen": 1.2100257873535156, "logits/rejected": 1.210863471031189, "logps/chosen": -333.85455322265625, "logps/ref_chosen": -70.57852935791016, "logps/ref_rejected": -84.73873901367188, "logps/rejected": -422.4052734375, "loss": 1.3863, "margin_dpo/margin_mean": 74.3905258178711, "margin_dpo/margin_std": 372.44439697265625, "step": 343 }, { "KL/chosen_KL_mean": -235.8098907470703, "KL/mean": -311.6300048828125, "KL/rejected_KL_mean": -387.4500732421875, "KL/std": 312.3414001464844, "epoch": 0.5200302343159486, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 151.64019775390625, "fcm_dpo/q_t": 0.49998533725738525, "grad_norm": 0.002630658680573106, "learning_rate": 2.7770954997525274e-07, "logits/chosen": 1.2740308046340942, "logits/rejected": 1.2405366897583008, "logps/chosen": -291.62091064453125, "logps/ref_chosen": -55.811004638671875, "logps/ref_rejected": -84.77637481689453, "logps/rejected": -472.2264404296875, "loss": 1.3862, "margin_dpo/margin_mean": 151.6402130126953, "margin_dpo/margin_std": 380.27703857421875, "step": 344 }, { "KL/chosen_KL_mean": -196.2252655029297, "KL/mean": -252.94921875, "KL/rejected_KL_mean": -309.6731872558594, "KL/std": 231.7432861328125, "epoch": 0.5215419501133787, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.44795227050781, "fcm_dpo/q_t": 0.49998903274536133, "grad_norm": 0.0021876078099012375, "learning_rate": 2.7639509632351927e-07, "logits/chosen": 1.2337894439697266, "logits/rejected": 1.21903395652771, "logps/chosen": -254.0113525390625, "logps/ref_chosen": -57.78609848022461, "logps/ref_rejected": -78.91847229003906, "logps/rejected": -388.5916748046875, "loss": 1.3863, "margin_dpo/margin_mean": 113.44795227050781, "margin_dpo/margin_std": 229.27005004882812, "step": 345 }, { "KL/chosen_KL_mean": -210.19422912597656, "KL/mean": -292.7418212890625, "KL/rejected_KL_mean": -375.2893981933594, "KL/std": 307.3424072265625, "epoch": 0.5230536659108088, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 165.09515380859375, "fcm_dpo/q_t": 0.4999840259552002, "grad_norm": 0.002935102442279458, "learning_rate": 2.7507990434420123e-07, "logits/chosen": 1.247345209121704, "logits/rejected": 1.2206707000732422, "logps/chosen": -266.4793701171875, "logps/ref_chosen": -56.285125732421875, "logps/ref_rejected": -91.15303039550781, "logps/rejected": -466.44244384765625, "loss": 1.3862, "margin_dpo/margin_mean": 165.09515380859375, "margin_dpo/margin_std": 327.62249755859375, "step": 346 }, { "KL/chosen_KL_mean": -215.6938018798828, "KL/mean": -268.2672424316406, "KL/rejected_KL_mean": -320.8406677246094, "KL/std": 256.3416442871094, "epoch": 0.5245653817082389, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.14689636230469, "fcm_dpo/q_t": 0.4999898076057434, "grad_norm": 0.001989042619243264, "learning_rate": 2.737640108260456e-07, "logits/chosen": 1.3327654600143433, "logits/rejected": 1.3226079940795898, "logps/chosen": -269.193359375, "logps/ref_chosen": -53.499542236328125, "logps/ref_rejected": -72.52565002441406, "logps/rejected": -393.3663330078125, "loss": 1.3863, "margin_dpo/margin_mean": 105.14688873291016, "margin_dpo/margin_std": 277.4522705078125, "step": 347 }, { "KL/chosen_KL_mean": -194.62255859375, "KL/mean": -241.3387908935547, "KL/rejected_KL_mean": -288.0550537109375, "KL/std": 223.19094848632812, "epoch": 0.5260770975056689, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.4324722290039, "fcm_dpo/q_t": 0.49999094009399414, "grad_norm": 0.0018931980011984706, "learning_rate": 2.724474525774229e-07, "logits/chosen": 1.2694389820098877, "logits/rejected": 1.2611613273620605, "logps/chosen": -245.409423828125, "logps/ref_chosen": -50.78684997558594, "logps/ref_rejected": -68.63732147216797, "logps/rejected": -356.6923522949219, "loss": 1.3863, "margin_dpo/margin_mean": 93.43247985839844, "margin_dpo/margin_std": 223.0704345703125, "step": 348 }, { "KL/chosen_KL_mean": -211.4373779296875, "KL/mean": -280.389404296875, "KL/rejected_KL_mean": -349.3414306640625, "KL/std": 292.76397705078125, "epoch": 0.527588813303099, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.904052734375, "fcm_dpo/q_t": 0.4999866485595703, "grad_norm": 0.0022273960057646036, "learning_rate": 2.711302664252973e-07, "logits/chosen": 1.2621893882751465, "logits/rejected": 1.2139443159103394, "logps/chosen": -264.76239013671875, "logps/ref_chosen": -53.325008392333984, "logps/ref_rejected": -83.21236419677734, "logps/rejected": -432.55377197265625, "loss": 1.3862, "margin_dpo/margin_mean": 137.904052734375, "margin_dpo/margin_std": 315.9725036621094, "step": 349 }, { "KL/chosen_KL_mean": -224.52761840820312, "KL/mean": -307.943359375, "KL/rejected_KL_mean": -391.3591003417969, "KL/std": 310.1175537109375, "epoch": 0.5291005291005291, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 166.83148193359375, "fcm_dpo/q_t": 0.49998384714126587, "grad_norm": 0.002880116691812873, "learning_rate": 2.698124892141971e-07, "logits/chosen": 1.258007526397705, "logits/rejected": 1.247691035270691, "logps/chosen": -286.15338134765625, "logps/ref_chosen": -61.625770568847656, "logps/ref_rejected": -87.63627624511719, "logps/rejected": -478.995361328125, "loss": 1.3862, "margin_dpo/margin_mean": 166.8314971923828, "margin_dpo/margin_std": 310.433837890625, "step": 350 }, { "KL/chosen_KL_mean": -202.2751007080078, "KL/mean": -260.5731506347656, "KL/rejected_KL_mean": -318.87115478515625, "KL/std": 201.6959991455078, "epoch": 0.5306122448979592, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.59608459472656, "fcm_dpo/q_t": 0.49998873472213745, "grad_norm": 0.0021962763275951147, "learning_rate": 2.6849415780518357e-07, "logits/chosen": 1.2232160568237305, "logits/rejected": 1.193381905555725, "logps/chosen": -258.53143310546875, "logps/ref_chosen": -56.2563362121582, "logps/ref_rejected": -79.11589813232422, "logps/rejected": -397.987060546875, "loss": 1.3862, "margin_dpo/margin_mean": 116.59608459472656, "margin_dpo/margin_std": 279.30438232421875, "step": 351 }, { "KL/chosen_KL_mean": -229.2626953125, "KL/mean": -293.04437255859375, "KL/rejected_KL_mean": -356.8260498046875, "KL/std": 303.4278564453125, "epoch": 0.5321239606953893, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.56336975097656, "fcm_dpo/q_t": 0.4999876618385315, "grad_norm": 0.002237893408164382, "learning_rate": 2.6717530907482024e-07, "logits/chosen": 1.2531712055206299, "logits/rejected": 1.2404941320419312, "logps/chosen": -292.31463623046875, "logps/ref_chosen": -63.05195236206055, "logps/ref_rejected": -85.52035522460938, "logps/rejected": -442.346435546875, "loss": 1.3862, "margin_dpo/margin_mean": 127.5633773803711, "margin_dpo/margin_std": 380.3623962402344, "step": 352 }, { "KL/chosen_KL_mean": -227.08648681640625, "KL/mean": -265.6427307128906, "KL/rejected_KL_mean": -304.198974609375, "KL/std": 260.7290954589844, "epoch": 0.5336356764928194, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.11247253417969, "fcm_dpo/q_t": 0.4999925494194031, "grad_norm": 0.0013403355842456222, "learning_rate": 2.658559799141411e-07, "logits/chosen": 1.2461869716644287, "logits/rejected": 1.2634531259536743, "logps/chosen": -296.0956726074219, "logps/ref_chosen": -69.00918579101562, "logps/ref_rejected": -72.65840148925781, "logps/rejected": -376.85736083984375, "loss": 1.3863, "margin_dpo/margin_mean": 77.11247253417969, "margin_dpo/margin_std": 259.31402587890625, "step": 353 }, { "KL/chosen_KL_mean": -149.09295654296875, "KL/mean": -222.05581665039062, "KL/rejected_KL_mean": -295.0186767578125, "KL/std": 179.0128936767578, "epoch": 0.5351473922902494, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.92572021484375, "fcm_dpo/q_t": 0.49998587369918823, "grad_norm": 0.002911218209192157, "learning_rate": 2.6453620722761895e-07, "logits/chosen": 1.290527105331421, "logits/rejected": 1.233759880065918, "logps/chosen": -188.8813018798828, "logps/ref_chosen": -39.78833770751953, "logps/ref_rejected": -69.56885528564453, "logps/rejected": -364.5875244140625, "loss": 1.3862, "margin_dpo/margin_mean": 145.92572021484375, "margin_dpo/margin_std": 210.240966796875, "step": 354 }, { "KL/chosen_KL_mean": -183.08291625976562, "KL/mean": -279.84710693359375, "KL/rejected_KL_mean": -376.611328125, "KL/std": 313.0096740722656, "epoch": 0.5366591080876795, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 193.52838134765625, "fcm_dpo/q_t": 0.49998125433921814, "grad_norm": 0.0035407356917858124, "learning_rate": 2.632160279321328e-07, "logits/chosen": 1.2896933555603027, "logits/rejected": 1.2377452850341797, "logps/chosen": -229.3383026123047, "logps/ref_chosen": -46.25537872314453, "logps/ref_rejected": -78.20236206054688, "logps/rejected": -454.8136901855469, "loss": 1.3862, "margin_dpo/margin_mean": 193.5283966064453, "margin_dpo/margin_std": 347.1357421875, "step": 355 }, { "KL/chosen_KL_mean": -173.40316772460938, "KL/mean": -241.93011474609375, "KL/rejected_KL_mean": -310.45709228515625, "KL/std": 237.36767578125, "epoch": 0.5381708238851096, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.0539093017578, "fcm_dpo/q_t": 0.4999867379665375, "grad_norm": 0.0023308955132961273, "learning_rate": 2.618954789559356e-07, "logits/chosen": 1.2501168251037598, "logits/rejected": 1.2139482498168945, "logps/chosen": -221.309326171875, "logps/ref_chosen": -47.906158447265625, "logps/ref_rejected": -74.29397583007812, "logps/rejected": -384.7510681152344, "loss": 1.3862, "margin_dpo/margin_mean": 137.0539093017578, "margin_dpo/margin_std": 294.431640625, "step": 356 }, { "KL/chosen_KL_mean": -230.25607299804688, "KL/mean": -282.31524658203125, "KL/rejected_KL_mean": -334.3744201660156, "KL/std": 269.92694091796875, "epoch": 0.5396825396825397, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.11831665039062, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.0023283795453608036, "learning_rate": 2.6057459723762076e-07, "logits/chosen": 1.2416390180587769, "logits/rejected": 1.2526752948760986, "logps/chosen": -292.8910827636719, "logps/ref_chosen": -62.63500213623047, "logps/ref_rejected": -65.11399841308594, "logps/rejected": -399.4884033203125, "loss": 1.3863, "margin_dpo/margin_mean": 104.11831665039062, "margin_dpo/margin_std": 328.5772705078125, "step": 357 }, { "KL/chosen_KL_mean": -257.3409423828125, "KL/mean": -282.57147216796875, "KL/rejected_KL_mean": -307.80206298828125, "KL/std": 286.5031433105469, "epoch": 0.5411942554799698, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 50.46110153198242, "fcm_dpo/q_t": 0.4999951124191284, "grad_norm": 0.001599806360900402, "learning_rate": 2.5925341972508954e-07, "logits/chosen": 1.2319351434707642, "logits/rejected": 1.2462717294692993, "logps/chosen": -324.550537109375, "logps/ref_chosen": -67.20960998535156, "logps/ref_rejected": -69.34715270996094, "logps/rejected": -377.1492004394531, "loss": 1.3863, "margin_dpo/margin_mean": 50.46110153198242, "margin_dpo/margin_std": 362.09979248046875, "step": 358 }, { "KL/chosen_KL_mean": -271.7464599609375, "KL/mean": -301.0337219238281, "KL/rejected_KL_mean": -330.32098388671875, "KL/std": 316.7093811035156, "epoch": 0.5427059712773998, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 58.574562072753906, "fcm_dpo/q_t": 0.49999430775642395, "grad_norm": 0.001978958724066615, "learning_rate": 2.579319833745169e-07, "logits/chosen": 1.267923355102539, "logits/rejected": 1.248403787612915, "logps/chosen": -334.272216796875, "logps/ref_chosen": -62.52578353881836, "logps/ref_rejected": -76.63114929199219, "logps/rejected": -406.9521484375, "loss": 1.3863, "margin_dpo/margin_mean": 58.574562072753906, "margin_dpo/margin_std": 401.56103515625, "step": 359 }, { "KL/chosen_KL_mean": -245.56210327148438, "KL/mean": -321.9141540527344, "KL/rejected_KL_mean": -398.26617431640625, "KL/std": 338.3564147949219, "epoch": 0.54421768707483, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.7041015625, "fcm_dpo/q_t": 0.4999852180480957, "grad_norm": 0.0029005235992372036, "learning_rate": 2.5661032514931834e-07, "logits/chosen": 1.2112646102905273, "logits/rejected": 1.183484673500061, "logps/chosen": -309.0498352050781, "logps/ref_chosen": -63.48772048950195, "logps/ref_rejected": -90.6891098022461, "logps/rejected": -488.9552917480469, "loss": 1.3862, "margin_dpo/margin_mean": 152.7041015625, "margin_dpo/margin_std": 392.829345703125, "step": 360 }, { "KL/chosen_KL_mean": -215.9862060546875, "KL/mean": -271.5434875488281, "KL/rejected_KL_mean": -327.1007385253906, "KL/std": 243.81903076171875, "epoch": 0.54572940287226, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.11454772949219, "fcm_dpo/q_t": 0.49998927116394043, "grad_norm": 0.0022065271623432636, "learning_rate": 2.552884820191154e-07, "logits/chosen": 1.2864508628845215, "logits/rejected": 1.2714757919311523, "logps/chosen": -273.9033508300781, "logps/ref_chosen": -57.917144775390625, "logps/ref_rejected": -72.39089965820312, "logps/rejected": -399.49163818359375, "loss": 1.3863, "margin_dpo/margin_mean": 111.11454772949219, "margin_dpo/margin_std": 294.0084228515625, "step": 361 }, { "KL/chosen_KL_mean": -237.84043884277344, "KL/mean": -358.2422790527344, "KL/rejected_KL_mean": -478.6441345214844, "KL/std": 465.93792724609375, "epoch": 0.54724111866969, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 240.80369567871094, "fcm_dpo/q_t": 0.4999766945838928, "grad_norm": 0.004349971190094948, "learning_rate": 2.53966490958702e-07, "logits/chosen": 1.2654731273651123, "logits/rejected": 1.2318336963653564, "logps/chosen": -301.2839050292969, "logps/ref_chosen": -63.4434700012207, "logps/ref_rejected": -103.45516967773438, "logps/rejected": -582.0993041992188, "loss": 1.3862, "margin_dpo/margin_mean": 240.8037109375, "margin_dpo/margin_std": 607.7028198242188, "step": 362 }, { "KL/chosen_KL_mean": -184.51370239257812, "KL/mean": -285.02001953125, "KL/rejected_KL_mean": -385.5263366699219, "KL/std": 271.80780029296875, "epoch": 0.5487528344671202, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 201.0126495361328, "fcm_dpo/q_t": 0.4999805688858032, "grad_norm": 0.003892081556841731, "learning_rate": 2.526443889470099e-07, "logits/chosen": 1.2803268432617188, "logits/rejected": 1.2195180654525757, "logps/chosen": -233.16552734375, "logps/ref_chosen": -48.65182876586914, "logps/ref_rejected": -88.65904235839844, "logps/rejected": -474.1853942871094, "loss": 1.3862, "margin_dpo/margin_mean": 201.0126495361328, "margin_dpo/margin_std": 300.37652587890625, "step": 363 }, { "KL/chosen_KL_mean": -215.3506622314453, "KL/mean": -287.34576416015625, "KL/rejected_KL_mean": -359.3408508300781, "KL/std": 300.54931640625, "epoch": 0.5502645502645502, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 143.99021911621094, "fcm_dpo/q_t": 0.49998608231544495, "grad_norm": 0.0026705956552177668, "learning_rate": 2.513222129660744e-07, "logits/chosen": 1.2083289623260498, "logits/rejected": 1.180536150932312, "logps/chosen": -273.22174072265625, "logps/ref_chosen": -57.87107467651367, "logps/ref_rejected": -80.95503234863281, "logps/rejected": -440.2958984375, "loss": 1.3862, "margin_dpo/margin_mean": 143.99021911621094, "margin_dpo/margin_std": 418.8135986328125, "step": 364 }, { "KL/chosen_KL_mean": -223.299072265625, "KL/mean": -262.5470275878906, "KL/rejected_KL_mean": -301.79498291015625, "KL/std": 329.4648742675781, "epoch": 0.5517762660619804, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 78.49591064453125, "fcm_dpo/q_t": 0.49999240040779114, "grad_norm": 0.001699400832876563, "learning_rate": 2.5e-07, "logits/chosen": 1.2776089906692505, "logits/rejected": 1.2857856750488281, "logps/chosen": -288.2412414550781, "logps/ref_chosen": -64.94217681884766, "logps/ref_rejected": -74.8599853515625, "logps/rejected": -376.65496826171875, "loss": 1.3863, "margin_dpo/margin_mean": 78.49592590332031, "margin_dpo/margin_std": 294.7704772949219, "step": 365 }, { "KL/chosen_KL_mean": -215.96405029296875, "KL/mean": -249.5860595703125, "KL/rejected_KL_mean": -283.2080383300781, "KL/std": 302.95806884765625, "epoch": 0.5532879818594104, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.2439956665039, "fcm_dpo/q_t": 0.4999935030937195, "grad_norm": 0.0016670352779328823, "learning_rate": 2.486777870339255e-07, "logits/chosen": 1.2348253726959229, "logits/rejected": 1.2384909391403198, "logps/chosen": -271.1300354003906, "logps/ref_chosen": -55.16598129272461, "logps/ref_rejected": -65.26121520996094, "logps/rejected": -348.46923828125, "loss": 1.3863, "margin_dpo/margin_mean": 67.2439956665039, "margin_dpo/margin_std": 308.7232666015625, "step": 366 }, { "KL/chosen_KL_mean": -227.30758666992188, "KL/mean": -292.5162048339844, "KL/rejected_KL_mean": -357.724853515625, "KL/std": 294.52685546875, "epoch": 0.5547996976568406, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.4172821044922, "fcm_dpo/q_t": 0.4999873638153076, "grad_norm": 0.0024976215790957212, "learning_rate": 2.4735561105299014e-07, "logits/chosen": 1.2854868173599243, "logits/rejected": 1.2474390268325806, "logps/chosen": -283.31805419921875, "logps/ref_chosen": -56.01046371459961, "logps/ref_rejected": -77.31010437011719, "logps/rejected": -435.03497314453125, "loss": 1.3862, "margin_dpo/margin_mean": 130.4172821044922, "margin_dpo/margin_std": 318.589111328125, "step": 367 }, { "KL/chosen_KL_mean": -271.27618408203125, "KL/mean": -293.41290283203125, "KL/rejected_KL_mean": -315.54962158203125, "KL/std": 280.5533447265625, "epoch": 0.5563114134542706, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.273414611816406, "fcm_dpo/q_t": 0.4999956488609314, "grad_norm": 0.0014242986217141151, "learning_rate": 2.46033509041298e-07, "logits/chosen": 1.193532109260559, "logits/rejected": 1.199931025505066, "logps/chosen": -346.10546875, "logps/ref_chosen": -74.82927703857422, "logps/ref_rejected": -76.11680603027344, "logps/rejected": -391.66644287109375, "loss": 1.3863, "margin_dpo/margin_mean": 44.2734260559082, "margin_dpo/margin_std": 339.95147705078125, "step": 368 }, { "KL/chosen_KL_mean": -228.98980712890625, "KL/mean": -280.967529296875, "KL/rejected_KL_mean": -332.9452209472656, "KL/std": 287.7408752441406, "epoch": 0.5578231292517006, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.9554214477539, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.002357279881834984, "learning_rate": 2.447115179808846e-07, "logits/chosen": 1.2877874374389648, "logits/rejected": 1.2739431858062744, "logps/chosen": -287.3160400390625, "logps/ref_chosen": -58.32621765136719, "logps/ref_rejected": -80.92183685302734, "logps/rejected": -413.8670654296875, "loss": 1.3863, "margin_dpo/margin_mean": 103.9554214477539, "margin_dpo/margin_std": 325.900634765625, "step": 369 }, { "KL/chosen_KL_mean": -201.3767852783203, "KL/mean": -259.1314392089844, "KL/rejected_KL_mean": -316.8861083984375, "KL/std": 232.7076873779297, "epoch": 0.5593348450491308, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.50931549072266, "fcm_dpo/q_t": 0.4999887943267822, "grad_norm": 0.002263600006699562, "learning_rate": 2.4338967485068164e-07, "logits/chosen": 1.2925589084625244, "logits/rejected": 1.2621781826019287, "logps/chosen": -254.260498046875, "logps/ref_chosen": -52.88372039794922, "logps/ref_rejected": -79.43692016601562, "logps/rejected": -396.322998046875, "loss": 1.3862, "margin_dpo/margin_mean": 115.50932312011719, "margin_dpo/margin_std": 267.6164855957031, "step": 370 }, { "KL/chosen_KL_mean": -184.48826599121094, "KL/mean": -229.01632690429688, "KL/rejected_KL_mean": -273.54437255859375, "KL/std": 192.02371215820312, "epoch": 0.5608465608465608, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.05611419677734, "fcm_dpo/q_t": 0.49999138712882996, "grad_norm": 0.0017809628043323755, "learning_rate": 2.420680166254831e-07, "logits/chosen": 1.3020191192626953, "logits/rejected": 1.2898714542388916, "logps/chosen": -233.7124786376953, "logps/ref_chosen": -49.224212646484375, "logps/ref_rejected": -63.348472595214844, "logps/rejected": -336.8928527832031, "loss": 1.3863, "margin_dpo/margin_mean": 89.05611419677734, "margin_dpo/margin_std": 218.2142333984375, "step": 371 }, { "KL/chosen_KL_mean": -195.20889282226562, "KL/mean": -253.14254760742188, "KL/rejected_KL_mean": -311.076171875, "KL/std": 244.80682373046875, "epoch": 0.562358276643991, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.86727142333984, "fcm_dpo/q_t": 0.49998876452445984, "grad_norm": 0.002640601247549057, "learning_rate": 2.4074658027491044e-07, "logits/chosen": 1.274523138999939, "logits/rejected": 1.2315776348114014, "logps/chosen": -247.47845458984375, "logps/ref_chosen": -52.269554138183594, "logps/ref_rejected": -72.99522399902344, "logps/rejected": -384.0714111328125, "loss": 1.3862, "margin_dpo/margin_mean": 115.86727905273438, "margin_dpo/margin_std": 293.59796142578125, "step": 372 }, { "KL/chosen_KL_mean": -264.95257568359375, "KL/mean": -320.5806884765625, "KL/rejected_KL_mean": -376.20880126953125, "KL/std": 349.17852783203125, "epoch": 0.563869992441421, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.25619506835938, "fcm_dpo/q_t": 0.49998921155929565, "grad_norm": 0.0023425230756402016, "learning_rate": 2.394254027623792e-07, "logits/chosen": 1.284815788269043, "logits/rejected": 1.2553908824920654, "logps/chosen": -326.0655822753906, "logps/ref_chosen": -61.112998962402344, "logps/ref_rejected": -76.24851989746094, "logps/rejected": -452.4573059082031, "loss": 1.3863, "margin_dpo/margin_mean": 111.25619506835938, "margin_dpo/margin_std": 393.34063720703125, "step": 373 }, { "KL/chosen_KL_mean": -264.5904541015625, "KL/mean": -314.9049072265625, "KL/rejected_KL_mean": -365.2193298339844, "KL/std": 318.2697448730469, "epoch": 0.5653817082388511, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.6288833618164, "fcm_dpo/q_t": 0.4999902546405792, "grad_norm": 0.0017044921405613422, "learning_rate": 2.381045210440644e-07, "logits/chosen": 1.231360912322998, "logits/rejected": 1.247937798500061, "logps/chosen": -337.2596435546875, "logps/ref_chosen": -72.66920471191406, "logps/ref_rejected": -76.83158874511719, "logps/rejected": -442.0509033203125, "loss": 1.3863, "margin_dpo/margin_mean": 100.6288833618164, "margin_dpo/margin_std": 454.28070068359375, "step": 374 }, { "KL/chosen_KL_mean": -232.35386657714844, "KL/mean": -285.4949951171875, "KL/rejected_KL_mean": -338.63616943359375, "KL/std": 343.39801025390625, "epoch": 0.5668934240362812, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.28231811523438, "fcm_dpo/q_t": 0.49998971819877625, "grad_norm": 0.002389972796663642, "learning_rate": 2.3678397206786715e-07, "logits/chosen": 1.2415052652359009, "logits/rejected": 1.22267484664917, "logps/chosen": -290.03717041015625, "logps/ref_chosen": -57.68330383300781, "logps/ref_rejected": -79.34097290039062, "logps/rejected": -417.9771423339844, "loss": 1.3863, "margin_dpo/margin_mean": 106.28231811523438, "margin_dpo/margin_std": 401.72052001953125, "step": 375 }, { "KL/chosen_KL_mean": -196.7454071044922, "KL/mean": -273.0303039550781, "KL/rejected_KL_mean": -349.315185546875, "KL/std": 331.4114990234375, "epoch": 0.5684051398337112, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.56979370117188, "fcm_dpo/q_t": 0.4999852180480957, "grad_norm": 0.0025670428294688463, "learning_rate": 2.3546379277238103e-07, "logits/chosen": 1.2817575931549072, "logits/rejected": 1.2636425495147705, "logps/chosen": -248.41946411132812, "logps/ref_chosen": -51.674072265625, "logps/ref_rejected": -75.69713592529297, "logps/rejected": -425.0123291015625, "loss": 1.3862, "margin_dpo/margin_mean": 152.56979370117188, "margin_dpo/margin_std": 369.35845947265625, "step": 376 }, { "KL/chosen_KL_mean": -192.5317840576172, "KL/mean": -222.54251098632812, "KL/rejected_KL_mean": -252.55325317382812, "KL/std": 172.49461364746094, "epoch": 0.5699168556311414, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 60.02146911621094, "fcm_dpo/q_t": 0.4999941885471344, "grad_norm": 0.0016488871769979596, "learning_rate": 2.3414402008585886e-07, "logits/chosen": 1.273218035697937, "logits/rejected": 1.2637633085250854, "logps/chosen": -238.7103271484375, "logps/ref_chosen": -46.17853546142578, "logps/ref_rejected": -57.756500244140625, "logps/rejected": -310.30975341796875, "loss": 1.3863, "margin_dpo/margin_mean": 60.02146911621094, "margin_dpo/margin_std": 183.21182250976562, "step": 377 }, { "KL/chosen_KL_mean": -234.7102813720703, "KL/mean": -283.94964599609375, "KL/rejected_KL_mean": -333.1890563964844, "KL/std": 307.97674560546875, "epoch": 0.5714285714285714, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.47879028320312, "fcm_dpo/q_t": 0.49999046325683594, "grad_norm": 0.0019810660742223263, "learning_rate": 2.3282469092517977e-07, "logits/chosen": 1.2981789112091064, "logits/rejected": 1.2841894626617432, "logps/chosen": -293.92913818359375, "logps/ref_chosen": -59.21887969970703, "logps/ref_rejected": -71.24818420410156, "logps/rejected": -404.437255859375, "loss": 1.3863, "margin_dpo/margin_mean": 98.47879028320312, "margin_dpo/margin_std": 306.84649658203125, "step": 378 }, { "KL/chosen_KL_mean": -273.62860107421875, "KL/mean": -351.3236083984375, "KL/rejected_KL_mean": -429.01861572265625, "KL/std": 358.9712829589844, "epoch": 0.5729402872260015, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.38998413085938, "fcm_dpo/q_t": 0.4999849498271942, "grad_norm": 0.003134517464786768, "learning_rate": 2.3150584219481643e-07, "logits/chosen": 1.227163553237915, "logits/rejected": 1.1967265605926514, "logps/chosen": -349.9451904296875, "logps/ref_chosen": -76.31658935546875, "logps/ref_rejected": -104.26200103759766, "logps/rejected": -533.2806396484375, "loss": 1.3862, "margin_dpo/margin_mean": 155.38998413085938, "margin_dpo/margin_std": 411.5760192871094, "step": 379 }, { "KL/chosen_KL_mean": -208.5601806640625, "KL/mean": -263.50933837890625, "KL/rejected_KL_mean": -318.45849609375, "KL/std": 250.38766479492188, "epoch": 0.5744520030234316, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.89828491210938, "fcm_dpo/q_t": 0.49998939037323, "grad_norm": 0.0019253168720752, "learning_rate": 2.3018751078580283e-07, "logits/chosen": 1.2732884883880615, "logits/rejected": 1.262009620666504, "logps/chosen": -269.8433532714844, "logps/ref_chosen": -61.283164978027344, "logps/ref_rejected": -72.38892364501953, "logps/rejected": -390.847412109375, "loss": 1.3863, "margin_dpo/margin_mean": 109.89830017089844, "margin_dpo/margin_std": 294.832275390625, "step": 380 }, { "KL/chosen_KL_mean": -244.27020263671875, "KL/mean": -272.90728759765625, "KL/rejected_KL_mean": -301.54437255859375, "KL/std": 301.01202392578125, "epoch": 0.5759637188208617, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.27418518066406, "fcm_dpo/q_t": 0.4999944567680359, "grad_norm": 0.0016863668570294976, "learning_rate": 2.288697335747027e-07, "logits/chosen": 1.2436158657073975, "logits/rejected": 1.240373134613037, "logps/chosen": -302.48419189453125, "logps/ref_chosen": -58.2139892578125, "logps/ref_rejected": -60.78669357299805, "logps/rejected": -362.3310546875, "loss": 1.3863, "margin_dpo/margin_mean": 57.27417755126953, "margin_dpo/margin_std": 403.81927490234375, "step": 381 }, { "KL/chosen_KL_mean": -229.37368774414062, "KL/mean": -292.00982666015625, "KL/rejected_KL_mean": -354.6460266113281, "KL/std": 275.9203796386719, "epoch": 0.5774754346182918, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.2723388671875, "fcm_dpo/q_t": 0.4999878704547882, "grad_norm": 0.0023127899039536715, "learning_rate": 2.2755254742257706e-07, "logits/chosen": 1.258483648300171, "logits/rejected": 1.2473549842834473, "logps/chosen": -291.1990051269531, "logps/ref_chosen": -61.82532501220703, "logps/ref_rejected": -83.0452880859375, "logps/rejected": -437.6913146972656, "loss": 1.3862, "margin_dpo/margin_mean": 125.27233123779297, "margin_dpo/margin_std": 331.326416015625, "step": 382 }, { "KL/chosen_KL_mean": -284.44842529296875, "KL/mean": -303.908447265625, "KL/rejected_KL_mean": -323.3684997558594, "KL/std": 256.37823486328125, "epoch": 0.5789871504157218, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 38.92008972167969, "fcm_dpo/q_t": 0.49999624490737915, "grad_norm": 0.002672591246664524, "learning_rate": 2.2623598917395436e-07, "logits/chosen": 1.2364006042480469, "logits/rejected": 1.2648537158966064, "logps/chosen": -365.01165771484375, "logps/ref_chosen": -80.56326293945312, "logps/ref_rejected": -74.62922668457031, "logps/rejected": -397.99774169921875, "loss": 1.3863, "margin_dpo/margin_mean": 38.92008590698242, "margin_dpo/margin_std": 323.5065612792969, "step": 383 }, { "KL/chosen_KL_mean": -215.20654296875, "KL/mean": -270.7156677246094, "KL/rejected_KL_mean": -326.22479248046875, "KL/std": 235.81796264648438, "epoch": 0.5804988662131519, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 111.01825714111328, "fcm_dpo/q_t": 0.49998927116394043, "grad_norm": 0.0019059223122894764, "learning_rate": 2.2492009565579875e-07, "logits/chosen": 1.2277659177780151, "logits/rejected": 1.224064588546753, "logps/chosen": -280.68170166015625, "logps/ref_chosen": -65.47514343261719, "logps/ref_rejected": -79.67378234863281, "logps/rejected": -405.8985595703125, "loss": 1.3863, "margin_dpo/margin_mean": 111.01826477050781, "margin_dpo/margin_std": 284.46221923828125, "step": 384 }, { "KL/chosen_KL_mean": -261.8004150390625, "KL/mean": -326.974365234375, "KL/rejected_KL_mean": -392.1483154296875, "KL/std": 286.7763977050781, "epoch": 0.582010582010582, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.34793090820312, "fcm_dpo/q_t": 0.49998739361763, "grad_norm": 0.002203842392191291, "learning_rate": 2.2360490367648084e-07, "logits/chosen": 1.246370792388916, "logits/rejected": 1.2299015522003174, "logps/chosen": -327.85693359375, "logps/ref_chosen": -66.0565185546875, "logps/ref_rejected": -86.68023681640625, "logps/rejected": -478.82855224609375, "loss": 1.3862, "margin_dpo/margin_mean": 130.34791564941406, "margin_dpo/margin_std": 344.5843811035156, "step": 385 }, { "KL/chosen_KL_mean": -280.88348388671875, "KL/mean": -343.2861328125, "KL/rejected_KL_mean": -405.68878173828125, "KL/std": 325.25909423828125, "epoch": 0.5835222978080121, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.80535125732422, "fcm_dpo/q_t": 0.4999879002571106, "grad_norm": 0.0024226726964116096, "learning_rate": 2.2229045002474724e-07, "logits/chosen": 1.2367877960205078, "logits/rejected": 1.2165486812591553, "logps/chosen": -356.50714111328125, "logps/ref_chosen": -75.6236572265625, "logps/ref_rejected": -92.62330627441406, "logps/rejected": -498.3121032714844, "loss": 1.3862, "margin_dpo/margin_mean": 124.80534362792969, "margin_dpo/margin_std": 393.1319580078125, "step": 386 }, { "KL/chosen_KL_mean": -202.7679443359375, "KL/mean": -288.2266540527344, "KL/rejected_KL_mean": -373.68536376953125, "KL/std": 276.45501708984375, "epoch": 0.5850340136054422, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.91744995117188, "fcm_dpo/q_t": 0.4999834895133972, "grad_norm": 0.0029783707577735186, "learning_rate": 2.209767714686924e-07, "logits/chosen": 1.2404074668884277, "logits/rejected": 1.191530704498291, "logps/chosen": -249.98965454101562, "logps/ref_chosen": -47.22170639038086, "logps/ref_rejected": -87.338134765625, "logps/rejected": -461.02349853515625, "loss": 1.3862, "margin_dpo/margin_mean": 170.9174346923828, "margin_dpo/margin_std": 304.63897705078125, "step": 387 }, { "KL/chosen_KL_mean": -288.9200439453125, "KL/mean": -319.6807861328125, "KL/rejected_KL_mean": -350.4415588378906, "KL/std": 325.09112548828125, "epoch": 0.5865457294028723, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.52149963378906, "fcm_dpo/q_t": 0.49999403953552246, "grad_norm": 0.00200459873303771, "learning_rate": 2.1966390475472954e-07, "logits/chosen": 1.2456412315368652, "logits/rejected": 1.248414158821106, "logps/chosen": -363.49951171875, "logps/ref_chosen": -74.5794677734375, "logps/ref_rejected": -79.92558288574219, "logps/rejected": -430.36712646484375, "loss": 1.3863, "margin_dpo/margin_mean": 61.52150344848633, "margin_dpo/margin_std": 357.44195556640625, "step": 388 }, { "KL/chosen_KL_mean": -228.92892456054688, "KL/mean": -281.46588134765625, "KL/rejected_KL_mean": -334.0028381347656, "KL/std": 260.0592041015625, "epoch": 0.5880574452003023, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.07390594482422, "fcm_dpo/q_t": 0.4999898374080658, "grad_norm": 0.0018365130526944995, "learning_rate": 2.1835188660656265e-07, "logits/chosen": 1.256210446357727, "logits/rejected": 1.2504541873931885, "logps/chosen": -290.55328369140625, "logps/ref_chosen": -61.624366760253906, "logps/ref_rejected": -76.50978088378906, "logps/rejected": -410.51263427734375, "loss": 1.3863, "margin_dpo/margin_mean": 105.07391357421875, "margin_dpo/margin_std": 320.30426025390625, "step": 389 }, { "KL/chosen_KL_mean": -165.7669677734375, "KL/mean": -217.87681579589844, "KL/rejected_KL_mean": -269.9866943359375, "KL/std": 203.6674346923828, "epoch": 0.5895691609977324, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.21971130371094, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.0020355647429823875, "learning_rate": 2.170407537241599e-07, "logits/chosen": 1.3178033828735352, "logits/rejected": 1.2974865436553955, "logps/chosen": -211.63882446289062, "logps/ref_chosen": -45.871864318847656, "logps/ref_rejected": -61.305999755859375, "logps/rejected": -331.29266357421875, "loss": 1.3863, "margin_dpo/margin_mean": 104.21971130371094, "margin_dpo/margin_std": 242.00851440429688, "step": 390 }, { "KL/chosen_KL_mean": -229.9234619140625, "KL/mean": -298.4404296875, "KL/rejected_KL_mean": -366.95745849609375, "KL/std": 266.13519287109375, "epoch": 0.5910808767951625, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.0339813232422, "fcm_dpo/q_t": 0.4999867081642151, "grad_norm": 0.0023842838127166033, "learning_rate": 2.1573054278272636e-07, "logits/chosen": 1.2694813013076782, "logits/rejected": 1.245664119720459, "logps/chosen": -288.1104736328125, "logps/ref_chosen": -58.18701171875, "logps/ref_rejected": -83.63442993164062, "logps/rejected": -450.59185791015625, "loss": 1.3862, "margin_dpo/margin_mean": 137.0339813232422, "margin_dpo/margin_std": 317.8896484375, "step": 391 }, { "KL/chosen_KL_mean": -249.9038848876953, "KL/mean": -310.838623046875, "KL/rejected_KL_mean": -371.7734069824219, "KL/std": 360.3361511230469, "epoch": 0.5925925925925926, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.86954498291016, "fcm_dpo/q_t": 0.4999881982803345, "grad_norm": 0.0024389002937823534, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 1.2220652103424072, "logits/rejected": 1.2007293701171875, "logps/chosen": -319.6484375, "logps/ref_chosen": -69.7445297241211, "logps/ref_rejected": -94.05877685546875, "logps/rejected": -465.8321838378906, "loss": 1.3862, "margin_dpo/margin_mean": 121.86953735351562, "margin_dpo/margin_std": 398.31768798828125, "step": 392 }, { "KL/chosen_KL_mean": -212.96188354492188, "KL/mean": -276.4071044921875, "KL/rejected_KL_mean": -339.852294921875, "KL/std": 308.44964599609375, "epoch": 0.5941043083900227, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.89041137695312, "fcm_dpo/q_t": 0.49998772144317627, "grad_norm": 0.00197499874047935, "learning_rate": 2.131130332936195e-07, "logits/chosen": 1.2748873233795166, "logits/rejected": 1.260786771774292, "logps/chosen": -265.2967834472656, "logps/ref_chosen": -52.33489990234375, "logps/ref_rejected": -74.33809661865234, "logps/rejected": -414.1903991699219, "loss": 1.3862, "margin_dpo/margin_mean": 126.89041137695312, "margin_dpo/margin_std": 273.0295104980469, "step": 393 }, { "KL/chosen_KL_mean": -218.35079956054688, "KL/mean": -262.255126953125, "KL/rejected_KL_mean": -306.1594543457031, "KL/std": 245.12030029296875, "epoch": 0.5956160241874527, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.80865478515625, "fcm_dpo/q_t": 0.4999915063381195, "grad_norm": 0.0014762998325750232, "learning_rate": 2.1180580796331323e-07, "logits/chosen": 1.280198097229004, "logits/rejected": 1.2725932598114014, "logps/chosen": -279.02691650390625, "logps/ref_chosen": -60.6761360168457, "logps/ref_rejected": -71.36074829101562, "logps/rejected": -377.52020263671875, "loss": 1.3863, "margin_dpo/margin_mean": 87.80865478515625, "margin_dpo/margin_std": 251.21902465820312, "step": 394 }, { "KL/chosen_KL_mean": -204.20767211914062, "KL/mean": -277.1258544921875, "KL/rejected_KL_mean": -350.0440673828125, "KL/std": 298.6371765136719, "epoch": 0.5971277399848829, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.8363800048828, "fcm_dpo/q_t": 0.49998587369918823, "grad_norm": 0.0028106472454965115, "learning_rate": 2.104996510066625e-07, "logits/chosen": 1.2618656158447266, "logits/rejected": 1.2272990942001343, "logps/chosen": -254.81199645996094, "logps/ref_chosen": -50.60432434082031, "logps/ref_rejected": -77.08731079101562, "logps/rejected": -427.13134765625, "loss": 1.3862, "margin_dpo/margin_mean": 145.8363800048828, "margin_dpo/margin_std": 384.5468444824219, "step": 395 }, { "KL/chosen_KL_mean": -191.80307006835938, "KL/mean": -259.073486328125, "KL/rejected_KL_mean": -326.34393310546875, "KL/std": 249.66404724121094, "epoch": 0.5986394557823129, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.54083251953125, "fcm_dpo/q_t": 0.49998700618743896, "grad_norm": 0.002482072915881872, "learning_rate": 2.0919459895968517e-07, "logits/chosen": 1.263901948928833, "logits/rejected": 1.2265499830245972, "logps/chosen": -243.16268920898438, "logps/ref_chosen": -51.35961151123047, "logps/ref_rejected": -79.89360046386719, "logps/rejected": -406.2375183105469, "loss": 1.3862, "margin_dpo/margin_mean": 134.54083251953125, "margin_dpo/margin_std": 234.10260009765625, "step": 396 }, { "KL/chosen_KL_mean": -259.5587463378906, "KL/mean": -302.8362121582031, "KL/rejected_KL_mean": -346.1136474609375, "KL/std": 284.9183654785156, "epoch": 0.600151171579743, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.55490112304688, "fcm_dpo/q_t": 0.49999162554740906, "grad_norm": 0.0018756233621388674, "learning_rate": 2.078906883274924e-07, "logits/chosen": 1.194448709487915, "logits/rejected": 1.1737489700317383, "logps/chosen": -326.0149841308594, "logps/ref_chosen": -66.45622253417969, "logps/ref_rejected": -85.74736785888672, "logps/rejected": -431.86102294921875, "loss": 1.3863, "margin_dpo/margin_mean": 86.55490112304688, "margin_dpo/margin_std": 322.10992431640625, "step": 397 }, { "KL/chosen_KL_mean": -200.04078674316406, "KL/mean": -269.54071044921875, "KL/rejected_KL_mean": -339.0406494140625, "KL/std": 272.4809875488281, "epoch": 0.6016628873771731, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.99984741210938, "fcm_dpo/q_t": 0.49998655915260315, "grad_norm": 0.0020650143269449472, "learning_rate": 2.065879555832674e-07, "logits/chosen": 1.2693488597869873, "logits/rejected": 1.2469160556793213, "logps/chosen": -249.2850341796875, "logps/ref_chosen": -49.244239807128906, "logps/ref_rejected": -75.18949127197266, "logps/rejected": -414.2301025390625, "loss": 1.3862, "margin_dpo/margin_mean": 138.99984741210938, "margin_dpo/margin_std": 358.0387268066406, "step": 398 }, { "KL/chosen_KL_mean": -245.84356689453125, "KL/mean": -352.4913330078125, "KL/rejected_KL_mean": -459.1390380859375, "KL/std": 331.2461242675781, "epoch": 0.6031746031746031, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 213.29550170898438, "fcm_dpo/q_t": 0.4999793469905853, "grad_norm": 0.0037792252842336893, "learning_rate": 2.052864371672457e-07, "logits/chosen": 1.2296762466430664, "logits/rejected": 1.1701855659484863, "logps/chosen": -314.15032958984375, "logps/ref_chosen": -68.30679321289062, "logps/ref_rejected": -113.2708511352539, "logps/rejected": -572.409912109375, "loss": 1.3862, "margin_dpo/margin_mean": 213.29551696777344, "margin_dpo/margin_std": 435.9686584472656, "step": 399 }, { "KL/chosen_KL_mean": -270.5350646972656, "KL/mean": -334.7283630371094, "KL/rejected_KL_mean": -398.9216613769531, "KL/std": 345.2818908691406, "epoch": 0.6046863189720333, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.38658142089844, "fcm_dpo/q_t": 0.49998757243156433, "grad_norm": 0.002531483769416809, "learning_rate": 2.0398616948569493e-07, "logits/chosen": 1.2615954875946045, "logits/rejected": 1.237792730331421, "logps/chosen": -342.16156005859375, "logps/ref_chosen": -71.62649536132812, "logps/ref_rejected": -90.98765563964844, "logps/rejected": -489.9093017578125, "loss": 1.3862, "margin_dpo/margin_mean": 128.3865966796875, "margin_dpo/margin_std": 375.5792541503906, "step": 400 }, { "KL/chosen_KL_mean": -182.42166137695312, "KL/mean": -258.8485107421875, "KL/rejected_KL_mean": -335.275390625, "KL/std": 285.74456787109375, "epoch": 0.6061980347694633, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.85372924804688, "fcm_dpo/q_t": 0.4999852180480957, "grad_norm": 0.002612342592328787, "learning_rate": 2.0268718890989752e-07, "logits/chosen": 1.272200584411621, "logits/rejected": 1.2434661388397217, "logps/chosen": -236.14662170410156, "logps/ref_chosen": -53.72495651245117, "logps/ref_rejected": -75.06304931640625, "logps/rejected": -410.33843994140625, "loss": 1.3862, "margin_dpo/margin_mean": 152.85372924804688, "margin_dpo/margin_std": 328.9865417480469, "step": 401 }, { "KL/chosen_KL_mean": -211.4462890625, "KL/mean": -261.5096435546875, "KL/rejected_KL_mean": -311.572998046875, "KL/std": 243.91207885742188, "epoch": 0.6077097505668935, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.12675476074219, "fcm_dpo/q_t": 0.4999902844429016, "grad_norm": 0.00198004231788218, "learning_rate": 2.013895317751323e-07, "logits/chosen": 1.2314531803131104, "logits/rejected": 1.2497401237487793, "logps/chosen": -273.3201904296875, "logps/ref_chosen": -61.873931884765625, "logps/ref_rejected": -66.15198516845703, "logps/rejected": -377.7249755859375, "loss": 1.3863, "margin_dpo/margin_mean": 100.12675476074219, "margin_dpo/margin_std": 330.08746337890625, "step": 402 }, { "KL/chosen_KL_mean": -185.16749572753906, "KL/mean": -260.25628662109375, "KL/rejected_KL_mean": -335.34503173828125, "KL/std": 266.97735595703125, "epoch": 0.6092214663643235, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 150.1775665283203, "fcm_dpo/q_t": 0.4999854564666748, "grad_norm": 0.002466583391651511, "learning_rate": 2.0009323437965898e-07, "logits/chosen": 1.3430713415145874, "logits/rejected": 1.302276611328125, "logps/chosen": -236.48899841308594, "logps/ref_chosen": -51.321502685546875, "logps/ref_rejected": -86.54010772705078, "logps/rejected": -421.8851623535156, "loss": 1.3862, "margin_dpo/margin_mean": 150.1775665283203, "margin_dpo/margin_std": 270.43560791015625, "step": 403 }, { "KL/chosen_KL_mean": -225.32257080078125, "KL/mean": -306.13555908203125, "KL/rejected_KL_mean": -386.9486083984375, "KL/std": 334.2081604003906, "epoch": 0.6107331821617535, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.62606811523438, "fcm_dpo/q_t": 0.49998435378074646, "grad_norm": 0.003134074853733182, "learning_rate": 1.9879833298370237e-07, "logits/chosen": 1.26384699344635, "logits/rejected": 1.219362735748291, "logps/chosen": -287.58544921875, "logps/ref_chosen": -62.26288604736328, "logps/ref_rejected": -95.19029998779297, "logps/rejected": -482.138916015625, "loss": 1.3862, "margin_dpo/margin_mean": 161.6260528564453, "margin_dpo/margin_std": 429.6391296386719, "step": 404 }, { "KL/chosen_KL_mean": -192.32791137695312, "KL/mean": -249.77906799316406, "KL/rejected_KL_mean": -307.230224609375, "KL/std": 221.09945678710938, "epoch": 0.6122448979591837, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.90231323242188, "fcm_dpo/q_t": 0.499988853931427, "grad_norm": 0.0018737346399575472, "learning_rate": 1.975048638084379e-07, "logits/chosen": 1.316763997077942, "logits/rejected": 1.30680513381958, "logps/chosen": -242.91226196289062, "logps/ref_chosen": -50.5843391418457, "logps/ref_rejected": -65.43156433105469, "logps/rejected": -372.66180419921875, "loss": 1.3862, "margin_dpo/margin_mean": 114.90231323242188, "margin_dpo/margin_std": 245.88070678710938, "step": 405 }, { "KL/chosen_KL_mean": -180.87625122070312, "KL/mean": -281.7327880859375, "KL/rejected_KL_mean": -382.58935546875, "KL/std": 283.203369140625, "epoch": 0.6137566137566137, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 201.71310424804688, "fcm_dpo/q_t": 0.49998044967651367, "grad_norm": 0.0038125943392515182, "learning_rate": 1.9621286303497914e-07, "logits/chosen": 1.2964931726455688, "logits/rejected": 1.2286221981048584, "logps/chosen": -229.87185668945312, "logps/ref_chosen": -48.99560546875, "logps/ref_rejected": -92.47774505615234, "logps/rejected": -475.06707763671875, "loss": 1.3862, "margin_dpo/margin_mean": 201.71310424804688, "margin_dpo/margin_std": 359.033447265625, "step": 406 }, { "KL/chosen_KL_mean": -312.56707763671875, "KL/mean": -398.27764892578125, "KL/rejected_KL_mean": -483.9881896972656, "KL/std": 455.59527587890625, "epoch": 0.6152683295540439, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 171.42111206054688, "fcm_dpo/q_t": 0.49998340010643005, "grad_norm": 0.00319691258482635, "learning_rate": 1.9492236680336483e-07, "logits/chosen": 1.2000561952590942, "logits/rejected": 1.1917808055877686, "logps/chosen": -401.9676513671875, "logps/ref_chosen": -89.40056610107422, "logps/ref_rejected": -99.28775024414062, "logps/rejected": -583.2759399414062, "loss": 1.3862, "margin_dpo/margin_mean": 171.42111206054688, "margin_dpo/margin_std": 497.2770080566406, "step": 407 }, { "KL/chosen_KL_mean": -188.9539794921875, "KL/mean": -266.2425842285156, "KL/rejected_KL_mean": -343.53118896484375, "KL/std": 264.1246337890625, "epoch": 0.6167800453514739, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.5772247314453, "fcm_dpo/q_t": 0.4999850392341614, "grad_norm": 0.0026424252428114414, "learning_rate": 1.9363341121154895e-07, "logits/chosen": 1.2529544830322266, "logits/rejected": 1.2337000370025635, "logps/chosen": -243.65789794921875, "logps/ref_chosen": -54.70391845703125, "logps/ref_rejected": -73.98648834228516, "logps/rejected": -417.5177001953125, "loss": 1.3862, "margin_dpo/margin_mean": 154.5772247314453, "margin_dpo/margin_std": 305.14788818359375, "step": 408 }, { "KL/chosen_KL_mean": -219.13699340820312, "KL/mean": -239.82040405273438, "KL/rejected_KL_mean": -260.5038146972656, "KL/std": 184.755859375, "epoch": 0.618291761148904, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.36683654785156, "fcm_dpo/q_t": 0.49999600648880005, "grad_norm": 0.0013175446074455976, "learning_rate": 1.9234603231438994e-07, "logits/chosen": 1.251279592514038, "logits/rejected": 1.262905240058899, "logps/chosen": -281.2552185058594, "logps/ref_chosen": -62.11822509765625, "logps/ref_rejected": -61.933509826660156, "logps/rejected": -322.43731689453125, "loss": 1.3863, "margin_dpo/margin_mean": 41.36684036254883, "margin_dpo/margin_std": 230.91448974609375, "step": 409 }, { "KL/chosen_KL_mean": -235.9901580810547, "KL/mean": -287.7513122558594, "KL/rejected_KL_mean": -339.5124206542969, "KL/std": 274.4139404296875, "epoch": 0.6198034769463341, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.52224731445312, "fcm_dpo/q_t": 0.49998998641967773, "grad_norm": 0.0023037977516651154, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 1.2742689847946167, "logits/rejected": 1.2695902585983276, "logps/chosen": -297.7928466796875, "logps/ref_chosen": -61.80266189575195, "logps/ref_rejected": -76.60002136230469, "logps/rejected": -416.1124267578125, "loss": 1.3863, "margin_dpo/margin_mean": 103.52224731445312, "margin_dpo/margin_std": 323.41107177734375, "step": 410 }, { "KL/chosen_KL_mean": -204.3567352294922, "KL/mean": -268.3748779296875, "KL/rejected_KL_mean": -332.39306640625, "KL/std": 259.58526611328125, "epoch": 0.6213151927437641, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.03631591796875, "fcm_dpo/q_t": 0.4999876022338867, "grad_norm": 0.002165102167055011, "learning_rate": 1.8977614860195296e-07, "logits/chosen": 1.2589843273162842, "logits/rejected": 1.2471109628677368, "logps/chosen": -258.8021240234375, "logps/ref_chosen": -54.44539260864258, "logps/ref_rejected": -74.5650863647461, "logps/rejected": -406.9581298828125, "loss": 1.3862, "margin_dpo/margin_mean": 128.03631591796875, "margin_dpo/margin_std": 278.3114013671875, "step": 411 }, { "KL/chosen_KL_mean": -205.3777313232422, "KL/mean": -259.4642028808594, "KL/rejected_KL_mean": -313.55072021484375, "KL/std": 227.35836791992188, "epoch": 0.6228269085411943, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.17296600341797, "fcm_dpo/q_t": 0.4999895393848419, "grad_norm": 0.0022113437298685312, "learning_rate": 1.8849371567184662e-07, "logits/chosen": 1.2673554420471191, "logits/rejected": 1.2555253505706787, "logps/chosen": -260.6258239746094, "logps/ref_chosen": -55.248085021972656, "logps/ref_rejected": -68.96623229980469, "logps/rejected": -382.51690673828125, "loss": 1.3863, "margin_dpo/margin_mean": 108.17295837402344, "margin_dpo/margin_std": 254.52096557617188, "step": 412 }, { "KL/chosen_KL_mean": -264.1756286621094, "KL/mean": -309.2933654785156, "KL/rejected_KL_mean": -354.4111328125, "KL/std": 302.36083984375, "epoch": 0.6243386243386243, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.23551940917969, "fcm_dpo/q_t": 0.4999912679195404, "grad_norm": 0.0019386338535696268, "learning_rate": 1.872130032047302e-07, "logits/chosen": 1.1930580139160156, "logits/rejected": 1.1896083354949951, "logps/chosen": -332.8963623046875, "logps/ref_chosen": -68.72074890136719, "logps/ref_rejected": -78.76539611816406, "logps/rejected": -433.176513671875, "loss": 1.3863, "margin_dpo/margin_mean": 90.23551940917969, "margin_dpo/margin_std": 384.3971862792969, "step": 413 }, { "KL/chosen_KL_mean": -200.8385009765625, "KL/mean": -264.8590087890625, "KL/rejected_KL_mean": -328.8795166015625, "KL/std": 333.3802490234375, "epoch": 0.6258503401360545, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.04103088378906, "fcm_dpo/q_t": 0.4999876022338867, "grad_norm": 0.0021054602693766356, "learning_rate": 1.8593404702488436e-07, "logits/chosen": 1.2829506397247314, "logits/rejected": 1.2689223289489746, "logps/chosen": -254.97671508789062, "logps/ref_chosen": -54.138214111328125, "logps/ref_rejected": -74.65741729736328, "logps/rejected": -403.53692626953125, "loss": 1.3862, "margin_dpo/margin_mean": 128.04103088378906, "margin_dpo/margin_std": 383.669189453125, "step": 414 }, { "KL/chosen_KL_mean": -208.382568359375, "KL/mean": -246.01028442382812, "KL/rejected_KL_mean": -283.6380310058594, "KL/std": 253.46176147460938, "epoch": 0.6273620559334845, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.25543212890625, "fcm_dpo/q_t": 0.4999927282333374, "grad_norm": 0.00155459635425359, "learning_rate": 1.846568829074628e-07, "logits/chosen": 1.2656569480895996, "logits/rejected": 1.2652559280395508, "logps/chosen": -264.3011169433594, "logps/ref_chosen": -55.91856002807617, "logps/ref_rejected": -61.747703552246094, "logps/rejected": -345.3857421875, "loss": 1.3863, "margin_dpo/margin_mean": 75.25544738769531, "margin_dpo/margin_std": 311.67486572265625, "step": 415 }, { "KL/chosen_KL_mean": -220.16928100585938, "KL/mean": -272.12646484375, "KL/rejected_KL_mean": -324.0836181640625, "KL/std": 320.10504150390625, "epoch": 0.6288737717309146, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.91433715820312, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.0020540771074593067, "learning_rate": 1.8338154657749128e-07, "logits/chosen": 1.2827048301696777, "logits/rejected": 1.2663986682891846, "logps/chosen": -274.8923645019531, "logps/ref_chosen": -54.72308349609375, "logps/ref_rejected": -69.17388916015625, "logps/rejected": -393.25750732421875, "loss": 1.3863, "margin_dpo/margin_mean": 103.91433715820312, "margin_dpo/margin_std": 338.7135925292969, "step": 416 }, { "KL/chosen_KL_mean": -224.02847290039062, "KL/mean": -288.44708251953125, "KL/rejected_KL_mean": -352.8656921386719, "KL/std": 303.87835693359375, "epoch": 0.6303854875283447, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.83721923828125, "fcm_dpo/q_t": 0.49998751282691956, "grad_norm": 0.002455303678289056, "learning_rate": 1.8210807370886849e-07, "logits/chosen": 1.3073334693908691, "logits/rejected": 1.2928953170776367, "logps/chosen": -280.8197326660156, "logps/ref_chosen": -56.791259765625, "logps/ref_rejected": -68.7791748046875, "logps/rejected": -421.6448974609375, "loss": 1.3862, "margin_dpo/margin_mean": 128.83721923828125, "margin_dpo/margin_std": 371.76605224609375, "step": 417 }, { "KL/chosen_KL_mean": -284.6827087402344, "KL/mean": -312.7228088378906, "KL/rejected_KL_mean": -340.7629089355469, "KL/std": 332.7872314453125, "epoch": 0.6318972033257747, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 56.08018112182617, "fcm_dpo/q_t": 0.49999457597732544, "grad_norm": 0.001611909014172852, "learning_rate": 1.8083649992336825e-07, "logits/chosen": 1.2963645458221436, "logits/rejected": 1.3024256229400635, "logps/chosen": -353.79071044921875, "logps/ref_chosen": -69.10798645019531, "logps/ref_rejected": -75.09132385253906, "logps/rejected": -415.854248046875, "loss": 1.3863, "margin_dpo/margin_mean": 56.080177307128906, "margin_dpo/margin_std": 397.62890625, "step": 418 }, { "KL/chosen_KL_mean": -200.85401916503906, "KL/mean": -274.2613830566406, "KL/rejected_KL_mean": -347.66876220703125, "KL/std": 276.50970458984375, "epoch": 0.6334089191232048, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 146.81472778320312, "fcm_dpo/q_t": 0.49998578429222107, "grad_norm": 0.002377911237999797, "learning_rate": 1.7956686078964255e-07, "logits/chosen": 1.2625758647918701, "logits/rejected": 1.2619942426681519, "logps/chosen": -259.0257873535156, "logps/ref_chosen": -58.1717643737793, "logps/ref_rejected": -71.67066955566406, "logps/rejected": -419.33941650390625, "loss": 1.3862, "margin_dpo/margin_mean": 146.81472778320312, "margin_dpo/margin_std": 333.00238037109375, "step": 419 }, { "KL/chosen_KL_mean": -232.24765014648438, "KL/mean": -268.002197265625, "KL/rejected_KL_mean": -303.7567443847656, "KL/std": 265.8175048828125, "epoch": 0.6349206349206349, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.50907897949219, "fcm_dpo/q_t": 0.49999308586120605, "grad_norm": 0.001728436560370028, "learning_rate": 1.782991918222275e-07, "logits/chosen": 1.2875854969024658, "logits/rejected": 1.2710034847259521, "logps/chosen": -289.3011474609375, "logps/ref_chosen": -57.05351257324219, "logps/ref_rejected": -62.670982360839844, "logps/rejected": -366.427734375, "loss": 1.3863, "margin_dpo/margin_mean": 71.50907897949219, "margin_dpo/margin_std": 340.4837951660156, "step": 420 }, { "KL/chosen_KL_mean": -230.90493774414062, "KL/mean": -285.153076171875, "KL/rejected_KL_mean": -339.4012145996094, "KL/std": 287.30902099609375, "epoch": 0.636432350718065, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.49628448486328, "fcm_dpo/q_t": 0.49998950958251953, "grad_norm": 0.002344191772863269, "learning_rate": 1.7703352848054887e-07, "logits/chosen": 1.237176775932312, "logits/rejected": 1.2174639701843262, "logps/chosen": -288.22821044921875, "logps/ref_chosen": -57.32324981689453, "logps/ref_rejected": -75.33782958984375, "logps/rejected": -414.7390441894531, "loss": 1.3863, "margin_dpo/margin_mean": 108.49629211425781, "margin_dpo/margin_std": 395.6588439941406, "step": 421 }, { "KL/chosen_KL_mean": -228.75091552734375, "KL/mean": -281.6344909667969, "KL/rejected_KL_mean": -334.51806640625, "KL/std": 274.93902587890625, "epoch": 0.6379440665154951, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 105.76715087890625, "fcm_dpo/q_t": 0.49998974800109863, "grad_norm": 0.0021835805382579565, "learning_rate": 1.7576990616793137e-07, "logits/chosen": 1.231833577156067, "logits/rejected": 1.2441257238388062, "logps/chosen": -295.8084716796875, "logps/ref_chosen": -67.05757141113281, "logps/ref_rejected": -72.12803649902344, "logps/rejected": -406.6460876464844, "loss": 1.3863, "margin_dpo/margin_mean": 105.76715087890625, "margin_dpo/margin_std": 315.6912841796875, "step": 422 }, { "KL/chosen_KL_mean": -189.46343994140625, "KL/mean": -263.8796691894531, "KL/rejected_KL_mean": -338.2958679199219, "KL/std": 262.6513671875, "epoch": 0.6394557823129252, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.83242797851562, "fcm_dpo/q_t": 0.49998557567596436, "grad_norm": 0.0025841707829385996, "learning_rate": 1.745083602306071e-07, "logits/chosen": 1.2950785160064697, "logits/rejected": 1.2744381427764893, "logps/chosen": -243.52511596679688, "logps/ref_chosen": -54.06167221069336, "logps/ref_rejected": -76.64092254638672, "logps/rejected": -414.9367980957031, "loss": 1.3862, "margin_dpo/margin_mean": 148.83242797851562, "margin_dpo/margin_std": 266.20477294921875, "step": 423 }, { "KL/chosen_KL_mean": -204.17025756835938, "KL/mean": -266.5606384277344, "KL/rejected_KL_mean": -328.95098876953125, "KL/std": 212.84597778320312, "epoch": 0.6409674981103552, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.78072357177734, "fcm_dpo/q_t": 0.499987930059433, "grad_norm": 0.002034224336966872, "learning_rate": 1.7324892595672804e-07, "logits/chosen": 1.2100038528442383, "logits/rejected": 1.197531819343567, "logps/chosen": -257.7791442871094, "logps/ref_chosen": -53.60887145996094, "logps/ref_rejected": -79.2139892578125, "logps/rejected": -408.16497802734375, "loss": 1.3862, "margin_dpo/margin_mean": 124.78071594238281, "margin_dpo/margin_std": 222.72096252441406, "step": 424 }, { "KL/chosen_KL_mean": -202.778564453125, "KL/mean": -247.59307861328125, "KL/rejected_KL_mean": -292.4075927734375, "KL/std": 240.92269897460938, "epoch": 0.6424792139077853, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.62903594970703, "fcm_dpo/q_t": 0.4999913275241852, "grad_norm": 0.0015491149388253689, "learning_rate": 1.7199163857537824e-07, "logits/chosen": 1.2798230648040771, "logits/rejected": 1.2812039852142334, "logps/chosen": -261.1932373046875, "logps/ref_chosen": -58.41468048095703, "logps/ref_rejected": -66.59054565429688, "logps/rejected": -358.9981384277344, "loss": 1.3863, "margin_dpo/margin_mean": 89.62903594970703, "margin_dpo/margin_std": 312.6791687011719, "step": 425 }, { "KL/chosen_KL_mean": -276.497314453125, "KL/mean": -286.79400634765625, "KL/rejected_KL_mean": -297.0906677246094, "KL/std": 287.28729248046875, "epoch": 0.6439909297052154, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.59334373474121, "fcm_dpo/q_t": 0.4999980032444, "grad_norm": 0.0016692699864506721, "learning_rate": 1.7073653325558828e-07, "logits/chosen": 1.2584002017974854, "logits/rejected": 1.2575640678405762, "logps/chosen": -348.20550537109375, "logps/ref_chosen": -71.70822143554688, "logps/ref_rejected": -73.57725524902344, "logps/rejected": -370.66790771484375, "loss": 1.3863, "margin_dpo/margin_mean": 20.593345642089844, "margin_dpo/margin_std": 402.493408203125, "step": 426 }, { "KL/chosen_KL_mean": -244.16412353515625, "KL/mean": -299.6019287109375, "KL/rejected_KL_mean": -355.03973388671875, "KL/std": 328.6190185546875, "epoch": 0.6455026455026455, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.87562561035156, "fcm_dpo/q_t": 0.49998927116394043, "grad_norm": 0.002250807359814644, "learning_rate": 1.6948364510535218e-07, "logits/chosen": 1.2549666166305542, "logits/rejected": 1.2232038974761963, "logps/chosen": -302.806884765625, "logps/ref_chosen": -58.64276885986328, "logps/ref_rejected": -86.25437927246094, "logps/rejected": -441.29412841796875, "loss": 1.3863, "margin_dpo/margin_mean": 110.87562561035156, "margin_dpo/margin_std": 329.8369140625, "step": 427 }, { "KL/chosen_KL_mean": -276.95233154296875, "KL/mean": -329.34759521484375, "KL/rejected_KL_mean": -381.74285888671875, "KL/std": 323.8177185058594, "epoch": 0.6470143613000756, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.79049682617188, "fcm_dpo/q_t": 0.4999898374080658, "grad_norm": 0.0020647216588258743, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 1.2424485683441162, "logits/rejected": 1.2366441488265991, "logps/chosen": -343.54840087890625, "logps/ref_chosen": -66.5960464477539, "logps/ref_rejected": -82.3941650390625, "logps/rejected": -464.13702392578125, "loss": 1.3863, "margin_dpo/margin_mean": 104.79049682617188, "margin_dpo/margin_std": 409.09844970703125, "step": 428 }, { "KL/chosen_KL_mean": -245.20016479492188, "KL/mean": -263.1362609863281, "KL/rejected_KL_mean": -281.0723571777344, "KL/std": 223.36959838867188, "epoch": 0.6485260770975056, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 35.87220764160156, "fcm_dpo/q_t": 0.499996542930603, "grad_norm": 0.0011962910648435354, "learning_rate": 1.669846604344412e-07, "logits/chosen": 1.259268879890442, "logits/rejected": 1.2742395401000977, "logps/chosen": -302.2098693847656, "logps/ref_chosen": -57.00970458984375, "logps/ref_rejected": -59.86549377441406, "logps/rejected": -340.9378662109375, "loss": 1.3863, "margin_dpo/margin_mean": 35.8722038269043, "margin_dpo/margin_std": 258.54791259765625, "step": 429 }, { "KL/chosen_KL_mean": -219.3946533203125, "KL/mean": -255.57508850097656, "KL/rejected_KL_mean": -291.75555419921875, "KL/std": 203.72706604003906, "epoch": 0.6500377928949358, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.36093139648438, "fcm_dpo/q_t": 0.4999930262565613, "grad_norm": 0.0014631770318374038, "learning_rate": 1.6573863381573954e-07, "logits/chosen": 1.1872947216033936, "logits/rejected": 1.187971591949463, "logps/chosen": -278.95782470703125, "logps/ref_chosen": -59.563194274902344, "logps/ref_rejected": -70.52289581298828, "logps/rejected": -362.2784423828125, "loss": 1.3863, "margin_dpo/margin_mean": 72.36092376708984, "margin_dpo/margin_std": 281.1922302246094, "step": 430 }, { "KL/chosen_KL_mean": -191.01605224609375, "KL/mean": -245.7452850341797, "KL/rejected_KL_mean": -300.47454833984375, "KL/std": 234.70254516601562, "epoch": 0.6515495086923658, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.45848083496094, "fcm_dpo/q_t": 0.49998942017555237, "grad_norm": 0.0020323162898421288, "learning_rate": 1.6449496416858282e-07, "logits/chosen": 1.2493000030517578, "logits/rejected": 1.2200860977172852, "logps/chosen": -241.21636962890625, "logps/ref_chosen": -50.20032501220703, "logps/ref_rejected": -77.81680297851562, "logps/rejected": -378.29132080078125, "loss": 1.3863, "margin_dpo/margin_mean": 109.45848846435547, "margin_dpo/margin_std": 307.2296142578125, "step": 431 }, { "KL/chosen_KL_mean": -256.33648681640625, "KL/mean": -303.86322021484375, "KL/rejected_KL_mean": -351.3900146484375, "KL/std": 295.88214111328125, "epoch": 0.6530612244897959, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.05351257324219, "fcm_dpo/q_t": 0.4999908208847046, "grad_norm": 0.0019163392717018723, "learning_rate": 1.632536862810844e-07, "logits/chosen": 1.286834716796875, "logits/rejected": 1.2581818103790283, "logps/chosen": -317.999267578125, "logps/ref_chosen": -61.662757873535156, "logps/ref_rejected": -83.94496154785156, "logps/rejected": -435.3349609375, "loss": 1.3863, "margin_dpo/margin_mean": 95.05350494384766, "margin_dpo/margin_std": 400.80767822265625, "step": 432 }, { "KL/chosen_KL_mean": -221.035400390625, "KL/mean": -255.6205291748047, "KL/rejected_KL_mean": -290.2056579589844, "KL/std": 240.65435791015625, "epoch": 0.654572940287226, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 69.17024993896484, "fcm_dpo/q_t": 0.49999332427978516, "grad_norm": 0.0015163794159889221, "learning_rate": 1.6201483487445515e-07, "logits/chosen": 1.273536205291748, "logits/rejected": 1.2836341857910156, "logps/chosen": -284.76458740234375, "logps/ref_chosen": -63.72917938232422, "logps/ref_rejected": -65.8391342163086, "logps/rejected": -356.0447998046875, "loss": 1.3863, "margin_dpo/margin_mean": 69.17024993896484, "margin_dpo/margin_std": 295.4766845703125, "step": 433 }, { "KL/chosen_KL_mean": -177.51564025878906, "KL/mean": -254.1595458984375, "KL/rejected_KL_mean": -330.803466796875, "KL/std": 256.6459655761719, "epoch": 0.656084656084656, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 153.2877960205078, "fcm_dpo/q_t": 0.4999851584434509, "grad_norm": 0.0024449967313557863, "learning_rate": 1.6077844460203204e-07, "logits/chosen": 1.3231778144836426, "logits/rejected": 1.300271987915039, "logps/chosen": -225.48895263671875, "logps/ref_chosen": -47.97331619262695, "logps/ref_rejected": -72.51132202148438, "logps/rejected": -403.31475830078125, "loss": 1.3862, "margin_dpo/margin_mean": 153.28781127929688, "margin_dpo/margin_std": 307.38153076171875, "step": 434 }, { "KL/chosen_KL_mean": -212.97402954101562, "KL/mean": -253.61410522460938, "KL/rejected_KL_mean": -294.25421142578125, "KL/std": 250.4377899169922, "epoch": 0.6575963718820862, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 81.28021240234375, "fcm_dpo/q_t": 0.49999213218688965, "grad_norm": 0.0016701404238119721, "learning_rate": 1.5954455004830878e-07, "logits/chosen": 1.3001594543457031, "logits/rejected": 1.285398244857788, "logps/chosen": -270.0342712402344, "logps/ref_chosen": -57.06024932861328, "logps/ref_rejected": -71.69146728515625, "logps/rejected": -365.9456787109375, "loss": 1.3863, "margin_dpo/margin_mean": 81.28020477294922, "margin_dpo/margin_std": 294.3331298828125, "step": 435 }, { "KL/chosen_KL_mean": -216.45889282226562, "KL/mean": -261.227294921875, "KL/rejected_KL_mean": -305.9957275390625, "KL/std": 265.82232666015625, "epoch": 0.6591080876795162, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.53681182861328, "fcm_dpo/q_t": 0.49999135732650757, "grad_norm": 0.0022236332297325134, "learning_rate": 1.5831318572796847e-07, "logits/chosen": 1.289149284362793, "logits/rejected": 1.2721104621887207, "logps/chosen": -272.616943359375, "logps/ref_chosen": -56.158050537109375, "logps/ref_rejected": -67.63787841796875, "logps/rejected": -373.63360595703125, "loss": 1.3863, "margin_dpo/margin_mean": 89.53681182861328, "margin_dpo/margin_std": 242.168701171875, "step": 436 }, { "KL/chosen_KL_mean": -215.29190063476562, "KL/mean": -292.97491455078125, "KL/rejected_KL_mean": -370.65789794921875, "KL/std": 285.3720397949219, "epoch": 0.6606198034769464, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 155.36599731445312, "fcm_dpo/q_t": 0.4999849498271942, "grad_norm": 0.0029099148232489824, "learning_rate": 1.5708438608491815e-07, "logits/chosen": 1.2747769355773926, "logits/rejected": 1.217381238937378, "logps/chosen": -272.2777099609375, "logps/ref_chosen": -56.98578643798828, "logps/ref_rejected": -85.61524963378906, "logps/rejected": -456.2731628417969, "loss": 1.3862, "margin_dpo/margin_mean": 155.36599731445312, "margin_dpo/margin_std": 385.7756042480469, "step": 437 }, { "KL/chosen_KL_mean": -179.04933166503906, "KL/mean": -239.98414611816406, "KL/rejected_KL_mean": -300.9189758300781, "KL/std": 274.84893798828125, "epoch": 0.6621315192743764, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.86964416503906, "fcm_dpo/q_t": 0.4999881982803345, "grad_norm": 0.002175640081986785, "learning_rate": 1.558581854913253e-07, "logits/chosen": 1.3229100704193115, "logits/rejected": 1.2967147827148438, "logps/chosen": -220.32711791992188, "logps/ref_chosen": -41.27777862548828, "logps/ref_rejected": -65.33840942382812, "logps/rejected": -366.25738525390625, "loss": 1.3862, "margin_dpo/margin_mean": 121.86964416503906, "margin_dpo/margin_std": 330.1829528808594, "step": 438 }, { "KL/chosen_KL_mean": -285.81622314453125, "KL/mean": -340.0804443359375, "KL/rejected_KL_mean": -394.3446350097656, "KL/std": 310.67144775390625, "epoch": 0.6636432350718064, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.52841186523438, "fcm_dpo/q_t": 0.49998950958251953, "grad_norm": 0.0019142036326229572, "learning_rate": 1.5463461824665658e-07, "logits/chosen": 1.188014268875122, "logits/rejected": 1.1888413429260254, "logps/chosen": -367.23388671875, "logps/ref_chosen": -81.41764831542969, "logps/ref_rejected": -94.72309875488281, "logps/rejected": -489.0677185058594, "loss": 1.3863, "margin_dpo/margin_mean": 108.52839660644531, "margin_dpo/margin_std": 330.1500549316406, "step": 439 }, { "KL/chosen_KL_mean": -170.18858337402344, "KL/mean": -235.81759643554688, "KL/rejected_KL_mean": -301.44659423828125, "KL/std": 246.02532958984375, "epoch": 0.6651549508692366, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.25802612304688, "fcm_dpo/q_t": 0.49998730421066284, "grad_norm": 0.002329205861315131, "learning_rate": 1.534137185767178e-07, "logits/chosen": 1.2724201679229736, "logits/rejected": 1.230487585067749, "logps/chosen": -212.72677612304688, "logps/ref_chosen": -42.538185119628906, "logps/ref_rejected": -69.78813934326172, "logps/rejected": -371.2347412109375, "loss": 1.3862, "margin_dpo/margin_mean": 131.25802612304688, "margin_dpo/margin_std": 252.58546447753906, "step": 440 }, { "KL/chosen_KL_mean": -212.27427673339844, "KL/mean": -289.75164794921875, "KL/rejected_KL_mean": -367.22900390625, "KL/std": 316.7421875, "epoch": 0.6666666666666666, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.95472717285156, "fcm_dpo/q_t": 0.499985009431839, "grad_norm": 0.0029717073775827885, "learning_rate": 1.521955206326976e-07, "logits/chosen": 1.239859938621521, "logits/rejected": 1.210754156112671, "logps/chosen": -269.86749267578125, "logps/ref_chosen": -57.593223571777344, "logps/ref_rejected": -84.82878875732422, "logps/rejected": -452.05780029296875, "loss": 1.3862, "margin_dpo/margin_mean": 154.95472717285156, "margin_dpo/margin_std": 341.307861328125, "step": 441 }, { "KL/chosen_KL_mean": -266.17144775390625, "KL/mean": -336.0986328125, "KL/rejected_KL_mean": -406.02587890625, "KL/std": 319.74578857421875, "epoch": 0.6681783824640968, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.8543701171875, "fcm_dpo/q_t": 0.499986469745636, "grad_norm": 0.0025126931723207235, "learning_rate": 1.5098005849021078e-07, "logits/chosen": 1.2591302394866943, "logits/rejected": 1.2514472007751465, "logps/chosen": -333.6326599121094, "logps/ref_chosen": -67.46121978759766, "logps/ref_rejected": -89.0693588256836, "logps/rejected": -495.09521484375, "loss": 1.3862, "margin_dpo/margin_mean": 139.85438537597656, "margin_dpo/margin_std": 342.5337219238281, "step": 442 }, { "KL/chosen_KL_mean": -193.10202026367188, "KL/mean": -278.1657409667969, "KL/rejected_KL_mean": -363.22943115234375, "KL/std": 328.15521240234375, "epoch": 0.6696900982615268, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.12744140625, "fcm_dpo/q_t": 0.499983549118042, "grad_norm": 0.0025645981077104807, "learning_rate": 1.4976736614834662e-07, "logits/chosen": 1.2513320446014404, "logits/rejected": 1.2338635921478271, "logps/chosen": -247.8981170654297, "logps/ref_chosen": -54.79610061645508, "logps/ref_rejected": -77.80781555175781, "logps/rejected": -441.0372619628906, "loss": 1.3862, "margin_dpo/margin_mean": 170.12744140625, "margin_dpo/margin_std": 411.92755126953125, "step": 443 }, { "KL/chosen_KL_mean": -238.94351196289062, "KL/mean": -313.384521484375, "KL/rejected_KL_mean": -387.8255615234375, "KL/std": 340.74688720703125, "epoch": 0.671201814058957, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.88204956054688, "fcm_dpo/q_t": 0.49998557567596436, "grad_norm": 0.0032997538801282644, "learning_rate": 1.4855747752871654e-07, "logits/chosen": 1.279803991317749, "logits/rejected": 1.249694585800171, "logps/chosen": -297.69256591796875, "logps/ref_chosen": -58.749061584472656, "logps/ref_rejected": -86.87396240234375, "logps/rejected": -474.69952392578125, "loss": 1.3862, "margin_dpo/margin_mean": 148.88204956054688, "margin_dpo/margin_std": 396.679443359375, "step": 444 }, { "KL/chosen_KL_mean": -241.49159240722656, "KL/mean": -290.87945556640625, "KL/rejected_KL_mean": -340.2673645019531, "KL/std": 260.7337341308594, "epoch": 0.672713529856387, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.77577209472656, "fcm_dpo/q_t": 0.49999046325683594, "grad_norm": 0.0014876207569614053, "learning_rate": 1.473504264745062e-07, "logits/chosen": 1.2597832679748535, "logits/rejected": 1.2682100534439087, "logps/chosen": -302.4090270996094, "logps/ref_chosen": -60.91743850708008, "logps/ref_rejected": -71.5637435913086, "logps/rejected": -411.83111572265625, "loss": 1.3863, "margin_dpo/margin_mean": 98.77577209472656, "margin_dpo/margin_std": 304.1396484375, "step": 445 }, { "KL/chosen_KL_mean": -183.92437744140625, "KL/mean": -257.93829345703125, "KL/rejected_KL_mean": -331.9521484375, "KL/std": 250.339111328125, "epoch": 0.674225245653817, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.02777099609375, "fcm_dpo/q_t": 0.4999856650829315, "grad_norm": 0.002402180340141058, "learning_rate": 1.461462467495284e-07, "logits/chosen": 1.2483906745910645, "logits/rejected": 1.225560188293457, "logps/chosen": -232.7236328125, "logps/ref_chosen": -48.79924774169922, "logps/ref_rejected": -71.8719482421875, "logps/rejected": -403.8240966796875, "loss": 1.3862, "margin_dpo/margin_mean": 148.02777099609375, "margin_dpo/margin_std": 283.2513427734375, "step": 446 }, { "KL/chosen_KL_mean": -195.73277282714844, "KL/mean": -288.6422424316406, "KL/rejected_KL_mean": -381.55169677734375, "KL/std": 270.0787353515625, "epoch": 0.6757369614512472, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 185.81895446777344, "fcm_dpo/q_t": 0.49998199939727783, "grad_norm": 0.0031413903925567865, "learning_rate": 1.4494497203727843e-07, "logits/chosen": 1.2971501350402832, "logits/rejected": 1.2553091049194336, "logps/chosen": -249.41549682617188, "logps/ref_chosen": -53.682716369628906, "logps/ref_rejected": -88.17315673828125, "logps/rejected": -469.724853515625, "loss": 1.3862, "margin_dpo/margin_mean": 185.81893920898438, "margin_dpo/margin_std": 336.22564697265625, "step": 447 }, { "KL/chosen_KL_mean": -236.30259704589844, "KL/mean": -296.205078125, "KL/rejected_KL_mean": -356.1075439453125, "KL/std": 286.213623046875, "epoch": 0.6772486772486772, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.80496215820312, "fcm_dpo/q_t": 0.4999884366989136, "grad_norm": 0.002245934447273612, "learning_rate": 1.4374663593999256e-07, "logits/chosen": 1.302297830581665, "logits/rejected": 1.282482624053955, "logps/chosen": -290.0538330078125, "logps/ref_chosen": -53.75125503540039, "logps/ref_rejected": -77.17623901367188, "logps/rejected": -433.2837829589844, "loss": 1.3862, "margin_dpo/margin_mean": 119.80496215820312, "margin_dpo/margin_std": 352.40093994140625, "step": 448 }, { "KL/chosen_KL_mean": -323.4554443359375, "KL/mean": -345.5867919921875, "KL/rejected_KL_mean": -367.7181701660156, "KL/std": 359.06500244140625, "epoch": 0.6787603930461074, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.262718200683594, "fcm_dpo/q_t": 0.49999570846557617, "grad_norm": 0.0019348947098478675, "learning_rate": 1.4255127197770707e-07, "logits/chosen": 1.22648024559021, "logits/rejected": 1.2325232028961182, "logps/chosen": -399.2828369140625, "logps/ref_chosen": -75.82737731933594, "logps/ref_rejected": -82.20687866210938, "logps/rejected": -449.925048828125, "loss": 1.3863, "margin_dpo/margin_mean": 44.262718200683594, "margin_dpo/margin_std": 400.443359375, "step": 449 }, { "KL/chosen_KL_mean": -181.9138946533203, "KL/mean": -270.99798583984375, "KL/rejected_KL_mean": -360.08209228515625, "KL/std": 310.10675048828125, "epoch": 0.6802721088435374, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 178.16824340820312, "fcm_dpo/q_t": 0.4999827742576599, "grad_norm": 0.0035590401384979486, "learning_rate": 1.4135891358732205e-07, "logits/chosen": 1.3165409564971924, "logits/rejected": 1.2763895988464355, "logps/chosen": -229.02960205078125, "logps/ref_chosen": -47.11572265625, "logps/ref_rejected": -78.7546615600586, "logps/rejected": -438.8367919921875, "loss": 1.3862, "margin_dpo/margin_mean": 178.16824340820312, "margin_dpo/margin_std": 364.7618408203125, "step": 450 }, { "KL/chosen_KL_mean": -227.50653076171875, "KL/mean": -281.16180419921875, "KL/rejected_KL_mean": -334.8171081542969, "KL/std": 289.9800720214844, "epoch": 0.6817838246409675, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.31055450439453, "fcm_dpo/q_t": 0.4999896287918091, "grad_norm": 0.002129608066752553, "learning_rate": 1.4016959412166437e-07, "logits/chosen": 1.2388477325439453, "logits/rejected": 1.225501298904419, "logps/chosen": -290.85699462890625, "logps/ref_chosen": -63.350440979003906, "logps/ref_rejected": -76.28530883789062, "logps/rejected": -411.1024169921875, "loss": 1.3863, "margin_dpo/margin_mean": 107.31056213378906, "margin_dpo/margin_std": 304.1910400390625, "step": 451 }, { "KL/chosen_KL_mean": -224.90155029296875, "KL/mean": -285.91082763671875, "KL/rejected_KL_mean": -346.92010498046875, "KL/std": 273.02587890625, "epoch": 0.6832955404383976, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.01853942871094, "fcm_dpo/q_t": 0.4999881982803345, "grad_norm": 0.002451541367918253, "learning_rate": 1.3898334684855645e-07, "logits/chosen": 1.2452630996704102, "logits/rejected": 1.216700553894043, "logps/chosen": -280.4873962402344, "logps/ref_chosen": -55.58583450317383, "logps/ref_rejected": -77.68738555908203, "logps/rejected": -424.60748291015625, "loss": 1.3862, "margin_dpo/margin_mean": 122.01854705810547, "margin_dpo/margin_std": 345.8335266113281, "step": 452 }, { "KL/chosen_KL_mean": -220.40438842773438, "KL/mean": -267.64581298828125, "KL/rejected_KL_mean": -314.8872375488281, "KL/std": 251.55581665039062, "epoch": 0.6848072562358276, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.48284149169922, "fcm_dpo/q_t": 0.49999088048934937, "grad_norm": 0.0015818747924640775, "learning_rate": 1.3780020494988445e-07, "logits/chosen": 1.2517281770706177, "logits/rejected": 1.2492544651031494, "logps/chosen": -282.1825866699219, "logps/ref_chosen": -61.778202056884766, "logps/ref_rejected": -71.51403045654297, "logps/rejected": -386.4012451171875, "loss": 1.3863, "margin_dpo/margin_mean": 94.48284149169922, "margin_dpo/margin_std": 269.7317810058594, "step": 453 }, { "KL/chosen_KL_mean": -197.87136840820312, "KL/mean": -252.01133728027344, "KL/rejected_KL_mean": -306.15130615234375, "KL/std": 242.91844177246094, "epoch": 0.6863189720332578, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.2799301147461, "fcm_dpo/q_t": 0.49998950958251953, "grad_norm": 0.001828188425861299, "learning_rate": 1.366202015206706e-07, "logits/chosen": 1.2671639919281006, "logits/rejected": 1.2640118598937988, "logps/chosen": -249.46652221679688, "logps/ref_chosen": -51.59515380859375, "logps/ref_rejected": -63.96732711791992, "logps/rejected": -370.11865234375, "loss": 1.3863, "margin_dpo/margin_mean": 108.27993774414062, "margin_dpo/margin_std": 284.8143615722656, "step": 454 }, { "KL/chosen_KL_mean": -279.669677734375, "KL/mean": -327.62640380859375, "KL/rejected_KL_mean": -375.5830993652344, "KL/std": 286.3306884765625, "epoch": 0.6878306878306878, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.91340637207031, "fcm_dpo/q_t": 0.49999070167541504, "grad_norm": 0.0017985355807468295, "learning_rate": 1.354433695681474e-07, "logits/chosen": 1.2243902683258057, "logits/rejected": 1.225219964981079, "logps/chosen": -350.3214111328125, "logps/ref_chosen": -70.65170288085938, "logps/ref_rejected": -77.44276428222656, "logps/rejected": -453.02587890625, "loss": 1.3863, "margin_dpo/margin_mean": 95.91340637207031, "margin_dpo/margin_std": 302.83038330078125, "step": 455 }, { "KL/chosen_KL_mean": -215.04669189453125, "KL/mean": -274.91461181640625, "KL/rejected_KL_mean": -334.78253173828125, "KL/std": 239.27294921875, "epoch": 0.6893424036281179, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.73584747314453, "fcm_dpo/q_t": 0.4999884366989136, "grad_norm": 0.0024633186403661966, "learning_rate": 1.3426974201083439e-07, "logits/chosen": 1.2097394466400146, "logits/rejected": 1.1886322498321533, "logps/chosen": -271.4449768066406, "logps/ref_chosen": -56.398284912109375, "logps/ref_rejected": -82.61642456054688, "logps/rejected": -417.3989562988281, "loss": 1.3862, "margin_dpo/margin_mean": 119.73583984375, "margin_dpo/margin_std": 301.7488098144531, "step": 456 }, { "KL/chosen_KL_mean": -213.508544921875, "KL/mean": -266.53350830078125, "KL/rejected_KL_mean": -319.5584411621094, "KL/std": 283.9473876953125, "epoch": 0.690854119425548, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.04988098144531, "fcm_dpo/q_t": 0.49998974800109863, "grad_norm": 0.0022488024551421404, "learning_rate": 1.3309935167761717e-07, "logits/chosen": 1.3623318672180176, "logits/rejected": 1.318712592124939, "logps/chosen": -258.2291259765625, "logps/ref_chosen": -44.72057342529297, "logps/ref_rejected": -68.1158676147461, "logps/rejected": -387.67431640625, "loss": 1.3863, "margin_dpo/margin_mean": 106.04988861083984, "margin_dpo/margin_std": 333.2089538574219, "step": 457 }, { "KL/chosen_KL_mean": -188.3619384765625, "KL/mean": -288.2777404785156, "KL/rejected_KL_mean": -388.19354248046875, "KL/std": 289.7039489746094, "epoch": 0.6923658352229781, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 199.83157348632812, "fcm_dpo/q_t": 0.4999806582927704, "grad_norm": 0.00349022657610476, "learning_rate": 1.3193223130682936e-07, "logits/chosen": 1.2601938247680664, "logits/rejected": 1.2150036096572876, "logps/chosen": -238.36764526367188, "logps/ref_chosen": -50.00569152832031, "logps/ref_rejected": -87.50015258789062, "logps/rejected": -475.69366455078125, "loss": 1.3862, "margin_dpo/margin_mean": 199.83157348632812, "margin_dpo/margin_std": 349.6432800292969, "step": 458 }, { "KL/chosen_KL_mean": -256.993896484375, "KL/mean": -320.4532470703125, "KL/rejected_KL_mean": -383.91259765625, "KL/std": 375.80279541015625, "epoch": 0.6938775510204082, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.91870880126953, "fcm_dpo/q_t": 0.49998772144317627, "grad_norm": 0.002192882588133216, "learning_rate": 1.3076841354533658e-07, "logits/chosen": 1.2736434936523438, "logits/rejected": 1.2665081024169922, "logps/chosen": -322.371826171875, "logps/ref_chosen": -65.37794494628906, "logps/ref_rejected": -88.19244384765625, "logps/rejected": -472.10504150390625, "loss": 1.3862, "margin_dpo/margin_mean": 126.918701171875, "margin_dpo/margin_std": 348.869140625, "step": 459 }, { "KL/chosen_KL_mean": -235.994384765625, "KL/mean": -324.6895751953125, "KL/rejected_KL_mean": -413.38482666015625, "KL/std": 302.2080993652344, "epoch": 0.6953892668178382, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 177.39044189453125, "fcm_dpo/q_t": 0.4999828338623047, "grad_norm": 0.00325951655395329, "learning_rate": 1.2960793094762345e-07, "logits/chosen": 1.2858896255493164, "logits/rejected": 1.2416361570358276, "logps/chosen": -300.5560302734375, "logps/ref_chosen": -64.5616683959961, "logps/ref_rejected": -88.67890167236328, "logps/rejected": -502.063720703125, "loss": 1.3862, "margin_dpo/margin_mean": 177.39044189453125, "margin_dpo/margin_std": 368.12908935546875, "step": 460 }, { "KL/chosen_KL_mean": -190.54605102539062, "KL/mean": -272.08990478515625, "KL/rejected_KL_mean": -353.6337585449219, "KL/std": 280.8585510253906, "epoch": 0.6969009826152683, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 163.08770751953125, "fcm_dpo/q_t": 0.4999842047691345, "grad_norm": 0.0028130814898759127, "learning_rate": 1.2845081597488286e-07, "logits/chosen": 1.2972333431243896, "logits/rejected": 1.270198106765747, "logps/chosen": -240.02395629882812, "logps/ref_chosen": -49.4779167175293, "logps/ref_rejected": -72.65262603759766, "logps/rejected": -426.286376953125, "loss": 1.3862, "margin_dpo/margin_mean": 163.08770751953125, "margin_dpo/margin_std": 284.0039978027344, "step": 461 }, { "KL/chosen_KL_mean": -188.55178833007812, "KL/mean": -252.95437622070312, "KL/rejected_KL_mean": -317.35693359375, "KL/std": 175.87823486328125, "epoch": 0.6984126984126984, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.80517578125, "fcm_dpo/q_t": 0.49998754262924194, "grad_norm": 0.002148764207959175, "learning_rate": 1.27297100994108e-07, "logits/chosen": 1.276878833770752, "logits/rejected": 1.2633922100067139, "logps/chosen": -249.04689025878906, "logps/ref_chosen": -60.4951171875, "logps/ref_rejected": -74.82136535644531, "logps/rejected": -392.1783142089844, "loss": 1.3862, "margin_dpo/margin_mean": 128.80517578125, "margin_dpo/margin_std": 224.19122314453125, "step": 462 }, { "KL/chosen_KL_mean": -273.58941650390625, "KL/mean": -293.7682800292969, "KL/rejected_KL_mean": -313.9471435546875, "KL/std": 260.3634033203125, "epoch": 0.6999244142101285, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 40.35773468017578, "fcm_dpo/q_t": 0.4999960660934448, "grad_norm": 0.0018327016150578856, "learning_rate": 1.2614681827718695e-07, "logits/chosen": 1.2572622299194336, "logits/rejected": 1.2583889961242676, "logps/chosen": -341.2745361328125, "logps/ref_chosen": -67.68511962890625, "logps/ref_rejected": -71.32196044921875, "logps/rejected": -385.26910400390625, "loss": 1.3863, "margin_dpo/margin_mean": 40.35773468017578, "margin_dpo/margin_std": 348.34228515625, "step": 463 }, { "KL/chosen_KL_mean": -230.39385986328125, "KL/mean": -282.303466796875, "KL/rejected_KL_mean": -334.21307373046875, "KL/std": 285.50360107421875, "epoch": 0.7014361300075586, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 103.8191909790039, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.0019776602275669575, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 1.2877094745635986, "logits/rejected": 1.2803895473480225, "logps/chosen": -289.55950927734375, "logps/ref_chosen": -59.16564178466797, "logps/ref_rejected": -69.56146240234375, "logps/rejected": -403.7745361328125, "loss": 1.3863, "margin_dpo/margin_mean": 103.81919860839844, "margin_dpo/margin_std": 348.489501953125, "step": 464 }, { "KL/chosen_KL_mean": -232.89459228515625, "KL/mean": -298.0108337402344, "KL/rejected_KL_mean": -363.1270751953125, "KL/std": 265.3921813964844, "epoch": 0.7029478458049887, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.23245239257812, "fcm_dpo/q_t": 0.4999874234199524, "grad_norm": 0.0023306291550397873, "learning_rate": 1.238566782415197e-07, "logits/chosen": 1.3072185516357422, "logits/rejected": 1.2831568717956543, "logps/chosen": -291.40826416015625, "logps/ref_chosen": -58.513671875, "logps/ref_rejected": -84.31745910644531, "logps/rejected": -447.44451904296875, "loss": 1.3862, "margin_dpo/margin_mean": 130.23245239257812, "margin_dpo/margin_std": 334.58380126953125, "step": 465 }, { "KL/chosen_KL_mean": -299.5732421875, "KL/mean": -299.2271728515625, "KL/rejected_KL_mean": -298.881103515625, "KL/std": 311.90045166015625, "epoch": 0.7044595616024187, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.6921577453613281, "fcm_dpo/q_t": 0.5000000596046448, "grad_norm": 0.001482538296841085, "learning_rate": 1.2271688498291334e-07, "logits/chosen": 1.261409878730774, "logits/rejected": 1.2607641220092773, "logps/chosen": -372.83905029296875, "logps/ref_chosen": -73.26580810546875, "logps/ref_rejected": -74.83621215820312, "logps/rejected": -373.71728515625, "loss": 1.3863, "margin_dpo/margin_mean": -0.6921501159667969, "margin_dpo/margin_std": 345.80474853515625, "step": 466 }, { "KL/chosen_KL_mean": -198.47096252441406, "KL/mean": -302.1253356933594, "KL/rejected_KL_mean": -405.77972412109375, "KL/std": 344.73516845703125, "epoch": 0.7059712773998488, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 207.30874633789062, "fcm_dpo/q_t": 0.4999799430370331, "grad_norm": 0.004299922846257687, "learning_rate": 1.2158065210664848e-07, "logits/chosen": 1.2816591262817383, "logits/rejected": 1.2331783771514893, "logps/chosen": -246.05044555664062, "logps/ref_chosen": -47.57947540283203, "logps/ref_rejected": -78.68522644042969, "logps/rejected": -484.4649353027344, "loss": 1.3862, "margin_dpo/margin_mean": 207.30874633789062, "margin_dpo/margin_std": 427.2309265136719, "step": 467 }, { "KL/chosen_KL_mean": -241.93890380859375, "KL/mean": -287.9136962890625, "KL/rejected_KL_mean": -333.88848876953125, "KL/std": 330.4139404296875, "epoch": 0.7074829931972789, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.94956970214844, "fcm_dpo/q_t": 0.4999910891056061, "grad_norm": 0.0016830574022606015, "learning_rate": 1.204480113956011e-07, "logits/chosen": 1.2580692768096924, "logits/rejected": 1.2581918239593506, "logps/chosen": -305.86669921875, "logps/ref_chosen": -63.92778778076172, "logps/ref_rejected": -76.51626586914062, "logps/rejected": -410.40472412109375, "loss": 1.3863, "margin_dpo/margin_mean": 91.94956970214844, "margin_dpo/margin_std": 405.082275390625, "step": 468 }, { "KL/chosen_KL_mean": -257.592529296875, "KL/mean": -292.6340026855469, "KL/rejected_KL_mean": -327.6754455566406, "KL/std": 307.759521484375, "epoch": 0.708994708994709, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.08292388916016, "fcm_dpo/q_t": 0.4999932050704956, "grad_norm": 0.0019712212961167097, "learning_rate": 1.1931899453216697e-07, "logits/chosen": 1.2918429374694824, "logits/rejected": 1.2837411165237427, "logps/chosen": -316.65069580078125, "logps/ref_chosen": -59.05818176269531, "logps/ref_rejected": -75.67672729492188, "logps/rejected": -403.3521728515625, "loss": 1.3863, "margin_dpo/margin_mean": 70.08292388916016, "margin_dpo/margin_std": 339.2579345703125, "step": 469 }, { "KL/chosen_KL_mean": -186.16371154785156, "KL/mean": -246.60861206054688, "KL/rejected_KL_mean": -307.05352783203125, "KL/std": 228.65818786621094, "epoch": 0.7105064247921391, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.88981628417969, "fcm_dpo/q_t": 0.499988317489624, "grad_norm": 0.002206693636253476, "learning_rate": 1.1819363309737438e-07, "logits/chosen": 1.2765345573425293, "logits/rejected": 1.256923794746399, "logps/chosen": -234.03114318847656, "logps/ref_chosen": -47.86743927001953, "logps/ref_rejected": -65.96859741210938, "logps/rejected": -373.0220947265625, "loss": 1.3862, "margin_dpo/margin_mean": 120.88981628417969, "margin_dpo/margin_std": 278.8352966308594, "step": 470 }, { "KL/chosen_KL_mean": -218.08656311035156, "KL/mean": -278.8359680175781, "KL/rejected_KL_mean": -339.5853576660156, "KL/std": 286.42633056640625, "epoch": 0.7120181405895691, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.498779296875, "fcm_dpo/q_t": 0.49998825788497925, "grad_norm": 0.0023722737096250057, "learning_rate": 1.1707195857000215e-07, "logits/chosen": 1.238015055656433, "logits/rejected": 1.219024658203125, "logps/chosen": -275.8644104003906, "logps/ref_chosen": -57.777854919433594, "logps/ref_rejected": -73.81172180175781, "logps/rejected": -413.3970947265625, "loss": 1.3862, "margin_dpo/margin_mean": 121.498779296875, "margin_dpo/margin_std": 337.14422607421875, "step": 471 }, { "KL/chosen_KL_mean": -214.7991485595703, "KL/mean": -261.65557861328125, "KL/rejected_KL_mean": -308.5119934082031, "KL/std": 277.4007568359375, "epoch": 0.7135298563869993, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 93.71286010742188, "fcm_dpo/q_t": 0.49999094009399414, "grad_norm": 0.0018467764602974057, "learning_rate": 1.1595400232569768e-07, "logits/chosen": 1.2192950248718262, "logits/rejected": 1.2010465860366821, "logps/chosen": -270.70782470703125, "logps/ref_chosen": -55.908668518066406, "logps/ref_rejected": -74.70294189453125, "logps/rejected": -383.2149353027344, "loss": 1.3863, "margin_dpo/margin_mean": 93.71286010742188, "margin_dpo/margin_std": 368.9359130859375, "step": 472 }, { "KL/chosen_KL_mean": -204.37344360351562, "KL/mean": -304.8914489746094, "KL/rejected_KL_mean": -405.40948486328125, "KL/std": 354.4217529296875, "epoch": 0.7150415721844293, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 201.0360107421875, "fcm_dpo/q_t": 0.49998053908348083, "grad_norm": 0.003650173544883728, "learning_rate": 1.1483979563610069e-07, "logits/chosen": 1.3033931255340576, "logits/rejected": 1.2516118288040161, "logps/chosen": -258.5343322753906, "logps/ref_chosen": -54.16088104248047, "logps/ref_rejected": -92.76789855957031, "logps/rejected": -498.1773681640625, "loss": 1.3862, "margin_dpo/margin_mean": 201.03599548339844, "margin_dpo/margin_std": 407.78076171875, "step": 473 }, { "KL/chosen_KL_mean": -185.13568115234375, "KL/mean": -263.2508544921875, "KL/rejected_KL_mean": -341.36602783203125, "KL/std": 283.1648254394531, "epoch": 0.7165532879818595, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.2303466796875, "fcm_dpo/q_t": 0.49998486042022705, "grad_norm": 0.0029000849463045597, "learning_rate": 1.1372936966796709e-07, "logits/chosen": 1.295190691947937, "logits/rejected": 1.2742294073104858, "logps/chosen": -231.82138061523438, "logps/ref_chosen": -46.685707092285156, "logps/ref_rejected": -71.44731903076172, "logps/rejected": -412.8133544921875, "loss": 1.3862, "margin_dpo/margin_mean": 156.2303466796875, "margin_dpo/margin_std": 273.30877685546875, "step": 474 }, { "KL/chosen_KL_mean": -244.22735595703125, "KL/mean": -307.16156005859375, "KL/rejected_KL_mean": -370.0957946777344, "KL/std": 284.45855712890625, "epoch": 0.7180650037792895, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 125.86846160888672, "fcm_dpo/q_t": 0.4999878406524658, "grad_norm": 0.002193914959207177, "learning_rate": 1.126227554822985e-07, "logits/chosen": 1.2467389106750488, "logits/rejected": 1.2255311012268066, "logps/chosen": -302.71466064453125, "logps/ref_chosen": -58.4873046875, "logps/ref_rejected": -87.00187683105469, "logps/rejected": -457.0976867675781, "loss": 1.3862, "margin_dpo/margin_mean": 125.86845397949219, "margin_dpo/margin_std": 340.77777099609375, "step": 475 }, { "KL/chosen_KL_mean": -278.6739501953125, "KL/mean": -314.0935363769531, "KL/rejected_KL_mean": -349.51312255859375, "KL/std": 314.4804382324219, "epoch": 0.7195767195767195, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 70.83914184570312, "fcm_dpo/q_t": 0.49999314546585083, "grad_norm": 0.001390301389619708, "learning_rate": 1.1151998403347243e-07, "logits/chosen": 1.2042824029922485, "logits/rejected": 1.2116657495498657, "logps/chosen": -354.05560302734375, "logps/ref_chosen": -75.38162231445312, "logps/ref_rejected": -76.99822235107422, "logps/rejected": -426.5113525390625, "loss": 1.3863, "margin_dpo/margin_mean": 70.83914184570312, "margin_dpo/margin_std": 338.26055908203125, "step": 476 }, { "KL/chosen_KL_mean": -237.08114624023438, "KL/mean": -295.21539306640625, "KL/rejected_KL_mean": -353.34967041015625, "KL/std": 313.53399658203125, "epoch": 0.7210884353741497, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.26850891113281, "fcm_dpo/q_t": 0.49998873472213745, "grad_norm": 0.0024357279762625694, "learning_rate": 1.1042108616837692e-07, "logits/chosen": 1.2554326057434082, "logits/rejected": 1.2339025735855103, "logps/chosen": -298.154541015625, "logps/ref_chosen": -61.073387145996094, "logps/ref_rejected": -81.34375, "logps/rejected": -434.69342041015625, "loss": 1.3862, "margin_dpo/margin_mean": 116.26850891113281, "margin_dpo/margin_std": 433.21453857421875, "step": 477 }, { "KL/chosen_KL_mean": -206.6198272705078, "KL/mean": -230.17007446289062, "KL/rejected_KL_mean": -253.7202911376953, "KL/std": 195.4527130126953, "epoch": 0.7226001511715797, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 47.100467681884766, "fcm_dpo/q_t": 0.4999954402446747, "grad_norm": 0.0013403815682977438, "learning_rate": 1.0932609262554746e-07, "logits/chosen": 1.2230312824249268, "logits/rejected": 1.2336747646331787, "logps/chosen": -263.7871398925781, "logps/ref_chosen": -57.16731643676758, "logps/ref_rejected": -53.30917739868164, "logps/rejected": -307.02947998046875, "loss": 1.3863, "margin_dpo/margin_mean": 47.1004638671875, "margin_dpo/margin_std": 252.40872192382812, "step": 478 }, { "KL/chosen_KL_mean": -214.67568969726562, "KL/mean": -258.00732421875, "KL/rejected_KL_mean": -301.3389892578125, "KL/std": 248.02645874023438, "epoch": 0.7241118669690099, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 86.66331481933594, "fcm_dpo/q_t": 0.49999159574508667, "grad_norm": 0.0024421438574790955, "learning_rate": 1.0823503403430734e-07, "logits/chosen": 1.2256994247436523, "logits/rejected": 1.2227530479431152, "logps/chosen": -273.5889892578125, "logps/ref_chosen": -58.91331481933594, "logps/ref_rejected": -63.7403450012207, "logps/rejected": -365.079345703125, "loss": 1.3863, "margin_dpo/margin_mean": 86.66331481933594, "margin_dpo/margin_std": 288.06201171875, "step": 479 }, { "KL/chosen_KL_mean": -232.02232360839844, "KL/mean": -267.8770446777344, "KL/rejected_KL_mean": -303.73175048828125, "KL/std": 240.46432495117188, "epoch": 0.7256235827664399, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.70942687988281, "fcm_dpo/q_t": 0.49999308586120605, "grad_norm": 0.0013092632871121168, "learning_rate": 1.0714794091391072e-07, "logits/chosen": 1.234184741973877, "logits/rejected": 1.2392432689666748, "logps/chosen": -294.82293701171875, "logps/ref_chosen": -62.80061340332031, "logps/ref_rejected": -67.58859252929688, "logps/rejected": -371.3203430175781, "loss": 1.3863, "margin_dpo/margin_mean": 71.70941925048828, "margin_dpo/margin_std": 270.3521728515625, "step": 480 }, { "KL/chosen_KL_mean": -257.1699523925781, "KL/mean": -295.2996826171875, "KL/rejected_KL_mean": -333.42950439453125, "KL/std": 286.9197082519531, "epoch": 0.72713529856387, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.25953674316406, "fcm_dpo/q_t": 0.49999263882637024, "grad_norm": 0.0014908494194969535, "learning_rate": 1.0606484367268906e-07, "logits/chosen": 1.2494676113128662, "logits/rejected": 1.257669448852539, "logps/chosen": -322.4564208984375, "logps/ref_chosen": -65.28649139404297, "logps/ref_rejected": -70.78668212890625, "logps/rejected": -404.2161865234375, "loss": 1.3863, "margin_dpo/margin_mean": 76.2595443725586, "margin_dpo/margin_std": 337.06689453125, "step": 481 }, { "KL/chosen_KL_mean": -263.18341064453125, "KL/mean": -354.302978515625, "KL/rejected_KL_mean": -445.4225158691406, "KL/std": 386.18914794921875, "epoch": 0.7286470143613001, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 182.2390594482422, "fcm_dpo/q_t": 0.4999823570251465, "grad_norm": 0.003877797396853566, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 1.230767011642456, "logits/rejected": 1.1534785032272339, "logps/chosen": -324.089599609375, "logps/ref_chosen": -60.906185150146484, "logps/ref_rejected": -103.44656372070312, "logps/rejected": -548.8690795898438, "loss": 1.3862, "margin_dpo/margin_mean": 182.23907470703125, "margin_dpo/margin_std": 488.2938232421875, "step": 482 }, { "KL/chosen_KL_mean": -213.94471740722656, "KL/mean": -296.33514404296875, "KL/rejected_KL_mean": -378.7255859375, "KL/std": 284.4780578613281, "epoch": 0.7301587301587301, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 164.78086853027344, "fcm_dpo/q_t": 0.4999840557575226, "grad_norm": 0.0031552868895232677, "learning_rate": 1.0391075790138232e-07, "logits/chosen": 1.2734465599060059, "logits/rejected": 1.2415012121200562, "logps/chosen": -267.13671875, "logps/ref_chosen": -53.192012786865234, "logps/ref_rejected": -81.83927154541016, "logps/rejected": -460.56488037109375, "loss": 1.3862, "margin_dpo/margin_mean": 164.78086853027344, "margin_dpo/margin_std": 341.28466796875, "step": 483 }, { "KL/chosen_KL_mean": -244.69627380371094, "KL/mean": -283.5141296386719, "KL/rejected_KL_mean": -322.3320007324219, "KL/std": 284.9932861328125, "epoch": 0.7316704459561603, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.6357421875, "fcm_dpo/q_t": 0.4999924898147583, "grad_norm": 0.0018454925157129765, "learning_rate": 1.0283982962570681e-07, "logits/chosen": 1.2972228527069092, "logits/rejected": 1.2862370014190674, "logps/chosen": -302.4657287597656, "logps/ref_chosen": -57.76945877075195, "logps/ref_rejected": -71.6829833984375, "logps/rejected": -394.0149841308594, "loss": 1.3863, "margin_dpo/margin_mean": 77.6357421875, "margin_dpo/margin_std": 290.02801513671875, "step": 484 }, { "KL/chosen_KL_mean": -232.70870971679688, "KL/mean": -280.1761779785156, "KL/rejected_KL_mean": -327.6436462402344, "KL/std": 277.9726257324219, "epoch": 0.7331821617535903, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.93495178222656, "fcm_dpo/q_t": 0.4999908208847046, "grad_norm": 0.0017475350759923458, "learning_rate": 1.0177301773633992e-07, "logits/chosen": 1.2424696683883667, "logits/rejected": 1.2406383752822876, "logps/chosen": -289.34454345703125, "logps/ref_chosen": -56.63584899902344, "logps/ref_rejected": -70.85614013671875, "logps/rejected": -398.4997863769531, "loss": 1.3863, "margin_dpo/margin_mean": 94.9349594116211, "margin_dpo/margin_std": 307.6720275878906, "step": 485 }, { "KL/chosen_KL_mean": -219.482666015625, "KL/mean": -304.8910827636719, "KL/rejected_KL_mean": -390.29949951171875, "KL/std": 327.8187255859375, "epoch": 0.7346938775510204, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.81683349609375, "fcm_dpo/q_t": 0.49998345971107483, "grad_norm": 0.00316282594576478, "learning_rate": 1.007103520743035e-07, "logits/chosen": 1.260047435760498, "logits/rejected": 1.2206038236618042, "logps/chosen": -275.8296813964844, "logps/ref_chosen": -56.347023010253906, "logps/ref_rejected": -85.97221374511719, "logps/rejected": -476.271728515625, "loss": 1.3862, "margin_dpo/margin_mean": 170.8168487548828, "margin_dpo/margin_std": 370.9302978515625, "step": 486 }, { "KL/chosen_KL_mean": -245.201416015625, "KL/mean": -311.9969787597656, "KL/rejected_KL_mean": -378.79254150390625, "KL/std": 356.8734130859375, "epoch": 0.7362055933484505, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.5911407470703, "fcm_dpo/q_t": 0.49998706579208374, "grad_norm": 0.0026520034298300743, "learning_rate": 9.965186236464046e-08, "logits/chosen": 1.3191529512405396, "logits/rejected": 1.297263503074646, "logps/chosen": -305.8186340332031, "logps/ref_chosen": -60.617218017578125, "logps/ref_rejected": -82.50975036621094, "logps/rejected": -461.3022766113281, "loss": 1.3862, "margin_dpo/margin_mean": 133.59112548828125, "margin_dpo/margin_std": 416.29364013671875, "step": 487 }, { "KL/chosen_KL_mean": -238.37918090820312, "KL/mean": -309.9017333984375, "KL/rejected_KL_mean": -381.42431640625, "KL/std": 288.9468078613281, "epoch": 0.7377173091458806, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 143.04513549804688, "fcm_dpo/q_t": 0.4999861717224121, "grad_norm": 0.0026024070102721453, "learning_rate": 9.859757821558337e-08, "logits/chosen": 1.2986162900924683, "logits/rejected": 1.2784430980682373, "logps/chosen": -301.48822021484375, "logps/ref_chosen": -63.10905075073242, "logps/ref_rejected": -82.49348449707031, "logps/rejected": -463.91778564453125, "loss": 1.3862, "margin_dpo/margin_mean": 143.04513549804688, "margin_dpo/margin_std": 348.89385986328125, "step": 488 }, { "KL/chosen_KL_mean": -275.52362060546875, "KL/mean": -324.05682373046875, "KL/rejected_KL_mean": -372.5899658203125, "KL/std": 305.4133605957031, "epoch": 0.7392290249433107, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 97.06634521484375, "fcm_dpo/q_t": 0.4999906122684479, "grad_norm": 0.0021042677108198404, "learning_rate": 9.754752911772615e-08, "logits/chosen": 1.2624127864837646, "logits/rejected": 1.245917797088623, "logps/chosen": -340.5125732421875, "logps/ref_chosen": -64.98896026611328, "logps/ref_rejected": -84.39607238769531, "logps/rejected": -456.98602294921875, "loss": 1.3863, "margin_dpo/margin_mean": 97.06634521484375, "margin_dpo/margin_std": 349.16461181640625, "step": 489 }, { "KL/chosen_KL_mean": -234.01315307617188, "KL/mean": -281.6185607910156, "KL/rejected_KL_mean": -329.2239990234375, "KL/std": 286.2935791015625, "epoch": 0.7407407407407407, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.21084594726562, "fcm_dpo/q_t": 0.4999907612800598, "grad_norm": 0.001769617898389697, "learning_rate": 9.650174444319956e-08, "logits/chosen": 1.2830909490585327, "logits/rejected": 1.288257360458374, "logps/chosen": -295.921875, "logps/ref_chosen": -61.90874481201172, "logps/ref_rejected": -70.58566284179688, "logps/rejected": -399.80963134765625, "loss": 1.3863, "margin_dpo/margin_mean": 95.21085357666016, "margin_dpo/margin_std": 358.0466613769531, "step": 490 }, { "KL/chosen_KL_mean": -204.4666748046875, "KL/mean": -257.9485778808594, "KL/rejected_KL_mean": -311.43048095703125, "KL/std": 246.65692138671875, "epoch": 0.7422524565381708, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 106.96379852294922, "fcm_dpo/q_t": 0.4999896287918091, "grad_norm": 0.0019353864481672645, "learning_rate": 9.546025344484868e-08, "logits/chosen": 1.2297401428222656, "logits/rejected": 1.202380895614624, "logps/chosen": -259.9423828125, "logps/ref_chosen": -55.47570037841797, "logps/ref_rejected": -78.70318603515625, "logps/rejected": -390.1336669921875, "loss": 1.3863, "margin_dpo/margin_mean": 106.96379852294922, "margin_dpo/margin_std": 309.4108581542969, "step": 491 }, { "KL/chosen_KL_mean": -274.214599609375, "KL/mean": -337.2662353515625, "KL/rejected_KL_mean": -400.31787109375, "KL/std": 328.02960205078125, "epoch": 0.7437641723356009, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.1032943725586, "fcm_dpo/q_t": 0.49998778104782104, "grad_norm": 0.002519700676202774, "learning_rate": 9.442308525541589e-08, "logits/chosen": 1.2626538276672363, "logits/rejected": 1.2385413646697998, "logps/chosen": -341.5009765625, "logps/ref_chosen": -67.28638458251953, "logps/ref_rejected": -82.78628540039062, "logps/rejected": -483.10418701171875, "loss": 1.3862, "margin_dpo/margin_mean": 126.1032943725586, "margin_dpo/margin_std": 430.36181640625, "step": 492 }, { "KL/chosen_KL_mean": -193.55007934570312, "KL/mean": -267.8668212890625, "KL/rejected_KL_mean": -342.18353271484375, "KL/std": 282.44659423828125, "epoch": 0.745275888133031, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.63345336914062, "fcm_dpo/q_t": 0.49998563528060913, "grad_norm": 0.0027879553381353617, "learning_rate": 9.339026888672468e-08, "logits/chosen": 1.2619478702545166, "logits/rejected": 1.2343031167984009, "logps/chosen": -249.47760009765625, "logps/ref_chosen": -55.92750549316406, "logps/ref_rejected": -79.12149810791016, "logps/rejected": -421.3050537109375, "loss": 1.3862, "margin_dpo/margin_mean": 148.63343811035156, "margin_dpo/margin_std": 357.33514404296875, "step": 493 }, { "KL/chosen_KL_mean": -263.4035949707031, "KL/mean": -321.43548583984375, "KL/rejected_KL_mean": -379.4674072265625, "KL/std": 349.48931884765625, "epoch": 0.7467876039304611, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.06379699707031, "fcm_dpo/q_t": 0.4999887943267822, "grad_norm": 0.002763553289696574, "learning_rate": 9.236183322886945e-08, "logits/chosen": 1.210706114768982, "logits/rejected": 1.1937711238861084, "logps/chosen": -331.3576965332031, "logps/ref_chosen": -67.95410919189453, "logps/ref_rejected": -90.50865173339844, "logps/rejected": -469.9760437011719, "loss": 1.3862, "margin_dpo/margin_mean": 116.06378936767578, "margin_dpo/margin_std": 462.98712158203125, "step": 494 }, { "KL/chosen_KL_mean": -195.76187133789062, "KL/mean": -256.2266540527344, "KL/rejected_KL_mean": -316.69140625, "KL/std": 313.70343017578125, "epoch": 0.7482993197278912, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.9295654296875, "fcm_dpo/q_t": 0.49998828768730164, "grad_norm": 0.002702318364754319, "learning_rate": 9.133780704940594e-08, "logits/chosen": 1.2848858833312988, "logits/rejected": 1.2627146244049072, "logps/chosen": -248.3873291015625, "logps/ref_chosen": -52.62546157836914, "logps/ref_rejected": -72.06781005859375, "logps/rejected": -388.75921630859375, "loss": 1.3862, "margin_dpo/margin_mean": 120.92955780029297, "margin_dpo/margin_std": 426.9603271484375, "step": 495 }, { "KL/chosen_KL_mean": -217.66781616210938, "KL/mean": -307.02685546875, "KL/rejected_KL_mean": -396.3858642578125, "KL/std": 350.0010070800781, "epoch": 0.7498110355253212, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 178.71804809570312, "fcm_dpo/q_t": 0.49998271465301514, "grad_norm": 0.003732392331585288, "learning_rate": 9.031821899254797e-08, "logits/chosen": 1.2494525909423828, "logits/rejected": 1.1938560009002686, "logps/chosen": -275.26513671875, "logps/ref_chosen": -57.597320556640625, "logps/ref_rejected": -94.36127471923828, "logps/rejected": -490.7471618652344, "loss": 1.3862, "margin_dpo/margin_mean": 178.71804809570312, "margin_dpo/margin_std": 492.61053466796875, "step": 496 }, { "KL/chosen_KL_mean": -261.3085021972656, "KL/mean": -313.65008544921875, "KL/rejected_KL_mean": -365.99163818359375, "KL/std": 278.7204284667969, "epoch": 0.7513227513227513, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.68314361572266, "fcm_dpo/q_t": 0.4999898672103882, "grad_norm": 0.002317574806511402, "learning_rate": 8.930309757836516e-08, "logits/chosen": 1.2523890733718872, "logits/rejected": 1.24021315574646, "logps/chosen": -334.09844970703125, "logps/ref_chosen": -72.78994750976562, "logps/ref_rejected": -89.48483276367188, "logps/rejected": -455.4764709472656, "loss": 1.3863, "margin_dpo/margin_mean": 104.68313598632812, "margin_dpo/margin_std": 338.248291015625, "step": 497 }, { "KL/chosen_KL_mean": -224.85238647460938, "KL/mean": -278.84197998046875, "KL/rejected_KL_mean": -332.83160400390625, "KL/std": 270.00634765625, "epoch": 0.7528344671201814, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.97920989990234, "fcm_dpo/q_t": 0.4999895691871643, "grad_norm": 0.0020106714218854904, "learning_rate": 8.829247120198563e-08, "logits/chosen": 1.2464903593063354, "logits/rejected": 1.2533658742904663, "logps/chosen": -293.21807861328125, "logps/ref_chosen": -68.36572265625, "logps/ref_rejected": -71.28846740722656, "logps/rejected": -404.1200866699219, "loss": 1.3863, "margin_dpo/margin_mean": 107.97921752929688, "margin_dpo/margin_std": 352.33111572265625, "step": 498 }, { "KL/chosen_KL_mean": -242.342529296875, "KL/mean": -309.7532653808594, "KL/rejected_KL_mean": -377.1640319824219, "KL/std": 302.4976806640625, "epoch": 0.7543461829176115, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.82147216796875, "fcm_dpo/q_t": 0.4999869465827942, "grad_norm": 0.0024030758067965508, "learning_rate": 8.728636813280163e-08, "logits/chosen": 1.269078016281128, "logits/rejected": 1.2355639934539795, "logps/chosen": -304.2513427734375, "logps/ref_chosen": -61.90882873535156, "logps/ref_rejected": -91.9411392211914, "logps/rejected": -469.10516357421875, "loss": 1.3862, "margin_dpo/margin_mean": 134.82147216796875, "margin_dpo/margin_std": 416.40582275390625, "step": 499 }, { "KL/chosen_KL_mean": -253.39688110351562, "KL/mean": -294.63592529296875, "KL/rejected_KL_mean": -335.87493896484375, "KL/std": 297.90802001953125, "epoch": 0.7558578987150416, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 82.47804260253906, "fcm_dpo/q_t": 0.4999920129776001, "grad_norm": 0.0017515573417767882, "learning_rate": 8.628481651367875e-08, "logits/chosen": 1.25115966796875, "logits/rejected": 1.267160415649414, "logps/chosen": -323.62274169921875, "logps/ref_chosen": -70.225830078125, "logps/ref_rejected": -71.72203063964844, "logps/rejected": -407.59698486328125, "loss": 1.3863, "margin_dpo/margin_mean": 82.47804260253906, "margin_dpo/margin_std": 377.62408447265625, "step": 500 }, { "KL/chosen_KL_mean": -261.0697326660156, "KL/mean": -288.06939697265625, "KL/rejected_KL_mean": -315.069091796875, "KL/std": 263.5559387207031, "epoch": 0.7573696145124716, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.99939727783203, "fcm_dpo/q_t": 0.49999478459358215, "grad_norm": 0.0011944005964323878, "learning_rate": 8.528784436016878e-08, "logits/chosen": 1.2886323928833008, "logits/rejected": 1.289060354232788, "logps/chosen": -325.66851806640625, "logps/ref_chosen": -64.59880828857422, "logps/ref_rejected": -70.59329223632812, "logps/rejected": -385.66241455078125, "loss": 1.3863, "margin_dpo/margin_mean": 53.99940490722656, "margin_dpo/margin_std": 239.36679077148438, "step": 501 }, { "KL/chosen_KL_mean": -269.3963317871094, "KL/mean": -318.6138916015625, "KL/rejected_KL_mean": -367.83148193359375, "KL/std": 324.662841796875, "epoch": 0.7588813303099018, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.43511962890625, "fcm_dpo/q_t": 0.49999046325683594, "grad_norm": 0.0020151876378804445, "learning_rate": 8.4295479559726e-08, "logits/chosen": 1.2497098445892334, "logits/rejected": 1.2299872636795044, "logps/chosen": -334.86297607421875, "logps/ref_chosen": -65.46662902832031, "logps/ref_rejected": -90.22233581542969, "logps/rejected": -458.0538024902344, "loss": 1.3863, "margin_dpo/margin_mean": 98.43512725830078, "margin_dpo/margin_std": 369.150634765625, "step": 502 }, { "KL/chosen_KL_mean": -190.64247131347656, "KL/mean": -217.7227783203125, "KL/rejected_KL_mean": -244.80307006835938, "KL/std": 184.62762451171875, "epoch": 0.7603930461073318, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 54.16061019897461, "fcm_dpo/q_t": 0.49999475479125977, "grad_norm": 0.0011236389400437474, "learning_rate": 8.330774987092712e-08, "logits/chosen": 1.2540428638458252, "logits/rejected": 1.2642710208892822, "logps/chosen": -242.47723388671875, "logps/ref_chosen": -51.83476257324219, "logps/ref_rejected": -57.62522506713867, "logps/rejected": -302.4283142089844, "loss": 1.3863, "margin_dpo/margin_mean": 54.160614013671875, "margin_dpo/margin_std": 232.08360290527344, "step": 503 }, { "KL/chosen_KL_mean": -230.22885131835938, "KL/mean": -277.79840087890625, "KL/rejected_KL_mean": -325.367919921875, "KL/std": 244.9968719482422, "epoch": 0.7619047619047619, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 95.13908386230469, "fcm_dpo/q_t": 0.4999907910823822, "grad_norm": 0.0015835659578442574, "learning_rate": 8.232468292269479e-08, "logits/chosen": 1.2332806587219238, "logits/rejected": 1.2388975620269775, "logps/chosen": -298.88006591796875, "logps/ref_chosen": -68.65119934082031, "logps/ref_rejected": -77.91394805908203, "logps/rejected": -403.2818908691406, "loss": 1.3863, "margin_dpo/margin_mean": 95.13908386230469, "margin_dpo/margin_std": 298.76800537109375, "step": 504 }, { "KL/chosen_KL_mean": -239.896484375, "KL/mean": -291.92791748046875, "KL/rejected_KL_mean": -343.9593200683594, "KL/std": 342.92205810546875, "epoch": 0.763416477702192, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.06285095214844, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.002278943546116352, "learning_rate": 8.134630621352483e-08, "logits/chosen": 1.2496414184570312, "logits/rejected": 1.2356963157653809, "logps/chosen": -299.89532470703125, "logps/ref_chosen": -59.99884796142578, "logps/ref_rejected": -76.88048553466797, "logps/rejected": -420.8398132324219, "loss": 1.3863, "margin_dpo/margin_mean": 104.06285095214844, "margin_dpo/margin_std": 399.6540222167969, "step": 505 }, { "KL/chosen_KL_mean": -271.675048828125, "KL/mean": -315.27374267578125, "KL/rejected_KL_mean": -358.87237548828125, "KL/std": 294.46759033203125, "epoch": 0.764928193499622, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.19731903076172, "fcm_dpo/q_t": 0.4999915361404419, "grad_norm": 0.0017523845890536904, "learning_rate": 8.037264711071698e-08, "logits/chosen": 1.2550417184829712, "logits/rejected": 1.2555642127990723, "logps/chosen": -341.746337890625, "logps/ref_chosen": -70.07130432128906, "logps/ref_rejected": -82.03775024414062, "logps/rejected": -440.9101257324219, "loss": 1.3863, "margin_dpo/margin_mean": 87.19731903076172, "margin_dpo/margin_std": 383.368896484375, "step": 506 }, { "KL/chosen_KL_mean": -286.790283203125, "KL/mean": -347.9385986328125, "KL/rejected_KL_mean": -409.0868835449219, "KL/std": 353.75408935546875, "epoch": 0.7664399092970522, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 122.29658508300781, "fcm_dpo/q_t": 0.4999881684780121, "grad_norm": 0.0025757874827831984, "learning_rate": 7.940373284960933e-08, "logits/chosen": 1.2367452383041382, "logits/rejected": 1.2185437679290771, "logps/chosen": -358.79730224609375, "logps/ref_chosen": -72.00703430175781, "logps/ref_rejected": -93.94987487792969, "logps/rejected": -503.0367431640625, "loss": 1.3862, "margin_dpo/margin_mean": 122.29658508300781, "margin_dpo/margin_std": 428.36688232421875, "step": 507 }, { "KL/chosen_KL_mean": -234.4129638671875, "KL/mean": -341.6184387207031, "KL/rejected_KL_mean": -448.82391357421875, "KL/std": 360.1383056640625, "epoch": 0.7679516250944822, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 214.41094970703125, "fcm_dpo/q_t": 0.49997925758361816, "grad_norm": 0.004092069808393717, "learning_rate": 7.843959053281663e-08, "logits/chosen": 1.2509568929672241, "logits/rejected": 1.2035400867462158, "logps/chosen": -294.63287353515625, "logps/ref_chosen": -60.21992492675781, "logps/ref_rejected": -95.9200668334961, "logps/rejected": -544.7439575195312, "loss": 1.3862, "margin_dpo/margin_mean": 214.41094970703125, "margin_dpo/margin_std": 459.3138122558594, "step": 508 }, { "KL/chosen_KL_mean": -234.1763458251953, "KL/mean": -279.1103210449219, "KL/rejected_KL_mean": -324.0443115234375, "KL/std": 251.64889526367188, "epoch": 0.7694633408919124, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.86798858642578, "fcm_dpo/q_t": 0.4999912977218628, "grad_norm": 0.001714211655780673, "learning_rate": 7.748024712947204e-08, "logits/chosen": 1.2467401027679443, "logits/rejected": 1.2466963529586792, "logps/chosen": -300.4465026855469, "logps/ref_chosen": -66.27017211914062, "logps/ref_rejected": -71.73065185546875, "logps/rejected": -395.77496337890625, "loss": 1.3863, "margin_dpo/margin_mean": 89.86798858642578, "margin_dpo/margin_std": 300.79638671875, "step": 509 }, { "KL/chosen_KL_mean": -217.1124725341797, "KL/mean": -314.4279479980469, "KL/rejected_KL_mean": -411.743408203125, "KL/std": 359.33087158203125, "epoch": 0.7709750566893424, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 194.6309814453125, "fcm_dpo/q_t": 0.499981164932251, "grad_norm": 0.0035733687691390514, "learning_rate": 7.652572947447272e-08, "logits/chosen": 1.3153700828552246, "logits/rejected": 1.2759437561035156, "logps/chosen": -270.6573486328125, "logps/ref_chosen": -53.54487609863281, "logps/ref_rejected": -91.36648559570312, "logps/rejected": -503.10992431640625, "loss": 1.3862, "margin_dpo/margin_mean": 194.6309814453125, "margin_dpo/margin_std": 484.9547119140625, "step": 510 }, { "KL/chosen_KL_mean": -246.47100830078125, "KL/mean": -316.29608154296875, "KL/rejected_KL_mean": -386.12115478515625, "KL/std": 318.93328857421875, "epoch": 0.7724867724867724, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.65016174316406, "fcm_dpo/q_t": 0.499986469745636, "grad_norm": 0.0027404821012169123, "learning_rate": 7.557606426772961e-08, "logits/chosen": 1.2811682224273682, "logits/rejected": 1.257398009300232, "logps/chosen": -302.31536865234375, "logps/ref_chosen": -55.844383239746094, "logps/ref_rejected": -86.49819946289062, "logps/rejected": -472.6193542480469, "loss": 1.3862, "margin_dpo/margin_mean": 139.65016174316406, "margin_dpo/margin_std": 339.78717041015625, "step": 511 }, { "KL/chosen_KL_mean": -258.85089111328125, "KL/mean": -287.54278564453125, "KL/rejected_KL_mean": -316.2346496582031, "KL/std": 276.65423583984375, "epoch": 0.7739984882842026, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.38376998901367, "fcm_dpo/q_t": 0.4999944269657135, "grad_norm": 0.0019444817444309592, "learning_rate": 7.463127807341966e-08, "logits/chosen": 1.254358172416687, "logits/rejected": 1.2565571069717407, "logps/chosen": -320.5039367675781, "logps/ref_chosen": -61.653038024902344, "logps/ref_rejected": -72.83148193359375, "logps/rejected": -389.066162109375, "loss": 1.3863, "margin_dpo/margin_mean": 57.383766174316406, "margin_dpo/margin_std": 347.9620361328125, "step": 512 }, { "KL/chosen_KL_mean": -200.0421142578125, "KL/mean": -261.5788269042969, "KL/rejected_KL_mean": -323.11553955078125, "KL/std": 258.5805969238281, "epoch": 0.7755102040816326, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.07341003417969, "fcm_dpo/q_t": 0.4999880790710449, "grad_norm": 0.002271485747769475, "learning_rate": 7.369139731924401e-08, "logits/chosen": 1.3097870349884033, "logits/rejected": 1.2971343994140625, "logps/chosen": -250.89468383789062, "logps/ref_chosen": -50.85256576538086, "logps/ref_rejected": -69.21754455566406, "logps/rejected": -392.33306884765625, "loss": 1.3862, "margin_dpo/margin_mean": 123.07341003417969, "margin_dpo/margin_std": 297.73004150390625, "step": 513 }, { "KL/chosen_KL_mean": -270.1363830566406, "KL/mean": -320.6453552246094, "KL/rejected_KL_mean": -371.15435791015625, "KL/std": 278.06121826171875, "epoch": 0.7770219198790628, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.01797485351562, "fcm_dpo/q_t": 0.49999022483825684, "grad_norm": 0.0019997938070446253, "learning_rate": 7.275644829568747e-08, "logits/chosen": 1.2445127964019775, "logits/rejected": 1.241532802581787, "logps/chosen": -339.52130126953125, "logps/ref_chosen": -69.38493347167969, "logps/ref_rejected": -83.32447814941406, "logps/rejected": -454.47882080078125, "loss": 1.3863, "margin_dpo/margin_mean": 101.01797485351562, "margin_dpo/margin_std": 324.2949523925781, "step": 514 }, { "KL/chosen_KL_mean": -241.34022521972656, "KL/mean": -301.35577392578125, "KL/rejected_KL_mean": -361.371337890625, "KL/std": 278.6277160644531, "epoch": 0.7785336356764928, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.03112030029297, "fcm_dpo/q_t": 0.4999883770942688, "grad_norm": 0.0023619269486516714, "learning_rate": 7.182645715528435e-08, "logits/chosen": 1.2922251224517822, "logits/rejected": 1.2539401054382324, "logps/chosen": -295.02728271484375, "logps/ref_chosen": -53.687034606933594, "logps/ref_rejected": -83.59614562988281, "logps/rejected": -444.96746826171875, "loss": 1.3862, "margin_dpo/margin_mean": 120.03111267089844, "margin_dpo/margin_std": 330.4995422363281, "step": 515 }, { "KL/chosen_KL_mean": -212.0453643798828, "KL/mean": -263.1299133300781, "KL/rejected_KL_mean": -314.2144470214844, "KL/std": 261.9797668457031, "epoch": 0.780045351473923, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.16908264160156, "fcm_dpo/q_t": 0.4999901056289673, "grad_norm": 0.0018633684376254678, "learning_rate": 7.090144991188568e-08, "logits/chosen": 1.2701607942581177, "logits/rejected": 1.2603992223739624, "logps/chosen": -268.94708251953125, "logps/ref_chosen": -56.9017219543457, "logps/ref_rejected": -67.83477783203125, "logps/rejected": -382.0492248535156, "loss": 1.3863, "margin_dpo/margin_mean": 102.16908264160156, "margin_dpo/margin_std": 321.8263244628906, "step": 516 }, { "KL/chosen_KL_mean": -254.77786254882812, "KL/mean": -277.159423828125, "KL/rejected_KL_mean": -299.541015625, "KL/std": 273.0831298828125, "epoch": 0.781557067271353, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.76313781738281, "fcm_dpo/q_t": 0.4999956488609314, "grad_norm": 0.001817610813304782, "learning_rate": 6.998145243993284e-08, "logits/chosen": 1.2920082807540894, "logits/rejected": 1.2981328964233398, "logps/chosen": -316.5530090332031, "logps/ref_chosen": -61.775142669677734, "logps/ref_rejected": -62.88270950317383, "logps/rejected": -362.4237365722656, "loss": 1.3863, "margin_dpo/margin_mean": 44.76313400268555, "margin_dpo/margin_std": 304.3832702636719, "step": 517 }, { "KL/chosen_KL_mean": -219.64892578125, "KL/mean": -284.0128173828125, "KL/rejected_KL_mean": -348.376708984375, "KL/std": 307.4132385253906, "epoch": 0.783068783068783, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.727783203125, "fcm_dpo/q_t": 0.49998754262924194, "grad_norm": 0.002293572761118412, "learning_rate": 6.906649047373245e-08, "logits/chosen": 1.2533849477767944, "logits/rejected": 1.2436450719833374, "logps/chosen": -281.6741638183594, "logps/ref_chosen": -62.02523422241211, "logps/ref_rejected": -79.06085205078125, "logps/rejected": -427.43756103515625, "loss": 1.3862, "margin_dpo/margin_mean": 128.72779846191406, "margin_dpo/margin_std": 379.77642822265625, "step": 518 }, { "KL/chosen_KL_mean": -248.82159423828125, "KL/mean": -290.4149169921875, "KL/rejected_KL_mean": -332.00823974609375, "KL/std": 269.19866943359375, "epoch": 0.7845804988662132, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 83.18663024902344, "fcm_dpo/q_t": 0.4999919533729553, "grad_norm": 0.002268343698233366, "learning_rate": 6.815658960673781e-08, "logits/chosen": 1.260763168334961, "logits/rejected": 1.2478408813476562, "logps/chosen": -310.427978515625, "logps/ref_chosen": -61.60636901855469, "logps/ref_rejected": -74.50727844238281, "logps/rejected": -406.5155029296875, "loss": 1.3863, "margin_dpo/margin_mean": 83.18663024902344, "margin_dpo/margin_std": 341.28594970703125, "step": 519 }, { "KL/chosen_KL_mean": -225.19485473632812, "KL/mean": -291.09344482421875, "KL/rejected_KL_mean": -356.9920349121094, "KL/std": 300.70770263671875, "epoch": 0.7860922146636432, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.7971649169922, "fcm_dpo/q_t": 0.49998724460601807, "grad_norm": 0.002785892691463232, "learning_rate": 6.725177529083209e-08, "logits/chosen": 1.3261933326721191, "logits/rejected": 1.3110194206237793, "logps/chosen": -288.06829833984375, "logps/ref_chosen": -62.87343215942383, "logps/ref_rejected": -76.505615234375, "logps/rejected": -433.4976501464844, "loss": 1.3862, "margin_dpo/margin_mean": 131.7971649169922, "margin_dpo/margin_std": 311.9014892578125, "step": 520 }, { "KL/chosen_KL_mean": -233.61395263671875, "KL/mean": -314.4349060058594, "KL/rejected_KL_mean": -395.255859375, "KL/std": 295.2518310546875, "epoch": 0.7876039304610734, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 161.64187622070312, "fcm_dpo/q_t": 0.49998435378074646, "grad_norm": 0.0028386113699525595, "learning_rate": 6.63520728356167e-08, "logits/chosen": 1.244354486465454, "logits/rejected": 1.2204021215438843, "logps/chosen": -297.82061767578125, "logps/ref_chosen": -64.20668029785156, "logps/ref_rejected": -92.28083038330078, "logps/rejected": -487.53668212890625, "loss": 1.3862, "margin_dpo/margin_mean": 161.6418914794922, "margin_dpo/margin_std": 372.21527099609375, "step": 521 }, { "KL/chosen_KL_mean": -254.05210876464844, "KL/mean": -273.76922607421875, "KL/rejected_KL_mean": -293.48638916015625, "KL/std": 242.01705932617188, "epoch": 0.7891156462585034, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.43425750732422, "fcm_dpo/q_t": 0.4999961853027344, "grad_norm": 0.001333137508481741, "learning_rate": 6.545750740770336e-08, "logits/chosen": 1.263893723487854, "logits/rejected": 1.2616896629333496, "logps/chosen": -312.42181396484375, "logps/ref_chosen": -58.369720458984375, "logps/ref_rejected": -68.79248046875, "logps/rejected": -362.27886962890625, "loss": 1.3863, "margin_dpo/margin_mean": 39.434261322021484, "margin_dpo/margin_std": 283.5724792480469, "step": 522 }, { "KL/chosen_KL_mean": -269.55242919921875, "KL/mean": -339.4023132324219, "KL/rejected_KL_mean": -409.252197265625, "KL/std": 313.99249267578125, "epoch": 0.7906273620559335, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 139.69979858398438, "fcm_dpo/q_t": 0.499986469745636, "grad_norm": 0.0031393070239573717, "learning_rate": 6.456810403001012e-08, "logits/chosen": 1.2486999034881592, "logits/rejected": 1.2007946968078613, "logps/chosen": -335.2656555175781, "logps/ref_chosen": -65.71324157714844, "logps/ref_rejected": -91.98896789550781, "logps/rejected": -501.2411804199219, "loss": 1.3862, "margin_dpo/margin_mean": 139.69979858398438, "margin_dpo/margin_std": 364.4853515625, "step": 523 }, { "KL/chosen_KL_mean": -258.45904541015625, "KL/mean": -302.37017822265625, "KL/rejected_KL_mean": -346.28131103515625, "KL/std": 282.6915283203125, "epoch": 0.7921390778533636, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 87.82229614257812, "fcm_dpo/q_t": 0.4999915063381195, "grad_norm": 0.0017077861120924354, "learning_rate": 6.368388758106134e-08, "logits/chosen": 1.2249318361282349, "logits/rejected": 1.2157388925552368, "logps/chosen": -334.8102722167969, "logps/ref_chosen": -76.35124969482422, "logps/ref_rejected": -89.96072387695312, "logps/rejected": -436.2420349121094, "loss": 1.3863, "margin_dpo/margin_mean": 87.82229614257812, "margin_dpo/margin_std": 297.6631774902344, "step": 524 }, { "KL/chosen_KL_mean": -292.2108154296875, "KL/mean": -318.42254638671875, "KL/rejected_KL_mean": -344.63427734375, "KL/std": 277.3336486816406, "epoch": 0.7936507936507936, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 52.42350387573242, "fcm_dpo/q_t": 0.4999949336051941, "grad_norm": 0.0015537795843556523, "learning_rate": 6.280488279429185e-08, "logits/chosen": 1.1935780048370361, "logits/rejected": 1.1954379081726074, "logps/chosen": -367.70660400390625, "logps/ref_chosen": -75.49578857421875, "logps/ref_rejected": -84.04852294921875, "logps/rejected": -428.68280029296875, "loss": 1.3863, "margin_dpo/margin_mean": 52.42350769042969, "margin_dpo/margin_std": 347.2845458984375, "step": 525 }, { "KL/chosen_KL_mean": -247.38699340820312, "KL/mean": -307.152587890625, "KL/rejected_KL_mean": -366.91815185546875, "KL/std": 318.8529968261719, "epoch": 0.7951625094482238, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.53118896484375, "fcm_dpo/q_t": 0.4999884366989136, "grad_norm": 0.0027431962080299854, "learning_rate": 6.193111425735515e-08, "logits/chosen": 1.2542002201080322, "logits/rejected": 1.226431131362915, "logps/chosen": -308.67938232421875, "logps/ref_chosen": -61.29241943359375, "logps/ref_rejected": -82.47763061523438, "logps/rejected": -449.39581298828125, "loss": 1.3862, "margin_dpo/margin_mean": 119.53117370605469, "margin_dpo/margin_std": 372.6949768066406, "step": 526 }, { "KL/chosen_KL_mean": -241.8243408203125, "KL/mean": -305.8216247558594, "KL/rejected_KL_mean": -369.81884765625, "KL/std": 304.98187255859375, "epoch": 0.7966742252456538, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 127.9945068359375, "fcm_dpo/q_t": 0.4999876022338867, "grad_norm": 0.0031173720490187407, "learning_rate": 6.106260641143546e-08, "logits/chosen": 1.2762367725372314, "logits/rejected": 1.2428832054138184, "logps/chosen": -303.2969970703125, "logps/ref_chosen": -61.472625732421875, "logps/ref_rejected": -90.52831268310547, "logps/rejected": -460.34716796875, "loss": 1.3862, "margin_dpo/margin_mean": 127.9945068359375, "margin_dpo/margin_std": 330.25439453125, "step": 527 }, { "KL/chosen_KL_mean": -213.7490234375, "KL/mean": -268.8468017578125, "KL/rejected_KL_mean": -323.944580078125, "KL/std": 265.69378662109375, "epoch": 0.7981859410430839, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.19552612304688, "fcm_dpo/q_t": 0.4999893605709076, "grad_norm": 0.0022039057221263647, "learning_rate": 6.019938355056422e-08, "logits/chosen": 1.2133557796478271, "logits/rejected": 1.192253589630127, "logps/chosen": -272.5410461425781, "logps/ref_chosen": -58.792015075683594, "logps/ref_rejected": -71.82516479492188, "logps/rejected": -395.76971435546875, "loss": 1.3863, "margin_dpo/margin_mean": 110.1955337524414, "margin_dpo/margin_std": 350.6766357421875, "step": 528 }, { "KL/chosen_KL_mean": -207.93161010742188, "KL/mean": -281.9962158203125, "KL/rejected_KL_mean": -356.06085205078125, "KL/std": 229.11720275878906, "epoch": 0.799697656840514, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.1292266845703, "fcm_dpo/q_t": 0.4999856650829315, "grad_norm": 0.0023845250252634287, "learning_rate": 5.934146982094049e-08, "logits/chosen": 1.2498539686203003, "logits/rejected": 1.2348082065582275, "logps/chosen": -263.0025634765625, "logps/ref_chosen": -55.070960998535156, "logps/ref_rejected": -75.44007873535156, "logps/rejected": -431.50091552734375, "loss": 1.3862, "margin_dpo/margin_mean": 148.1292266845703, "margin_dpo/margin_std": 242.88185119628906, "step": 529 }, { "KL/chosen_KL_mean": -232.45703125, "KL/mean": -286.29730224609375, "KL/rejected_KL_mean": -340.1375732421875, "KL/std": 313.3363037109375, "epoch": 0.8012093726379441, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 107.68051147460938, "fcm_dpo/q_t": 0.4999895691871643, "grad_norm": 0.002098920289427042, "learning_rate": 5.848888922025552e-08, "logits/chosen": 1.2603845596313477, "logits/rejected": 1.2442349195480347, "logps/chosen": -289.20086669921875, "logps/ref_chosen": -56.743812561035156, "logps/ref_rejected": -76.6692123413086, "logps/rejected": -416.8067626953125, "loss": 1.3863, "margin_dpo/margin_mean": 107.68052673339844, "margin_dpo/margin_std": 359.12908935546875, "step": 530 }, { "KL/chosen_KL_mean": -207.04556274414062, "KL/mean": -264.3824462890625, "KL/rejected_KL_mean": -321.7193603515625, "KL/std": 270.8686218261719, "epoch": 0.8027210884353742, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.67379760742188, "fcm_dpo/q_t": 0.4999889135360718, "grad_norm": 0.0024485730100423098, "learning_rate": 5.7641665597021435e-08, "logits/chosen": 1.262247085571289, "logits/rejected": 1.2219009399414062, "logps/chosen": -258.1620178222656, "logps/ref_chosen": -51.116455078125, "logps/ref_rejected": -79.52884674072266, "logps/rejected": -401.2481689453125, "loss": 1.3863, "margin_dpo/margin_mean": 114.67379760742188, "margin_dpo/margin_std": 330.855712890625, "step": 531 }, { "KL/chosen_KL_mean": -252.53512573242188, "KL/mean": -311.3507080078125, "KL/rejected_KL_mean": -370.16632080078125, "KL/std": 310.71435546875, "epoch": 0.8042328042328042, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.63117980957031, "fcm_dpo/q_t": 0.4999886155128479, "grad_norm": 0.0020603465382009745, "learning_rate": 5.679982264990424e-08, "logits/chosen": 1.233213186264038, "logits/rejected": 1.2175933122634888, "logps/chosen": -310.8150634765625, "logps/ref_chosen": -58.279945373535156, "logps/ref_rejected": -78.05426788330078, "logps/rejected": -448.2205810546875, "loss": 1.3862, "margin_dpo/margin_mean": 117.63117980957031, "margin_dpo/margin_std": 353.5859375, "step": 532 }, { "KL/chosen_KL_mean": -186.49844360351562, "KL/mean": -257.16241455078125, "KL/rejected_KL_mean": -327.8263854980469, "KL/std": 312.2849426269531, "epoch": 0.8057445200302343, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 141.3279571533203, "fcm_dpo/q_t": 0.49998632073402405, "grad_norm": 0.0025260313414037228, "learning_rate": 5.596338392706076e-08, "logits/chosen": 1.2709081172943115, "logits/rejected": 1.2562451362609863, "logps/chosen": -242.9164581298828, "logps/ref_chosen": -56.41801071166992, "logps/ref_rejected": -73.89324951171875, "logps/rejected": -401.7196350097656, "loss": 1.3862, "margin_dpo/margin_mean": 141.3279571533203, "margin_dpo/margin_std": 353.8362121582031, "step": 533 }, { "KL/chosen_KL_mean": -237.41729736328125, "KL/mean": -282.9198913574219, "KL/rejected_KL_mean": -328.4224853515625, "KL/std": 276.8403625488281, "epoch": 0.8072562358276644, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.00517272949219, "fcm_dpo/q_t": 0.49999117851257324, "grad_norm": 0.002167914994060993, "learning_rate": 5.513237282548033e-08, "logits/chosen": 1.2200748920440674, "logits/rejected": 1.2163264751434326, "logps/chosen": -298.1659851074219, "logps/ref_chosen": -60.748687744140625, "logps/ref_rejected": -73.8623046875, "logps/rejected": -402.2847900390625, "loss": 1.3863, "margin_dpo/margin_mean": 91.00517272949219, "margin_dpo/margin_std": 360.96624755859375, "step": 534 }, { "KL/chosen_KL_mean": -243.32843017578125, "KL/mean": -293.2642822265625, "KL/rejected_KL_mean": -343.20013427734375, "KL/std": 254.28857421875, "epoch": 0.8087679516250945, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.87169647216797, "fcm_dpo/q_t": 0.4999903440475464, "grad_norm": 0.0023620380088686943, "learning_rate": 5.430681259032957e-08, "logits/chosen": 1.2484328746795654, "logits/rejected": 1.2265081405639648, "logps/chosen": -304.9658508300781, "logps/ref_chosen": -61.637413024902344, "logps/ref_rejected": -80.93138885498047, "logps/rejected": -424.13153076171875, "loss": 1.3863, "margin_dpo/margin_mean": 99.8717041015625, "margin_dpo/margin_std": 285.54364013671875, "step": 535 }, { "KL/chosen_KL_mean": -181.73826599121094, "KL/mean": -265.3481140136719, "KL/rejected_KL_mean": -348.95794677734375, "KL/std": 246.2275390625, "epoch": 0.8102796674225246, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 167.21969604492188, "fcm_dpo/q_t": 0.4999838173389435, "grad_norm": 0.003139457432553172, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 1.2669060230255127, "logits/rejected": 1.2405221462249756, "logps/chosen": -233.6272430419922, "logps/ref_chosen": -51.88897705078125, "logps/ref_rejected": -73.34864044189453, "logps/rejected": -422.30657958984375, "loss": 1.3862, "margin_dpo/margin_mean": 167.21969604492188, "margin_dpo/margin_std": 304.58843994140625, "step": 536 }, { "KL/chosen_KL_mean": -222.4708251953125, "KL/mean": -305.9654541015625, "KL/rejected_KL_mean": -389.4600830078125, "KL/std": 298.34405517578125, "epoch": 0.8117913832199547, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 166.9892578125, "fcm_dpo/q_t": 0.49998384714126587, "grad_norm": 0.0030098857823759317, "learning_rate": 5.267213693697695e-08, "logits/chosen": 1.298835277557373, "logits/rejected": 1.2577811479568481, "logps/chosen": -276.7194519042969, "logps/ref_chosen": -54.248619079589844, "logps/ref_rejected": -94.94343566894531, "logps/rejected": -484.40350341796875, "loss": 1.3862, "margin_dpo/margin_mean": 166.9892578125, "margin_dpo/margin_std": 360.7626953125, "step": 537 }, { "KL/chosen_KL_mean": -288.58770751953125, "KL/mean": -338.69049072265625, "KL/rejected_KL_mean": -388.79327392578125, "KL/std": 344.57171630859375, "epoch": 0.8133030990173847, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 100.20556640625, "fcm_dpo/q_t": 0.499990314245224, "grad_norm": 0.002249750541523099, "learning_rate": 5.1863067244167144e-08, "logits/chosen": 1.2919939756393433, "logits/rejected": 1.285117268562317, "logps/chosen": -358.6812438964844, "logps/ref_chosen": -70.09353637695312, "logps/ref_rejected": -79.49833679199219, "logps/rejected": -468.2916259765625, "loss": 1.3863, "margin_dpo/margin_mean": 100.20558166503906, "margin_dpo/margin_std": 351.8709411621094, "step": 538 }, { "KL/chosen_KL_mean": -225.29541015625, "KL/mean": -294.580078125, "KL/rejected_KL_mean": -363.8647766113281, "KL/std": 290.5321044921875, "epoch": 0.8148148148148148, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.56936645507812, "fcm_dpo/q_t": 0.49998658895492554, "grad_norm": 0.0030424855649471283, "learning_rate": 5.105953986729195e-08, "logits/chosen": 1.2382125854492188, "logits/rejected": 1.2072124481201172, "logps/chosen": -287.22711181640625, "logps/ref_chosen": -61.93169403076172, "logps/ref_rejected": -84.08946228027344, "logps/rejected": -447.9542236328125, "loss": 1.3862, "margin_dpo/margin_mean": 138.56936645507812, "margin_dpo/margin_std": 335.16851806640625, "step": 539 }, { "KL/chosen_KL_mean": -242.7515869140625, "KL/mean": -336.0493469238281, "KL/rejected_KL_mean": -429.34710693359375, "KL/std": 340.7681884765625, "epoch": 0.8163265306122449, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 186.59555053710938, "fcm_dpo/q_t": 0.49998193979263306, "grad_norm": 0.003513064468279481, "learning_rate": 5.026157728273966e-08, "logits/chosen": 1.3039121627807617, "logits/rejected": 1.2665743827819824, "logps/chosen": -305.455810546875, "logps/ref_chosen": -62.704254150390625, "logps/ref_rejected": -95.63597106933594, "logps/rejected": -524.9830932617188, "loss": 1.3862, "margin_dpo/margin_mean": 186.59555053710938, "margin_dpo/margin_std": 353.68701171875, "step": 540 }, { "KL/chosen_KL_mean": -221.798095703125, "KL/mean": -250.46072387695312, "KL/rejected_KL_mean": -279.1233825683594, "KL/std": 239.862548828125, "epoch": 0.817838246409675, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.32527160644531, "fcm_dpo/q_t": 0.4999944567680359, "grad_norm": 0.0014312553685158491, "learning_rate": 4.9469201811239035e-08, "logits/chosen": 1.290186882019043, "logits/rejected": 1.3193426132202148, "logps/chosen": -284.2789306640625, "logps/ref_chosen": -62.48084259033203, "logps/ref_rejected": -57.55541229248047, "logps/rejected": -336.67877197265625, "loss": 1.3863, "margin_dpo/margin_mean": 57.32527160644531, "margin_dpo/margin_std": 238.53594970703125, "step": 541 }, { "KL/chosen_KL_mean": -184.5562744140625, "KL/mean": -236.01080322265625, "KL/rejected_KL_mean": -287.46533203125, "KL/std": 224.02711486816406, "epoch": 0.8193499622071051, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.90904235839844, "fcm_dpo/q_t": 0.4999900460243225, "grad_norm": 0.0018258477794006467, "learning_rate": 4.868243561723534e-08, "logits/chosen": 1.2936313152313232, "logits/rejected": 1.2742774486541748, "logps/chosen": -234.01116943359375, "logps/ref_chosen": -49.454891204833984, "logps/ref_rejected": -65.33275604248047, "logps/rejected": -352.798095703125, "loss": 1.3863, "margin_dpo/margin_mean": 102.90904998779297, "margin_dpo/margin_std": 270.83367919921875, "step": 542 }, { "KL/chosen_KL_mean": -195.629638671875, "KL/mean": -286.21685791015625, "KL/rejected_KL_mean": -376.8040771484375, "KL/std": 335.13543701171875, "epoch": 0.8208616780045351, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 181.17446899414062, "fcm_dpo/q_t": 0.49998247623443604, "grad_norm": 0.003246963955461979, "learning_rate": 4.790130070827028e-08, "logits/chosen": 1.257094383239746, "logits/rejected": 1.228837490081787, "logps/chosen": -246.73049926757812, "logps/ref_chosen": -51.100860595703125, "logps/ref_rejected": -76.06130981445312, "logps/rejected": -452.86541748046875, "loss": 1.3862, "margin_dpo/margin_mean": 181.1744842529297, "margin_dpo/margin_std": 412.4785461425781, "step": 543 }, { "KL/chosen_KL_mean": -238.99652099609375, "KL/mean": -313.9105529785156, "KL/rejected_KL_mean": -388.8245544433594, "KL/std": 292.81646728515625, "epoch": 0.8223733938019653, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 149.82806396484375, "fcm_dpo/q_t": 0.4999855160713196, "grad_norm": 0.003041402902454138, "learning_rate": 4.7125818934366454e-08, "logits/chosen": 1.2706331014633179, "logits/rejected": 1.2441833019256592, "logps/chosen": -299.27374267578125, "logps/ref_chosen": -60.2772331237793, "logps/ref_rejected": -88.40553283691406, "logps/rejected": -477.2301025390625, "loss": 1.3862, "margin_dpo/margin_mean": 149.8280487060547, "margin_dpo/margin_std": 375.4241943359375, "step": 544 }, { "KL/chosen_KL_mean": -241.99879455566406, "KL/mean": -287.8915100097656, "KL/rejected_KL_mean": -333.7841796875, "KL/std": 267.11920166015625, "epoch": 0.8238851095993953, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.78538513183594, "fcm_dpo/q_t": 0.49999111890792847, "grad_norm": 0.0022016982547938824, "learning_rate": 4.635601198741607e-08, "logits/chosen": 1.2439241409301758, "logits/rejected": 1.217350959777832, "logps/chosen": -303.6140441894531, "logps/ref_chosen": -61.61524963378906, "logps/ref_rejected": -78.71266174316406, "logps/rejected": -412.496826171875, "loss": 1.3863, "margin_dpo/margin_mean": 91.78538513183594, "margin_dpo/margin_std": 318.565673828125, "step": 545 }, { "KL/chosen_KL_mean": -221.55914306640625, "KL/mean": -255.8831787109375, "KL/rejected_KL_mean": -290.20721435546875, "KL/std": 262.83538818359375, "epoch": 0.8253968253968254, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.64806365966797, "fcm_dpo/q_t": 0.49999332427978516, "grad_norm": 0.0015192232094705105, "learning_rate": 4.559190140057428e-08, "logits/chosen": 1.302754282951355, "logits/rejected": 1.3086268901824951, "logps/chosen": -280.8724060058594, "logps/ref_chosen": -59.313262939453125, "logps/ref_rejected": -64.73631286621094, "logps/rejected": -354.9435119628906, "loss": 1.3863, "margin_dpo/margin_mean": 68.6480712890625, "margin_dpo/margin_std": 304.9132080078125, "step": 546 }, { "KL/chosen_KL_mean": -201.32754516601562, "KL/mean": -267.2884826660156, "KL/rejected_KL_mean": -333.24945068359375, "KL/std": 234.06785583496094, "epoch": 0.8269085411942555, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 131.92190551757812, "fcm_dpo/q_t": 0.49998724460601807, "grad_norm": 0.00217438698746264, "learning_rate": 4.483350854765672e-08, "logits/chosen": 1.2100231647491455, "logits/rejected": 1.1951963901519775, "logps/chosen": -256.3042907714844, "logps/ref_chosen": -54.97674560546875, "logps/ref_rejected": -75.35922241210938, "logps/rejected": -408.608642578125, "loss": 1.3862, "margin_dpo/margin_mean": 131.92190551757812, "margin_dpo/margin_std": 245.69558715820312, "step": 547 }, { "KL/chosen_KL_mean": -246.73431396484375, "KL/mean": -302.9718017578125, "KL/rejected_KL_mean": -359.20928955078125, "KL/std": 298.39251708984375, "epoch": 0.8284202569916855, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 112.47500610351562, "fcm_dpo/q_t": 0.4999891221523285, "grad_norm": 0.0023801657371222973, "learning_rate": 4.4080854642541826e-08, "logits/chosen": 1.2188358306884766, "logits/rejected": 1.203696846961975, "logps/chosen": -309.9449768066406, "logps/ref_chosen": -63.21067428588867, "logps/ref_rejected": -81.23347473144531, "logps/rejected": -440.4427795410156, "loss": 1.3863, "margin_dpo/margin_mean": 112.47501373291016, "margin_dpo/margin_std": 335.791015625, "step": 548 }, { "KL/chosen_KL_mean": -246.50332641601562, "KL/mean": -307.3105773925781, "KL/rejected_KL_mean": -368.1178283691406, "KL/std": 339.2923583984375, "epoch": 0.8299319727891157, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.61450958251953, "fcm_dpo/q_t": 0.49998822808265686, "grad_norm": 0.00257733091711998, "learning_rate": 4.333396073857723e-08, "logits/chosen": 1.2848644256591797, "logits/rejected": 1.2579424381256104, "logps/chosen": -310.77685546875, "logps/ref_chosen": -64.27351379394531, "logps/ref_rejected": -92.31663513183594, "logps/rejected": -460.4344482421875, "loss": 1.3862, "margin_dpo/margin_mean": 121.61450958251953, "margin_dpo/margin_std": 392.3224792480469, "step": 549 }, { "KL/chosen_KL_mean": -218.34458923339844, "KL/mean": -256.52752685546875, "KL/rejected_KL_mean": -294.71044921875, "KL/std": 242.93511962890625, "epoch": 0.8314436885865457, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.36582946777344, "fcm_dpo/q_t": 0.49999260902404785, "grad_norm": 0.0018148054368793964, "learning_rate": 4.259284772799099e-08, "logits/chosen": 1.2855148315429688, "logits/rejected": 1.2823785543441772, "logps/chosen": -274.57501220703125, "logps/ref_chosen": -56.230438232421875, "logps/ref_rejected": -62.59788513183594, "logps/rejected": -357.3083190917969, "loss": 1.3863, "margin_dpo/margin_mean": 76.3658218383789, "margin_dpo/margin_std": 288.6811218261719, "step": 550 }, { "KL/chosen_KL_mean": -248.3517303466797, "KL/mean": -309.2677917480469, "KL/rejected_KL_mean": -370.1838684082031, "KL/std": 306.270751953125, "epoch": 0.8329554043839759, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 121.83216094970703, "fcm_dpo/q_t": 0.4999881982803345, "grad_norm": 0.0023011192679405212, "learning_rate": 4.1857536341307176e-08, "logits/chosen": 1.2524220943450928, "logits/rejected": 1.2448160648345947, "logps/chosen": -316.09893798828125, "logps/ref_chosen": -67.74720764160156, "logps/ref_rejected": -87.04285430908203, "logps/rejected": -457.22674560546875, "loss": 1.3862, "margin_dpo/margin_mean": 121.83216094970703, "margin_dpo/margin_std": 341.08929443359375, "step": 551 }, { "KL/chosen_KL_mean": -232.7744140625, "KL/mean": -290.295166015625, "KL/rejected_KL_mean": -347.8158874511719, "KL/std": 296.06097412109375, "epoch": 0.8344671201814059, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.04149627685547, "fcm_dpo/q_t": 0.4999888837337494, "grad_norm": 0.0021020383574068546, "learning_rate": 4.112804714676593e-08, "logits/chosen": 1.2442529201507568, "logits/rejected": 1.2225983142852783, "logps/chosen": -295.70068359375, "logps/ref_chosen": -62.92625427246094, "logps/ref_rejected": -82.98365783691406, "logps/rejected": -430.799560546875, "loss": 1.3862, "margin_dpo/margin_mean": 115.04149627685547, "margin_dpo/margin_std": 302.3883361816406, "step": 552 }, { "KL/chosen_KL_mean": -223.17971801757812, "KL/mean": -299.51727294921875, "KL/rejected_KL_mean": -375.8548583984375, "KL/std": 325.5655517578125, "epoch": 0.8359788359788359, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.67514038085938, "fcm_dpo/q_t": 0.4999852180480957, "grad_norm": 0.0032611123751848936, "learning_rate": 4.0404400549748144e-08, "logits/chosen": 1.2355235815048218, "logits/rejected": 1.1960480213165283, "logps/chosen": -279.21820068359375, "logps/ref_chosen": -56.038490295410156, "logps/ref_rejected": -84.48454284667969, "logps/rejected": -460.3393859863281, "loss": 1.3862, "margin_dpo/margin_mean": 152.6751251220703, "margin_dpo/margin_std": 377.767822265625, "step": 553 }, { "KL/chosen_KL_mean": -241.2162628173828, "KL/mean": -287.3515625, "KL/rejected_KL_mean": -333.48687744140625, "KL/std": 261.48486328125, "epoch": 0.8374905517762661, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 92.27063751220703, "fcm_dpo/q_t": 0.4999910593032837, "grad_norm": 0.0016775699332356453, "learning_rate": 3.968661679220467e-08, "logits/chosen": 1.247375726699829, "logits/rejected": 1.2545955181121826, "logps/chosen": -305.7468566894531, "logps/ref_chosen": -64.53059387207031, "logps/ref_rejected": -71.2155990600586, "logps/rejected": -404.7025146484375, "loss": 1.3863, "margin_dpo/margin_mean": 92.27063751220703, "margin_dpo/margin_std": 262.7148742675781, "step": 554 }, { "KL/chosen_KL_mean": -242.61669921875, "KL/mean": -278.1309814453125, "KL/rejected_KL_mean": -313.645263671875, "KL/std": 235.6993408203125, "epoch": 0.8390022675736961, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 71.02857971191406, "fcm_dpo/q_t": 0.49999311566352844, "grad_norm": 0.0012565868673846126, "learning_rate": 3.89747159520904e-08, "logits/chosen": 1.2524785995483398, "logits/rejected": 1.2564916610717773, "logps/chosen": -309.26861572265625, "logps/ref_chosen": -66.65191650390625, "logps/ref_rejected": -68.6667251586914, "logps/rejected": -382.3119812011719, "loss": 1.3863, "margin_dpo/margin_mean": 71.02857971191406, "margin_dpo/margin_std": 269.95416259765625, "step": 555 }, { "KL/chosen_KL_mean": -205.25283813476562, "KL/mean": -254.348876953125, "KL/rejected_KL_mean": -303.4449462890625, "KL/std": 214.45046997070312, "epoch": 0.8405139833711263, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.19207763671875, "fcm_dpo/q_t": 0.49999046325683594, "grad_norm": 0.001899282680824399, "learning_rate": 3.826871794280192e-08, "logits/chosen": 1.2756855487823486, "logits/rejected": 1.2640533447265625, "logps/chosen": -258.085205078125, "logps/ref_chosen": -52.832366943359375, "logps/ref_rejected": -64.49044036865234, "logps/rejected": -367.93536376953125, "loss": 1.3863, "margin_dpo/margin_mean": 98.19209289550781, "margin_dpo/margin_std": 222.29449462890625, "step": 556 }, { "KL/chosen_KL_mean": -227.02926635742188, "KL/mean": -313.59149169921875, "KL/rejected_KL_mean": -400.1536865234375, "KL/std": 321.18719482421875, "epoch": 0.8420256991685563, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 173.12440490722656, "fcm_dpo/q_t": 0.4999832510948181, "grad_norm": 0.003363779978826642, "learning_rate": 3.756864251262143e-08, "logits/chosen": 1.310492992401123, "logits/rejected": 1.288170576095581, "logps/chosen": -282.06524658203125, "logps/ref_chosen": -55.03598403930664, "logps/ref_rejected": -75.80644989013672, "logps/rejected": -475.96014404296875, "loss": 1.3862, "margin_dpo/margin_mean": 173.12442016601562, "margin_dpo/margin_std": 381.8446350097656, "step": 557 }, { "KL/chosen_KL_mean": -248.1762237548828, "KL/mean": -312.669189453125, "KL/rejected_KL_mean": -377.16217041015625, "KL/std": 295.711669921875, "epoch": 0.8435374149659864, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 128.98593139648438, "fcm_dpo/q_t": 0.49998751282691956, "grad_norm": 0.0022693185601383448, "learning_rate": 3.687450924416341e-08, "logits/chosen": 1.2714734077453613, "logits/rejected": 1.2482508420944214, "logps/chosen": -311.402587890625, "logps/ref_chosen": -63.226348876953125, "logps/ref_rejected": -91.46881866455078, "logps/rejected": -468.6309814453125, "loss": 1.3862, "margin_dpo/margin_mean": 128.98593139648438, "margin_dpo/margin_std": 365.5770263671875, "step": 558 }, { "KL/chosen_KL_mean": -236.52352905273438, "KL/mean": -312.6187744140625, "KL/rejected_KL_mean": -388.71405029296875, "KL/std": 306.2005615234375, "epoch": 0.8450491307634165, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.19052124023438, "fcm_dpo/q_t": 0.4999852776527405, "grad_norm": 0.0024620580952614546, "learning_rate": 3.6186337553827743e-08, "logits/chosen": 1.2394496202468872, "logits/rejected": 1.2185451984405518, "logps/chosen": -298.045166015625, "logps/ref_chosen": -61.521644592285156, "logps/ref_rejected": -82.83859252929688, "logps/rejected": -471.5526428222656, "loss": 1.3862, "margin_dpo/margin_mean": 152.1905059814453, "margin_dpo/margin_std": 371.47833251953125, "step": 559 }, { "KL/chosen_KL_mean": -240.15634155273438, "KL/mean": -291.6183776855469, "KL/rejected_KL_mean": -343.08038330078125, "KL/std": 260.51849365234375, "epoch": 0.8465608465608465, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.924072265625, "fcm_dpo/q_t": 0.4999900460243225, "grad_norm": 0.002105724299326539, "learning_rate": 3.550414669125573e-08, "logits/chosen": 1.3063299655914307, "logits/rejected": 1.2882559299468994, "logps/chosen": -300.79754638671875, "logps/ref_chosen": -60.64122009277344, "logps/ref_rejected": -78.75474548339844, "logps/rejected": -421.83514404296875, "loss": 1.3863, "margin_dpo/margin_mean": 102.924072265625, "margin_dpo/margin_std": 279.7295227050781, "step": 560 }, { "KL/chosen_KL_mean": -231.18161010742188, "KL/mean": -277.0013427734375, "KL/rejected_KL_mean": -322.821044921875, "KL/std": 236.68751525878906, "epoch": 0.8480725623582767, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.63945770263672, "fcm_dpo/q_t": 0.49999114871025085, "grad_norm": 0.0019574423786252737, "learning_rate": 3.482795573879241e-08, "logits/chosen": 1.2543569803237915, "logits/rejected": 1.2415952682495117, "logps/chosen": -293.68023681640625, "logps/ref_chosen": -62.49859619140625, "logps/ref_rejected": -78.72064208984375, "logps/rejected": -401.54168701171875, "loss": 1.3863, "margin_dpo/margin_mean": 91.63945770263672, "margin_dpo/margin_std": 286.7827453613281, "step": 561 }, { "KL/chosen_KL_mean": -279.95501708984375, "KL/mean": -352.92193603515625, "KL/rejected_KL_mean": -425.88885498046875, "KL/std": 363.0885009765625, "epoch": 0.8495842781557067, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.93382263183594, "fcm_dpo/q_t": 0.49998587369918823, "grad_norm": 0.002555001061409712, "learning_rate": 3.415778361095226e-08, "logits/chosen": 1.256950855255127, "logits/rejected": 1.2576011419296265, "logps/chosen": -354.73675537109375, "logps/ref_chosen": -74.78173828125, "logps/ref_rejected": -92.63499450683594, "logps/rejected": -518.5238037109375, "loss": 1.3862, "margin_dpo/margin_mean": 145.93382263183594, "margin_dpo/margin_std": 366.14715576171875, "step": 562 }, { "KL/chosen_KL_mean": -186.45376586914062, "KL/mean": -236.38587951660156, "KL/rejected_KL_mean": -286.3179931640625, "KL/std": 214.30816650390625, "epoch": 0.8510959939531368, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.86424255371094, "fcm_dpo/q_t": 0.4999903440475464, "grad_norm": 0.0017422273522242904, "learning_rate": 3.349364905389032e-08, "logits/chosen": 1.329614281654358, "logits/rejected": 1.3034515380859375, "logps/chosen": -236.6522674560547, "logps/ref_chosen": -50.19850158691406, "logps/ref_rejected": -66.76687622070312, "logps/rejected": -353.0848693847656, "loss": 1.3863, "margin_dpo/margin_mean": 99.86424255371094, "margin_dpo/margin_std": 297.21343994140625, "step": 563 }, { "KL/chosen_KL_mean": -198.27935791015625, "KL/mean": -253.13412475585938, "KL/rejected_KL_mean": -307.9888610839844, "KL/std": 239.6055145263672, "epoch": 0.8526077097505669, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.70948028564453, "fcm_dpo/q_t": 0.49998939037323, "grad_norm": 0.002160602016374469, "learning_rate": 3.283557064487785e-08, "logits/chosen": 1.2637401819229126, "logits/rejected": 1.250982642173767, "logps/chosen": -254.0202178955078, "logps/ref_chosen": -55.7408447265625, "logps/ref_rejected": -74.82323455810547, "logps/rejected": -382.81207275390625, "loss": 1.3863, "margin_dpo/margin_mean": 109.70947265625, "margin_dpo/margin_std": 332.83740234375, "step": 564 }, { "KL/chosen_KL_mean": -264.5958557128906, "KL/mean": -321.555908203125, "KL/rejected_KL_mean": -378.5159912109375, "KL/std": 297.69415283203125, "epoch": 0.854119425547997, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 113.92015075683594, "fcm_dpo/q_t": 0.49998897314071655, "grad_norm": 0.002675557043403387, "learning_rate": 3.218356679178252e-08, "logits/chosen": 1.3032476902008057, "logits/rejected": 1.2836694717407227, "logps/chosen": -322.9332275390625, "logps/ref_chosen": -58.33738327026367, "logps/ref_rejected": -78.31776428222656, "logps/rejected": -456.833740234375, "loss": 1.3863, "margin_dpo/margin_mean": 113.92015075683594, "margin_dpo/margin_std": 335.97607421875, "step": 565 }, { "KL/chosen_KL_mean": -251.61575317382812, "KL/mean": -298.7529296875, "KL/rejected_KL_mean": -345.89007568359375, "KL/std": 310.39801025390625, "epoch": 0.8556311413454271, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.27432250976562, "fcm_dpo/q_t": 0.49999088048934937, "grad_norm": 0.0015177810564637184, "learning_rate": 3.1537655732553764e-08, "logits/chosen": 1.2642093896865845, "logits/rejected": 1.2712171077728271, "logps/chosen": -322.8394775390625, "logps/ref_chosen": -71.22373962402344, "logps/ref_rejected": -71.11601257324219, "logps/rejected": -417.006103515625, "loss": 1.3863, "margin_dpo/margin_mean": 94.27432250976562, "margin_dpo/margin_std": 391.6617431640625, "step": 566 }, { "KL/chosen_KL_mean": -201.491943359375, "KL/mean": -286.70196533203125, "KL/rejected_KL_mean": -371.9119873046875, "KL/std": 300.2903747558594, "epoch": 0.8571428571428571, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 170.42001342773438, "fcm_dpo/q_t": 0.4999834895133972, "grad_norm": 0.002934924094006419, "learning_rate": 3.089785553471233e-08, "logits/chosen": 1.2649750709533691, "logits/rejected": 1.2357361316680908, "logps/chosen": -254.1612091064453, "logps/ref_chosen": -52.669273376464844, "logps/ref_rejected": -74.34785461425781, "logps/rejected": -446.25982666015625, "loss": 1.3862, "margin_dpo/margin_mean": 170.42002868652344, "margin_dpo/margin_std": 392.0222473144531, "step": 567 }, { "KL/chosen_KL_mean": -201.22650146484375, "KL/mean": -296.65826416015625, "KL/rejected_KL_mean": -392.0899963378906, "KL/std": 302.1213073730469, "epoch": 0.8586545729402872, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 190.86349487304688, "fcm_dpo/q_t": 0.49998152256011963, "grad_norm": 0.003850526176393032, "learning_rate": 3.026418409484513e-08, "logits/chosen": 1.2919840812683105, "logits/rejected": 1.2600350379943848, "logps/chosen": -253.4044952392578, "logps/ref_chosen": -52.178001403808594, "logps/ref_rejected": -85.8277587890625, "logps/rejected": -477.9177551269531, "loss": 1.3862, "margin_dpo/margin_mean": 190.86349487304688, "margin_dpo/margin_std": 386.70355224609375, "step": 568 }, { "KL/chosen_KL_mean": -245.7794189453125, "KL/mean": -283.582275390625, "KL/rejected_KL_mean": -321.3851318359375, "KL/std": 323.57220458984375, "epoch": 0.8601662887377173, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 75.60567474365234, "fcm_dpo/q_t": 0.4999926686286926, "grad_norm": 0.0020971940830349922, "learning_rate": 2.963665913810451e-08, "logits/chosen": 1.2494065761566162, "logits/rejected": 1.2419638633728027, "logps/chosen": -308.4286804199219, "logps/ref_chosen": -62.649261474609375, "logps/ref_rejected": -75.4298324584961, "logps/rejected": -396.81494140625, "loss": 1.3863, "margin_dpo/margin_mean": 75.60567474365234, "margin_dpo/margin_std": 389.1278381347656, "step": 569 }, { "KL/chosen_KL_mean": -213.00286865234375, "KL/mean": -280.4457702636719, "KL/rejected_KL_mean": -347.888671875, "KL/std": 267.0771789550781, "epoch": 0.8616780045351474, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 134.8858184814453, "fcm_dpo/q_t": 0.4999869465827942, "grad_norm": 0.0022408810909837484, "learning_rate": 2.9015298217712453e-08, "logits/chosen": 1.2825806140899658, "logits/rejected": 1.2437119483947754, "logps/chosen": -263.044677734375, "logps/ref_chosen": -50.04179382324219, "logps/ref_rejected": -78.27146911621094, "logps/rejected": -426.16015625, "loss": 1.3862, "margin_dpo/margin_mean": 134.88583374023438, "margin_dpo/margin_std": 310.875, "step": 570 }, { "KL/chosen_KL_mean": -230.393310546875, "KL/mean": -274.6932678222656, "KL/rejected_KL_mean": -318.99322509765625, "KL/std": 289.2368469238281, "epoch": 0.8631897203325775, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.59991455078125, "fcm_dpo/q_t": 0.49999141693115234, "grad_norm": 0.0022626707796007395, "learning_rate": 2.840011871446962e-08, "logits/chosen": 1.280132532119751, "logits/rejected": 1.2787832021713257, "logps/chosen": -284.05010986328125, "logps/ref_chosen": -53.65681457519531, "logps/ref_rejected": -66.13298034667969, "logps/rejected": -385.126220703125, "loss": 1.3863, "margin_dpo/margin_mean": 88.59992980957031, "margin_dpo/margin_std": 364.80145263671875, "step": 571 }, { "KL/chosen_KL_mean": -287.53131103515625, "KL/mean": -291.213623046875, "KL/rejected_KL_mean": -294.89599609375, "KL/std": 277.07366943359375, "epoch": 0.8647014361300076, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 7.364675521850586, "fcm_dpo/q_t": 0.4999992847442627, "grad_norm": 0.001692647230811417, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 1.258278250694275, "logits/rejected": 1.2805957794189453, "logps/chosen": -362.3492431640625, "logps/ref_chosen": -74.81792449951172, "logps/ref_rejected": -65.88681030273438, "logps/rejected": -360.7828063964844, "loss": 1.3863, "margin_dpo/margin_mean": 7.364673614501953, "margin_dpo/margin_std": 344.2757568359375, "step": 572 }, { "KL/chosen_KL_mean": -281.252685546875, "KL/mean": -336.67999267578125, "KL/rejected_KL_mean": -392.10736083984375, "KL/std": 340.51861572265625, "epoch": 0.8662131519274376, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 110.85468292236328, "fcm_dpo/q_t": 0.4999893009662628, "grad_norm": 0.002122138626873493, "learning_rate": 2.718837261761528e-08, "logits/chosen": 1.2949762344360352, "logits/rejected": 1.2791988849639893, "logps/chosen": -349.97833251953125, "logps/ref_chosen": -68.72564697265625, "logps/ref_rejected": -88.16201782226562, "logps/rejected": -480.26934814453125, "loss": 1.3863, "margin_dpo/margin_mean": 110.85468292236328, "margin_dpo/margin_std": 395.3820495605469, "step": 573 }, { "KL/chosen_KL_mean": -224.64120483398438, "KL/mean": -303.6715087890625, "KL/rejected_KL_mean": -382.7017822265625, "KL/std": 301.89447021484375, "epoch": 0.8677248677248677, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 158.06060791015625, "fcm_dpo/q_t": 0.4999846816062927, "grad_norm": 0.0023631516378372908, "learning_rate": 2.659183991914696e-08, "logits/chosen": 1.285936713218689, "logits/rejected": 1.2656428813934326, "logps/chosen": -280.95458984375, "logps/ref_chosen": -56.31340026855469, "logps/ref_rejected": -83.91553497314453, "logps/rejected": -466.6173095703125, "loss": 1.3862, "margin_dpo/margin_mean": 158.0605926513672, "margin_dpo/margin_std": 359.18701171875, "step": 574 }, { "KL/chosen_KL_mean": -240.16212463378906, "KL/mean": -306.2142333984375, "KL/rejected_KL_mean": -372.2663269042969, "KL/std": 327.98846435546875, "epoch": 0.8692365835222978, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 132.1042022705078, "fcm_dpo/q_t": 0.4999872148036957, "grad_norm": 0.00271924352273345, "learning_rate": 2.600155642716606e-08, "logits/chosen": 1.2739206552505493, "logits/rejected": 1.2386579513549805, "logps/chosen": -304.74627685546875, "logps/ref_chosen": -64.5841293334961, "logps/ref_rejected": -93.47034454345703, "logps/rejected": -465.7366638183594, "loss": 1.3862, "margin_dpo/margin_mean": 132.1042022705078, "margin_dpo/margin_std": 406.8048095703125, "step": 575 }, { "KL/chosen_KL_mean": -185.20242309570312, "KL/mean": -287.28302001953125, "KL/rejected_KL_mean": -389.36358642578125, "KL/std": 299.2473449707031, "epoch": 0.8707482993197279, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 204.16119384765625, "fcm_dpo/q_t": 0.49998024106025696, "grad_norm": 0.0033409777097404003, "learning_rate": 2.5417538653170754e-08, "logits/chosen": 1.2817810773849487, "logits/rejected": 1.2451581954956055, "logps/chosen": -238.48294067382812, "logps/ref_chosen": -53.28052520751953, "logps/ref_rejected": -84.2000503540039, "logps/rejected": -473.56365966796875, "loss": 1.3862, "margin_dpo/margin_mean": 204.16119384765625, "margin_dpo/margin_std": 395.5772705078125, "step": 576 }, { "KL/chosen_KL_mean": -230.99880981445312, "KL/mean": -275.36846923828125, "KL/rejected_KL_mean": -319.7381591796875, "KL/std": 301.64703369140625, "epoch": 0.872260015117158, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 88.7393569946289, "fcm_dpo/q_t": 0.49999138712882996, "grad_norm": 0.002223484218120575, "learning_rate": 2.4839802933393607e-08, "logits/chosen": 1.2827671766281128, "logits/rejected": 1.2928658723831177, "logps/chosen": -293.323486328125, "logps/ref_chosen": -62.32468795776367, "logps/ref_rejected": -67.300537109375, "logps/rejected": -387.0386962890625, "loss": 1.3863, "margin_dpo/margin_mean": 88.73936462402344, "margin_dpo/margin_std": 334.8846435546875, "step": 577 }, { "KL/chosen_KL_mean": -238.35610961914062, "KL/mean": -270.2100830078125, "KL/rejected_KL_mean": -302.0640869140625, "KL/std": 326.62493896484375, "epoch": 0.873771730914588, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 63.7080078125, "fcm_dpo/q_t": 0.49999380111694336, "grad_norm": 0.0015426744939759374, "learning_rate": 2.4268365428344733e-08, "logits/chosen": 1.3032362461090088, "logits/rejected": 1.3008297681808472, "logps/chosen": -295.0116882324219, "logps/ref_chosen": -56.65557861328125, "logps/ref_rejected": -68.21835327148438, "logps/rejected": -370.282470703125, "loss": 1.3863, "margin_dpo/margin_mean": 63.7080078125, "margin_dpo/margin_std": 349.22344970703125, "step": 578 }, { "KL/chosen_KL_mean": -219.2360076904297, "KL/mean": -270.0174255371094, "KL/rejected_KL_mean": -320.7988586425781, "KL/std": 243.76840209960938, "epoch": 0.8752834467120182, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.56283569335938, "fcm_dpo/q_t": 0.49999016523361206, "grad_norm": 0.001675861538387835, "learning_rate": 2.3703242122359357e-08, "logits/chosen": 1.258755087852478, "logits/rejected": 1.2550430297851562, "logps/chosen": -276.0456848144531, "logps/ref_chosen": -56.809661865234375, "logps/ref_rejected": -68.09613037109375, "logps/rejected": -388.8949890136719, "loss": 1.3863, "margin_dpo/margin_mean": 101.56283569335938, "margin_dpo/margin_std": 307.77728271484375, "step": 579 }, { "KL/chosen_KL_mean": -229.52764892578125, "KL/mean": -310.90203857421875, "KL/rejected_KL_mean": -392.2764587402344, "KL/std": 354.71600341796875, "epoch": 0.8767951625094482, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 162.74879455566406, "fcm_dpo/q_t": 0.4999842643737793, "grad_norm": 0.0027315111365169287, "learning_rate": 2.3144448823151392e-08, "logits/chosen": 1.252476453781128, "logits/rejected": 1.2458863258361816, "logps/chosen": -287.227783203125, "logps/ref_chosen": -57.70011520385742, "logps/ref_rejected": -77.90664672851562, "logps/rejected": -470.18310546875, "loss": 1.3862, "margin_dpo/margin_mean": 162.74879455566406, "margin_dpo/margin_std": 423.9322204589844, "step": 580 }, { "KL/chosen_KL_mean": -243.62631225585938, "KL/mean": -305.4908447265625, "KL/rejected_KL_mean": -367.3553466796875, "KL/std": 283.1879577636719, "epoch": 0.8783068783068783, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.72905731201172, "fcm_dpo/q_t": 0.49998801946640015, "grad_norm": 0.002351470058783889, "learning_rate": 2.259200116137039e-08, "logits/chosen": 1.2822654247283936, "logits/rejected": 1.2544565200805664, "logps/chosen": -302.95867919921875, "logps/ref_chosen": -59.332359313964844, "logps/ref_rejected": -83.64482116699219, "logps/rejected": -451.00018310546875, "loss": 1.3862, "margin_dpo/margin_mean": 123.72904968261719, "margin_dpo/margin_std": 322.47772216796875, "step": 581 }, { "KL/chosen_KL_mean": -234.3776092529297, "KL/mean": -258.0681457519531, "KL/rejected_KL_mean": -281.7586669921875, "KL/std": 228.97186279296875, "epoch": 0.8798185941043084, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 47.381065368652344, "fcm_dpo/q_t": 0.4999954104423523, "grad_norm": 0.0013384189223870635, "learning_rate": 2.204591459016525e-08, "logits/chosen": 1.285982370376587, "logits/rejected": 1.3120609521865845, "logps/chosen": -298.54046630859375, "logps/ref_chosen": -64.16285705566406, "logps/ref_rejected": -58.632896423339844, "logps/rejected": -340.3916015625, "loss": 1.3863, "margin_dpo/margin_mean": 47.38106918334961, "margin_dpo/margin_std": 315.0589904785156, "step": 582 }, { "KL/chosen_KL_mean": -187.56515502929688, "KL/mean": -304.24945068359375, "KL/rejected_KL_mean": -420.93377685546875, "KL/std": 375.9952087402344, "epoch": 0.8813303099017384, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 233.36862182617188, "fcm_dpo/q_t": 0.4999774098396301, "grad_norm": 0.0039016844239085913, "learning_rate": 2.1506204384751064e-08, "logits/chosen": 1.322705626487732, "logits/rejected": 1.2900676727294922, "logps/chosen": -239.4375457763672, "logps/ref_chosen": -51.87239456176758, "logps/ref_rejected": -83.86331176757812, "logps/rejected": -504.7970886230469, "loss": 1.3862, "margin_dpo/margin_mean": 233.36859130859375, "margin_dpo/margin_std": 436.15399169921875, "step": 583 }, { "KL/chosen_KL_mean": -171.99374389648438, "KL/mean": -252.2381591796875, "KL/rejected_KL_mean": -332.48260498046875, "KL/std": 261.9239807128906, "epoch": 0.8828420256991686, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 160.48883056640625, "fcm_dpo/q_t": 0.499984472990036, "grad_norm": 0.003471849486231804, "learning_rate": 2.09728856419826e-08, "logits/chosen": 1.3081614971160889, "logits/rejected": 1.2692360877990723, "logps/chosen": -218.5651397705078, "logps/ref_chosen": -46.571388244628906, "logps/ref_rejected": -80.67969512939453, "logps/rejected": -413.16229248046875, "loss": 1.3862, "margin_dpo/margin_mean": 160.48883056640625, "margin_dpo/margin_std": 291.8946228027344, "step": 584 }, { "KL/chosen_KL_mean": -242.6631622314453, "KL/mean": -276.41943359375, "KL/rejected_KL_mean": -310.1756896972656, "KL/std": 275.22735595703125, "epoch": 0.8843537414965986, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.51254272460938, "fcm_dpo/q_t": 0.4999934732913971, "grad_norm": 0.001750921132043004, "learning_rate": 2.044597327993153e-08, "logits/chosen": 1.2745296955108643, "logits/rejected": 1.2461663484573364, "logps/chosen": -300.7877197265625, "logps/ref_chosen": -58.124534606933594, "logps/ref_rejected": -79.00538635253906, "logps/rejected": -389.18109130859375, "loss": 1.3863, "margin_dpo/margin_mean": 67.51254272460938, "margin_dpo/margin_std": 317.5940246582031, "step": 585 }, { "KL/chosen_KL_mean": -195.4407958984375, "KL/mean": -253.28018188476562, "KL/rejected_KL_mean": -311.11956787109375, "KL/std": 251.697021484375, "epoch": 0.8858654572940288, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 115.67878723144531, "fcm_dpo/q_t": 0.4999887943267822, "grad_norm": 0.0022813198156654835, "learning_rate": 1.9925482037469187e-08, "logits/chosen": 1.2979401350021362, "logits/rejected": 1.2939859628677368, "logps/chosen": -249.54241943359375, "logps/ref_chosen": -54.10163879394531, "logps/ref_rejected": -63.72113037109375, "logps/rejected": -374.8406982421875, "loss": 1.3862, "margin_dpo/margin_mean": 115.67877960205078, "margin_dpo/margin_std": 308.5834655761719, "step": 586 }, { "KL/chosen_KL_mean": -257.64990234375, "KL/mean": -289.8480224609375, "KL/rejected_KL_mean": -322.046142578125, "KL/std": 334.7420959472656, "epoch": 0.8873771730914588, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 64.39622497558594, "fcm_dpo/q_t": 0.4999937415122986, "grad_norm": 0.0018675602041184902, "learning_rate": 1.9411426473854687e-08, "logits/chosen": 1.2932814359664917, "logits/rejected": 1.2983059883117676, "logps/chosen": -321.0671081542969, "logps/ref_chosen": -63.41719436645508, "logps/ref_rejected": -63.47003936767578, "logps/rejected": -385.5162048339844, "loss": 1.3863, "margin_dpo/margin_mean": 64.39622497558594, "margin_dpo/margin_std": 453.559814453125, "step": 587 }, { "KL/chosen_KL_mean": -256.47723388671875, "KL/mean": -315.72064208984375, "KL/rejected_KL_mean": -374.96405029296875, "KL/std": 312.8167724609375, "epoch": 0.8888888888888888, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 118.48678588867188, "fcm_dpo/q_t": 0.4999885559082031, "grad_norm": 0.0022324968595057726, "learning_rate": 1.890382096832699e-08, "logits/chosen": 1.307366132736206, "logits/rejected": 1.2943463325500488, "logps/chosen": -318.67828369140625, "logps/ref_chosen": -62.20103454589844, "logps/ref_rejected": -82.10249328613281, "logps/rejected": -457.0665283203125, "loss": 1.3862, "margin_dpo/margin_mean": 118.48678588867188, "margin_dpo/margin_std": 407.2249755859375, "step": 588 }, { "KL/chosen_KL_mean": -240.05502319335938, "KL/mean": -276.25872802734375, "KL/rejected_KL_mean": -312.46246337890625, "KL/std": 287.0700378417969, "epoch": 0.890400604686319, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 72.40746307373047, "fcm_dpo/q_t": 0.4999929666519165, "grad_norm": 0.001997460378333926, "learning_rate": 1.840267971970344e-08, "logits/chosen": 1.2659380435943604, "logits/rejected": 1.2496306896209717, "logps/chosen": -296.76861572265625, "logps/ref_chosen": -56.71361541748047, "logps/ref_rejected": -76.7366943359375, "logps/rejected": -389.19915771484375, "loss": 1.3863, "margin_dpo/margin_mean": 72.40746307373047, "margin_dpo/margin_std": 305.57440185546875, "step": 589 }, { "KL/chosen_KL_mean": -246.395263671875, "KL/mean": -291.65802001953125, "KL/rejected_KL_mean": -336.9208068847656, "KL/std": 262.71783447265625, "epoch": 0.891912320483749, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 90.52552032470703, "fcm_dpo/q_t": 0.499991238117218, "grad_norm": 0.002160015283152461, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 1.2421910762786865, "logits/rejected": 1.2292957305908203, "logps/chosen": -312.9090881347656, "logps/ref_chosen": -66.5138168334961, "logps/ref_rejected": -85.70820617675781, "logps/rejected": -422.6290283203125, "loss": 1.3863, "margin_dpo/margin_mean": 90.52552795410156, "margin_dpo/margin_std": 272.5157470703125, "step": 590 }, { "KL/chosen_KL_mean": -197.66827392578125, "KL/mean": -279.355224609375, "KL/rejected_KL_mean": -361.042236328125, "KL/std": 297.4459228515625, "epoch": 0.8934240362811792, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 163.37396240234375, "fcm_dpo/q_t": 0.4999842047691345, "grad_norm": 0.0024286925327032804, "learning_rate": 1.7419845883949098e-08, "logits/chosen": 1.2815182209014893, "logits/rejected": 1.266688585281372, "logps/chosen": -258.3654479980469, "logps/ref_chosen": -60.697181701660156, "logps/ref_rejected": -86.12278747558594, "logps/rejected": -447.1650390625, "loss": 1.3862, "margin_dpo/margin_mean": 163.37396240234375, "margin_dpo/margin_std": 392.95819091796875, "step": 591 }, { "KL/chosen_KL_mean": -201.080322265625, "KL/mean": -273.27728271484375, "KL/rejected_KL_mean": -345.4742736816406, "KL/std": 250.97198486328125, "epoch": 0.8949357520786092, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 144.39395141601562, "fcm_dpo/q_t": 0.4999859929084778, "grad_norm": 0.0026749190874397755, "learning_rate": 1.6938180788793556e-08, "logits/chosen": 1.2822060585021973, "logits/rejected": 1.2332913875579834, "logps/chosen": -252.31764221191406, "logps/ref_chosen": -51.237327575683594, "logps/ref_rejected": -81.60242462158203, "logps/rejected": -427.0766906738281, "loss": 1.3862, "margin_dpo/margin_mean": 144.39395141601562, "margin_dpo/margin_std": 278.5047607421875, "step": 592 }, { "KL/chosen_KL_mean": -165.1071319580078, "KL/mean": -229.7209014892578, "KL/rejected_KL_mean": -294.33465576171875, "KL/std": 204.59747314453125, "epoch": 0.8964474678760394, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 129.22750854492188, "fcm_dpo/q_t": 0.49998748302459717, "grad_norm": 0.002245598705485463, "learning_rate": 1.6463034933723336e-08, "logits/chosen": 1.2915213108062744, "logits/rejected": 1.2537872791290283, "logps/chosen": -207.1871337890625, "logps/ref_chosen": -42.08000183105469, "logps/ref_rejected": -68.47499084472656, "logps/rejected": -362.80963134765625, "loss": 1.3862, "margin_dpo/margin_mean": 129.22750854492188, "margin_dpo/margin_std": 228.52536010742188, "step": 593 }, { "KL/chosen_KL_mean": -243.26272583007812, "KL/mean": -281.5732421875, "KL/rejected_KL_mean": -319.88372802734375, "KL/std": 282.73321533203125, "epoch": 0.8979591836734694, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.62098693847656, "fcm_dpo/q_t": 0.49999260902404785, "grad_norm": 0.0013759155990555882, "learning_rate": 1.5994421609589385e-08, "logits/chosen": 1.2534310817718506, "logits/rejected": 1.2528616189956665, "logps/chosen": -306.92138671875, "logps/ref_chosen": -63.658668518066406, "logps/ref_rejected": -70.35597229003906, "logps/rejected": -390.23968505859375, "loss": 1.3863, "margin_dpo/margin_mean": 76.62098693847656, "margin_dpo/margin_std": 327.9439697265625, "step": 594 }, { "KL/chosen_KL_mean": -198.01931762695312, "KL/mean": -272.7323913574219, "KL/rejected_KL_mean": -347.44549560546875, "KL/std": 273.14923095703125, "epoch": 0.8994708994708994, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 149.4261474609375, "fcm_dpo/q_t": 0.4999855160713196, "grad_norm": 0.0027599758468568325, "learning_rate": 1.553235392451377e-08, "logits/chosen": 1.3034560680389404, "logits/rejected": 1.2654231786727905, "logps/chosen": -254.2380828857422, "logps/ref_chosen": -56.21875762939453, "logps/ref_rejected": -83.95773315429688, "logps/rejected": -431.4031982421875, "loss": 1.3862, "margin_dpo/margin_mean": 149.4261474609375, "margin_dpo/margin_std": 291.680908203125, "step": 595 }, { "KL/chosen_KL_mean": -268.5811767578125, "KL/mean": -279.02777099609375, "KL/rejected_KL_mean": -289.474365234375, "KL/std": 269.3888244628906, "epoch": 0.9009826152683296, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.893169403076172, "fcm_dpo/q_t": 0.49999797344207764, "grad_norm": 0.001436872873455286, "learning_rate": 1.507684480352292e-08, "logits/chosen": 1.215804100036621, "logits/rejected": 1.2338910102844238, "logps/chosen": -337.06207275390625, "logps/ref_chosen": -68.48088073730469, "logps/ref_rejected": -61.732967376708984, "logps/rejected": -351.20733642578125, "loss": 1.3863, "margin_dpo/margin_mean": 20.893165588378906, "margin_dpo/margin_std": 239.36436462402344, "step": 596 }, { "KL/chosen_KL_mean": -190.21514892578125, "KL/mean": -211.99703979492188, "KL/rejected_KL_mean": -233.7789306640625, "KL/std": 190.4124755859375, "epoch": 0.9024943310657596, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 43.563785552978516, "fcm_dpo/q_t": 0.49999576807022095, "grad_norm": 0.0016540532233193517, "learning_rate": 1.4627906988186111e-08, "logits/chosen": 1.276237964630127, "logits/rejected": 1.267054557800293, "logps/chosen": -239.07266235351562, "logps/ref_chosen": -48.85750961303711, "logps/ref_rejected": -55.068084716796875, "logps/rejected": -288.8470153808594, "loss": 1.3863, "margin_dpo/margin_mean": 43.56378173828125, "margin_dpo/margin_std": 183.7689208984375, "step": 597 }, { "KL/chosen_KL_mean": -242.9158935546875, "KL/mean": -291.18408203125, "KL/rejected_KL_mean": -339.4522705078125, "KL/std": 272.9280090332031, "epoch": 0.9040060468631897, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.53633880615234, "fcm_dpo/q_t": 0.49999064207077026, "grad_norm": 0.0021814981009811163, "learning_rate": 1.4185553036259095e-08, "logits/chosen": 1.2822654247283936, "logits/rejected": 1.2468507289886475, "logps/chosen": -301.80303955078125, "logps/ref_chosen": -58.88715362548828, "logps/ref_rejected": -81.43145751953125, "logps/rejected": -420.88372802734375, "loss": 1.3863, "margin_dpo/margin_mean": 96.53633117675781, "margin_dpo/margin_std": 309.15411376953125, "step": 598 }, { "KL/chosen_KL_mean": -241.9964599609375, "KL/mean": -287.59735107421875, "KL/rejected_KL_mean": -333.1982421875, "KL/std": 273.47198486328125, "epoch": 0.9055177626606198, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.2018051147461, "fcm_dpo/q_t": 0.49999117851257324, "grad_norm": 0.001978973625227809, "learning_rate": 1.3749795321332885e-08, "logits/chosen": 1.3052904605865479, "logits/rejected": 1.2893433570861816, "logps/chosen": -299.6036376953125, "logps/ref_chosen": -57.60719299316406, "logps/ref_rejected": -71.80469512939453, "logps/rejected": -405.0029296875, "loss": 1.3863, "margin_dpo/margin_mean": 91.20179748535156, "margin_dpo/margin_std": 326.43243408203125, "step": 599 }, { "KL/chosen_KL_mean": -229.87130737304688, "KL/mean": -281.87884521484375, "KL/rejected_KL_mean": -333.8863525390625, "KL/std": 281.9407958984375, "epoch": 0.9070294784580499, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.01502227783203, "fcm_dpo/q_t": 0.49998992681503296, "grad_norm": 0.00212049949914217, "learning_rate": 1.3320646032487393e-08, "logits/chosen": 1.2342650890350342, "logits/rejected": 1.2097012996673584, "logps/chosen": -288.3136291503906, "logps/ref_chosen": -58.44231414794922, "logps/ref_rejected": -83.64639282226562, "logps/rejected": -417.53271484375, "loss": 1.3863, "margin_dpo/margin_mean": 104.01502227783203, "margin_dpo/margin_std": 371.31842041015625, "step": 600 }, { "KL/chosen_KL_mean": -204.21005249023438, "KL/mean": -272.1788635253906, "KL/rejected_KL_mean": -340.147705078125, "KL/std": 289.89752197265625, "epoch": 0.90854119425548, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.9376220703125, "fcm_dpo/q_t": 0.499986857175827, "grad_norm": 0.002290458418428898, "learning_rate": 1.2898117173950868e-08, "logits/chosen": 1.2456085681915283, "logits/rejected": 1.209405541419983, "logps/chosen": -259.80438232421875, "logps/ref_chosen": -55.59432601928711, "logps/ref_rejected": -83.68630981445312, "logps/rejected": -423.833984375, "loss": 1.3862, "margin_dpo/margin_mean": 135.9376220703125, "margin_dpo/margin_std": 325.400634765625, "step": 601 }, { "KL/chosen_KL_mean": -210.0914764404297, "KL/mean": -256.0052185058594, "KL/rejected_KL_mean": -301.9189453125, "KL/std": 259.4412841796875, "epoch": 0.91005291005291, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 91.82748413085938, "fcm_dpo/q_t": 0.49999111890792847, "grad_norm": 0.002178723691031337, "learning_rate": 1.2482220564763667e-08, "logits/chosen": 1.2279410362243652, "logits/rejected": 1.2131808996200562, "logps/chosen": -266.440673828125, "logps/ref_chosen": -56.349185943603516, "logps/ref_rejected": -71.9959716796875, "logps/rejected": -373.9149169921875, "loss": 1.3863, "margin_dpo/margin_mean": 91.82748413085938, "margin_dpo/margin_std": 305.39068603515625, "step": 602 }, { "KL/chosen_KL_mean": -200.24911499023438, "KL/mean": -269.5469665527344, "KL/rejected_KL_mean": -338.8448486328125, "KL/std": 285.01885986328125, "epoch": 0.9115646258503401, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 138.59573364257812, "fcm_dpo/q_t": 0.49998658895492554, "grad_norm": 0.0024140651803463697, "learning_rate": 1.2072967838448051e-08, "logits/chosen": 1.2549041509628296, "logits/rejected": 1.244372844696045, "logps/chosen": -253.41749572753906, "logps/ref_chosen": -53.16838836669922, "logps/ref_rejected": -73.8604736328125, "logps/rejected": -412.705322265625, "loss": 1.3862, "margin_dpo/margin_mean": 138.59573364257812, "margin_dpo/margin_std": 345.45501708984375, "step": 603 }, { "KL/chosen_KL_mean": -227.04763793945312, "KL/mean": -257.6112365722656, "KL/rejected_KL_mean": -288.1748046875, "KL/std": 230.52130126953125, "epoch": 0.9130763416477702, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 61.12716293334961, "fcm_dpo/q_t": 0.49999406933784485, "grad_norm": 0.0016172666801139712, "learning_rate": 1.1670370442682459e-08, "logits/chosen": 1.210902214050293, "logits/rejected": 1.2236418724060059, "logps/chosen": -299.69708251953125, "logps/ref_chosen": -72.64942169189453, "logps/ref_rejected": -69.8792724609375, "logps/rejected": -358.0540771484375, "loss": 1.3863, "margin_dpo/margin_mean": 61.127159118652344, "margin_dpo/margin_std": 327.2840576171875, "step": 604 }, { "KL/chosen_KL_mean": -239.35777282714844, "KL/mean": -294.0518798828125, "KL/rejected_KL_mean": -348.7460632324219, "KL/std": 237.30853271484375, "epoch": 0.9145880574452003, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 109.38829040527344, "fcm_dpo/q_t": 0.49998942017555237, "grad_norm": 0.002217090455815196, "learning_rate": 1.1274439638981532e-08, "logits/chosen": 1.2849633693695068, "logits/rejected": 1.2634557485580444, "logps/chosen": -300.9706115722656, "logps/ref_chosen": -61.61284637451172, "logps/ref_rejected": -79.34398651123047, "logps/rejected": -428.09002685546875, "loss": 1.3863, "margin_dpo/margin_mean": 109.38829040527344, "margin_dpo/margin_std": 277.03143310546875, "step": 605 }, { "KL/chosen_KL_mean": -208.34193420410156, "KL/mean": -277.22564697265625, "KL/rejected_KL_mean": -346.10943603515625, "KL/std": 279.93414306640625, "epoch": 0.9160997732426304, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 137.76748657226562, "fcm_dpo/q_t": 0.4999866485595703, "grad_norm": 0.00246420968323946, "learning_rate": 1.0885186502381016e-08, "logits/chosen": 1.2400498390197754, "logits/rejected": 1.221116542816162, "logps/chosen": -262.80615234375, "logps/ref_chosen": -54.46424102783203, "logps/ref_rejected": -79.62708282470703, "logps/rejected": -425.73651123046875, "loss": 1.3862, "margin_dpo/margin_mean": 137.76748657226562, "margin_dpo/margin_std": 318.7607727050781, "step": 606 }, { "KL/chosen_KL_mean": -250.8245391845703, "KL/mean": -299.26092529296875, "KL/rejected_KL_mean": -347.69732666015625, "KL/std": 316.63677978515625, "epoch": 0.9176114890400605, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 96.87279510498047, "fcm_dpo/q_t": 0.49999064207077026, "grad_norm": 0.0016430618707090616, "learning_rate": 1.0502621921127774e-08, "logits/chosen": 1.2330322265625, "logits/rejected": 1.2289059162139893, "logps/chosen": -313.6854248046875, "logps/ref_chosen": -62.86086654663086, "logps/ref_rejected": -72.5501937866211, "logps/rejected": -420.2475280761719, "loss": 1.3863, "margin_dpo/margin_mean": 96.87281036376953, "margin_dpo/margin_std": 384.65234375, "step": 607 }, { "KL/chosen_KL_mean": -240.77908325195312, "KL/mean": -311.6595458984375, "KL/rejected_KL_mean": -382.5400390625, "KL/std": 341.69573974609375, "epoch": 0.9191232048374905, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 141.76089477539062, "fcm_dpo/q_t": 0.49998629093170166, "grad_norm": 0.002434444148093462, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 1.211251974105835, "logits/rejected": 1.174188494682312, "logps/chosen": -303.9598083496094, "logps/ref_chosen": -63.18071746826172, "logps/ref_rejected": -99.15888214111328, "logps/rejected": -481.69891357421875, "loss": 1.3862, "margin_dpo/margin_mean": 141.7609100341797, "margin_dpo/margin_std": 400.3000793457031, "step": 608 }, { "KL/chosen_KL_mean": -180.31060791015625, "KL/mean": -238.63388061523438, "KL/rejected_KL_mean": -296.9571533203125, "KL/std": 239.36196899414062, "epoch": 0.9206349206349206, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.64653015136719, "fcm_dpo/q_t": 0.49998870491981506, "grad_norm": 0.001932095387019217, "learning_rate": 9.757601041885694e-09, "logits/chosen": 1.271007776260376, "logits/rejected": 1.262525200843811, "logps/chosen": -228.933837890625, "logps/ref_chosen": -48.62322235107422, "logps/ref_rejected": -68.28271484375, "logps/rejected": -365.2398681640625, "loss": 1.3862, "margin_dpo/margin_mean": 116.64653015136719, "margin_dpo/margin_std": 259.3739318847656, "step": 609 }, { "KL/chosen_KL_mean": -303.2109680175781, "KL/mean": -337.4134521484375, "KL/rejected_KL_mean": -371.615966796875, "KL/std": 341.649658203125, "epoch": 0.9221466364323507, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 68.40501403808594, "fcm_dpo/q_t": 0.49999338388442993, "grad_norm": 0.0016822778852656484, "learning_rate": 9.395165583732379e-09, "logits/chosen": 1.248294711112976, "logits/rejected": 1.2520670890808105, "logps/chosen": -375.8760986328125, "logps/ref_chosen": -72.66513061523438, "logps/ref_rejected": -87.15310668945312, "logps/rejected": -458.7690734863281, "loss": 1.3863, "margin_dpo/margin_mean": 68.40501403808594, "margin_dpo/margin_std": 410.22003173828125, "step": 610 }, { "KL/chosen_KL_mean": -202.11181640625, "KL/mean": -256.2706604003906, "KL/rejected_KL_mean": -310.42950439453125, "KL/std": 272.47833251953125, "epoch": 0.9236583522297808, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.31767272949219, "fcm_dpo/q_t": 0.49998950958251953, "grad_norm": 0.0027471468783915043, "learning_rate": 9.03946036001449e-09, "logits/chosen": 1.3028149604797363, "logits/rejected": 1.2825827598571777, "logps/chosen": -250.42039489746094, "logps/ref_chosen": -48.30857849121094, "logps/ref_rejected": -70.6141128540039, "logps/rejected": -381.0436096191406, "loss": 1.3863, "margin_dpo/margin_mean": 108.31765747070312, "margin_dpo/margin_std": 253.734375, "step": 611 }, { "KL/chosen_KL_mean": -241.52801513671875, "KL/mean": -312.7408447265625, "KL/rejected_KL_mean": -383.95361328125, "KL/std": 254.8455810546875, "epoch": 0.9251700680272109, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 142.42562866210938, "fcm_dpo/q_t": 0.4999862313270569, "grad_norm": 0.0026474539190530777, "learning_rate": 8.690495320571839e-09, "logits/chosen": 1.2259531021118164, "logits/rejected": 1.1953879594802856, "logps/chosen": -302.75958251953125, "logps/ref_chosen": -61.23155975341797, "logps/ref_rejected": -94.37979888916016, "logps/rejected": -478.33343505859375, "loss": 1.3862, "margin_dpo/margin_mean": 142.42559814453125, "margin_dpo/margin_std": 282.14910888671875, "step": 612 }, { "KL/chosen_KL_mean": -215.14187622070312, "KL/mean": -248.74453735351562, "KL/rejected_KL_mean": -282.34716796875, "KL/std": 317.31195068359375, "epoch": 0.926681783824641, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 67.20530700683594, "fcm_dpo/q_t": 0.4999935030937195, "grad_norm": 0.0015928486827760935, "learning_rate": 8.348280226706722e-09, "logits/chosen": 1.2564080953598022, "logits/rejected": 1.26479971408844, "logps/chosen": -269.125, "logps/ref_chosen": -53.98310852050781, "logps/ref_rejected": -58.32208251953125, "logps/rejected": -340.66925048828125, "loss": 1.3863, "margin_dpo/margin_mean": 67.20530700683594, "margin_dpo/margin_std": 282.4429626464844, "step": 613 }, { "KL/chosen_KL_mean": -235.35379028320312, "KL/mean": -268.393798828125, "KL/rejected_KL_mean": -301.4338073730469, "KL/std": 251.66497802734375, "epoch": 0.9281934996220711, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 66.08002471923828, "fcm_dpo/q_t": 0.49999362230300903, "grad_norm": 0.0015550341922789812, "learning_rate": 8.012824650910937e-09, "logits/chosen": 1.275475025177002, "logits/rejected": 1.2758376598358154, "logps/chosen": -295.5968322753906, "logps/ref_chosen": -60.24303436279297, "logps/ref_rejected": -72.26258850097656, "logps/rejected": -373.6964111328125, "loss": 1.3863, "margin_dpo/margin_mean": 66.08003234863281, "margin_dpo/margin_std": 330.7596435546875, "step": 614 }, { "KL/chosen_KL_mean": -288.22857666015625, "KL/mean": -350.49859619140625, "KL/rejected_KL_mean": -412.76861572265625, "KL/std": 364.74468994140625, "epoch": 0.9297052154195011, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 124.53999328613281, "fcm_dpo/q_t": 0.49998795986175537, "grad_norm": 0.0024001309648156166, "learning_rate": 7.684137976598088e-09, "logits/chosen": 1.2872897386550903, "logits/rejected": 1.2527488470077515, "logps/chosen": -360.3232727050781, "logps/ref_chosen": -72.09467315673828, "logps/ref_rejected": -104.02980041503906, "logps/rejected": -516.7984008789062, "loss": 1.3862, "margin_dpo/margin_mean": 124.53999328613281, "margin_dpo/margin_std": 407.09320068359375, "step": 615 }, { "KL/chosen_KL_mean": -214.70602416992188, "KL/mean": -280.04010009765625, "KL/rejected_KL_mean": -345.3741455078125, "KL/std": 338.1444091796875, "epoch": 0.9312169312169312, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.6681365966797, "fcm_dpo/q_t": 0.4999873638153076, "grad_norm": 0.002586161717772484, "learning_rate": 7.36222939784098e-09, "logits/chosen": 1.3120701313018799, "logits/rejected": 1.2948602437973022, "logps/chosen": -273.23675537109375, "logps/ref_chosen": -58.530723571777344, "logps/ref_rejected": -75.48025512695312, "logps/rejected": -420.85443115234375, "loss": 1.3862, "margin_dpo/margin_mean": 130.6681365966797, "margin_dpo/margin_std": 311.08026123046875, "step": 616 }, { "KL/chosen_KL_mean": -231.34402465820312, "KL/mean": -275.96807861328125, "KL/rejected_KL_mean": -320.59210205078125, "KL/std": 250.75045776367188, "epoch": 0.9327286470143613, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 89.24805450439453, "fcm_dpo/q_t": 0.49999135732650757, "grad_norm": 0.0019385057967156172, "learning_rate": 7.047107919114586e-09, "logits/chosen": 1.3058488368988037, "logits/rejected": 1.2810614109039307, "logps/chosen": -288.95269775390625, "logps/ref_chosen": -57.608673095703125, "logps/ref_rejected": -81.22109985351562, "logps/rejected": -401.81317138671875, "loss": 1.3863, "margin_dpo/margin_mean": 89.24806213378906, "margin_dpo/margin_std": 294.38568115234375, "step": 617 }, { "KL/chosen_KL_mean": -219.1179656982422, "KL/mean": -298.6544189453125, "KL/rejected_KL_mean": -378.19085693359375, "KL/std": 302.0130310058594, "epoch": 0.9342403628117913, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 159.0728759765625, "fcm_dpo/q_t": 0.49998462200164795, "grad_norm": 0.003072551218792796, "learning_rate": 6.738782355044048e-09, "logits/chosen": 1.2910728454589844, "logits/rejected": 1.2586588859558105, "logps/chosen": -275.81390380859375, "logps/ref_chosen": -56.69594192504883, "logps/ref_rejected": -85.92362976074219, "logps/rejected": -464.1144714355469, "loss": 1.3862, "margin_dpo/margin_mean": 159.0728759765625, "margin_dpo/margin_std": 343.8255920410156, "step": 618 }, { "KL/chosen_KL_mean": -202.87046813964844, "KL/mean": -281.81170654296875, "KL/rejected_KL_mean": -360.75299072265625, "KL/std": 335.5523681640625, "epoch": 0.9357520786092215, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 157.8825225830078, "fcm_dpo/q_t": 0.4999847114086151, "grad_norm": 0.002635043114423752, "learning_rate": 6.437261330158206e-09, "logits/chosen": 1.328965425491333, "logits/rejected": 1.297224760055542, "logps/chosen": -256.92889404296875, "logps/ref_chosen": -54.05841827392578, "logps/ref_rejected": -83.55493927001953, "logps/rejected": -444.30792236328125, "loss": 1.3862, "margin_dpo/margin_mean": 157.8825225830078, "margin_dpo/margin_std": 431.3692626953125, "step": 619 }, { "KL/chosen_KL_mean": -240.50149536132812, "KL/mean": -257.7876281738281, "KL/rejected_KL_mean": -275.0737609863281, "KL/std": 267.0464782714844, "epoch": 0.9372637944066515, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 34.57228088378906, "fcm_dpo/q_t": 0.4999966323375702, "grad_norm": 0.0017283577471971512, "learning_rate": 6.142553278648238e-09, "logits/chosen": 1.2990144491195679, "logits/rejected": 1.3006207942962646, "logps/chosen": -303.8712158203125, "logps/ref_chosen": -63.36971664428711, "logps/ref_rejected": -65.68269348144531, "logps/rejected": -340.7564392089844, "loss": 1.3863, "margin_dpo/margin_mean": 34.57228088378906, "margin_dpo/margin_std": 325.6532287597656, "step": 620 }, { "KL/chosen_KL_mean": -205.8194580078125, "KL/mean": -296.57342529296875, "KL/rejected_KL_mean": -387.327392578125, "KL/std": 351.635498046875, "epoch": 0.9387755102040817, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 181.50796508789062, "fcm_dpo/q_t": 0.49998241662979126, "grad_norm": 0.0033599112648516893, "learning_rate": 5.854666444131934e-09, "logits/chosen": 1.2818564176559448, "logits/rejected": 1.236130952835083, "logps/chosen": -258.14068603515625, "logps/ref_chosen": -52.321224212646484, "logps/ref_rejected": -88.09001159667969, "logps/rejected": -475.41741943359375, "loss": 1.3862, "margin_dpo/margin_mean": 181.50796508789062, "margin_dpo/margin_std": 466.3568115234375, "step": 621 }, { "KL/chosen_KL_mean": -252.88168334960938, "KL/mean": -307.3717956542969, "KL/rejected_KL_mean": -361.8619384765625, "KL/std": 309.87310791015625, "epoch": 0.9402872260015117, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 108.98025512695312, "fcm_dpo/q_t": 0.49998944997787476, "grad_norm": 0.0022065993398427963, "learning_rate": 5.573608879422875e-09, "logits/chosen": 1.2621958255767822, "logits/rejected": 1.2421605587005615, "logps/chosen": -312.74713134765625, "logps/ref_chosen": -59.86545944213867, "logps/ref_rejected": -81.86668395996094, "logps/rejected": -443.7286071777344, "loss": 1.3863, "margin_dpo/margin_mean": 108.98025512695312, "margin_dpo/margin_std": 352.85650634765625, "step": 622 }, { "KL/chosen_KL_mean": -274.98095703125, "KL/mean": -340.2034606933594, "KL/rejected_KL_mean": -405.4259338378906, "KL/std": 373.55999755859375, "epoch": 0.9417989417989417, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 130.4449462890625, "fcm_dpo/q_t": 0.4999873638153076, "grad_norm": 0.002324384171515703, "learning_rate": 5.299388446305342e-09, "logits/chosen": 1.2612360715866089, "logits/rejected": 1.245605707168579, "logps/chosen": -342.34942626953125, "logps/ref_chosen": -67.36846160888672, "logps/ref_rejected": -82.02733612060547, "logps/rejected": -487.4532470703125, "loss": 1.3862, "margin_dpo/margin_mean": 130.4449462890625, "margin_dpo/margin_std": 484.2095642089844, "step": 623 }, { "KL/chosen_KL_mean": -195.49905395507812, "KL/mean": -272.87152099609375, "KL/rejected_KL_mean": -350.2440490722656, "KL/std": 312.8592834472656, "epoch": 0.9433106575963719, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 154.7449951171875, "fcm_dpo/q_t": 0.4999850392341614, "grad_norm": 0.002652752446010709, "learning_rate": 5.03201281531429e-09, "logits/chosen": 1.2726714611053467, "logits/rejected": 1.2336527109146118, "logps/chosen": -246.52560424804688, "logps/ref_chosen": -51.02655029296875, "logps/ref_rejected": -76.49203491210938, "logps/rejected": -426.736083984375, "loss": 1.3862, "margin_dpo/margin_mean": 154.7449951171875, "margin_dpo/margin_std": 322.3939514160156, "step": 624 }, { "KL/chosen_KL_mean": -213.73834228515625, "KL/mean": -271.16351318359375, "KL/rejected_KL_mean": -328.5887145996094, "KL/std": 293.5513000488281, "epoch": 0.9448223733938019, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 114.85034942626953, "fcm_dpo/q_t": 0.4999888837337494, "grad_norm": 0.0026168376207351685, "learning_rate": 4.7714894655209174e-09, "logits/chosen": 1.3009436130523682, "logits/rejected": 1.2550803422927856, "logps/chosen": -267.9459533691406, "logps/ref_chosen": -54.20761489868164, "logps/ref_rejected": -84.93669128417969, "logps/rejected": -413.525390625, "loss": 1.3862, "margin_dpo/margin_mean": 114.85035705566406, "margin_dpo/margin_std": 311.7130126953125, "step": 625 }, { "KL/chosen_KL_mean": -169.8229522705078, "KL/mean": -267.14849853515625, "KL/rejected_KL_mean": -364.47406005859375, "KL/std": 340.929931640625, "epoch": 0.9463340891912321, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 194.651123046875, "fcm_dpo/q_t": 0.499981164932251, "grad_norm": 0.0032611822243779898, "learning_rate": 4.517825684323323e-09, "logits/chosen": 1.2939801216125488, "logits/rejected": 1.2345411777496338, "logps/chosen": -214.88497924804688, "logps/ref_chosen": -45.06201934814453, "logps/ref_rejected": -89.66368103027344, "logps/rejected": -454.13775634765625, "loss": 1.3862, "margin_dpo/margin_mean": 194.651123046875, "margin_dpo/margin_std": 399.38238525390625, "step": 626 }, { "KL/chosen_KL_mean": -223.85272216796875, "KL/mean": -312.509033203125, "KL/rejected_KL_mean": -401.1653747558594, "KL/std": 350.2436218261719, "epoch": 0.9478458049886621, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 177.31263732910156, "fcm_dpo/q_t": 0.4999828338623047, "grad_norm": 0.0033359085209667683, "learning_rate": 4.271028567242818e-09, "logits/chosen": 1.2410776615142822, "logits/rejected": 1.183640718460083, "logps/chosen": -282.643798828125, "logps/ref_chosen": -58.791053771972656, "logps/ref_rejected": -94.90802001953125, "logps/rejected": -496.0733947753906, "loss": 1.3862, "margin_dpo/margin_mean": 177.3126220703125, "margin_dpo/margin_std": 358.99542236328125, "step": 627 }, { "KL/chosen_KL_mean": -220.1020050048828, "KL/mean": -270.9043273925781, "KL/rejected_KL_mean": -321.7066650390625, "KL/std": 269.397705078125, "epoch": 0.9493575207860923, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 101.60467529296875, "fcm_dpo/q_t": 0.49999016523361206, "grad_norm": 0.0020546908490359783, "learning_rate": 4.0311050177251895e-09, "logits/chosen": 1.292949914932251, "logits/rejected": 1.277388572692871, "logps/chosen": -272.90557861328125, "logps/ref_chosen": -52.80357360839844, "logps/ref_rejected": -76.49468994140625, "logps/rejected": -398.20135498046875, "loss": 1.3863, "margin_dpo/margin_mean": 101.60467529296875, "margin_dpo/margin_std": 336.76214599609375, "step": 628 }, { "KL/chosen_KL_mean": -258.51788330078125, "KL/mean": -296.933349609375, "KL/rejected_KL_mean": -335.34881591796875, "KL/std": 292.3125, "epoch": 0.9508692365835223, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 76.83094787597656, "fcm_dpo/q_t": 0.4999925494194031, "grad_norm": 0.0015813360223546624, "learning_rate": 3.798061746947995e-09, "logits/chosen": 1.2685928344726562, "logits/rejected": 1.2776813507080078, "logps/chosen": -329.2353820800781, "logps/ref_chosen": -70.71749877929688, "logps/ref_rejected": -78.96273803710938, "logps/rejected": -414.31158447265625, "loss": 1.3863, "margin_dpo/margin_mean": 76.83094787597656, "margin_dpo/margin_std": 334.86175537109375, "step": 629 }, { "KL/chosen_KL_mean": -204.28152465820312, "KL/mean": -270.9383544921875, "KL/rejected_KL_mean": -337.5951843261719, "KL/std": 271.621826171875, "epoch": 0.9523809523809523, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.31365966796875, "fcm_dpo/q_t": 0.49998709559440613, "grad_norm": 0.0021148163359612226, "learning_rate": 3.5719052736323806e-09, "logits/chosen": 1.272836685180664, "logits/rejected": 1.2661714553833008, "logps/chosen": -260.4829406738281, "logps/ref_chosen": -56.201412200927734, "logps/ref_rejected": -74.69807434082031, "logps/rejected": -412.29327392578125, "loss": 1.3862, "margin_dpo/margin_mean": 133.31365966796875, "margin_dpo/margin_std": 350.02862548828125, "step": 630 }, { "KL/chosen_KL_mean": -239.1774444580078, "KL/mean": -327.0312805175781, "KL/rejected_KL_mean": -414.8851318359375, "KL/std": 334.594482421875, "epoch": 0.9538926681783825, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 175.70767211914062, "fcm_dpo/q_t": 0.4999829828739166, "grad_norm": 0.0031651852186769247, "learning_rate": 3.352641923861144e-09, "logits/chosen": 1.3142149448394775, "logits/rejected": 1.264453411102295, "logps/chosen": -297.998046875, "logps/ref_chosen": -58.82059860229492, "logps/ref_rejected": -96.51437377929688, "logps/rejected": -511.3995056152344, "loss": 1.3862, "margin_dpo/margin_mean": 175.70765686035156, "margin_dpo/margin_std": 397.9634094238281, "step": 631 }, { "KL/chosen_KL_mean": -214.4776153564453, "KL/mean": -264.37371826171875, "KL/rejected_KL_mean": -314.269775390625, "KL/std": 248.91575622558594, "epoch": 0.9554043839758125, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.79216003417969, "fcm_dpo/q_t": 0.4999903440475464, "grad_norm": 0.0018715175101533532, "learning_rate": 3.140277830901428e-09, "logits/chosen": 1.289548397064209, "logits/rejected": 1.2905744314193726, "logps/chosen": -273.263671875, "logps/ref_chosen": -58.786048889160156, "logps/ref_rejected": -67.21923828125, "logps/rejected": -381.489013671875, "loss": 1.3863, "margin_dpo/margin_mean": 99.79216003417969, "margin_dpo/margin_std": 307.36798095703125, "step": 632 }, { "KL/chosen_KL_mean": -178.64141845703125, "KL/mean": -238.59205627441406, "KL/rejected_KL_mean": -298.5426940917969, "KL/std": 253.90550231933594, "epoch": 0.9569160997732427, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 119.90129089355469, "fcm_dpo/q_t": 0.4999883770942688, "grad_norm": 0.002127637853845954, "learning_rate": 2.9348189350335007e-09, "logits/chosen": 1.2887210845947266, "logits/rejected": 1.2725958824157715, "logps/chosen": -230.77162170410156, "logps/ref_chosen": -52.13019561767578, "logps/ref_rejected": -67.23016357421875, "logps/rejected": -365.7728576660156, "loss": 1.3862, "margin_dpo/margin_mean": 119.90128326416016, "margin_dpo/margin_std": 286.3564453125, "step": 633 }, { "KL/chosen_KL_mean": -235.68812561035156, "KL/mean": -247.49952697753906, "KL/rejected_KL_mean": -259.3109436035156, "KL/std": 264.06658935546875, "epoch": 0.9584278155706727, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.622833251953125, "fcm_dpo/q_t": 0.49999770522117615, "grad_norm": 0.0016156296478584409, "learning_rate": 2.736270983384276e-09, "logits/chosen": 1.3023512363433838, "logits/rejected": 1.3197746276855469, "logps/chosen": -296.66790771484375, "logps/ref_chosen": -60.97979736328125, "logps/ref_rejected": -58.50825119018555, "logps/rejected": -317.8191833496094, "loss": 1.3863, "margin_dpo/margin_mean": 23.62282943725586, "margin_dpo/margin_std": 295.5430908203125, "step": 634 }, { "KL/chosen_KL_mean": -270.3648376464844, "KL/mean": -338.20135498046875, "KL/rejected_KL_mean": -406.0378723144531, "KL/std": 269.56549072265625, "epoch": 0.9599395313681028, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 135.67300415039062, "fcm_dpo/q_t": 0.4999868869781494, "grad_norm": 0.0024493515957146883, "learning_rate": 2.5446395297668287e-09, "logits/chosen": 1.2402102947235107, "logits/rejected": 1.2276395559310913, "logps/chosen": -336.3378601074219, "logps/ref_chosen": -65.9730224609375, "logps/ref_rejected": -85.61317443847656, "logps/rejected": -491.6510314941406, "loss": 1.3862, "margin_dpo/margin_mean": 135.67300415039062, "margin_dpo/margin_std": 318.53192138671875, "step": 635 }, { "KL/chosen_KL_mean": -190.3470001220703, "KL/mean": -266.31427001953125, "KL/rejected_KL_mean": -342.281494140625, "KL/std": 254.54660034179688, "epoch": 0.9614512471655329, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 151.93450927734375, "fcm_dpo/q_t": 0.4999852776527405, "grad_norm": 0.002561971079558134, "learning_rate": 2.359929934524829e-09, "logits/chosen": 1.2839725017547607, "logits/rejected": 1.2400085926055908, "logps/chosen": -239.48716735839844, "logps/ref_chosen": -49.140167236328125, "logps/ref_rejected": -81.26971435546875, "logps/rejected": -423.55120849609375, "loss": 1.3862, "margin_dpo/margin_mean": 151.93450927734375, "margin_dpo/margin_std": 261.3985595703125, "step": 636 }, { "KL/chosen_KL_mean": -266.4497985839844, "KL/mean": -329.82757568359375, "KL/rejected_KL_mean": -393.205322265625, "KL/std": 320.78643798828125, "epoch": 0.9629629629629629, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 126.7555160522461, "fcm_dpo/q_t": 0.49998772144317627, "grad_norm": 0.002606100868433714, "learning_rate": 2.1821473643827137e-09, "logits/chosen": 1.2443512678146362, "logits/rejected": 1.230696201324463, "logps/chosen": -340.1463928222656, "logps/ref_chosen": -73.69658660888672, "logps/ref_rejected": -83.01487731933594, "logps/rejected": -476.22021484375, "loss": 1.3862, "margin_dpo/margin_mean": 126.75550842285156, "margin_dpo/margin_std": 355.1866149902344, "step": 637 }, { "KL/chosen_KL_mean": -243.36972045898438, "KL/mean": -302.26416015625, "KL/rejected_KL_mean": -361.1585388183594, "KL/std": 264.62188720703125, "epoch": 0.9644746787603931, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 117.788818359375, "fcm_dpo/q_t": 0.4999886155128479, "grad_norm": 0.0024036893155425787, "learning_rate": 2.0112967923011646e-09, "logits/chosen": 1.290708065032959, "logits/rejected": 1.2749075889587402, "logps/chosen": -306.15130615234375, "logps/ref_chosen": -62.78158187866211, "logps/ref_rejected": -85.40478515625, "logps/rejected": -446.5633239746094, "loss": 1.3862, "margin_dpo/margin_mean": 117.788818359375, "margin_dpo/margin_std": 344.58038330078125, "step": 638 }, { "KL/chosen_KL_mean": -215.73281860351562, "KL/mean": -291.6164855957031, "KL/rejected_KL_mean": -367.5001220703125, "KL/std": 315.470947265625, "epoch": 0.9659863945578231, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 151.767333984375, "fcm_dpo/q_t": 0.49998530745506287, "grad_norm": 0.002579744439572096, "learning_rate": 1.847382997337943e-09, "logits/chosen": 1.2794743776321411, "logits/rejected": 1.251741886138916, "logps/chosen": -269.4994201660156, "logps/ref_chosen": -53.76658630371094, "logps/ref_rejected": -72.30009460449219, "logps/rejected": -439.80023193359375, "loss": 1.3862, "margin_dpo/margin_mean": 151.767333984375, "margin_dpo/margin_std": 374.64910888671875, "step": 639 }, { "KL/chosen_KL_mean": -203.71878051757812, "KL/mean": -270.4925842285156, "KL/rejected_KL_mean": -337.2663879394531, "KL/std": 287.63861083984375, "epoch": 0.9674981103552532, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 133.547607421875, "fcm_dpo/q_t": 0.49998706579208374, "grad_norm": 0.0021755900233983994, "learning_rate": 1.690410564514244e-09, "logits/chosen": 1.3102037906646729, "logits/rejected": 1.282835602760315, "logps/chosen": -255.13656616210938, "logps/ref_chosen": -51.41777801513672, "logps/ref_rejected": -77.27879333496094, "logps/rejected": -414.545166015625, "loss": 1.3862, "margin_dpo/margin_mean": 133.54759216308594, "margin_dpo/margin_std": 318.0604248046875, "step": 640 }, { "KL/chosen_KL_mean": -253.23318481445312, "KL/mean": -302.85894775390625, "KL/rejected_KL_mean": -352.48468017578125, "KL/std": 273.1927795410156, "epoch": 0.9690098261526833, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 99.2514877319336, "fcm_dpo/q_t": 0.49999040365219116, "grad_norm": 0.0019294384401291609, "learning_rate": 1.5403838846864692e-09, "logits/chosen": 1.2644659280776978, "logits/rejected": 1.2682955265045166, "logps/chosen": -324.287841796875, "logps/ref_chosen": -71.0546646118164, "logps/ref_rejected": -82.2440185546875, "logps/rejected": -434.72869873046875, "loss": 1.3863, "margin_dpo/margin_mean": 99.2514877319336, "margin_dpo/margin_std": 286.9643859863281, "step": 641 }, { "KL/chosen_KL_mean": -276.5420227050781, "KL/mean": -290.9453430175781, "KL/rejected_KL_mean": -305.3486633300781, "KL/std": 272.9807434082031, "epoch": 0.9705215419501134, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 28.806636810302734, "fcm_dpo/q_t": 0.49999719858169556, "grad_norm": 0.001907353405840695, "learning_rate": 1.3973071544233218e-09, "logits/chosen": 1.2331547737121582, "logits/rejected": 1.250314474105835, "logps/chosen": -345.4713134765625, "logps/ref_chosen": -68.92927551269531, "logps/ref_rejected": -70.85682678222656, "logps/rejected": -376.20550537109375, "loss": 1.3863, "margin_dpo/margin_mean": 28.8066349029541, "margin_dpo/margin_std": 318.6273193359375, "step": 642 }, { "KL/chosen_KL_mean": -229.24893188476562, "KL/mean": -313.97808837890625, "KL/rejected_KL_mean": -398.70721435546875, "KL/std": 334.5547790527344, "epoch": 0.9720332577475435, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 169.45823669433594, "fcm_dpo/q_t": 0.4999835789203644, "grad_norm": 0.003335776040330529, "learning_rate": 1.261184375888541e-09, "logits/chosen": 1.2515963315963745, "logits/rejected": 1.2289414405822754, "logps/chosen": -294.5579833984375, "logps/ref_chosen": -65.30903625488281, "logps/ref_rejected": -83.61613464355469, "logps/rejected": -482.3233337402344, "loss": 1.3862, "margin_dpo/margin_mean": 169.458251953125, "margin_dpo/margin_std": 398.0389099121094, "step": 643 }, { "KL/chosen_KL_mean": -195.06884765625, "KL/mean": -232.06622314453125, "KL/rejected_KL_mean": -269.0635986328125, "KL/std": 229.22683715820312, "epoch": 0.9735449735449735, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 73.99472045898438, "fcm_dpo/q_t": 0.49999281764030457, "grad_norm": 0.0016697602113708854, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 1.319830060005188, "logits/rejected": 1.3086210489273071, "logps/chosen": -246.0714569091797, "logps/ref_chosen": -51.002601623535156, "logps/ref_rejected": -64.46372985839844, "logps/rejected": -333.5273132324219, "loss": 1.3863, "margin_dpo/margin_mean": 73.99470520019531, "margin_dpo/margin_std": 218.09716796875, "step": 644 }, { "KL/chosen_KL_mean": -238.21824645996094, "KL/mean": -267.15887451171875, "KL/rejected_KL_mean": -296.0995178222656, "KL/std": 247.69064331054688, "epoch": 0.9750566893424036, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 57.881282806396484, "fcm_dpo/q_t": 0.4999943971633911, "grad_norm": 0.001644918229430914, "learning_rate": 1.0098157099674987e-09, "logits/chosen": 1.2519395351409912, "logits/rejected": 1.2480509281158447, "logps/chosen": -299.181640625, "logps/ref_chosen": -60.963409423828125, "logps/ref_rejected": -69.73353576660156, "logps/rejected": -365.83306884765625, "loss": 1.3863, "margin_dpo/margin_mean": 57.88128662109375, "margin_dpo/margin_std": 300.2716064453125, "step": 645 }, { "KL/chosen_KL_mean": -262.1314697265625, "KL/mean": -314.58929443359375, "KL/rejected_KL_mean": -367.047119140625, "KL/std": 342.90087890625, "epoch": 0.9765684051398337, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 104.91563415527344, "fcm_dpo/q_t": 0.4999898374080658, "grad_norm": 0.0021173881832510233, "learning_rate": 8.945768539031783e-10, "logits/chosen": 1.2763549089431763, "logits/rejected": 1.2538020610809326, "logps/chosen": -324.42156982421875, "logps/ref_chosen": -62.290069580078125, "logps/ref_rejected": -85.54812622070312, "logps/rejected": -452.59521484375, "loss": 1.3863, "margin_dpo/margin_mean": 104.91563415527344, "margin_dpo/margin_std": 296.7239990234375, "step": 646 }, { "KL/chosen_KL_mean": -277.2899475097656, "KL/mean": -347.98150634765625, "KL/rejected_KL_mean": -418.67303466796875, "KL/std": 369.2318115234375, "epoch": 0.9780801209372638, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 141.3831024169922, "fcm_dpo/q_t": 0.49998632073402405, "grad_norm": 0.002953104441985488, "learning_rate": 7.863060120144316e-10, "logits/chosen": 1.2947700023651123, "logits/rejected": 1.2425816059112549, "logps/chosen": -344.8058166503906, "logps/ref_chosen": -67.515869140625, "logps/ref_rejected": -101.50871276855469, "logps/rejected": -520.1817626953125, "loss": 1.3862, "margin_dpo/margin_mean": 141.3831024169922, "margin_dpo/margin_std": 402.0182189941406, "step": 647 }, { "KL/chosen_KL_mean": -248.36538696289062, "KL/mean": -310.21771240234375, "KL/rejected_KL_mean": -372.0700378417969, "KL/std": 275.6451416015625, "epoch": 0.9795918367346939, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 123.70465087890625, "fcm_dpo/q_t": 0.49998801946640015, "grad_norm": 0.002356282202526927, "learning_rate": 6.850062128694045e-10, "logits/chosen": 1.2679743766784668, "logits/rejected": 1.2473814487457275, "logps/chosen": -312.9613037109375, "logps/ref_chosen": -64.59593963623047, "logps/ref_rejected": -83.384033203125, "logps/rejected": -455.4540710449219, "loss": 1.3862, "margin_dpo/margin_mean": 123.70465087890625, "margin_dpo/margin_std": 370.0848693847656, "step": 648 }, { "KL/chosen_KL_mean": -202.95803833007812, "KL/mean": -254.43443298339844, "KL/rejected_KL_mean": -305.91082763671875, "KL/std": 240.42996215820312, "epoch": 0.981103552532124, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.95277404785156, "fcm_dpo/q_t": 0.4999900460243225, "grad_norm": 0.0025812452659010887, "learning_rate": 5.906802900412788e-10, "logits/chosen": 1.308830738067627, "logits/rejected": 1.2813698053359985, "logps/chosen": -252.2677001953125, "logps/ref_chosen": -49.30964660644531, "logps/ref_rejected": -73.73710632324219, "logps/rejected": -379.64794921875, "loss": 1.3863, "margin_dpo/margin_mean": 102.95277404785156, "margin_dpo/margin_std": 293.2994384765625, "step": 649 }, { "KL/chosen_KL_mean": -175.028076171875, "KL/mean": -253.37965393066406, "KL/rejected_KL_mean": -331.731201171875, "KL/std": 274.1990661621094, "epoch": 0.982615268329554, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.70315551757812, "fcm_dpo/q_t": 0.49998483061790466, "grad_norm": 0.002816020045429468, "learning_rate": 5.033308820289184e-10, "logits/chosen": 1.3074793815612793, "logits/rejected": 1.2863340377807617, "logps/chosen": -230.09133911132812, "logps/ref_chosen": -55.06325912475586, "logps/ref_rejected": -77.39610290527344, "logps/rejected": -409.1273193359375, "loss": 1.3862, "margin_dpo/margin_mean": 156.70314025878906, "margin_dpo/margin_std": 366.322998046875, "step": 650 }, { "KL/chosen_KL_mean": -214.81884765625, "KL/mean": -288.9599609375, "KL/rejected_KL_mean": -363.10113525390625, "KL/std": 292.44451904296875, "epoch": 0.9841269841269841, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 148.2822723388672, "fcm_dpo/q_t": 0.49998563528060913, "grad_norm": 0.002445911755785346, "learning_rate": 4.2296043218295606e-10, "logits/chosen": 1.288893461227417, "logits/rejected": 1.2606956958770752, "logps/chosen": -268.8840026855469, "logps/ref_chosen": -54.065162658691406, "logps/ref_rejected": -77.79080200195312, "logps/rejected": -440.89190673828125, "loss": 1.3862, "margin_dpo/margin_mean": 148.2822723388672, "margin_dpo/margin_std": 326.09967041015625, "step": 651 }, { "KL/chosen_KL_mean": -275.9385986328125, "KL/mean": -314.86370849609375, "KL/rejected_KL_mean": -353.788818359375, "KL/std": 326.7114562988281, "epoch": 0.9856386999244142, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 77.85020446777344, "fcm_dpo/q_t": 0.4999924600124359, "grad_norm": 0.0021558792795985937, "learning_rate": 3.4957118863768176e-10, "logits/chosen": 1.2859222888946533, "logits/rejected": 1.265965223312378, "logps/chosen": -339.57891845703125, "logps/ref_chosen": -63.64030456542969, "logps/ref_rejected": -78.86882019042969, "logps/rejected": -432.6576232910156, "loss": 1.3863, "margin_dpo/margin_mean": 77.85020446777344, "margin_dpo/margin_std": 423.6152648925781, "step": 652 }, { "KL/chosen_KL_mean": -228.88973999023438, "KL/mean": -280.29022216796875, "KL/rejected_KL_mean": -331.6907043457031, "KL/std": 262.5857849121094, "epoch": 0.9871504157218443, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 102.80097961425781, "fcm_dpo/q_t": 0.4999900460243225, "grad_norm": 0.001885903999209404, "learning_rate": 2.831652042480093e-10, "logits/chosen": 1.2586016654968262, "logits/rejected": 1.2437849044799805, "logps/chosen": -290.55810546875, "logps/ref_chosen": -61.668373107910156, "logps/ref_rejected": -73.83012390136719, "logps/rejected": -405.52081298828125, "loss": 1.3863, "margin_dpo/margin_mean": 102.80097961425781, "margin_dpo/margin_std": 331.70751953125, "step": 653 }, { "KL/chosen_KL_mean": -235.8380126953125, "KL/mean": -312.2591552734375, "KL/rejected_KL_mean": -388.6803283691406, "KL/std": 327.74395751953125, "epoch": 0.9886621315192744, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 152.84230041503906, "fcm_dpo/q_t": 0.4999852180480957, "grad_norm": 0.002882143948227167, "learning_rate": 2.2374433653205016e-10, "logits/chosen": 1.2653387784957886, "logits/rejected": 1.2258384227752686, "logps/chosen": -293.4062805175781, "logps/ref_chosen": -57.568267822265625, "logps/ref_rejected": -87.74789428710938, "logps/rejected": -476.42822265625, "loss": 1.3862, "margin_dpo/margin_mean": 152.84230041503906, "margin_dpo/margin_std": 319.22900390625, "step": 654 }, { "KL/chosen_KL_mean": -178.378173828125, "KL/mean": -272.81280517578125, "KL/rejected_KL_mean": -367.2474670410156, "KL/std": 287.7238464355469, "epoch": 0.9901738473167044, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 188.86932373046875, "fcm_dpo/q_t": 0.49998173117637634, "grad_norm": 0.003305543912574649, "learning_rate": 1.7131024761923852e-10, "logits/chosen": 1.2574398517608643, "logits/rejected": 1.2300739288330078, "logps/chosen": -230.5253143310547, "logps/ref_chosen": -52.14714813232422, "logps/ref_rejected": -80.85014343261719, "logps/rejected": -448.09759521484375, "loss": 1.3862, "margin_dpo/margin_mean": 188.86932373046875, "margin_dpo/margin_std": 355.13861083984375, "step": 655 }, { "KL/chosen_KL_mean": -268.1697692871094, "KL/mean": -328.5416564941406, "KL/rejected_KL_mean": -388.91357421875, "KL/std": 299.40435791015625, "epoch": 0.9916855631141346, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 120.74380493164062, "fcm_dpo/q_t": 0.499988317489624, "grad_norm": 0.0024166665971279144, "learning_rate": 1.2586440420372934e-10, "logits/chosen": 1.2543549537658691, "logits/rejected": 1.2443668842315674, "logps/chosen": -341.4264831542969, "logps/ref_chosen": -73.25672912597656, "logps/ref_rejected": -85.35127258300781, "logps/rejected": -474.26483154296875, "loss": 1.3862, "margin_dpo/margin_mean": 120.74380493164062, "margin_dpo/margin_std": 330.2796630859375, "step": 656 }, { "KL/chosen_KL_mean": -185.50474548339844, "KL/mean": -276.64654541015625, "KL/rejected_KL_mean": -367.788330078125, "KL/std": 325.72955322265625, "epoch": 0.9931972789115646, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 182.28355407714844, "fcm_dpo/q_t": 0.4999823570251465, "grad_norm": 0.00303459819406271, "learning_rate": 8.740807750345913e-11, "logits/chosen": 1.3113363981246948, "logits/rejected": 1.2885608673095703, "logps/chosen": -235.2281494140625, "logps/ref_chosen": -49.72339630126953, "logps/ref_rejected": -75.1568603515625, "logps/rejected": -442.9451904296875, "loss": 1.3862, "margin_dpo/margin_mean": 182.28355407714844, "margin_dpo/margin_std": 396.7044677734375, "step": 657 }, { "KL/chosen_KL_mean": -248.78866577148438, "KL/mean": -298.27093505859375, "KL/rejected_KL_mean": -347.7532043457031, "KL/std": 330.69586181640625, "epoch": 0.9947089947089947, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 98.96455383300781, "fcm_dpo/q_t": 0.49999043345451355, "grad_norm": 0.0021189674735069275, "learning_rate": 5.594234322453539e-11, "logits/chosen": 1.2897260189056396, "logits/rejected": 1.271322250366211, "logps/chosen": -311.83502197265625, "logps/ref_chosen": -63.04634094238281, "logps/ref_rejected": -83.44963073730469, "logps/rejected": -431.2028503417969, "loss": 1.3863, "margin_dpo/margin_mean": 98.96455383300781, "margin_dpo/margin_std": 376.11456298828125, "step": 658 }, { "KL/chosen_KL_mean": -244.0128936767578, "KL/mean": -291.29986572265625, "KL/rejected_KL_mean": -338.5868225097656, "KL/std": 314.4420166015625, "epoch": 0.9962207105064248, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 94.57395935058594, "fcm_dpo/q_t": 0.4999908208847046, "grad_norm": 0.0024397147353738546, "learning_rate": 3.146808153123293e-11, "logits/chosen": 1.3078527450561523, "logits/rejected": 1.281451940536499, "logps/chosen": -299.09307861328125, "logps/ref_chosen": -55.0802001953125, "logps/ref_rejected": -71.91049194335938, "logps/rejected": -410.497314453125, "loss": 1.3863, "margin_dpo/margin_mean": 94.5739517211914, "margin_dpo/margin_std": 354.7186279296875, "step": 659 }, { "KL/chosen_KL_mean": -202.13861083984375, "KL/mean": -280.438720703125, "KL/rejected_KL_mean": -358.7388610839844, "KL/std": 307.3833923339844, "epoch": 0.9977324263038548, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 156.60025024414062, "fcm_dpo/q_t": 0.49998486042022705, "grad_norm": 0.002643043175339699, "learning_rate": 1.3985977021235829e-11, "logits/chosen": 1.329068660736084, "logits/rejected": 1.3038125038146973, "logps/chosen": -256.6645202636719, "logps/ref_chosen": -54.525917053222656, "logps/ref_rejected": -81.23604583740234, "logps/rejected": -439.97491455078125, "loss": 1.3862, "margin_dpo/margin_mean": 156.60025024414062, "margin_dpo/margin_std": 385.9468688964844, "step": 660 }, { "KL/chosen_KL_mean": -255.35760498046875, "KL/mean": -313.8293762207031, "KL/rejected_KL_mean": -372.3011474609375, "KL/std": 311.5862121582031, "epoch": 0.999244142101285, "fcm_dpo/beta": 3.8711289107595803e-07, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 116.94351959228516, "fcm_dpo/q_t": 0.4999886751174927, "grad_norm": 0.0025908024981617928, "learning_rate": 3.4965187065971735e-12, "logits/chosen": 1.2794947624206543, "logits/rejected": 1.2472293376922607, "logps/chosen": -315.7302551269531, "logps/ref_chosen": -60.37263870239258, "logps/ref_rejected": -77.42874145507812, "logps/rejected": -449.7298583984375, "loss": 1.3862, "margin_dpo/margin_mean": 116.94351959228516, "margin_dpo/margin_std": 371.20477294921875, "step": 661 }, { "epoch": 0.999244142101285, "step": 661, "total_flos": 0.0, "train_loss": 1.370076598991244, "train_runtime": 1650.2951, "train_samples_per_second": 25.654, "train_steps_per_second": 0.401 } ], "logging_steps": 1, "max_steps": 661, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }