{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.999244142101285, "eval_steps": 200, "global_step": 661, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.02867889404296875, "KL/mean": 0.029354453086853027, "KL/rejected_KL_mean": 0.030029296875, "KL/std": 0.2071000635623932, "epoch": 0.0015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013532638549804688, "fcm_dpo/q_t": 0.5000336766242981, "grad_norm": 28.21967887878418, "learning_rate": 0.0, "logits/chosen": 0.13337239623069763, "logits/rejected": 0.12492949515581131, "logps/chosen": -64.5841293334961, "logps/ref_chosen": -64.61280822753906, "logps/ref_rejected": -64.17195129394531, "logps/rejected": -64.14192199707031, "loss": 1.3866, "margin_dpo/margin_mean": -0.0013527870178222656, "margin_dpo/margin_std": 0.2561596930027008, "step": 1 }, { "KL/chosen_KL_mean": -0.00289154052734375, "KL/mean": -0.021616414189338684, "KL/rejected_KL_mean": -0.04033660888671875, "KL/std": 0.19624735414981842, "epoch": 0.0030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.037450045347213745, "fcm_dpo/q_t": 0.4990639090538025, "grad_norm": 27.828706741333008, "learning_rate": 7.462686567164179e-09, "logits/chosen": 0.09414851665496826, "logits/rejected": 0.07363267242908478, "logps/chosen": -56.101890563964844, "logps/ref_chosen": -56.0989990234375, "logps/ref_rejected": -66.59971618652344, "logps/rejected": -66.64006042480469, "loss": 1.3828, "margin_dpo/margin_mean": 0.03744968771934509, "margin_dpo/margin_std": 0.27811938524246216, "step": 2 }, { "KL/chosen_KL_mean": 0.007659912109375, "KL/mean": 0.032633185386657715, "KL/rejected_KL_mean": 0.0576019287109375, "KL/std": 0.2508915066719055, "epoch": 0.0045351473922902496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04993748664855957, "fcm_dpo/q_t": 0.501247763633728, "grad_norm": 31.35702133178711, "learning_rate": 1.4925373134328357e-08, "logits/chosen": 0.09931470453739166, "logits/rejected": 0.06133907288312912, "logps/chosen": -65.44960021972656, "logps/ref_chosen": -65.45726013183594, "logps/ref_rejected": -90.82853698730469, "logps/rejected": -90.77093505859375, "loss": 1.3916, "margin_dpo/margin_mean": -0.04993721842765808, "margin_dpo/margin_std": 0.33015739917755127, "step": 3 }, { "KL/chosen_KL_mean": 0.003841400146484375, "KL/mean": -0.005494982004165649, "KL/rejected_KL_mean": -0.014835357666015625, "KL/std": 0.22913160920143127, "epoch": 0.006046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01867443323135376, "fcm_dpo/q_t": 0.49953362345695496, "grad_norm": 34.08740997314453, "learning_rate": 2.2388059701492534e-08, "logits/chosen": 0.1126205176115036, "logits/rejected": 0.09632235765457153, "logps/chosen": -76.85633850097656, "logps/ref_chosen": -76.86018371582031, "logps/ref_rejected": -79.91523742675781, "logps/rejected": -79.93006896972656, "loss": 1.3847, "margin_dpo/margin_mean": 0.018673986196517944, "margin_dpo/margin_std": 0.3117932677268982, "step": 4 }, { "KL/chosen_KL_mean": -0.014787673950195312, "KL/mean": -0.016955822706222534, "KL/rejected_KL_mean": -0.0191192626953125, "KL/std": 0.222198024392128, "epoch": 0.007558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.004332542419433594, "fcm_dpo/q_t": 0.4998916983604431, "grad_norm": 29.587570190429688, "learning_rate": 2.9850746268656714e-08, "logits/chosen": 0.0821952074766159, "logits/rejected": 0.04323691129684448, "logps/chosen": -62.98612976074219, "logps/ref_chosen": -62.97134017944336, "logps/ref_rejected": -79.9192123413086, "logps/rejected": -79.9383316040039, "loss": 1.3861, "margin_dpo/margin_mean": 0.004333585500717163, "margin_dpo/margin_std": 0.30388158559799194, "step": 5 }, { "KL/chosen_KL_mean": -0.020517349243164062, "KL/mean": 0.00726713240146637, "KL/rejected_KL_mean": 0.03505706787109375, "KL/std": 0.30582496523857117, "epoch": 0.009070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05557325482368469, "fcm_dpo/q_t": 0.5013865232467651, "grad_norm": 29.830081939697266, "learning_rate": 3.731343283582089e-08, "logits/chosen": 0.14537031948566437, "logits/rejected": 0.1057555228471756, "logps/chosen": -51.327880859375, "logps/ref_chosen": -51.30736541748047, "logps/ref_rejected": -82.77239227294922, "logps/rejected": -82.73733520507812, "loss": 1.3923, "margin_dpo/margin_mean": -0.05557364225387573, "margin_dpo/margin_std": 0.4328301250934601, "step": 6 }, { "KL/chosen_KL_mean": 0.017831802368164062, "KL/mean": 0.02575281262397766, "KL/rejected_KL_mean": 0.033672332763671875, "KL/std": 0.20926260948181152, "epoch": 0.010582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01584029197692871, "fcm_dpo/q_t": 0.5003958940505981, "grad_norm": 27.20305633544922, "learning_rate": 4.477611940298507e-08, "logits/chosen": 0.007591056637465954, "logits/rejected": -0.035748470574617386, "logps/chosen": -51.44158172607422, "logps/ref_chosen": -51.45941162109375, "logps/ref_rejected": -66.3828125, "logps/rejected": -66.34913635253906, "loss": 1.388, "margin_dpo/margin_mean": -0.015840649604797363, "margin_dpo/margin_std": 0.22362905740737915, "step": 7 }, { "KL/chosen_KL_mean": 0.002826690673828125, "KL/mean": -0.014976292848587036, "KL/rejected_KL_mean": -0.03277397155761719, "KL/std": 0.2553374171257019, "epoch": 0.012093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03559964895248413, "fcm_dpo/q_t": 0.49911004304885864, "grad_norm": 28.653799057006836, "learning_rate": 5.223880597014925e-08, "logits/chosen": 0.06839226186275482, "logits/rejected": 0.04627063870429993, "logps/chosen": -62.19471740722656, "logps/ref_chosen": -62.197547912597656, "logps/ref_rejected": -74.66180419921875, "logps/rejected": -74.694580078125, "loss": 1.383, "margin_dpo/margin_mean": 0.03559872508049011, "margin_dpo/margin_std": 0.33308953046798706, "step": 8 }, { "KL/chosen_KL_mean": -0.03289794921875, "KL/mean": -0.015181422233581543, "KL/rejected_KL_mean": 0.002529144287109375, "KL/std": 0.23224589228630066, "epoch": 0.013605442176870748, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03543180227279663, "fcm_dpo/q_t": 0.5008852481842041, "grad_norm": 31.527856826782227, "learning_rate": 5.970149253731343e-08, "logits/chosen": 0.17893418669700623, "logits/rejected": 0.1187577098608017, "logps/chosen": -55.662620544433594, "logps/ref_chosen": -55.629722595214844, "logps/ref_rejected": -86.21221923828125, "logps/rejected": -86.2096939086914, "loss": 1.3901, "margin_dpo/margin_mean": -0.035430908203125, "margin_dpo/margin_std": 0.3422740697860718, "step": 9 }, { "KL/chosen_KL_mean": 0.03826141357421875, "KL/mean": 0.021607831120491028, "KL/rejected_KL_mean": 0.00495147705078125, "KL/std": 0.270057737827301, "epoch": 0.015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03330746293067932, "fcm_dpo/q_t": 0.49916791915893555, "grad_norm": 29.675212860107422, "learning_rate": 6.71641791044776e-08, "logits/chosen": 0.1317688524723053, "logits/rejected": 0.10075588524341583, "logps/chosen": -62.652339935302734, "logps/ref_chosen": -62.69060134887695, "logps/ref_rejected": -90.610107421875, "logps/rejected": -90.60516357421875, "loss": 1.3834, "margin_dpo/margin_mean": 0.033307403326034546, "margin_dpo/margin_std": 0.38862723112106323, "step": 10 }, { "KL/chosen_KL_mean": -0.004878997802734375, "KL/mean": -0.020426854491233826, "KL/rejected_KL_mean": -0.035980224609375, "KL/std": 0.1961967498064041, "epoch": 0.016628873771730914, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.031100362539291382, "fcm_dpo/q_t": 0.4992234408855438, "grad_norm": 29.25577163696289, "learning_rate": 7.462686567164178e-08, "logits/chosen": 0.11657217144966125, "logits/rejected": 0.10957114398479462, "logps/chosen": -65.7719955444336, "logps/ref_chosen": -65.76712036132812, "logps/ref_rejected": -72.4764633178711, "logps/rejected": -72.51244354248047, "loss": 1.3834, "margin_dpo/margin_mean": 0.031100690364837646, "margin_dpo/margin_std": 0.2950153946876526, "step": 11 }, { "KL/chosen_KL_mean": -0.008037567138671875, "KL/mean": -0.016360193490982056, "KL/rejected_KL_mean": -0.02468109130859375, "KL/std": 0.21698607504367828, "epoch": 0.018140589569160998, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.016642391681671143, "fcm_dpo/q_t": 0.49958401918411255, "grad_norm": 28.2968692779541, "learning_rate": 8.208955223880596e-08, "logits/chosen": 0.04076576605439186, "logits/rejected": 0.024239014834165573, "logps/chosen": -60.712928771972656, "logps/ref_chosen": -60.704891204833984, "logps/ref_rejected": -69.41564178466797, "logps/rejected": -69.44032287597656, "loss": 1.3849, "margin_dpo/margin_mean": 0.016642063856124878, "margin_dpo/margin_std": 0.3236123323440552, "step": 12 }, { "KL/chosen_KL_mean": -0.025043487548828125, "KL/mean": 0.008205235004425049, "KL/rejected_KL_mean": 0.04146575927734375, "KL/std": 0.23161034286022186, "epoch": 0.019652305366591082, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06650584936141968, "fcm_dpo/q_t": 0.5016615986824036, "grad_norm": 29.276233673095703, "learning_rate": 8.955223880597014e-08, "logits/chosen": 0.10737905651330948, "logits/rejected": 0.04509834945201874, "logps/chosen": -49.93429946899414, "logps/ref_chosen": -49.90925598144531, "logps/ref_rejected": -92.37818145751953, "logps/rejected": -92.33671569824219, "loss": 1.3933, "margin_dpo/margin_mean": -0.06650650501251221, "margin_dpo/margin_std": 0.34030449390411377, "step": 13 }, { "KL/chosen_KL_mean": 0.05045509338378906, "KL/mean": 0.0025360435247421265, "KL/rejected_KL_mean": -0.045379638671875, "KL/std": 0.19650276005268097, "epoch": 0.021164021164021163, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.095831960439682, "fcm_dpo/q_t": 0.4976050853729248, "grad_norm": 29.4014835357666, "learning_rate": 9.701492537313432e-08, "logits/chosen": 0.08358614891767502, "logits/rejected": 0.06596517562866211, "logps/chosen": -60.568336486816406, "logps/ref_chosen": -60.61879348754883, "logps/ref_rejected": -71.79306030273438, "logps/rejected": -71.83843994140625, "loss": 1.3769, "margin_dpo/margin_mean": 0.09583166241645813, "margin_dpo/margin_std": 0.2745246887207031, "step": 14 }, { "KL/chosen_KL_mean": -0.04099273681640625, "KL/mean": -0.02316710352897644, "KL/rejected_KL_mean": -0.005344390869140625, "KL/std": 0.2433144450187683, "epoch": 0.022675736961451247, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03565239906311035, "fcm_dpo/q_t": 0.5008911490440369, "grad_norm": 33.39137649536133, "learning_rate": 1.044776119402985e-07, "logits/chosen": 0.06936109066009521, "logits/rejected": 0.02610601857304573, "logps/chosen": -63.510528564453125, "logps/ref_chosen": -63.46953582763672, "logps/ref_rejected": -88.88951110839844, "logps/rejected": -88.89485931396484, "loss": 1.3902, "margin_dpo/margin_mean": -0.03565263748168945, "margin_dpo/margin_std": 0.3474429249763489, "step": 15 }, { "KL/chosen_KL_mean": -0.024089813232421875, "KL/mean": -0.025951489806175232, "KL/rejected_KL_mean": -0.0278167724609375, "KL/std": 0.20087632536888123, "epoch": 0.02418745275888133, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003727465867996216, "fcm_dpo/q_t": 0.49990740418434143, "grad_norm": 26.88129997253418, "learning_rate": 1.1194029850746268e-07, "logits/chosen": 0.11440499126911163, "logits/rejected": 0.07738970220088959, "logps/chosen": -46.55638885498047, "logps/ref_chosen": -46.53229904174805, "logps/ref_rejected": -74.27533721923828, "logps/rejected": -74.30315399169922, "loss": 1.3861, "margin_dpo/margin_mean": 0.003727525472640991, "margin_dpo/margin_std": 0.2645993232727051, "step": 16 }, { "KL/chosen_KL_mean": 0.02545928955078125, "KL/mean": 0.028340786695480347, "KL/rejected_KL_mean": 0.031219482421875, "KL/std": 0.23972105979919434, "epoch": 0.025699168556311415, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.005758881568908691, "fcm_dpo/q_t": 0.5001437067985535, "grad_norm": 32.905975341796875, "learning_rate": 1.1940298507462686e-07, "logits/chosen": 0.031329765915870667, "logits/rejected": 0.013444792479276657, "logps/chosen": -64.05237579345703, "logps/ref_chosen": -64.07783508300781, "logps/ref_rejected": -86.40876770019531, "logps/rejected": -86.37754821777344, "loss": 1.3871, "margin_dpo/margin_mean": -0.005759298801422119, "margin_dpo/margin_std": 0.31801456212997437, "step": 17 }, { "KL/chosen_KL_mean": 0.02117156982421875, "KL/mean": -0.0042324066162109375, "KL/rejected_KL_mean": -0.02964019775390625, "KL/std": 0.2313615083694458, "epoch": 0.027210884353741496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.050807178020477295, "fcm_dpo/q_t": 0.49873000383377075, "grad_norm": 27.679706573486328, "learning_rate": 1.2686567164179106e-07, "logits/chosen": 0.08802352845668793, "logits/rejected": 0.042935267090797424, "logps/chosen": -44.85316467285156, "logps/ref_chosen": -44.87433624267578, "logps/ref_rejected": -70.97604370117188, "logps/rejected": -71.00569152832031, "loss": 1.3815, "margin_dpo/margin_mean": 0.05080723762512207, "margin_dpo/margin_std": 0.3414297103881836, "step": 18 }, { "KL/chosen_KL_mean": 0.05913352966308594, "KL/mean": 0.02635572850704193, "KL/rejected_KL_mean": -0.006427764892578125, "KL/std": 0.2230260968208313, "epoch": 0.02872260015117158, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06556209921836853, "fcm_dpo/q_t": 0.49836117029190063, "grad_norm": 30.85642433166504, "learning_rate": 1.343283582089552e-07, "logits/chosen": 0.06814460456371307, "logits/rejected": 0.054654691368341446, "logps/chosen": -68.10067749023438, "logps/ref_chosen": -68.1598129272461, "logps/ref_rejected": -81.17138671875, "logps/rejected": -81.17781829833984, "loss": 1.38, "margin_dpo/margin_mean": 0.06556195020675659, "margin_dpo/margin_std": 0.2927999496459961, "step": 19 }, { "KL/chosen_KL_mean": 0.027414321899414062, "KL/mean": 0.022625096142292023, "KL/rejected_KL_mean": 0.0178375244140625, "KL/std": 0.22135794162750244, "epoch": 0.030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.009577527642250061, "fcm_dpo/q_t": 0.4997604191303253, "grad_norm": 29.147907257080078, "learning_rate": 1.4179104477611938e-07, "logits/chosen": 0.12029474228620529, "logits/rejected": 0.09780436754226685, "logps/chosen": -53.651145935058594, "logps/ref_chosen": -53.67856216430664, "logps/ref_rejected": -74.16911315917969, "logps/rejected": -74.15127563476562, "loss": 1.3855, "margin_dpo/margin_mean": 0.009577080607414246, "margin_dpo/margin_std": 0.27818232774734497, "step": 20 }, { "KL/chosen_KL_mean": -0.01338958740234375, "KL/mean": -0.00590062141418457, "KL/rejected_KL_mean": 0.0015869140625, "KL/std": 0.24868792295455933, "epoch": 0.031746031746031744, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.014971256256103516, "fcm_dpo/q_t": 0.5003751516342163, "grad_norm": 29.285295486450195, "learning_rate": 1.4925373134328355e-07, "logits/chosen": 0.12290213257074356, "logits/rejected": 0.09662627428770065, "logps/chosen": -64.7149429321289, "logps/ref_chosen": -64.70155334472656, "logps/ref_rejected": -81.02095031738281, "logps/rejected": -81.01936340332031, "loss": 1.3881, "margin_dpo/margin_mean": -0.014971524477005005, "margin_dpo/margin_std": 0.3347770571708679, "step": 21 }, { "KL/chosen_KL_mean": -0.007518768310546875, "KL/mean": -0.034809038043022156, "KL/rejected_KL_mean": -0.06209564208984375, "KL/std": 0.20913906395435333, "epoch": 0.03325774754346183, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05458232760429382, "fcm_dpo/q_t": 0.4986357092857361, "grad_norm": 28.887042999267578, "learning_rate": 1.5671641791044775e-07, "logits/chosen": 0.02207999862730503, "logits/rejected": 0.00017184100579470396, "logps/chosen": -58.04351043701172, "logps/ref_chosen": -58.03599166870117, "logps/ref_rejected": -80.72721862792969, "logps/rejected": -80.78931427001953, "loss": 1.381, "margin_dpo/margin_mean": 0.054582446813583374, "margin_dpo/margin_std": 0.2853773832321167, "step": 22 }, { "KL/chosen_KL_mean": 0.012264251708984375, "KL/mean": 0.009754911065101624, "KL/rejected_KL_mean": 0.00724029541015625, "KL/std": 0.26380670070648193, "epoch": 0.03476946334089191, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005020111799240112, "fcm_dpo/q_t": 0.4998743534088135, "grad_norm": 33.045352935791016, "learning_rate": 1.6417910447761193e-07, "logits/chosen": 0.12751388549804688, "logits/rejected": 0.10264043509960175, "logps/chosen": -66.34382629394531, "logps/ref_chosen": -66.35608673095703, "logps/ref_rejected": -93.02769470214844, "logps/rejected": -93.02044677734375, "loss": 1.3861, "margin_dpo/margin_mean": 0.005019426345825195, "margin_dpo/margin_std": 0.3555549383163452, "step": 23 }, { "KL/chosen_KL_mean": -0.004886627197265625, "KL/mean": -0.0031652823090553284, "KL/rejected_KL_mean": -0.0014438629150390625, "KL/std": 0.2178821861743927, "epoch": 0.036281179138321996, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.003436669707298279, "fcm_dpo/q_t": 0.5000854730606079, "grad_norm": 26.289295196533203, "learning_rate": 1.716417910447761e-07, "logits/chosen": 0.15002357959747314, "logits/rejected": 0.1167023628950119, "logps/chosen": -54.46612548828125, "logps/ref_chosen": -54.461238861083984, "logps/ref_rejected": -68.33817291259766, "logps/rejected": -68.33961486816406, "loss": 1.3869, "margin_dpo/margin_mean": -0.003437221050262451, "margin_dpo/margin_std": 0.29839447140693665, "step": 24 }, { "KL/chosen_KL_mean": -0.03343772888183594, "KL/mean": -0.024197697639465332, "KL/rejected_KL_mean": -0.01496124267578125, "KL/std": 0.23071929812431335, "epoch": 0.03779289493575208, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01847773790359497, "fcm_dpo/q_t": 0.5004626512527466, "grad_norm": 29.7857666015625, "learning_rate": 1.7910447761194027e-07, "logits/chosen": 0.11460703611373901, "logits/rejected": 0.06244586408138275, "logps/chosen": -60.03764343261719, "logps/ref_chosen": -60.00420379638672, "logps/ref_rejected": -90.47376251220703, "logps/rejected": -90.48872375488281, "loss": 1.3884, "margin_dpo/margin_mean": -0.01847824454307556, "margin_dpo/margin_std": 0.2980327606201172, "step": 25 }, { "KL/chosen_KL_mean": -0.00635528564453125, "KL/mean": -0.007910311222076416, "KL/rejected_KL_mean": -0.009464263916015625, "KL/std": 0.1977149397134781, "epoch": 0.039304610733182165, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003108978271484375, "fcm_dpo/q_t": 0.499922513961792, "grad_norm": 29.830995559692383, "learning_rate": 1.8656716417910447e-07, "logits/chosen": 0.0936226099729538, "logits/rejected": 0.07567183673381805, "logps/chosen": -56.82550811767578, "logps/ref_chosen": -56.81915283203125, "logps/ref_rejected": -77.84333038330078, "logps/rejected": -77.85279846191406, "loss": 1.3862, "margin_dpo/margin_mean": 0.0031086504459381104, "margin_dpo/margin_std": 0.2862233519554138, "step": 26 }, { "KL/chosen_KL_mean": -0.016384124755859375, "KL/mean": -0.016014084219932556, "KL/rejected_KL_mean": -0.015649795532226562, "KL/std": 0.2392708659172058, "epoch": 0.04081632653061224, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.000739365816116333, "fcm_dpo/q_t": 0.5000181198120117, "grad_norm": 28.958324432373047, "learning_rate": 1.9402985074626865e-07, "logits/chosen": 0.11648187786340714, "logits/rejected": 0.09103013575077057, "logps/chosen": -62.893409729003906, "logps/ref_chosen": -62.87702560424805, "logps/ref_rejected": -71.34437561035156, "logps/rejected": -71.36002349853516, "loss": 1.3867, "margin_dpo/margin_mean": -0.0007395446300506592, "margin_dpo/margin_std": 0.36002424359321594, "step": 27 }, { "KL/chosen_KL_mean": -0.012584686279296875, "KL/mean": -0.004347294569015503, "KL/rejected_KL_mean": 0.0038890838623046875, "KL/std": 0.23575863242149353, "epoch": 0.042328042328042326, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.016472816467285156, "fcm_dpo/q_t": 0.5004111528396606, "grad_norm": 27.684032440185547, "learning_rate": 2.0149253731343282e-07, "logits/chosen": 0.07097271829843521, "logits/rejected": 0.06192881613969803, "logps/chosen": -59.84596252441406, "logps/ref_chosen": -59.8333740234375, "logps/ref_rejected": -70.39804077148438, "logps/rejected": -70.39414978027344, "loss": 1.3882, "margin_dpo/margin_mean": -0.016472458839416504, "margin_dpo/margin_std": 0.3243771195411682, "step": 28 }, { "KL/chosen_KL_mean": -0.018489837646484375, "KL/mean": -0.031423747539520264, "KL/rejected_KL_mean": -0.04434967041015625, "KL/std": 0.20472386479377747, "epoch": 0.04383975812547241, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025858044624328613, "fcm_dpo/q_t": 0.4993535876274109, "grad_norm": 32.789634704589844, "learning_rate": 2.08955223880597e-07, "logits/chosen": 0.15825878083705902, "logits/rejected": 0.1399884819984436, "logps/chosen": -74.13868713378906, "logps/ref_chosen": -74.12020111083984, "logps/ref_rejected": -83.33099365234375, "logps/rejected": -83.37533569335938, "loss": 1.3839, "margin_dpo/margin_mean": 0.025858759880065918, "margin_dpo/margin_std": 0.2755180895328522, "step": 29 }, { "KL/chosen_KL_mean": -0.0454254150390625, "KL/mean": -0.06478771567344666, "KL/rejected_KL_mean": -0.0841522216796875, "KL/std": 0.20571255683898926, "epoch": 0.045351473922902494, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.038724422454833984, "fcm_dpo/q_t": 0.49903199076652527, "grad_norm": 30.31739616394043, "learning_rate": 2.1641791044776117e-07, "logits/chosen": 0.11089472472667694, "logits/rejected": 0.05757853388786316, "logps/chosen": -50.796714782714844, "logps/ref_chosen": -50.75128936767578, "logps/ref_rejected": -89.29063415527344, "logps/rejected": -89.37478637695312, "loss": 1.3827, "margin_dpo/margin_mean": 0.038724422454833984, "margin_dpo/margin_std": 0.30037564039230347, "step": 30 }, { "KL/chosen_KL_mean": 0.008935928344726562, "KL/mean": -0.04200010001659393, "KL/rejected_KL_mean": -0.09293365478515625, "KL/std": 0.27889418601989746, "epoch": 0.04686318972033258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10187044739723206, "fcm_dpo/q_t": 0.4974536597728729, "grad_norm": 34.08384323120117, "learning_rate": 2.2388059701492537e-07, "logits/chosen": 0.111326202750206, "logits/rejected": 0.0648956447839737, "logps/chosen": -65.32781982421875, "logps/ref_chosen": -65.33675384521484, "logps/ref_rejected": -100.76666259765625, "logps/rejected": -100.8595962524414, "loss": 1.3766, "margin_dpo/margin_mean": 0.10187077522277832, "margin_dpo/margin_std": 0.39277029037475586, "step": 31 }, { "KL/chosen_KL_mean": -0.01952362060546875, "KL/mean": -0.024612531065940857, "KL/rejected_KL_mean": -0.029697418212890625, "KL/std": 0.22407123446464539, "epoch": 0.04837490551776266, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01017645001411438, "fcm_dpo/q_t": 0.49974536895751953, "grad_norm": 30.059810638427734, "learning_rate": 2.3134328358208954e-07, "logits/chosen": 0.07288508117198944, "logits/rejected": 0.06522229313850403, "logps/chosen": -67.20285034179688, "logps/ref_chosen": -67.18333435058594, "logps/ref_rejected": -82.80763244628906, "logps/rejected": -82.83733367919922, "loss": 1.3856, "margin_dpo/margin_mean": 0.010176807641983032, "margin_dpo/margin_std": 0.31959301233291626, "step": 32 }, { "KL/chosen_KL_mean": -0.026327133178710938, "KL/mean": -0.03532563894987106, "KL/rejected_KL_mean": -0.044322967529296875, "KL/std": 0.258139967918396, "epoch": 0.049886621315192746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.017992421984672546, "fcm_dpo/q_t": 0.4995502829551697, "grad_norm": 31.032793045043945, "learning_rate": 2.388059701492537e-07, "logits/chosen": 0.0272968877106905, "logits/rejected": 0.0015150085091590881, "logps/chosen": -64.06581115722656, "logps/ref_chosen": -64.03948211669922, "logps/ref_rejected": -75.68357849121094, "logps/rejected": -75.7279052734375, "loss": 1.3848, "margin_dpo/margin_mean": 0.0179927796125412, "margin_dpo/margin_std": 0.3556821942329407, "step": 33 }, { "KL/chosen_KL_mean": -0.025463104248046875, "KL/mean": -0.06123238801956177, "KL/rejected_KL_mean": -0.09700202941894531, "KL/std": 0.20554988086223602, "epoch": 0.05139833711262283, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07154315710067749, "fcm_dpo/q_t": 0.49821144342422485, "grad_norm": 28.452770233154297, "learning_rate": 2.4626865671641786e-07, "logits/chosen": 0.11309738457202911, "logits/rejected": 0.08256950974464417, "logps/chosen": -53.68975830078125, "logps/ref_chosen": -53.6642951965332, "logps/ref_rejected": -65.77989959716797, "logps/rejected": -65.87689971923828, "loss": 1.3793, "margin_dpo/margin_mean": 0.07154309749603271, "margin_dpo/margin_std": 0.269156813621521, "step": 34 }, { "KL/chosen_KL_mean": -0.09962081909179688, "KL/mean": -0.09243303537368774, "KL/rejected_KL_mean": -0.08524322509765625, "KL/std": 0.23130691051483154, "epoch": 0.05291005291005291, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01437610387802124, "fcm_dpo/q_t": 0.5003600120544434, "grad_norm": 27.835346221923828, "learning_rate": 2.537313432835821e-07, "logits/chosen": 0.05176283419132233, "logits/rejected": 0.029404601082205772, "logps/chosen": -61.116485595703125, "logps/ref_chosen": -61.01686096191406, "logps/ref_rejected": -72.78598022460938, "logps/rejected": -72.87123107910156, "loss": 1.388, "margin_dpo/margin_mean": -0.014376223087310791, "margin_dpo/margin_std": 0.30166250467300415, "step": 35 }, { "KL/chosen_KL_mean": -0.10001564025878906, "KL/mean": -0.09694293141365051, "KL/rejected_KL_mean": -0.0938720703125, "KL/std": 0.2522987723350525, "epoch": 0.05442176870748299, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006146609783172607, "fcm_dpo/q_t": 0.5001544952392578, "grad_norm": 28.722679138183594, "learning_rate": 2.611940298507462e-07, "logits/chosen": 0.08311149477958679, "logits/rejected": 0.031588129699230194, "logps/chosen": -50.63737487792969, "logps/ref_chosen": -50.53736114501953, "logps/ref_rejected": -78.11678314208984, "logps/rejected": -78.21065521240234, "loss": 1.3873, "margin_dpo/margin_mean": -0.006146460771560669, "margin_dpo/margin_std": 0.3678821325302124, "step": 36 }, { "KL/chosen_KL_mean": -0.05405426025390625, "KL/mean": -0.0969603955745697, "KL/rejected_KL_mean": -0.13987350463867188, "KL/std": 0.2613227665424347, "epoch": 0.055933484504913075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08581572771072388, "fcm_dpo/q_t": 0.4978557825088501, "grad_norm": 37.60136413574219, "learning_rate": 2.686567164179104e-07, "logits/chosen": 0.09612583369016647, "logits/rejected": 0.017332565039396286, "logps/chosen": -59.608001708984375, "logps/ref_chosen": -59.55394744873047, "logps/ref_rejected": -108.27702331542969, "logps/rejected": -108.41690063476562, "loss": 1.3781, "margin_dpo/margin_mean": 0.08581539988517761, "margin_dpo/margin_std": 0.36218589544296265, "step": 37 }, { "KL/chosen_KL_mean": -0.07653617858886719, "KL/mean": -0.09487441182136536, "KL/rejected_KL_mean": -0.11320877075195312, "KL/std": 0.23041307926177979, "epoch": 0.05744520030234316, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03667086362838745, "fcm_dpo/q_t": 0.499083936214447, "grad_norm": 29.755857467651367, "learning_rate": 2.761194029850746e-07, "logits/chosen": 0.0761660635471344, "logits/rejected": 0.0621175691485405, "logps/chosen": -65.8648910522461, "logps/ref_chosen": -65.78836059570312, "logps/ref_rejected": -76.1619873046875, "logps/rejected": -76.27519226074219, "loss": 1.3829, "margin_dpo/margin_mean": 0.03667101263999939, "margin_dpo/margin_std": 0.30828237533569336, "step": 38 }, { "KL/chosen_KL_mean": -0.1410198211669922, "KL/mean": -0.12570250034332275, "KL/rejected_KL_mean": -0.110382080078125, "KL/std": 0.28514018654823303, "epoch": 0.05895691609977324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.030639827251434326, "fcm_dpo/q_t": 0.5007643699645996, "grad_norm": 29.475297927856445, "learning_rate": 2.8358208955223876e-07, "logits/chosen": 0.17340002954006195, "logits/rejected": 0.14621543884277344, "logps/chosen": -57.31782913208008, "logps/ref_chosen": -57.17681121826172, "logps/ref_rejected": -79.486328125, "logps/rejected": -79.59671020507812, "loss": 1.3897, "margin_dpo/margin_mean": -0.03063926100730896, "margin_dpo/margin_std": 0.3815447986125946, "step": 39 }, { "KL/chosen_KL_mean": -0.135894775390625, "KL/mean": -0.1299644410610199, "KL/rejected_KL_mean": -0.1240386962890625, "KL/std": 0.2603451907634735, "epoch": 0.06046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01186208426952362, "fcm_dpo/q_t": 0.5002955198287964, "grad_norm": 31.529159545898438, "learning_rate": 2.9104477611940296e-07, "logits/chosen": 0.11634434759616852, "logits/rejected": 0.06645578145980835, "logps/chosen": -61.470062255859375, "logps/ref_chosen": -61.33416748046875, "logps/ref_rejected": -79.10697174072266, "logps/rejected": -79.23101043701172, "loss": 1.3878, "margin_dpo/margin_mean": -0.011862263083457947, "margin_dpo/margin_std": 0.34227511286735535, "step": 40 }, { "KL/chosen_KL_mean": -0.101654052734375, "KL/mean": -0.15321721136569977, "KL/rejected_KL_mean": -0.20477676391601562, "KL/std": 0.2765595316886902, "epoch": 0.06198034769463341, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10311892628669739, "fcm_dpo/q_t": 0.4974229335784912, "grad_norm": 30.425931930541992, "learning_rate": 2.985074626865671e-07, "logits/chosen": 0.06092709302902222, "logits/rejected": 0.040676962584257126, "logps/chosen": -67.64838409423828, "logps/ref_chosen": -67.5467300415039, "logps/ref_rejected": -83.87788391113281, "logps/rejected": -84.0826644897461, "loss": 1.3763, "margin_dpo/margin_mean": 0.10311943292617798, "margin_dpo/margin_std": 0.34685975313186646, "step": 41 }, { "KL/chosen_KL_mean": -0.1368083953857422, "KL/mean": -0.14792399108409882, "KL/rejected_KL_mean": -0.15903854370117188, "KL/std": 0.2869614362716675, "epoch": 0.06349206349206349, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.022232264280319214, "fcm_dpo/q_t": 0.49944519996643066, "grad_norm": 29.266164779663086, "learning_rate": 3.059701492537313e-07, "logits/chosen": 0.07238396257162094, "logits/rejected": 0.05022376403212547, "logps/chosen": -61.40166473388672, "logps/ref_chosen": -61.26485824584961, "logps/ref_rejected": -76.3629150390625, "logps/rejected": -76.52195739746094, "loss": 1.3845, "margin_dpo/margin_mean": 0.02223220467567444, "margin_dpo/margin_std": 0.38739311695098877, "step": 42 }, { "KL/chosen_KL_mean": -0.08211898803710938, "KL/mean": -0.10783997178077698, "KL/rejected_KL_mean": -0.1335601806640625, "KL/std": 0.28459784388542175, "epoch": 0.06500377928949358, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.051439881324768066, "fcm_dpo/q_t": 0.49871477484703064, "grad_norm": 34.59758758544922, "learning_rate": 3.134328358208955e-07, "logits/chosen": 0.08342611789703369, "logits/rejected": 0.07251065224409103, "logps/chosen": -71.89114379882812, "logps/ref_chosen": -71.80902862548828, "logps/ref_rejected": -81.12464141845703, "logps/rejected": -81.2582015991211, "loss": 1.3815, "margin_dpo/margin_mean": 0.051440536975860596, "margin_dpo/margin_std": 0.35654395818710327, "step": 43 }, { "KL/chosen_KL_mean": -0.14848709106445312, "KL/mean": -0.1632252335548401, "KL/rejected_KL_mean": -0.17796707153320312, "KL/std": 0.3100023865699768, "epoch": 0.06651549508692366, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.029478788375854492, "fcm_dpo/q_t": 0.4992641508579254, "grad_norm": 32.544593811035156, "learning_rate": 3.2089552238805965e-07, "logits/chosen": 0.022504672408103943, "logits/rejected": -0.006912057753652334, "logps/chosen": -66.69891357421875, "logps/ref_chosen": -66.55043029785156, "logps/ref_rejected": -85.06198120117188, "logps/rejected": -85.23994445800781, "loss": 1.3839, "margin_dpo/margin_mean": 0.029478758573532104, "margin_dpo/margin_std": 0.44489896297454834, "step": 44 }, { "KL/chosen_KL_mean": -0.1548919677734375, "KL/mean": -0.20103472471237183, "KL/rejected_KL_mean": -0.24718093872070312, "KL/std": 0.2796245217323303, "epoch": 0.06802721088435375, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09229524433612823, "fcm_dpo/q_t": 0.4976927936077118, "grad_norm": 31.892377853393555, "learning_rate": 3.2835820895522385e-07, "logits/chosen": 0.13807475566864014, "logits/rejected": 0.08398568630218506, "logps/chosen": -62.398746490478516, "logps/ref_chosen": -62.24385452270508, "logps/ref_rejected": -92.96665954589844, "logps/rejected": -93.2138442993164, "loss": 1.3774, "margin_dpo/margin_mean": 0.09229600429534912, "margin_dpo/margin_std": 0.3383764326572418, "step": 45 }, { "KL/chosen_KL_mean": -0.08540916442871094, "KL/mean": -0.16423317790031433, "KL/rejected_KL_mean": -0.24305343627929688, "KL/std": 0.3250676989555359, "epoch": 0.06953892668178382, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.15764117240905762, "fcm_dpo/q_t": 0.4960617423057556, "grad_norm": 31.140756607055664, "learning_rate": 3.3582089552238805e-07, "logits/chosen": 0.12817460298538208, "logits/rejected": 0.0819612368941307, "logps/chosen": -61.584312438964844, "logps/ref_chosen": -61.498905181884766, "logps/ref_rejected": -78.91172790527344, "logps/rejected": -79.15478515625, "loss": 1.3711, "margin_dpo/margin_mean": 0.15764087438583374, "margin_dpo/margin_std": 0.43353208899497986, "step": 46 }, { "KL/chosen_KL_mean": -0.10969352722167969, "KL/mean": -0.1963786482810974, "KL/rejected_KL_mean": -0.2830657958984375, "KL/std": 0.2953278422355652, "epoch": 0.0710506424792139, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17337125539779663, "fcm_dpo/q_t": 0.4956666827201843, "grad_norm": 28.431907653808594, "learning_rate": 3.432835820895522e-07, "logits/chosen": 0.0386110283434391, "logits/rejected": -0.0044650724157691, "logps/chosen": -51.68804168701172, "logps/ref_chosen": -51.578346252441406, "logps/ref_rejected": -68.2215576171875, "logps/rejected": -68.50462341308594, "loss": 1.3694, "margin_dpo/margin_mean": 0.17337137460708618, "margin_dpo/margin_std": 0.39710623025894165, "step": 47 }, { "KL/chosen_KL_mean": -0.22099685668945312, "KL/mean": -0.24017362296581268, "KL/rejected_KL_mean": -0.2593536376953125, "KL/std": 0.31508952379226685, "epoch": 0.07256235827664399, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03835412859916687, "fcm_dpo/q_t": 0.49904191493988037, "grad_norm": 26.774696350097656, "learning_rate": 3.507462686567164e-07, "logits/chosen": 0.1500009000301361, "logits/rejected": 0.12023096531629562, "logps/chosen": -52.0146484375, "logps/ref_chosen": -51.79365158081055, "logps/ref_rejected": -64.22503662109375, "logps/rejected": -64.48439025878906, "loss": 1.3829, "margin_dpo/margin_mean": 0.03835412859916687, "margin_dpo/margin_std": 0.4297327399253845, "step": 48 }, { "KL/chosen_KL_mean": -0.18651199340820312, "KL/mean": -0.25922006368637085, "KL/rejected_KL_mean": -0.3319263458251953, "KL/std": 0.33801430463790894, "epoch": 0.07407407407407407, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1454138606786728, "fcm_dpo/q_t": 0.49636557698249817, "grad_norm": 27.162975311279297, "learning_rate": 3.5820895522388055e-07, "logits/chosen": 0.026051603257656097, "logits/rejected": 0.004764057695865631, "logps/chosen": -58.32111358642578, "logps/ref_chosen": -58.13460159301758, "logps/ref_rejected": -64.63206481933594, "logps/rejected": -64.96399688720703, "loss": 1.3723, "margin_dpo/margin_mean": 0.14541450142860413, "margin_dpo/margin_std": 0.43787360191345215, "step": 49 }, { "KL/chosen_KL_mean": -0.30684661865234375, "KL/mean": -0.3562437891960144, "KL/rejected_KL_mean": -0.4056396484375, "KL/std": 0.35121792554855347, "epoch": 0.07558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09879663586616516, "fcm_dpo/q_t": 0.497530460357666, "grad_norm": 28.057424545288086, "learning_rate": 3.6567164179104475e-07, "logits/chosen": 0.10507173836231232, "logits/rejected": 0.07558847963809967, "logps/chosen": -53.16328430175781, "logps/ref_chosen": -52.85643768310547, "logps/ref_rejected": -72.17460632324219, "logps/rejected": -72.58024597167969, "loss": 1.3768, "margin_dpo/margin_mean": 0.09879684448242188, "margin_dpo/margin_std": 0.38183924555778503, "step": 50 }, { "KL/chosen_KL_mean": -0.2800483703613281, "KL/mean": -0.3949533998966217, "KL/rejected_KL_mean": -0.5098648071289062, "KL/std": 0.39111822843551636, "epoch": 0.07709750566893424, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22981137037277222, "fcm_dpo/q_t": 0.4942595958709717, "grad_norm": 31.003662109375, "learning_rate": 3.7313432835820895e-07, "logits/chosen": 0.059184394776821136, "logits/rejected": 0.03226791322231293, "logps/chosen": -63.936492919921875, "logps/ref_chosen": -63.65644073486328, "logps/ref_rejected": -86.13229370117188, "logps/rejected": -86.64216613769531, "loss": 1.364, "margin_dpo/margin_mean": 0.22981131076812744, "margin_dpo/margin_std": 0.4713187515735626, "step": 51 }, { "KL/chosen_KL_mean": -0.3123207092285156, "KL/mean": -0.42203959822654724, "KL/rejected_KL_mean": -0.5317573547363281, "KL/std": 0.42343559861183167, "epoch": 0.07860922146636433, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21943794190883636, "fcm_dpo/q_t": 0.49452149868011475, "grad_norm": 32.59117126464844, "learning_rate": 3.805970149253731e-07, "logits/chosen": 0.08497665822505951, "logits/rejected": 0.03561322018504143, "logps/chosen": -68.15253448486328, "logps/ref_chosen": -67.8402099609375, "logps/ref_rejected": -96.97090911865234, "logps/rejected": -97.50267028808594, "loss": 1.3652, "margin_dpo/margin_mean": 0.21943779289722443, "margin_dpo/margin_std": 0.5385311841964722, "step": 52 }, { "KL/chosen_KL_mean": -0.3397502899169922, "KL/mean": -0.4259885549545288, "KL/rejected_KL_mean": -0.5122280120849609, "KL/std": 0.391630619764328, "epoch": 0.0801209372637944, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17248192429542542, "fcm_dpo/q_t": 0.4956892132759094, "grad_norm": 27.551605224609375, "learning_rate": 3.880597014925373e-07, "logits/chosen": 0.10184863209724426, "logits/rejected": 0.09087856113910675, "logps/chosen": -57.21788787841797, "logps/ref_chosen": -56.87813949584961, "logps/ref_rejected": -60.75569152832031, "logps/rejected": -61.267921447753906, "loss": 1.3698, "margin_dpo/margin_mean": 0.17248201370239258, "margin_dpo/margin_std": 0.4895680248737335, "step": 53 }, { "KL/chosen_KL_mean": -0.39040374755859375, "KL/mean": -0.47420793771743774, "KL/rejected_KL_mean": -0.5580081939697266, "KL/std": 0.3961087465286255, "epoch": 0.08163265306122448, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16760483384132385, "fcm_dpo/q_t": 0.4958149492740631, "grad_norm": 26.449312210083008, "learning_rate": 3.9552238805970144e-07, "logits/chosen": 0.07982660830020905, "logits/rejected": 0.06419865787029266, "logps/chosen": -47.657325744628906, "logps/ref_chosen": -47.26692199707031, "logps/ref_rejected": -62.19426727294922, "logps/rejected": -62.75227355957031, "loss": 1.3703, "margin_dpo/margin_mean": 0.16760462522506714, "margin_dpo/margin_std": 0.5214763879776001, "step": 54 }, { "KL/chosen_KL_mean": -0.3712482452392578, "KL/mean": -0.4966890513896942, "KL/rejected_KL_mean": -0.62213134765625, "KL/std": 0.4974730610847473, "epoch": 0.08314436885865457, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2508871555328369, "fcm_dpo/q_t": 0.493741512298584, "grad_norm": 31.378108978271484, "learning_rate": 4.0298507462686564e-07, "logits/chosen": 0.03751315549015999, "logits/rejected": -0.03892657905817032, "logps/chosen": -50.69744110107422, "logps/ref_chosen": -50.32619094848633, "logps/ref_rejected": -92.44389343261719, "logps/rejected": -93.06602478027344, "loss": 1.3625, "margin_dpo/margin_mean": 0.2508869171142578, "margin_dpo/margin_std": 0.6618380546569824, "step": 55 }, { "KL/chosen_KL_mean": -0.41105079650878906, "KL/mean": -0.4723691940307617, "KL/rejected_KL_mean": -0.5336799621582031, "KL/std": 0.5030519962310791, "epoch": 0.08465608465608465, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12263241410255432, "fcm_dpo/q_t": 0.49693045020103455, "grad_norm": 27.65058708190918, "learning_rate": 4.1044776119402984e-07, "logits/chosen": 0.14740391075611115, "logits/rejected": 0.12471330165863037, "logps/chosen": -57.17802429199219, "logps/ref_chosen": -56.766971588134766, "logps/ref_rejected": -66.30504608154297, "logps/rejected": -66.83872985839844, "loss": 1.3752, "margin_dpo/margin_mean": 0.12263259291648865, "margin_dpo/margin_std": 0.6547993421554565, "step": 56 }, { "KL/chosen_KL_mean": -0.5095005035400391, "KL/mean": -0.6674191951751709, "KL/rejected_KL_mean": -0.8253402709960938, "KL/std": 0.6238527297973633, "epoch": 0.08616780045351474, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.315839022397995, "fcm_dpo/q_t": 0.4921194911003113, "grad_norm": 30.4954833984375, "learning_rate": 4.17910447761194e-07, "logits/chosen": 0.09336856752634048, "logits/rejected": 0.029725098982453346, "logps/chosen": -58.277244567871094, "logps/ref_chosen": -57.76774597167969, "logps/ref_rejected": -82.75698852539062, "logps/rejected": -83.58232879638672, "loss": 1.3562, "margin_dpo/margin_mean": 0.3158394694328308, "margin_dpo/margin_std": 0.6920211911201477, "step": 57 }, { "KL/chosen_KL_mean": -0.5488929748535156, "KL/mean": -0.6903192400932312, "KL/rejected_KL_mean": -0.8317451477050781, "KL/std": 0.6478506326675415, "epoch": 0.08767951625094482, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.282858282327652, "fcm_dpo/q_t": 0.49298739433288574, "grad_norm": 30.036136627197266, "learning_rate": 4.253731343283582e-07, "logits/chosen": 0.0658133402466774, "logits/rejected": 0.05019402876496315, "logps/chosen": -73.31298065185547, "logps/ref_chosen": -72.76408386230469, "logps/ref_rejected": -84.49275207519531, "logps/rejected": -85.32449340820312, "loss": 1.3607, "margin_dpo/margin_mean": 0.28285834193229675, "margin_dpo/margin_std": 0.9295713305473328, "step": 58 }, { "KL/chosen_KL_mean": -0.5454654693603516, "KL/mean": -0.6809777021408081, "KL/rejected_KL_mean": -0.8164939880371094, "KL/std": 0.6243307590484619, "epoch": 0.08919123204837491, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.27102532982826233, "fcm_dpo/q_t": 0.49324700236320496, "grad_norm": 26.647247314453125, "learning_rate": 4.3283582089552234e-07, "logits/chosen": 0.13273830711841583, "logits/rejected": 0.0665852427482605, "logps/chosen": -50.366241455078125, "logps/ref_chosen": -49.820777893066406, "logps/ref_rejected": -77.14368438720703, "logps/rejected": -77.96017456054688, "loss": 1.3612, "margin_dpo/margin_mean": 0.2710251212120056, "margin_dpo/margin_std": 0.7743154168128967, "step": 59 }, { "KL/chosen_KL_mean": -0.5872421264648438, "KL/mean": -0.6434541940689087, "KL/rejected_KL_mean": -0.6996688842773438, "KL/std": 0.6261066198348999, "epoch": 0.09070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11242368817329407, "fcm_dpo/q_t": 0.4971911907196045, "grad_norm": 29.517229080200195, "learning_rate": 4.4029850746268654e-07, "logits/chosen": 0.12953968346118927, "logits/rejected": 0.12811188399791718, "logps/chosen": -63.812015533447266, "logps/ref_chosen": -63.22477340698242, "logps/ref_rejected": -61.360477447509766, "logps/rejected": -62.06014633178711, "loss": 1.3769, "margin_dpo/margin_mean": 0.11242353916168213, "margin_dpo/margin_std": 0.850089430809021, "step": 60 }, { "KL/chosen_KL_mean": -0.7440834045410156, "KL/mean": -0.7743111848831177, "KL/rejected_KL_mean": -0.8045387268066406, "KL/std": 0.6986474990844727, "epoch": 0.09221466364323508, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06045445799827576, "fcm_dpo/q_t": 0.4984866976737976, "grad_norm": 28.10788917541504, "learning_rate": 4.4776119402985074e-07, "logits/chosen": 0.14267820119857788, "logits/rejected": 0.1098393127322197, "logps/chosen": -49.76087951660156, "logps/ref_chosen": -49.01679992675781, "logps/ref_rejected": -74.90817260742188, "logps/rejected": -75.71270751953125, "loss": 1.3827, "margin_dpo/margin_mean": 0.06045445799827576, "margin_dpo/margin_std": 0.9751712083816528, "step": 61 }, { "KL/chosen_KL_mean": -0.7253894805908203, "KL/mean": -0.8400178551673889, "KL/rejected_KL_mean": -0.9546432495117188, "KL/std": 0.623093843460083, "epoch": 0.09372637944066516, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.229254812002182, "fcm_dpo/q_t": 0.4942789077758789, "grad_norm": 28.548622131347656, "learning_rate": 4.552238805970149e-07, "logits/chosen": 0.13225194811820984, "logits/rejected": 0.09242310374975204, "logps/chosen": -63.477256774902344, "logps/ref_chosen": -62.751869201660156, "logps/ref_rejected": -78.93360900878906, "logps/rejected": -79.88825225830078, "loss": 1.3653, "margin_dpo/margin_mean": 0.22925478219985962, "margin_dpo/margin_std": 0.8522534370422363, "step": 62 }, { "KL/chosen_KL_mean": -0.5486431121826172, "KL/mean": -0.8511289358139038, "KL/rejected_KL_mean": -1.1536216735839844, "KL/std": 0.7331008911132812, "epoch": 0.09523809523809523, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6049745082855225, "fcm_dpo/q_t": 0.48493385314941406, "grad_norm": 31.141754150390625, "learning_rate": 4.626865671641791e-07, "logits/chosen": 0.18645809590816498, "logits/rejected": 0.16159288585186005, "logps/chosen": -61.06389236450195, "logps/ref_chosen": -60.51525115966797, "logps/ref_rejected": -85.11021423339844, "logps/rejected": -86.26383972167969, "loss": 1.3289, "margin_dpo/margin_mean": 0.6049743890762329, "margin_dpo/margin_std": 0.9378039836883545, "step": 63 }, { "KL/chosen_KL_mean": -0.8053035736083984, "KL/mean": -0.84546959400177, "KL/rejected_KL_mean": -0.8856353759765625, "KL/std": 0.6875101327896118, "epoch": 0.09674981103552532, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08033087849617004, "fcm_dpo/q_t": 0.49798619747161865, "grad_norm": 26.139707565307617, "learning_rate": 4.701492537313433e-07, "logits/chosen": 0.09058545529842377, "logits/rejected": 0.06569143384695053, "logps/chosen": -52.012149810791016, "logps/ref_chosen": -51.20684814453125, "logps/ref_rejected": -66.93081665039062, "logps/rejected": -67.81645202636719, "loss": 1.38, "margin_dpo/margin_mean": 0.08033090829849243, "margin_dpo/margin_std": 0.8200113773345947, "step": 64 }, { "KL/chosen_KL_mean": -0.8628063201904297, "KL/mean": -1.1255594491958618, "KL/rejected_KL_mean": -1.3883171081542969, "KL/std": 0.9253497123718262, "epoch": 0.0982615268329554, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5255085229873657, "fcm_dpo/q_t": 0.48692017793655396, "grad_norm": 30.805511474609375, "learning_rate": 4.776119402985074e-07, "logits/chosen": 0.22027479112148285, "logits/rejected": 0.18991133570671082, "logps/chosen": -68.15150451660156, "logps/ref_chosen": -67.2886962890625, "logps/ref_rejected": -74.44281005859375, "logps/rejected": -75.83112335205078, "loss": 1.338, "margin_dpo/margin_mean": 0.5255081057548523, "margin_dpo/margin_std": 1.190438985824585, "step": 65 }, { "KL/chosen_KL_mean": -0.9165191650390625, "KL/mean": -1.099397897720337, "KL/rejected_KL_mean": -1.2822761535644531, "KL/std": 0.8203333616256714, "epoch": 0.09977324263038549, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3657585382461548, "fcm_dpo/q_t": 0.4908965229988098, "grad_norm": 29.26482582092285, "learning_rate": 4.850746268656717e-07, "logits/chosen": 0.09936878085136414, "logits/rejected": 0.07552456855773926, "logps/chosen": -71.65992736816406, "logps/ref_chosen": -70.743408203125, "logps/ref_rejected": -77.26499938964844, "logps/rejected": -78.54727935791016, "loss": 1.353, "margin_dpo/margin_mean": 0.3657584488391876, "margin_dpo/margin_std": 1.0532437562942505, "step": 66 }, { "KL/chosen_KL_mean": -0.7958889007568359, "KL/mean": -0.9142911434173584, "KL/rejected_KL_mean": -1.0326957702636719, "KL/std": 0.7134413123130798, "epoch": 0.10128495842781557, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.23680096864700317, "fcm_dpo/q_t": 0.49409249424934387, "grad_norm": 29.31564712524414, "learning_rate": 4.925373134328357e-07, "logits/chosen": 0.07396933436393738, "logits/rejected": 0.01928994059562683, "logps/chosen": -61.39849090576172, "logps/ref_chosen": -60.60260009765625, "logps/ref_rejected": -75.22235870361328, "logps/rejected": -76.25505065917969, "loss": 1.3646, "margin_dpo/margin_mean": 0.23680099844932556, "margin_dpo/margin_std": 0.8555749654769897, "step": 67 }, { "KL/chosen_KL_mean": -1.2452201843261719, "KL/mean": -1.4087040424346924, "KL/rejected_KL_mean": -1.5721893310546875, "KL/std": 0.9630727767944336, "epoch": 0.10279667422524566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3269740343093872, "fcm_dpo/q_t": 0.49186715483665466, "grad_norm": 31.29288101196289, "learning_rate": 5e-07, "logits/chosen": 0.08105947077274323, "logits/rejected": 0.05047762021422386, "logps/chosen": -78.77359008789062, "logps/ref_chosen": -77.52836608886719, "logps/ref_rejected": -93.17778015136719, "logps/rejected": -94.74996948242188, "loss": 1.358, "margin_dpo/margin_mean": 0.3269742727279663, "margin_dpo/margin_std": 1.267851710319519, "step": 68 }, { "KL/chosen_KL_mean": -1.13970947265625, "KL/mean": -1.466085433959961, "KL/rejected_KL_mean": -1.7924575805664062, "KL/std": 1.0306477546691895, "epoch": 0.10430839002267574, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6527523994445801, "fcm_dpo/q_t": 0.4837765097618103, "grad_norm": 30.93454933166504, "learning_rate": 4.999965034812934e-07, "logits/chosen": 0.0892995297908783, "logits/rejected": 0.04694270342588425, "logps/chosen": -67.082763671875, "logps/ref_chosen": -65.94305419921875, "logps/ref_rejected": -89.7735595703125, "logps/rejected": -91.5660171508789, "loss": 1.3259, "margin_dpo/margin_mean": 0.6527519226074219, "margin_dpo/margin_std": 1.201098918914795, "step": 69 }, { "KL/chosen_KL_mean": -1.238992691040039, "KL/mean": -1.4067550897598267, "KL/rejected_KL_mean": -1.5745105743408203, "KL/std": 0.8888180255889893, "epoch": 0.10582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.33551764488220215, "fcm_dpo/q_t": 0.49164047837257385, "grad_norm": 28.678653717041016, "learning_rate": 4.999860140229787e-07, "logits/chosen": 0.15405255556106567, "logits/rejected": 0.13066557049751282, "logps/chosen": -63.19690704345703, "logps/ref_chosen": -61.95791244506836, "logps/ref_rejected": -75.80945587158203, "logps/rejected": -77.38397216796875, "loss": 1.3565, "margin_dpo/margin_mean": 0.3355180025100708, "margin_dpo/margin_std": 1.1736544370651245, "step": 70 }, { "KL/chosen_KL_mean": -1.4223155975341797, "KL/mean": -1.555302381515503, "KL/rejected_KL_mean": -1.6882915496826172, "KL/std": 1.0254878997802734, "epoch": 0.1073318216175359, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26597607135772705, "fcm_dpo/q_t": 0.49335426092147827, "grad_norm": 28.391786575317383, "learning_rate": 4.999685319184688e-07, "logits/chosen": 0.09895138442516327, "logits/rejected": 0.08333157747983932, "logps/chosen": -64.7698974609375, "logps/ref_chosen": -63.34757995605469, "logps/ref_rejected": -67.49658203125, "logps/rejected": -69.18487548828125, "loss": 1.3653, "margin_dpo/margin_mean": 0.2659757733345032, "margin_dpo/margin_std": 1.4624427556991577, "step": 71 }, { "KL/chosen_KL_mean": -1.1982994079589844, "KL/mean": -1.54570472240448, "KL/rejected_KL_mean": -1.8931045532226562, "KL/std": 1.0516983270645142, "epoch": 0.10884353741496598, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6948050260543823, "fcm_dpo/q_t": 0.4827173948287964, "grad_norm": 30.34983253479004, "learning_rate": 4.999440576567755e-07, "logits/chosen": 0.08275315165519714, "logits/rejected": 0.022004300728440285, "logps/chosen": -57.05759811401367, "logps/ref_chosen": -55.85929870605469, "logps/ref_rejected": -68.45423889160156, "logps/rejected": -70.34735107421875, "loss": 1.3223, "margin_dpo/margin_mean": 0.6948049068450928, "margin_dpo/margin_std": 1.2801237106323242, "step": 72 }, { "KL/chosen_KL_mean": -1.8044281005859375, "KL/mean": -1.856170892715454, "KL/rejected_KL_mean": -1.9079151153564453, "KL/std": 1.1962889432907104, "epoch": 0.11035525321239607, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10349225997924805, "fcm_dpo/q_t": 0.49755388498306274, "grad_norm": 32.140106201171875, "learning_rate": 4.999125919224965e-07, "logits/chosen": 0.10440310835838318, "logits/rejected": 0.09044543653726578, "logps/chosen": -70.9432373046875, "logps/ref_chosen": -69.13880920410156, "logps/ref_rejected": -79.04586791992188, "logps/rejected": -80.95378112792969, "loss": 1.3822, "margin_dpo/margin_mean": 0.10349264740943909, "margin_dpo/margin_std": 1.502673625946045, "step": 73 }, { "KL/chosen_KL_mean": -1.2395572662353516, "KL/mean": -1.5856974124908447, "KL/rejected_KL_mean": -1.9318351745605469, "KL/std": 1.0054885149002075, "epoch": 0.11186696900982615, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6922777891159058, "fcm_dpo/q_t": 0.48279356956481934, "grad_norm": 28.01102638244629, "learning_rate": 4.998741355957963e-07, "logits/chosen": 0.13102754950523376, "logits/rejected": 0.07868199050426483, "logps/chosen": -51.163291931152344, "logps/ref_chosen": -49.923736572265625, "logps/ref_rejected": -81.73213958740234, "logps/rejected": -83.66397094726562, "loss": 1.3225, "margin_dpo/margin_mean": 0.6922776699066162, "margin_dpo/margin_std": 1.2654963731765747, "step": 74 }, { "KL/chosen_KL_mean": -1.4122562408447266, "KL/mean": -1.7794251441955566, "KL/rejected_KL_mean": -2.1465892791748047, "KL/std": 1.0802136659622192, "epoch": 0.11337868480725624, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.734336793422699, "fcm_dpo/q_t": 0.4817506670951843, "grad_norm": 26.342166900634766, "learning_rate": 4.998286897523808e-07, "logits/chosen": 0.10357685387134552, "logits/rejected": 0.07225304841995239, "logps/chosen": -47.48101043701172, "logps/ref_chosen": -46.06875228881836, "logps/ref_rejected": -66.1181411743164, "logps/rejected": -68.26473236083984, "loss": 1.3185, "margin_dpo/margin_mean": 0.7343367338180542, "margin_dpo/margin_std": 1.30611252784729, "step": 75 }, { "KL/chosen_KL_mean": -1.6146888732910156, "KL/mean": -1.7545742988586426, "KL/rejected_KL_mean": -1.8944587707519531, "KL/std": 1.1566370725631714, "epoch": 0.11489040060468632, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2797727584838867, "fcm_dpo/q_t": 0.49302801489830017, "grad_norm": 29.376850128173828, "learning_rate": 4.997762556634679e-07, "logits/chosen": 0.13626576960086823, "logits/rejected": 0.09158352017402649, "logps/chosen": -55.67743682861328, "logps/ref_chosen": -54.06275177001953, "logps/ref_rejected": -74.87464141845703, "logps/rejected": -76.76910400390625, "loss": 1.3642, "margin_dpo/margin_mean": 0.2797728478908539, "margin_dpo/margin_std": 1.5069594383239746, "step": 76 }, { "KL/chosen_KL_mean": -1.6369781494140625, "KL/mean": -2.009381055831909, "KL/rejected_KL_mean": -2.3817825317382812, "KL/std": 1.2048678398132324, "epoch": 0.1164021164021164, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7448074817657471, "fcm_dpo/q_t": 0.48149538040161133, "grad_norm": 29.28656768798828, "learning_rate": 4.99716834795752e-07, "logits/chosen": 0.1588551104068756, "logits/rejected": 0.11727854609489441, "logps/chosen": -54.713069915771484, "logps/ref_chosen": -53.07609176635742, "logps/ref_rejected": -74.45601654052734, "logps/rejected": -76.83779907226562, "loss": 1.3183, "margin_dpo/margin_mean": 0.7448074817657471, "margin_dpo/margin_std": 1.3909995555877686, "step": 77 }, { "KL/chosen_KL_mean": -1.8279800415039062, "KL/mean": -2.0209426879882812, "KL/rejected_KL_mean": -2.2139129638671875, "KL/std": 1.2771286964416504, "epoch": 0.11791383219954649, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.38592809438705444, "fcm_dpo/q_t": 0.490395188331604, "grad_norm": 29.480390548706055, "learning_rate": 4.996504288113623e-07, "logits/chosen": 0.10726994276046753, "logits/rejected": 0.08700601756572723, "logps/chosen": -69.55340576171875, "logps/ref_chosen": -67.72541809082031, "logps/ref_rejected": -79.03926849365234, "logps/rejected": -81.25318145751953, "loss": 1.3534, "margin_dpo/margin_mean": 0.3859279155731201, "margin_dpo/margin_std": 1.4653525352478027, "step": 78 }, { "KL/chosen_KL_mean": -1.7102622985839844, "KL/mean": -2.2839438915252686, "KL/rejected_KL_mean": -2.8576278686523438, "KL/std": 1.5375041961669922, "epoch": 0.11942554799697656, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1473674774169922, "fcm_dpo/q_t": 0.4718601703643799, "grad_norm": 30.79814910888672, "learning_rate": 4.995770395678171e-07, "logits/chosen": 0.1881793886423111, "logits/rejected": 0.12662510573863983, "logps/chosen": -53.870906829833984, "logps/ref_chosen": -52.16064453125, "logps/ref_rejected": -83.31062316894531, "logps/rejected": -86.16824340820312, "loss": 1.2847, "margin_dpo/margin_mean": 1.1473678350448608, "margin_dpo/margin_std": 1.9917948246002197, "step": 79 }, { "KL/chosen_KL_mean": -2.041339874267578, "KL/mean": -2.3201375007629395, "KL/rejected_KL_mean": -2.5989341735839844, "KL/std": 1.3568034172058105, "epoch": 0.12093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5575928688049316, "fcm_dpo/q_t": 0.4862174987792969, "grad_norm": 28.497337341308594, "learning_rate": 4.994966691179711e-07, "logits/chosen": 0.14524391293525696, "logits/rejected": 0.08540673553943634, "logps/chosen": -63.451900482177734, "logps/ref_chosen": -61.410560607910156, "logps/ref_rejected": -78.66004943847656, "logps/rejected": -81.25898742675781, "loss": 1.3409, "margin_dpo/margin_mean": 0.5575932264328003, "margin_dpo/margin_std": 1.8942968845367432, "step": 80 }, { "KL/chosen_KL_mean": -2.136859893798828, "KL/mean": -2.5757455825805664, "KL/rejected_KL_mean": -3.0146255493164062, "KL/std": 1.4461383819580078, "epoch": 0.12244897959183673, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8777664303779602, "fcm_dpo/q_t": 0.47836926579475403, "grad_norm": 28.98223876953125, "learning_rate": 4.994093197099587e-07, "logits/chosen": 0.13092893362045288, "logits/rejected": 0.09700986742973328, "logps/chosen": -65.94123077392578, "logps/ref_chosen": -63.80437088012695, "logps/ref_rejected": -79.3484115600586, "logps/rejected": -82.363037109375, "loss": 1.3086, "margin_dpo/margin_mean": 0.8777658939361572, "margin_dpo/margin_std": 1.7367991209030151, "step": 81 }, { "KL/chosen_KL_mean": -1.7457637786865234, "KL/mean": -2.286520481109619, "KL/rejected_KL_mean": -2.8272781372070312, "KL/std": 1.3924403190612793, "epoch": 0.12396069538926682, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.081512451171875, "fcm_dpo/q_t": 0.4731561541557312, "grad_norm": 26.511188507080078, "learning_rate": 4.993149937871306e-07, "logits/chosen": 0.10917188227176666, "logits/rejected": 0.04636561498045921, "logps/chosen": -50.563655853271484, "logps/ref_chosen": -48.817893981933594, "logps/ref_rejected": -70.31497955322266, "logps/rejected": -73.14225769042969, "loss": 1.2874, "margin_dpo/margin_mean": 1.081512212753296, "margin_dpo/margin_std": 1.579821228981018, "step": 82 }, { "KL/chosen_KL_mean": -2.0326290130615234, "KL/mean": -2.6406869888305664, "KL/rejected_KL_mean": -3.248748779296875, "KL/std": 1.456777811050415, "epoch": 0.1254724111866969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2161188125610352, "fcm_dpo/q_t": 0.4699665307998657, "grad_norm": 29.091482162475586, "learning_rate": 4.992136939879856e-07, "logits/chosen": 0.22049690783023834, "logits/rejected": 0.16898763179779053, "logps/chosen": -59.18340301513672, "logps/ref_chosen": -57.15077209472656, "logps/ref_rejected": -75.1710205078125, "logps/rejected": -78.41976928710938, "loss": 1.2762, "margin_dpo/margin_mean": 1.216118335723877, "margin_dpo/margin_std": 1.7663187980651855, "step": 83 }, { "KL/chosen_KL_mean": -2.5208396911621094, "KL/mean": -2.948389768600464, "KL/rejected_KL_mean": -3.375946044921875, "KL/std": 1.8345149755477905, "epoch": 0.12698412698412698, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.855107307434082, "fcm_dpo/q_t": 0.47885656356811523, "grad_norm": 30.742326736450195, "learning_rate": 4.991054231460969e-07, "logits/chosen": 0.14221562445163727, "logits/rejected": 0.10221521556377411, "logps/chosen": -67.29814147949219, "logps/ref_chosen": -64.77729797363281, "logps/ref_rejected": -84.71949768066406, "logps/rejected": -88.09544372558594, "loss": 1.3119, "margin_dpo/margin_mean": 0.8551075458526611, "margin_dpo/margin_std": 1.9248778820037842, "step": 84 }, { "KL/chosen_KL_mean": -2.2519493103027344, "KL/mean": -2.8956236839294434, "KL/rejected_KL_mean": -3.5392990112304688, "KL/std": 1.7731411457061768, "epoch": 0.12849584278155707, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2873495817184448, "fcm_dpo/q_t": 0.468211829662323, "grad_norm": 26.718862533569336, "learning_rate": 4.989901842900325e-07, "logits/chosen": 0.16839201748371124, "logits/rejected": 0.12411106377840042, "logps/chosen": -52.50364685058594, "logps/ref_chosen": -50.25169372558594, "logps/ref_rejected": -66.55439758300781, "logps/rejected": -70.09368896484375, "loss": 1.272, "margin_dpo/margin_mean": 1.2873497009277344, "margin_dpo/margin_std": 1.9850984811782837, "step": 85 }, { "KL/chosen_KL_mean": -2.819011688232422, "KL/mean": -3.3019821643829346, "KL/rejected_KL_mean": -3.7849502563476562, "KL/std": 1.8966737985610962, "epoch": 0.13000755857898716, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9659372568130493, "fcm_dpo/q_t": 0.4761325418949127, "grad_norm": 27.210853576660156, "learning_rate": 4.988679806432711e-07, "logits/chosen": 0.21716086566448212, "logits/rejected": 0.19776871800422668, "logps/chosen": -63.54819107055664, "logps/ref_chosen": -60.72917938232422, "logps/ref_rejected": -72.30961608886719, "logps/rejected": -76.09455871582031, "loss": 1.3025, "margin_dpo/margin_mean": 0.9659373760223389, "margin_dpo/margin_std": 2.055220603942871, "step": 86 }, { "KL/chosen_KL_mean": -2.9924278259277344, "KL/mean": -3.5514774322509766, "KL/rejected_KL_mean": -4.110523223876953, "KL/std": 2.083639144897461, "epoch": 0.13151927437641722, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1181044578552246, "fcm_dpo/q_t": 0.47244584560394287, "grad_norm": 30.3636474609375, "learning_rate": 4.987388156241114e-07, "logits/chosen": 0.1789131462574005, "logits/rejected": 0.11889512836933136, "logps/chosen": -68.75039672851562, "logps/ref_chosen": -65.75796508789062, "logps/ref_rejected": -84.81159973144531, "logps/rejected": -88.922119140625, "loss": 1.2933, "margin_dpo/margin_mean": 1.1181050539016724, "margin_dpo/margin_std": 2.491665840148926, "step": 87 }, { "KL/chosen_KL_mean": -3.064952850341797, "KL/mean": -3.4871764183044434, "KL/rejected_KL_mean": -3.9093971252441406, "KL/std": 2.3427746295928955, "epoch": 0.1330309901738473, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.84444260597229, "fcm_dpo/q_t": 0.4793391227722168, "grad_norm": 30.429597854614258, "learning_rate": 4.986026928455767e-07, "logits/chosen": 0.24011830985546112, "logits/rejected": 0.21302275359630585, "logps/chosen": -65.88897705078125, "logps/ref_chosen": -62.82402801513672, "logps/ref_rejected": -74.9607162475586, "logps/rejected": -78.8701171875, "loss": 1.327, "margin_dpo/margin_mean": 0.84444260597229, "margin_dpo/margin_std": 2.9619760513305664, "step": 88 }, { "KL/chosen_KL_mean": -2.5539302825927734, "KL/mean": -3.2578611373901367, "KL/rejected_KL_mean": -3.9617958068847656, "KL/std": 2.1723883152008057, "epoch": 0.1345427059712774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4078590869903564, "fcm_dpo/q_t": 0.4656612277030945, "grad_norm": 29.212039947509766, "learning_rate": 4.984596161153135e-07, "logits/chosen": 0.24471764266490936, "logits/rejected": 0.16260842978954315, "logps/chosen": -43.745365142822266, "logps/ref_chosen": -41.191436767578125, "logps/ref_rejected": -85.44769287109375, "logps/rejected": -89.40949249267578, "loss": 1.2662, "margin_dpo/margin_mean": 1.4078595638275146, "margin_dpo/margin_std": 2.449413776397705, "step": 89 }, { "KL/chosen_KL_mean": -3.128904342651367, "KL/mean": -3.782576560974121, "KL/rejected_KL_mean": -4.436252593994141, "KL/std": 2.440657615661621, "epoch": 0.1360544217687075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3073492050170898, "fcm_dpo/q_t": 0.4679213762283325, "grad_norm": 29.046615600585938, "learning_rate": 4.983095894354857e-07, "logits/chosen": 0.1570890247821808, "logits/rejected": 0.10190249979496002, "logps/chosen": -59.71281433105469, "logps/ref_chosen": -56.58390808105469, "logps/ref_rejected": -86.86978149414062, "logps/rejected": -91.3060302734375, "loss": 1.276, "margin_dpo/margin_mean": 1.307349443435669, "margin_dpo/margin_std": 2.514946460723877, "step": 90 }, { "KL/chosen_KL_mean": -3.1567344665527344, "KL/mean": -3.703597068786621, "KL/rejected_KL_mean": -4.250457763671875, "KL/std": 2.2865095138549805, "epoch": 0.13756613756613756, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0937232971191406, "fcm_dpo/q_t": 0.47355741262435913, "grad_norm": 25.90226936340332, "learning_rate": 4.98152617002662e-07, "logits/chosen": 0.15374761819839478, "logits/rejected": 0.11025101691484451, "logps/chosen": -55.53907775878906, "logps/ref_chosen": -52.38234329223633, "logps/ref_rejected": -72.17642211914062, "logps/rejected": -76.4268798828125, "loss": 1.2993, "margin_dpo/margin_mean": 1.0937230587005615, "margin_dpo/margin_std": 2.792675018310547, "step": 91 }, { "KL/chosen_KL_mean": -3.2463722229003906, "KL/mean": -4.036555290222168, "KL/rejected_KL_mean": -4.826740264892578, "KL/std": 2.6802730560302734, "epoch": 0.13907785336356765, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5803602933883667, "fcm_dpo/q_t": 0.4618731141090393, "grad_norm": 27.01513671875, "learning_rate": 4.979887032076988e-07, "logits/chosen": 0.22086292505264282, "logits/rejected": 0.18025824427604675, "logps/chosen": -56.25507354736328, "logps/ref_chosen": -53.00870132446289, "logps/ref_rejected": -79.77812957763672, "logps/rejected": -84.60487365722656, "loss": 1.2586, "margin_dpo/margin_mean": 1.5803604125976562, "margin_dpo/margin_std": 3.117692470550537, "step": 92 }, { "KL/chosen_KL_mean": -3.5561141967773438, "KL/mean": -4.080088138580322, "KL/rejected_KL_mean": -4.604063034057617, "KL/std": 2.6413497924804688, "epoch": 0.14058956916099774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0479507446289062, "fcm_dpo/q_t": 0.47497549653053284, "grad_norm": 24.47339630126953, "learning_rate": 4.978178526356172e-07, "logits/chosen": 0.18242627382278442, "logits/rejected": 0.15430913865566254, "logps/chosen": -48.463165283203125, "logps/ref_chosen": -44.90705108642578, "logps/ref_rejected": -58.7879524230957, "logps/rejected": -63.39201354980469, "loss": 1.3097, "margin_dpo/margin_mean": 1.0479507446289062, "margin_dpo/margin_std": 3.2303943634033203, "step": 93 }, { "KL/chosen_KL_mean": -3.2976722717285156, "KL/mean": -4.257216453552246, "KL/rejected_KL_mean": -5.216762542724609, "KL/std": 3.2533302307128906, "epoch": 0.1421012849584278, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9190917015075684, "fcm_dpo/q_t": 0.45400530099868774, "grad_norm": 27.097318649291992, "learning_rate": 4.976400700654751e-07, "logits/chosen": 0.2787092328071594, "logits/rejected": 0.23703975975513458, "logps/chosen": -63.235443115234375, "logps/ref_chosen": -59.93777084350586, "logps/ref_rejected": -79.3138427734375, "logps/rejected": -84.53060150146484, "loss": 1.2433, "margin_dpo/margin_mean": 1.9190914630889893, "margin_dpo/margin_std": 4.036993026733398, "step": 94 }, { "KL/chosen_KL_mean": -4.280851364135742, "KL/mean": -5.057154655456543, "KL/rejected_KL_mean": -5.833454132080078, "KL/std": 2.690345525741577, "epoch": 0.1436130007558579, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.552598476409912, "fcm_dpo/q_t": 0.46246445178985596, "grad_norm": 29.381898880004883, "learning_rate": 4.974553604702332e-07, "logits/chosen": 0.1680109202861786, "logits/rejected": 0.10245092958211899, "logps/chosen": -64.4493408203125, "logps/ref_chosen": -60.168487548828125, "logps/ref_rejected": -90.73665618896484, "logps/rejected": -96.57011413574219, "loss": 1.2654, "margin_dpo/margin_mean": 1.552598237991333, "margin_dpo/margin_std": 3.3945610523223877, "step": 95 }, { "KL/chosen_KL_mean": -4.168708801269531, "KL/mean": -5.101164817810059, "KL/rejected_KL_mean": -6.033618927001953, "KL/std": 3.3781328201293945, "epoch": 0.14512471655328799, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8649134635925293, "fcm_dpo/q_t": 0.4566536247730255, "grad_norm": 27.34356117248535, "learning_rate": 4.972637290166157e-07, "logits/chosen": 0.20887598395347595, "logits/rejected": 0.16518929600715637, "logps/chosen": -64.83748626708984, "logps/ref_chosen": -60.66877746582031, "logps/ref_rejected": -88.30673217773438, "logps/rejected": -94.34034729003906, "loss": 1.2566, "margin_dpo/margin_mean": 1.8649135828018188, "margin_dpo/margin_std": 4.4204630851745605, "step": 96 }, { "KL/chosen_KL_mean": -5.003776550292969, "KL/mean": -5.500189304351807, "KL/rejected_KL_mean": -5.996604919433594, "KL/std": 3.620288848876953, "epoch": 0.14663643235071808, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.992835283279419, "fcm_dpo/q_t": 0.4764229357242584, "grad_norm": 36.290287017822266, "learning_rate": 4.970651810649666e-07, "logits/chosen": 0.15075242519378662, "logits/rejected": 0.10647913813591003, "logps/chosen": -70.04789733886719, "logps/ref_chosen": -65.04412078857422, "logps/ref_rejected": -78.42092895507812, "logps/rejected": -84.41753387451172, "loss": 1.3485, "margin_dpo/margin_mean": 0.9928349256515503, "margin_dpo/margin_std": 4.941525459289551, "step": 97 }, { "KL/chosen_KL_mean": -4.657588958740234, "KL/mean": -5.016092300415039, "KL/rejected_KL_mean": -5.374599456787109, "KL/std": 3.3176703453063965, "epoch": 0.14814814814814814, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7170079946517944, "fcm_dpo/q_t": 0.48207545280456543, "grad_norm": 30.212032318115234, "learning_rate": 4.968597221690985e-07, "logits/chosen": 0.2352330982685089, "logits/rejected": 0.20778122544288635, "logps/chosen": -60.16082000732422, "logps/ref_chosen": -55.503231048583984, "logps/ref_rejected": -72.81553649902344, "logps/rejected": -78.19013977050781, "loss": 1.3422, "margin_dpo/margin_mean": 0.7170075178146362, "margin_dpo/margin_std": 3.2238073348999023, "step": 98 }, { "KL/chosen_KL_mean": -4.7126617431640625, "KL/mean": -5.202777862548828, "KL/rejected_KL_mean": -5.692897796630859, "KL/std": 3.4897851943969727, "epoch": 0.14965986394557823, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9802349209785461, "fcm_dpo/q_t": 0.47680675983428955, "grad_norm": 32.75977325439453, "learning_rate": 4.966473580761389e-07, "logits/chosen": 0.26266586780548096, "logits/rejected": 0.22229918837547302, "logps/chosen": -63.288299560546875, "logps/ref_chosen": -58.57563781738281, "logps/ref_rejected": -78.693603515625, "logps/rejected": -84.38650512695312, "loss": 1.3412, "margin_dpo/margin_mean": 0.9802342057228088, "margin_dpo/margin_std": 4.541085720062256, "step": 99 }, { "KL/chosen_KL_mean": -5.3377838134765625, "KL/mean": -6.065300941467285, "KL/rejected_KL_mean": -6.792819976806641, "KL/std": 4.165156841278076, "epoch": 0.15117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.455038070678711, "fcm_dpo/q_t": 0.4675733149051666, "grad_norm": 33.132652282714844, "learning_rate": 4.964280947263676e-07, "logits/chosen": 0.24497297406196594, "logits/rejected": 0.23685091733932495, "logps/chosen": -84.92121887207031, "logps/ref_chosen": -79.58343505859375, "logps/ref_rejected": -92.152587890625, "logps/rejected": -98.94540405273438, "loss": 1.3314, "margin_dpo/margin_mean": 1.4550374746322632, "margin_dpo/margin_std": 5.805309772491455, "step": 100 }, { "KL/chosen_KL_mean": -4.292753219604492, "KL/mean": -5.332766532897949, "KL/rejected_KL_mean": -6.372776031494141, "KL/std": 3.6866824626922607, "epoch": 0.15268329554043839, "fcm_dpo/beta": 0.10244277864694595, "fcm_dpo/delta": 0.07949934154748917, "fcm_dpo/margin": 2.0800251960754395, "fcm_dpo/q_t": 0.4498664438724518, "grad_norm": 26.55908203125, "learning_rate": 4.96201938253052e-07, "logits/chosen": 0.2531740963459015, "logits/rejected": 0.21294589340686798, "logps/chosen": -56.62554168701172, "logps/ref_chosen": -52.332786560058594, "logps/ref_rejected": -69.55589294433594, "logps/rejected": -75.92867279052734, "loss": 1.2299, "margin_dpo/margin_mean": 2.0800249576568604, "margin_dpo/margin_std": 4.159194469451904, "step": 101 }, { "KL/chosen_KL_mean": -5.356512069702148, "KL/mean": -5.766068458557129, "KL/rejected_KL_mean": -6.175632476806641, "KL/std": 3.7042946815490723, "epoch": 0.15419501133786848, "fcm_dpo/beta": 0.10488555580377579, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8191168308258057, "fcm_dpo/q_t": 0.4826941192150116, "grad_norm": 34.83320617675781, "learning_rate": 4.959688949822748e-07, "logits/chosen": 0.17034727334976196, "logits/rejected": 0.13114404678344727, "logps/chosen": -70.09999084472656, "logps/ref_chosen": -64.74348449707031, "logps/ref_rejected": -69.06132507324219, "logps/rejected": -75.2369613647461, "loss": 1.38, "margin_dpo/margin_mean": 0.8191161751747131, "margin_dpo/margin_std": 5.501483917236328, "step": 102 }, { "KL/chosen_KL_mean": -4.785610198974609, "KL/mean": -5.598519325256348, "KL/rejected_KL_mean": -6.411426544189453, "KL/std": 3.8898379802703857, "epoch": 0.15570672713529857, "fcm_dpo/beta": 0.10772866755723953, "fcm_dpo/delta": 0.08799183368682861, "fcm_dpo/margin": 1.6258132457733154, "fcm_dpo/q_t": 0.459196537733078, "grad_norm": 31.35845184326172, "learning_rate": 4.957289714327572e-07, "logits/chosen": 0.26058486104011536, "logits/rejected": 0.22672635316848755, "logps/chosen": -68.62225341796875, "logps/ref_chosen": -63.83664321899414, "logps/ref_rejected": -79.32362365722656, "logps/rejected": -85.73505401611328, "loss": 1.2559, "margin_dpo/margin_mean": 1.6258125305175781, "margin_dpo/margin_std": 3.6502134799957275, "step": 103 }, { "KL/chosen_KL_mean": -4.706455230712891, "KL/mean": -5.858922004699707, "KL/rejected_KL_mean": -7.011390686035156, "KL/std": 4.994261741638184, "epoch": 0.15721844293272866, "fcm_dpo/beta": 0.112154021859169, "fcm_dpo/delta": 0.04702896624803543, "fcm_dpo/margin": 2.3049371242523193, "fcm_dpo/q_t": 0.44699156284332275, "grad_norm": 34.770877838134766, "learning_rate": 4.954821743156767e-07, "logits/chosen": 0.24318566918373108, "logits/rejected": 0.1570195108652115, "logps/chosen": -65.7056655883789, "logps/ref_chosen": -60.99920654296875, "logps/ref_rejected": -98.84645080566406, "logps/rejected": -105.85784912109375, "loss": 1.2607, "margin_dpo/margin_mean": 2.304936408996582, "margin_dpo/margin_std": 6.377119064331055, "step": 104 }, { "KL/chosen_KL_mean": -5.352252960205078, "KL/mean": -6.269779205322266, "KL/rejected_KL_mean": -7.187309265136719, "KL/std": 3.987088680267334, "epoch": 0.15873015873015872, "fcm_dpo/beta": 0.11373626440763474, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8350532054901123, "fcm_dpo/q_t": 0.45318853855133057, "grad_norm": 34.859764099121094, "learning_rate": 4.952285105344791e-07, "logits/chosen": 0.20636920630931854, "logits/rejected": 0.15100428462028503, "logps/chosen": -76.30252075195312, "logps/ref_chosen": -70.95027160644531, "logps/ref_rejected": -87.88340759277344, "logps/rejected": -95.07072448730469, "loss": 1.2632, "margin_dpo/margin_mean": 1.8350532054901123, "margin_dpo/margin_std": 4.919508457183838, "step": 105 }, { "KL/chosen_KL_mean": -5.029964447021484, "KL/mean": -5.7955522537231445, "KL/rejected_KL_mean": -6.561138153076172, "KL/std": 3.89530611038208, "epoch": 0.1602418745275888, "fcm_dpo/beta": 0.11373626440763474, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5311670303344727, "fcm_dpo/q_t": 0.4586039185523987, "grad_norm": 36.046531677246094, "learning_rate": 4.949679871846857e-07, "logits/chosen": 0.24574297666549683, "logits/rejected": 0.2315644919872284, "logps/chosen": -67.48929595947266, "logps/ref_chosen": -62.45933151245117, "logps/ref_rejected": -67.00595092773438, "logps/rejected": -73.56708526611328, "loss": 1.3057, "margin_dpo/margin_mean": 1.5311667919158936, "margin_dpo/margin_std": 5.281202793121338, "step": 106 }, { "KL/chosen_KL_mean": -6.49530029296875, "KL/mean": -6.9094672203063965, "KL/rejected_KL_mean": -7.323631286621094, "KL/std": 4.887589454650879, "epoch": 0.1617535903250189, "fcm_dpo/beta": 0.11373626440763474, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8283355236053467, "fcm_dpo/q_t": 0.47569483518600464, "grad_norm": 48.95326232910156, "learning_rate": 4.947006115536947e-07, "logits/chosen": 0.17019329965114594, "logits/rejected": 0.1465989649295807, "logps/chosen": -82.33326721191406, "logps/ref_chosen": -75.83796691894531, "logps/ref_rejected": -87.74038696289062, "logps/rejected": -95.06401062011719, "loss": 1.409, "margin_dpo/margin_mean": 0.828335702419281, "margin_dpo/margin_std": 6.10739803314209, "step": 107 }, { "KL/chosen_KL_mean": -5.463619232177734, "KL/mean": -6.383543014526367, "KL/rejected_KL_mean": -7.303462982177734, "KL/std": 4.103908061981201, "epoch": 0.16326530612244897, "fcm_dpo/beta": 0.11373626440763474, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.839846134185791, "fcm_dpo/q_t": 0.4533940255641937, "grad_norm": 32.70323944091797, "learning_rate": 4.944263911205772e-07, "logits/chosen": 0.19207993149757385, "logits/rejected": 0.16023018956184387, "logps/chosen": -73.85685729980469, "logps/ref_chosen": -68.39323425292969, "logps/ref_rejected": -83.24267578125, "logps/rejected": -90.54613494873047, "loss": 1.2767, "margin_dpo/margin_mean": 1.8398463726043701, "margin_dpo/margin_std": 5.364116668701172, "step": 108 }, { "KL/chosen_KL_mean": -4.823814392089844, "KL/mean": -6.310299873352051, "KL/rejected_KL_mean": -7.796791076660156, "KL/std": 4.97265100479126, "epoch": 0.16477702191987906, "fcm_dpo/beta": 0.11705504357814789, "fcm_dpo/delta": 0.05780094116926193, "fcm_dpo/margin": 2.9729766845703125, "fcm_dpo/q_t": 0.426702082157135, "grad_norm": 32.38801193237305, "learning_rate": 4.941453335558681e-07, "logits/chosen": 0.17428874969482422, "logits/rejected": 0.12089354544878006, "logps/chosen": -60.351295471191406, "logps/ref_chosen": -55.52748107910156, "logps/ref_rejected": -83.55218505859375, "logps/rejected": -91.34898376464844, "loss": 1.1535, "margin_dpo/margin_mean": 2.9729764461517334, "margin_dpo/margin_std": 5.388824462890625, "step": 109 }, { "KL/chosen_KL_mean": -6.182544708251953, "KL/mean": -6.312487602233887, "KL/rejected_KL_mean": -6.442424774169922, "KL/std": 3.990682363510132, "epoch": 0.16628873771730915, "fcm_dpo/beta": 0.1177498996257782, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2598845958709717, "fcm_dpo/q_t": 0.4898316264152527, "grad_norm": 53.71846008300781, "learning_rate": 4.938574467213517e-07, "logits/chosen": 0.17680279910564423, "logits/rejected": 0.18614478409290314, "logps/chosen": -87.34129333496094, "logps/ref_chosen": -81.15874481201172, "logps/ref_rejected": -72.56021118164062, "logps/rejected": -79.00263977050781, "loss": 1.4483, "margin_dpo/margin_mean": 0.25988438725471497, "margin_dpo/margin_std": 5.361874103546143, "step": 110 }, { "KL/chosen_KL_mean": -5.583719253540039, "KL/mean": -6.557001113891602, "KL/rejected_KL_mean": -7.5302886962890625, "KL/std": 4.138578414916992, "epoch": 0.16780045351473924, "fcm_dpo/beta": 0.1203727126121521, "fcm_dpo/delta": 0.07264174520969391, "fcm_dpo/margin": 1.9465702772140503, "fcm_dpo/q_t": 0.44942060112953186, "grad_norm": 30.87441062927246, "learning_rate": 4.935627386698418e-07, "logits/chosen": 0.2694501280784607, "logits/rejected": 0.23262867331504822, "logps/chosen": -57.94270324707031, "logps/ref_chosen": -52.358985900878906, "logps/ref_rejected": -77.06150817871094, "logps/rejected": -84.591796875, "loss": 1.2512, "margin_dpo/margin_mean": 1.9465705156326294, "margin_dpo/margin_std": 5.017286777496338, "step": 111 }, { "KL/chosen_KL_mean": -5.246620178222656, "KL/mean": -6.449799537658691, "KL/rejected_KL_mean": -7.652973175048828, "KL/std": 4.185157775878906, "epoch": 0.1693121693121693, "fcm_dpo/beta": 0.12910687923431396, "fcm_dpo/delta": 0.09781448543071747, "fcm_dpo/margin": 2.4063587188720703, "fcm_dpo/q_t": 0.4301964044570923, "grad_norm": 40.442134857177734, "learning_rate": 4.932612176449559e-07, "logits/chosen": 0.19383251667022705, "logits/rejected": 0.12825272977352142, "logps/chosen": -68.26668548583984, "logps/ref_chosen": -63.02006530761719, "logps/ref_rejected": -111.36941528320312, "logps/rejected": -119.02238464355469, "loss": 1.1838, "margin_dpo/margin_mean": 2.4063572883605957, "margin_dpo/margin_std": 4.644060134887695, "step": 112 }, { "KL/chosen_KL_mean": -5.255073547363281, "KL/mean": -5.934301376342773, "KL/rejected_KL_mean": -6.613533020019531, "KL/std": 3.480978488922119, "epoch": 0.1708238851095994, "fcm_dpo/beta": 0.13342835009098053, "fcm_dpo/delta": 0.07491803169250488, "fcm_dpo/margin": 1.3584556579589844, "fcm_dpo/q_t": 0.4581720232963562, "grad_norm": 49.05155944824219, "learning_rate": 4.929528920808854e-07, "logits/chosen": 0.19861599802970886, "logits/rejected": 0.15952877700328827, "logps/chosen": -61.06273651123047, "logps/ref_chosen": -55.80766296386719, "logps/ref_rejected": -69.84014129638672, "logps/rejected": -76.45367431640625, "loss": 1.2973, "margin_dpo/margin_mean": 1.3584556579589844, "margin_dpo/margin_std": 4.468038558959961, "step": 113 }, { "KL/chosen_KL_mean": -4.988502502441406, "KL/mean": -6.088685035705566, "KL/rejected_KL_mean": -7.188873291015625, "KL/std": 3.774844169616699, "epoch": 0.17233560090702948, "fcm_dpo/beta": 0.14195749163627625, "fcm_dpo/delta": 0.09694860130548477, "fcm_dpo/margin": 2.200364351272583, "fcm_dpo/q_t": 0.4333820343017578, "grad_norm": 42.0706901550293, "learning_rate": 4.92637770602159e-07, "logits/chosen": 0.24624261260032654, "logits/rejected": 0.1857389360666275, "logps/chosen": -71.32127380371094, "logps/ref_chosen": -66.33277130126953, "logps/ref_rejected": -71.61489868164062, "logps/rejected": -78.80377197265625, "loss": 1.1946, "margin_dpo/margin_mean": 2.200364828109741, "margin_dpo/margin_std": 4.570239067077637, "step": 114 }, { "KL/chosen_KL_mean": -5.369632720947266, "KL/mean": -6.250866889953613, "KL/rejected_KL_mean": -7.132101058959961, "KL/std": 3.4158787727355957, "epoch": 0.17384731670445955, "fcm_dpo/beta": 0.14961527287960052, "fcm_dpo/delta": 0.056850727647542953, "fcm_dpo/margin": 1.7624667882919312, "fcm_dpo/q_t": 0.44253766536712646, "grad_norm": 42.88827133178711, "learning_rate": 4.923158620234019e-07, "logits/chosen": 0.23988944292068481, "logits/rejected": 0.17627336084842682, "logps/chosen": -61.11867141723633, "logps/ref_chosen": -55.74903869628906, "logps/ref_rejected": -79.59849548339844, "logps/rejected": -86.73059844970703, "loss": 1.2282, "margin_dpo/margin_mean": 1.7624670267105103, "margin_dpo/margin_std": 4.0995025634765625, "step": 115 }, { "KL/chosen_KL_mean": -4.933021545410156, "KL/mean": -6.309746742248535, "KL/rejected_KL_mean": -7.6864776611328125, "KL/std": 3.5162925720214844, "epoch": 0.17535903250188964, "fcm_dpo/beta": 0.15041202306747437, "fcm_dpo/delta": -0.017097879201173782, "fcm_dpo/margin": 2.7534589767456055, "fcm_dpo/q_t": 0.4049130082130432, "grad_norm": 35.12335968017578, "learning_rate": 4.91987175349089e-07, "logits/chosen": 0.2185758650302887, "logits/rejected": 0.15106023848056793, "logps/chosen": -54.298187255859375, "logps/ref_chosen": -49.36516571044922, "logps/ref_rejected": -72.84671020507812, "logps/rejected": -80.53318786621094, "loss": 1.0934, "margin_dpo/margin_mean": 2.7534589767456055, "margin_dpo/margin_std": 3.808335542678833, "step": 116 }, { "KL/chosen_KL_mean": -4.931730270385742, "KL/mean": -5.858213901519775, "KL/rejected_KL_mean": -6.784702301025391, "KL/std": 3.3140063285827637, "epoch": 0.17687074829931973, "fcm_dpo/beta": 0.14422379434108734, "fcm_dpo/delta": -0.04407034069299698, "fcm_dpo/margin": 1.8529614210128784, "fcm_dpo/q_t": 0.43684980273246765, "grad_norm": 40.910606384277344, "learning_rate": 4.916517197732933e-07, "logits/chosen": 0.2307669222354889, "logits/rejected": 0.19402286410331726, "logps/chosen": -62.64262771606445, "logps/ref_chosen": -57.710899353027344, "logps/ref_rejected": -69.77253723144531, "logps/rejected": -76.55723571777344, "loss": 1.2279, "margin_dpo/margin_mean": 1.8529608249664307, "margin_dpo/margin_std": 4.091181755065918, "step": 117 }, { "KL/chosen_KL_mean": -4.755271911621094, "KL/mean": -5.991068363189697, "KL/rejected_KL_mean": -7.226863861083984, "KL/std": 3.593959093093872, "epoch": 0.17838246409674982, "fcm_dpo/beta": 0.14586882293224335, "fcm_dpo/delta": 0.0441410131752491, "fcm_dpo/margin": 2.471588611602783, "fcm_dpo/q_t": 0.4175887703895569, "grad_norm": 38.68694305419922, "learning_rate": 4.913095046794281e-07, "logits/chosen": 0.30474239587783813, "logits/rejected": 0.2626001238822937, "logps/chosen": -57.23516845703125, "logps/ref_chosen": -52.479896545410156, "logps/ref_rejected": -81.359130859375, "logps/rejected": -88.58599090576172, "loss": 1.1265, "margin_dpo/margin_mean": 2.471587896347046, "margin_dpo/margin_std": 3.647130012512207, "step": 118 }, { "KL/chosen_KL_mean": -6.017473220825195, "KL/mean": -6.920298099517822, "KL/rejected_KL_mean": -7.823123931884766, "KL/std": 4.139496803283691, "epoch": 0.17989417989417988, "fcm_dpo/beta": 0.14982125163078308, "fcm_dpo/delta": 0.038419194519519806, "fcm_dpo/margin": 1.8056480884552002, "fcm_dpo/q_t": 0.44211965799331665, "grad_norm": 44.770748138427734, "learning_rate": 4.909605396399855e-07, "logits/chosen": 0.20789092779159546, "logits/rejected": 0.1685742735862732, "logps/chosen": -67.3751449584961, "logps/ref_chosen": -61.35767364501953, "logps/ref_rejected": -75.71510314941406, "logps/rejected": -83.53822326660156, "loss": 1.2659, "margin_dpo/margin_mean": 1.8056479692459106, "margin_dpo/margin_std": 5.146032333374023, "step": 119 }, { "KL/chosen_KL_mean": -4.956226348876953, "KL/mean": -6.255353927612305, "KL/rejected_KL_mean": -7.5544891357421875, "KL/std": 3.479482650756836, "epoch": 0.18140589569160998, "fcm_dpo/beta": 0.15262466669082642, "fcm_dpo/delta": 0.0036748070269823074, "fcm_dpo/margin": 2.5982601642608643, "fcm_dpo/q_t": 0.4099358320236206, "grad_norm": 40.47877502441406, "learning_rate": 4.906048344162676e-07, "logits/chosen": 0.19425508379936218, "logits/rejected": 0.13444660604000092, "logps/chosen": -64.8637924194336, "logps/ref_chosen": -59.907569885253906, "logps/ref_rejected": -79.6910629272461, "logps/rejected": -87.24555206298828, "loss": 1.0971, "margin_dpo/margin_mean": 2.5982608795166016, "margin_dpo/margin_std": 3.5266871452331543, "step": 120 }, { "KL/chosen_KL_mean": -5.358709335327148, "KL/mean": -6.739768028259277, "KL/rejected_KL_mean": -8.120830535888672, "KL/std": 3.996863842010498, "epoch": 0.18291761148904007, "fcm_dpo/beta": 0.1501387357711792, "fcm_dpo/delta": -0.016977794468402863, "fcm_dpo/margin": 2.7621190547943115, "fcm_dpo/q_t": 0.4117959439754486, "grad_norm": 37.1208610534668, "learning_rate": 4.902423989581143e-07, "logits/chosen": 0.30443626642227173, "logits/rejected": 0.2181735634803772, "logps/chosen": -61.024749755859375, "logps/ref_chosen": -55.66604232788086, "logps/ref_rejected": -101.56233978271484, "logps/rejected": -109.68316650390625, "loss": 1.1077, "margin_dpo/margin_mean": 2.7621188163757324, "margin_dpo/margin_std": 4.442630290985107, "step": 121 }, { "KL/chosen_KL_mean": -5.777549743652344, "KL/mean": -7.061952114105225, "KL/rejected_KL_mean": -8.346351623535156, "KL/std": 4.270412445068359, "epoch": 0.18442932728647016, "fcm_dpo/beta": 0.15251445770263672, "fcm_dpo/delta": 0.008429568260908127, "fcm_dpo/margin": 2.5688040256500244, "fcm_dpo/q_t": 0.42014241218566895, "grad_norm": 46.91085433959961, "learning_rate": 4.898732434036243e-07, "logits/chosen": 0.2145451307296753, "logits/rejected": 0.1792818009853363, "logps/chosen": -69.11192321777344, "logps/ref_chosen": -63.334373474121094, "logps/ref_rejected": -73.67523193359375, "logps/rejected": -82.02159118652344, "loss": 1.1596, "margin_dpo/margin_mean": 2.568803548812866, "margin_dpo/margin_std": 5.07568359375, "step": 122 }, { "KL/chosen_KL_mean": -5.459968566894531, "KL/mean": -6.718021392822266, "KL/rejected_KL_mean": -7.97607421875, "KL/std": 3.7030413150787354, "epoch": 0.18594104308390022, "fcm_dpo/beta": 0.1532004177570343, "fcm_dpo/delta": 0.01600516401231289, "fcm_dpo/margin": 2.5161049365997314, "fcm_dpo/q_t": 0.41378509998321533, "grad_norm": 38.989585876464844, "learning_rate": 4.894973780788722e-07, "logits/chosen": 0.23768547177314758, "logits/rejected": 0.19618701934814453, "logps/chosen": -62.35871124267578, "logps/ref_chosen": -56.89874267578125, "logps/ref_rejected": -78.97028350830078, "logps/rejected": -86.94635772705078, "loss": 1.123, "margin_dpo/margin_mean": 2.5161049365997314, "margin_dpo/margin_std": 3.9700427055358887, "step": 123 }, { "KL/chosen_KL_mean": -5.782169342041016, "KL/mean": -7.465737342834473, "KL/rejected_KL_mean": -9.149307250976562, "KL/std": 4.042940616607666, "epoch": 0.1874527588813303, "fcm_dpo/beta": 0.14460201561450958, "fcm_dpo/delta": -0.10314086824655533, "fcm_dpo/margin": 3.367136240005493, "fcm_dpo/q_t": 0.3885384202003479, "grad_norm": 36.7084846496582, "learning_rate": 4.89114813497619e-07, "logits/chosen": 0.28056734800338745, "logits/rejected": 0.21720805764198303, "logps/chosen": -62.89825439453125, "logps/ref_chosen": -57.116085052490234, "logps/ref_rejected": -87.93074035644531, "logps/rejected": -97.08004760742188, "loss": 1.0417, "margin_dpo/margin_mean": 3.3671364784240723, "margin_dpo/margin_std": 4.220187187194824, "step": 124 }, { "KL/chosen_KL_mean": -6.223566055297852, "KL/mean": -7.6594648361206055, "KL/rejected_KL_mean": -9.095359802246094, "KL/std": 4.320698261260986, "epoch": 0.1889644746787604, "fcm_dpo/beta": 0.1405811309814453, "fcm_dpo/delta": -0.006451927125453949, "fcm_dpo/margin": 2.8717947006225586, "fcm_dpo/q_t": 0.410498708486557, "grad_norm": 38.985382080078125, "learning_rate": 4.887255603610184e-07, "logits/chosen": 0.29075124859809875, "logits/rejected": 0.22918814420700073, "logps/chosen": -71.92974853515625, "logps/ref_chosen": -65.7061767578125, "logps/ref_rejected": -91.72711944580078, "logps/rejected": -100.82247924804688, "loss": 1.1143, "margin_dpo/margin_mean": 2.8717949390411377, "margin_dpo/margin_std": 4.4547529220581055, "step": 125 }, { "KL/chosen_KL_mean": -5.821783065795898, "KL/mean": -6.677826404571533, "KL/rejected_KL_mean": -7.53387451171875, "KL/std": 4.0686540603637695, "epoch": 0.19047619047619047, "fcm_dpo/beta": 0.15313303470611572, "fcm_dpo/delta": 0.14905323088169098, "fcm_dpo/margin": 1.7120908498764038, "fcm_dpo/q_t": 0.4439869523048401, "grad_norm": 42.53828430175781, "learning_rate": 4.883296295573176e-07, "logits/chosen": 0.09436048567295074, "logits/rejected": 0.08861444890499115, "logps/chosen": -73.99787139892578, "logps/ref_chosen": -68.17608642578125, "logps/ref_rejected": -65.1175537109375, "logps/rejected": -72.65142822265625, "loss": 1.2429, "margin_dpo/margin_mean": 1.7120912075042725, "margin_dpo/margin_std": 4.371842384338379, "step": 126 }, { "KL/chosen_KL_mean": -5.998382568359375, "KL/mean": -7.5352582931518555, "KL/rejected_KL_mean": -9.072135925292969, "KL/std": 3.6161179542541504, "epoch": 0.19198790627362056, "fcm_dpo/beta": 0.1484401971101761, "fcm_dpo/delta": -0.06965281069278717, "fcm_dpo/margin": 3.0737504959106445, "fcm_dpo/q_t": 0.3940262198448181, "grad_norm": 37.3619270324707, "learning_rate": 4.87927032161552e-07, "logits/chosen": 0.22243531048297882, "logits/rejected": 0.1914488971233368, "logps/chosen": -67.87861633300781, "logps/ref_chosen": -61.88023376464844, "logps/ref_rejected": -68.46012878417969, "logps/rejected": -77.53225708007812, "loss": 1.0581, "margin_dpo/margin_mean": 3.0737502574920654, "margin_dpo/margin_std": 3.76737117767334, "step": 127 }, { "KL/chosen_KL_mean": -6.474908828735352, "KL/mean": -8.036764144897461, "KL/rejected_KL_mean": -9.598621368408203, "KL/std": 4.666970252990723, "epoch": 0.19349962207105065, "fcm_dpo/beta": 0.14405301213264465, "fcm_dpo/delta": -0.05977921932935715, "fcm_dpo/margin": 3.123711109161377, "fcm_dpo/q_t": 0.4026954770088196, "grad_norm": 37.78749084472656, "learning_rate": 4.875177794352363e-07, "logits/chosen": 0.2339283525943756, "logits/rejected": 0.17583772540092468, "logps/chosen": -73.18389129638672, "logps/ref_chosen": -66.708984375, "logps/ref_rejected": -94.97969055175781, "logps/rejected": -104.57830810546875, "loss": 1.1297, "margin_dpo/margin_mean": 3.123711585998535, "margin_dpo/margin_std": 5.568070411682129, "step": 128 }, { "KL/chosen_KL_mean": -7.183927536010742, "KL/mean": -8.192344665527344, "KL/rejected_KL_mean": -9.200759887695312, "KL/std": 4.225727558135986, "epoch": 0.19501133786848074, "fcm_dpo/beta": 0.15166246891021729, "fcm_dpo/delta": 0.10337048768997192, "fcm_dpo/margin": 2.016836404800415, "fcm_dpo/q_t": 0.43475955724716187, "grad_norm": 50.885433197021484, "learning_rate": 4.871018828260491e-07, "logits/chosen": 0.2141018807888031, "logits/rejected": 0.20807525515556335, "logps/chosen": -72.52275085449219, "logps/ref_chosen": -65.33882904052734, "logps/ref_rejected": -68.06109619140625, "logps/rejected": -77.26185607910156, "loss": 1.2322, "margin_dpo/margin_mean": 2.0168368816375732, "margin_dpo/margin_std": 4.955423355102539, "step": 129 }, { "KL/chosen_KL_mean": -6.767240524291992, "KL/mean": -8.247897148132324, "KL/rejected_KL_mean": -9.72854995727539, "KL/std": 4.21795129776001, "epoch": 0.1965230536659108, "fcm_dpo/beta": 0.1474171131849289, "fcm_dpo/delta": -0.04654637351632118, "fcm_dpo/margin": 2.9613113403320312, "fcm_dpo/q_t": 0.39949163794517517, "grad_norm": 43.239646911621094, "learning_rate": 4.866793539675126e-07, "logits/chosen": 0.1729985773563385, "logits/rejected": 0.12204181402921677, "logps/chosen": -65.42798614501953, "logps/ref_chosen": -58.660743713378906, "logps/ref_rejected": -79.24510192871094, "logps/rejected": -88.97364807128906, "loss": 1.0837, "margin_dpo/margin_mean": 2.9613113403320312, "margin_dpo/margin_std": 4.048201084136963, "step": 130 }, { "KL/chosen_KL_mean": -6.502410888671875, "KL/mean": -8.361289978027344, "KL/rejected_KL_mean": -10.220169067382812, "KL/std": 4.88187313079834, "epoch": 0.1980347694633409, "fcm_dpo/beta": 0.1393418312072754, "fcm_dpo/delta": -0.14160066843032837, "fcm_dpo/margin": 3.717766761779785, "fcm_dpo/q_t": 0.3856920897960663, "grad_norm": 34.58555603027344, "learning_rate": 4.86250204678667e-07, "logits/chosen": 0.19449803233146667, "logits/rejected": 0.12917035818099976, "logps/chosen": -59.016944885253906, "logps/ref_chosen": -52.51453399658203, "logps/ref_rejected": -85.18299865722656, "logps/rejected": -95.40316772460938, "loss": 1.0653, "margin_dpo/margin_mean": 3.7177672386169434, "margin_dpo/margin_std": 5.4574785232543945, "step": 131 }, { "KL/chosen_KL_mean": -7.104595184326172, "KL/mean": -8.693756103515625, "KL/rejected_KL_mean": -10.282918930053711, "KL/std": 4.658677101135254, "epoch": 0.19954648526077098, "fcm_dpo/beta": 0.1348930299282074, "fcm_dpo/delta": -0.033193182200193405, "fcm_dpo/margin": 3.1783313751220703, "fcm_dpo/q_t": 0.4000505208969116, "grad_norm": 36.00300979614258, "learning_rate": 4.858144469637408e-07, "logits/chosen": 0.26692995429039, "logits/rejected": 0.23427237570285797, "logps/chosen": -72.78972625732422, "logps/ref_chosen": -65.68513488769531, "logps/ref_rejected": -69.54120635986328, "logps/rejected": -79.82412719726562, "loss": 1.0949, "margin_dpo/margin_mean": 3.1783313751220703, "margin_dpo/margin_std": 4.635673522949219, "step": 132 }, { "KL/chosen_KL_mean": -7.397819519042969, "KL/mean": -8.735570907592773, "KL/rejected_KL_mean": -10.073318481445312, "KL/std": 4.610377311706543, "epoch": 0.20105820105820105, "fcm_dpo/beta": 0.13705599308013916, "fcm_dpo/delta": 0.037375159561634064, "fcm_dpo/margin": 2.675502061843872, "fcm_dpo/q_t": 0.41829919815063477, "grad_norm": 41.43574905395508, "learning_rate": 4.853720930118138e-07, "logits/chosen": 0.20174159109592438, "logits/rejected": 0.1917198747396469, "logps/chosen": -70.99593353271484, "logps/ref_chosen": -63.598114013671875, "logps/ref_rejected": -73.72798156738281, "logps/rejected": -83.80130004882812, "loss": 1.1554, "margin_dpo/margin_mean": 2.675502300262451, "margin_dpo/margin_std": 4.851496696472168, "step": 133 }, { "KL/chosen_KL_mean": -6.886777877807617, "KL/mean": -9.030449867248535, "KL/rejected_KL_mean": -11.174121856689453, "KL/std": 5.263517379760742, "epoch": 0.20256991685563114, "fcm_dpo/beta": 0.12709717452526093, "fcm_dpo/delta": -0.17415767908096313, "fcm_dpo/margin": 4.287341117858887, "fcm_dpo/q_t": 0.3749292492866516, "grad_norm": 30.688871383666992, "learning_rate": 4.849231551964771e-07, "logits/chosen": 0.29332664608955383, "logits/rejected": 0.23596280813217163, "logps/chosen": -60.68135070800781, "logps/ref_chosen": -53.79457092285156, "logps/ref_rejected": -74.16741943359375, "logps/rejected": -85.34153747558594, "loss": 1.0112, "margin_dpo/margin_mean": 4.287341594696045, "margin_dpo/margin_std": 5.231392860412598, "step": 134 }, { "KL/chosen_KL_mean": -6.964086532592773, "KL/mean": -8.33128547668457, "KL/rejected_KL_mean": -9.698490142822266, "KL/std": 4.9238457679748535, "epoch": 0.20408163265306123, "fcm_dpo/beta": 0.12767961621284485, "fcm_dpo/delta": 0.05650739371776581, "fcm_dpo/margin": 2.73439884185791, "fcm_dpo/q_t": 0.42211827635765076, "grad_norm": 31.442127227783203, "learning_rate": 4.844676460754862e-07, "logits/chosen": 0.22777020931243896, "logits/rejected": 0.19519740343093872, "logps/chosen": -56.4051628112793, "logps/ref_chosen": -49.441078186035156, "logps/ref_rejected": -65.96878051757812, "logps/rejected": -75.66726684570312, "loss": 1.1643, "margin_dpo/margin_mean": 2.734398603439331, "margin_dpo/margin_std": 5.071935653686523, "step": 135 }, { "KL/chosen_KL_mean": -8.580795288085938, "KL/mean": -10.33868408203125, "KL/rejected_KL_mean": -12.096576690673828, "KL/std": 5.4477667808532715, "epoch": 0.20559334845049132, "fcm_dpo/beta": 0.1240832507610321, "fcm_dpo/delta": -0.043839938938617706, "fcm_dpo/margin": 3.5157880783081055, "fcm_dpo/q_t": 0.40707147121429443, "grad_norm": 37.98371505737305, "learning_rate": 4.840055783904106e-07, "logits/chosen": 0.23468932509422302, "logits/rejected": 0.1596575379371643, "logps/chosen": -75.34005737304688, "logps/ref_chosen": -66.75926208496094, "logps/ref_rejected": -94.61787414550781, "logps/rejected": -106.71444702148438, "loss": 1.1626, "margin_dpo/margin_mean": 3.515789031982422, "margin_dpo/margin_std": 6.921201705932617, "step": 136 }, { "KL/chosen_KL_mean": -8.011341094970703, "KL/mean": -9.820709228515625, "KL/rejected_KL_mean": -11.630073547363281, "KL/std": 5.151564598083496, "epoch": 0.20710506424792138, "fcm_dpo/beta": 0.12334209680557251, "fcm_dpo/delta": -0.05368944630026817, "fcm_dpo/margin": 3.6187355518341064, "fcm_dpo/q_t": 0.4003620445728302, "grad_norm": 30.845598220825195, "learning_rate": 4.835369650662767e-07, "logits/chosen": 0.25939062237739563, "logits/rejected": 0.22949111461639404, "logps/chosen": -64.79513549804688, "logps/ref_chosen": -56.78379821777344, "logps/ref_rejected": -69.89952087402344, "logps/rejected": -81.52959442138672, "loss": 1.0943, "margin_dpo/margin_mean": 3.6187360286712646, "margin_dpo/margin_std": 5.483453750610352, "step": 137 }, { "KL/chosen_KL_mean": -9.439451217651367, "KL/mean": -10.996476173400879, "KL/rejected_KL_mean": -12.55350112915039, "KL/std": 5.402915000915527, "epoch": 0.20861678004535147, "fcm_dpo/beta": 0.12223243713378906, "fcm_dpo/delta": 0.021854639053344727, "fcm_dpo/margin": 3.1140499114990234, "fcm_dpo/q_t": 0.4149552583694458, "grad_norm": 32.88602066040039, "learning_rate": 4.830618192112065e-07, "logits/chosen": 0.26229894161224365, "logits/rejected": 0.22495831549167633, "logps/chosen": -68.20545959472656, "logps/ref_chosen": -58.766014099121094, "logps/ref_rejected": -68.12371826171875, "logps/rejected": -80.67721557617188, "loss": 1.1688, "margin_dpo/margin_mean": 3.1140503883361816, "margin_dpo/margin_std": 6.036888599395752, "step": 138 }, { "KL/chosen_KL_mean": -9.852619171142578, "KL/mean": -11.74112319946289, "KL/rejected_KL_mean": -13.629623413085938, "KL/std": 5.306262969970703, "epoch": 0.21012849584278157, "fcm_dpo/beta": 0.12008741497993469, "fcm_dpo/delta": -0.06189559027552605, "fcm_dpo/margin": 3.7770047187805176, "fcm_dpo/q_t": 0.3965948224067688, "grad_norm": 34.94479751586914, "learning_rate": 4.825801541160509e-07, "logits/chosen": 0.21446341276168823, "logits/rejected": 0.18413154780864716, "logps/chosen": -81.07820892333984, "logps/ref_chosen": -71.2255859375, "logps/ref_rejected": -82.1834716796875, "logps/rejected": -95.81309509277344, "loss": 1.0784, "margin_dpo/margin_mean": 3.7770049571990967, "margin_dpo/margin_std": 5.333680629730225, "step": 139 }, { "KL/chosen_KL_mean": -9.090442657470703, "KL/mean": -11.611061096191406, "KL/rejected_KL_mean": -14.13167953491211, "KL/std": 7.010415077209473, "epoch": 0.21164021164021163, "fcm_dpo/beta": 0.1083516851067543, "fcm_dpo/delta": -0.1772288680076599, "fcm_dpo/margin": 5.04123592376709, "fcm_dpo/q_t": 0.3767069876194, "grad_norm": 32.844200134277344, "learning_rate": 4.820919832540181e-07, "logits/chosen": 0.2197141796350479, "logits/rejected": 0.17169150710105896, "logps/chosen": -72.36810302734375, "logps/ref_chosen": -63.27766418457031, "logps/ref_rejected": -83.30647277832031, "logps/rejected": -97.43815612792969, "loss": 1.0681, "margin_dpo/margin_mean": 5.041234970092773, "margin_dpo/margin_std": 7.689390182495117, "step": 140 }, { "KL/chosen_KL_mean": -9.954669952392578, "KL/mean": -12.317865371704102, "KL/rejected_KL_mean": -14.681060791015625, "KL/std": 5.889538764953613, "epoch": 0.21315192743764172, "fcm_dpo/beta": 0.09989713877439499, "fcm_dpo/delta": -0.08965474367141724, "fcm_dpo/margin": 4.726386070251465, "fcm_dpo/q_t": 0.3953888714313507, "grad_norm": 28.874492645263672, "learning_rate": 4.815973202802966e-07, "logits/chosen": 0.25134333968162537, "logits/rejected": 0.20534633100032806, "logps/chosen": -71.721435546875, "logps/ref_chosen": -61.76676940917969, "logps/ref_rejected": -88.60601806640625, "logps/rejected": -103.28707885742188, "loss": 1.0964, "margin_dpo/margin_mean": 4.726386070251465, "margin_dpo/margin_std": 7.404045104980469, "step": 141 }, { "KL/chosen_KL_mean": -9.856754302978516, "KL/mean": -11.797260284423828, "KL/rejected_KL_mean": -13.73776626586914, "KL/std": 5.778664588928223, "epoch": 0.2146636432350718, "fcm_dpo/beta": 0.10087822377681732, "fcm_dpo/delta": 0.009604006074368954, "fcm_dpo/margin": 3.8810181617736816, "fcm_dpo/q_t": 0.41101446747779846, "grad_norm": 27.98453140258789, "learning_rate": 4.810961790316729e-07, "logits/chosen": 0.255909264087677, "logits/rejected": 0.23032045364379883, "logps/chosen": -75.13153076171875, "logps/ref_chosen": -65.2747802734375, "logps/ref_rejected": -81.1378173828125, "logps/rejected": -94.87557983398438, "loss": 1.1152, "margin_dpo/margin_mean": 3.8810176849365234, "margin_dpo/margin_std": 5.909215927124023, "step": 142 }, { "KL/chosen_KL_mean": -10.505781173706055, "KL/mean": -12.216259002685547, "KL/rejected_KL_mean": -13.92673110961914, "KL/std": 6.674481391906738, "epoch": 0.2161753590325019, "fcm_dpo/beta": 0.10392451286315918, "fcm_dpo/delta": 0.04947058483958244, "fcm_dpo/margin": 3.4209537506103516, "fcm_dpo/q_t": 0.4193747937679291, "grad_norm": 37.11439895629883, "learning_rate": 4.805885735261454e-07, "logits/chosen": 0.2696152329444885, "logits/rejected": 0.25197485089302063, "logps/chosen": -73.12361145019531, "logps/ref_chosen": -62.617828369140625, "logps/ref_rejected": -70.39239501953125, "logps/rejected": -84.31912231445312, "loss": 1.2105, "margin_dpo/margin_mean": 3.4209539890289307, "margin_dpo/margin_std": 7.7674102783203125, "step": 143 }, { "KL/chosen_KL_mean": -10.830635070800781, "KL/mean": -12.981417655944824, "KL/rejected_KL_mean": -15.132190704345703, "KL/std": 6.750263214111328, "epoch": 0.21768707482993196, "fcm_dpo/beta": 0.1010802835226059, "fcm_dpo/delta": -0.04181172698736191, "fcm_dpo/margin": 4.301558971405029, "fcm_dpo/q_t": 0.40543052554130554, "grad_norm": 29.94562530517578, "learning_rate": 4.800745179625307e-07, "logits/chosen": 0.23211714625358582, "logits/rejected": 0.20083823800086975, "logps/chosen": -71.6333236694336, "logps/ref_chosen": -60.80268859863281, "logps/ref_rejected": -79.07284545898438, "logps/rejected": -94.20503234863281, "loss": 1.1339, "margin_dpo/margin_mean": 4.301558494567871, "margin_dpo/margin_std": 7.674080848693848, "step": 144 }, { "KL/chosen_KL_mean": -10.08212661743164, "KL/mean": -12.36398696899414, "KL/rejected_KL_mean": -14.645843505859375, "KL/std": 6.890389442443848, "epoch": 0.21919879062736206, "fcm_dpo/beta": 0.09865110367536545, "fcm_dpo/delta": -0.05853622406721115, "fcm_dpo/margin": 4.563719749450684, "fcm_dpo/q_t": 0.39978501200675964, "grad_norm": 32.027320861816406, "learning_rate": 4.795540267200686e-07, "logits/chosen": 0.20314472913742065, "logits/rejected": 0.21950089931488037, "logps/chosen": -84.69358825683594, "logps/ref_chosen": -74.61146545410156, "logps/ref_rejected": -83.24461364746094, "logps/rejected": -97.89045715332031, "loss": 1.1284, "margin_dpo/margin_mean": 4.563719749450684, "margin_dpo/margin_std": 8.117656707763672, "step": 145 }, { "KL/chosen_KL_mean": -9.851871490478516, "KL/mean": -12.232227325439453, "KL/rejected_KL_mean": -14.612586975097656, "KL/std": 6.808660507202148, "epoch": 0.22071050642479215, "fcm_dpo/beta": 0.09611174464225769, "fcm_dpo/delta": -0.06661855429410934, "fcm_dpo/margin": 4.760718822479248, "fcm_dpo/q_t": 0.3982863426208496, "grad_norm": 27.760150909423828, "learning_rate": 4.790271143580173e-07, "logits/chosen": 0.22264136373996735, "logits/rejected": 0.2081959843635559, "logps/chosen": -67.69285583496094, "logps/ref_chosen": -57.84098434448242, "logps/ref_rejected": -67.47422790527344, "logps/rejected": -82.0868148803711, "loss": 1.0995, "margin_dpo/margin_mean": 4.76071834564209, "margin_dpo/margin_std": 7.512134075164795, "step": 146 }, { "KL/chosen_KL_mean": -11.579130172729492, "KL/mean": -13.471564292907715, "KL/rejected_KL_mean": -15.364002227783203, "KL/std": 7.3946404457092285, "epoch": 0.2222222222222222, "fcm_dpo/beta": 0.09622933715581894, "fcm_dpo/delta": 0.040078941732645035, "fcm_dpo/margin": 3.78487491607666, "fcm_dpo/q_t": 0.4198254346847534, "grad_norm": 34.06290817260742, "learning_rate": 4.784937956152489e-07, "logits/chosen": 0.24301037192344666, "logits/rejected": 0.1984240710735321, "logps/chosen": -78.3926010131836, "logps/ref_chosen": -66.81346893310547, "logps/ref_rejected": -81.1796875, "logps/rejected": -96.54369354248047, "loss": 1.1853, "margin_dpo/margin_mean": 3.784874439239502, "margin_dpo/margin_std": 7.882870674133301, "step": 147 }, { "KL/chosen_KL_mean": -10.643348693847656, "KL/mean": -13.073385238647461, "KL/rejected_KL_mean": -15.50341796875, "KL/std": 6.8529863357543945, "epoch": 0.2237339380196523, "fcm_dpo/beta": 0.09473659843206406, "fcm_dpo/delta": -0.07015595585107803, "fcm_dpo/margin": 4.860075950622559, "fcm_dpo/q_t": 0.3961476683616638, "grad_norm": 23.567968368530273, "learning_rate": 4.779540854098347e-07, "logits/chosen": 0.36633625626564026, "logits/rejected": 0.2913614511489868, "logps/chosen": -59.33110427856445, "logps/ref_chosen": -48.6877555847168, "logps/ref_rejected": -67.50503540039062, "logps/rejected": -83.00845336914062, "loss": 1.1018, "margin_dpo/margin_mean": 4.860075950622559, "margin_dpo/margin_std": 7.616449356079102, "step": 148 }, { "KL/chosen_KL_mean": -12.942892074584961, "KL/mean": -15.660242080688477, "KL/rejected_KL_mean": -18.37759017944336, "KL/std": 7.375458717346191, "epoch": 0.2252456538170824, "fcm_dpo/beta": 0.08893537521362305, "fcm_dpo/delta": -0.09736001491546631, "fcm_dpo/margin": 5.434703350067139, "fcm_dpo/q_t": 0.3881411850452423, "grad_norm": 24.1125431060791, "learning_rate": 4.774079988386296e-07, "logits/chosen": 0.21307995915412903, "logits/rejected": 0.16604451835155487, "logps/chosen": -68.086669921875, "logps/ref_chosen": -55.143775939941406, "logps/ref_rejected": -64.79888916015625, "logps/rejected": -83.17648315429688, "loss": 1.0482, "margin_dpo/margin_mean": 5.4347028732299805, "margin_dpo/margin_std": 6.959212303161621, "step": 149 }, { "KL/chosen_KL_mean": -10.891637802124023, "KL/mean": -14.504773139953613, "KL/rejected_KL_mean": -18.117908477783203, "KL/std": 8.100428581237793, "epoch": 0.22675736961451248, "fcm_dpo/beta": 0.07920362055301666, "fcm_dpo/delta": -0.21006450057029724, "fcm_dpo/margin": 7.226279258728027, "fcm_dpo/q_t": 0.36848869919776917, "grad_norm": 21.305072784423828, "learning_rate": 4.768555511768486e-07, "logits/chosen": 0.24731308221817017, "logits/rejected": 0.202421173453331, "logps/chosen": -78.36238098144531, "logps/ref_chosen": -67.47074890136719, "logps/ref_rejected": -89.21170806884766, "logps/rejected": -107.32962036132812, "loss": 0.9942, "margin_dpo/margin_mean": 7.226279258728027, "margin_dpo/margin_std": 8.23183822631836, "step": 150 }, { "KL/chosen_KL_mean": -9.562496185302734, "KL/mean": -13.620951652526855, "KL/rejected_KL_mean": -17.679405212402344, "KL/std": 8.622806549072266, "epoch": 0.22826908541194255, "fcm_dpo/beta": 0.07037020474672318, "fcm_dpo/delta": -0.2068619728088379, "fcm_dpo/margin": 8.116912841796875, "fcm_dpo/q_t": 0.3663896322250366, "grad_norm": 19.14907455444336, "learning_rate": 4.762967578776406e-07, "logits/chosen": 0.2455185353755951, "logits/rejected": 0.1907639503479004, "logps/chosen": -62.02204132080078, "logps/ref_chosen": -52.45954132080078, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -96.74241638183594, "loss": 0.9791, "margin_dpo/margin_mean": 8.116911888122559, "margin_dpo/margin_std": 8.724565505981445, "step": 151 }, { "KL/chosen_KL_mean": -12.31747817993164, "KL/mean": -14.943049430847168, "KL/rejected_KL_mean": -17.568626403808594, "KL/std": 8.21334171295166, "epoch": 0.22978080120937264, "fcm_dpo/beta": 0.07021472603082657, "fcm_dpo/delta": 0.0327349528670311, "fcm_dpo/margin": 5.2511467933654785, "fcm_dpo/q_t": 0.4175514876842499, "grad_norm": 21.101016998291016, "learning_rate": 4.757316345716553e-07, "logits/chosen": 0.3428304195404053, "logits/rejected": 0.2872798442840576, "logps/chosen": -68.87130737304688, "logps/ref_chosen": -56.5538330078125, "logps/ref_rejected": -76.55074310302734, "logps/rejected": -94.11936950683594, "loss": 1.1541, "margin_dpo/margin_mean": 5.25114631652832, "margin_dpo/margin_std": 9.311445236206055, "step": 152 }, { "KL/chosen_KL_mean": -11.60400390625, "KL/mean": -14.734918594360352, "KL/rejected_KL_mean": -17.865833282470703, "KL/std": 8.638544082641602, "epoch": 0.23129251700680273, "fcm_dpo/beta": 0.06840331852436066, "fcm_dpo/delta": -0.03252699226140976, "fcm_dpo/margin": 6.261826515197754, "fcm_dpo/q_t": 0.4008117616176605, "grad_norm": 20.02750587463379, "learning_rate": 4.751601970666064e-07, "logits/chosen": 0.24029231071472168, "logits/rejected": 0.203395813703537, "logps/chosen": -79.61090087890625, "logps/ref_chosen": -68.00689697265625, "logps/ref_rejected": -74.83482360839844, "logps/rejected": -92.70065307617188, "loss": 1.0756, "margin_dpo/margin_mean": 6.261826515197754, "margin_dpo/margin_std": 8.131772994995117, "step": 153 }, { "KL/chosen_KL_mean": -12.812858581542969, "KL/mean": -15.129095077514648, "KL/rejected_KL_mean": -17.445331573486328, "KL/std": 8.995925903320312, "epoch": 0.2328042328042328, "fcm_dpo/beta": 0.0704408809542656, "fcm_dpo/delta": 0.08183015882968903, "fcm_dpo/margin": 4.63247537612915, "fcm_dpo/q_t": 0.42660561203956604, "grad_norm": 20.98277473449707, "learning_rate": 4.745824613468292e-07, "logits/chosen": 0.3132394552230835, "logits/rejected": 0.3103998899459839, "logps/chosen": -72.035400390625, "logps/ref_chosen": -59.222537994384766, "logps/ref_rejected": -64.19131469726562, "logps/rejected": -81.63665008544922, "loss": 1.1868, "margin_dpo/margin_mean": 4.63247537612915, "margin_dpo/margin_std": 9.253047943115234, "step": 154 }, { "KL/chosen_KL_mean": -13.649024963378906, "KL/mean": -16.94133949279785, "KL/rejected_KL_mean": -20.233646392822266, "KL/std": 8.983402252197266, "epoch": 0.23431594860166288, "fcm_dpo/beta": 0.06851907074451447, "fcm_dpo/delta": -0.06188790500164032, "fcm_dpo/margin": 6.584621906280518, "fcm_dpo/q_t": 0.39981919527053833, "grad_norm": 20.74521255493164, "learning_rate": 4.7399844357283393e-07, "logits/chosen": 0.3174290060997009, "logits/rejected": 0.29693371057510376, "logps/chosen": -82.10372161865234, "logps/ref_chosen": -68.45469665527344, "logps/ref_rejected": -77.91763305664062, "logps/rejected": -98.15128326416016, "loss": 1.1242, "margin_dpo/margin_mean": 6.584621429443359, "margin_dpo/margin_std": 11.35929012298584, "step": 155 }, { "KL/chosen_KL_mean": -13.674962997436523, "KL/mean": -17.565982818603516, "KL/rejected_KL_mean": -21.45700454711914, "KL/std": 10.152885437011719, "epoch": 0.23582766439909297, "fcm_dpo/beta": 0.06530015915632248, "fcm_dpo/delta": -0.12741731107234955, "fcm_dpo/margin": 7.782038688659668, "fcm_dpo/q_t": 0.38315021991729736, "grad_norm": 20.189838409423828, "learning_rate": 4.7340816008085305e-07, "logits/chosen": 0.26459452509880066, "logits/rejected": 0.21891286969184875, "logps/chosen": -80.94456481933594, "logps/ref_chosen": -67.26959991455078, "logps/ref_rejected": -86.95914459228516, "logps/rejected": -108.41615295410156, "loss": 1.0288, "margin_dpo/margin_mean": 7.782039642333984, "margin_dpo/margin_std": 9.416492462158203, "step": 156 }, { "KL/chosen_KL_mean": -12.285833358764648, "KL/mean": -15.452082633972168, "KL/rejected_KL_mean": -18.618331909179688, "KL/std": 9.885778427124023, "epoch": 0.23733938019652306, "fcm_dpo/beta": 0.06017298251390457, "fcm_dpo/delta": -0.09840401262044907, "fcm_dpo/margin": 6.332496643066406, "fcm_dpo/q_t": 0.40991830825805664, "grad_norm": 18.193811416625977, "learning_rate": 4.728116273823847e-07, "logits/chosen": 0.2879556715488434, "logits/rejected": 0.26909253001213074, "logps/chosen": -67.05870819091797, "logps/ref_chosen": -54.77287292480469, "logps/ref_rejected": -63.87866973876953, "logps/rejected": -82.49700164794922, "loss": 1.1274, "margin_dpo/margin_mean": 6.332496166229248, "margin_dpo/margin_std": 9.572912216186523, "step": 157 }, { "KL/chosen_KL_mean": -13.837478637695312, "KL/mean": -17.254711151123047, "KL/rejected_KL_mean": -20.671947479248047, "KL/std": 10.21955680847168, "epoch": 0.23885109599395313, "fcm_dpo/beta": 0.060004714876413345, "fcm_dpo/delta": -0.011527525261044502, "fcm_dpo/margin": 6.8344645500183105, "fcm_dpo/q_t": 0.4079589247703552, "grad_norm": 18.990568161010742, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 0.30475467443466187, "logits/rejected": 0.266677588224411, "logps/chosen": -78.76019287109375, "logps/ref_chosen": -64.92271423339844, "logps/ref_rejected": -82.23789978027344, "logps/rejected": -102.90984344482422, "loss": 1.116, "margin_dpo/margin_mean": 6.8344645500183105, "margin_dpo/margin_std": 10.858131408691406, "step": 158 }, { "KL/chosen_KL_mean": -14.046161651611328, "KL/mean": -17.997652053833008, "KL/rejected_KL_mean": -21.949138641357422, "KL/std": 11.05732250213623, "epoch": 0.24036281179138322, "fcm_dpo/beta": 0.05865692347288132, "fcm_dpo/delta": -0.07415425777435303, "fcm_dpo/margin": 7.902973651885986, "fcm_dpo/q_t": 0.39414629340171814, "grad_norm": 19.708227157592773, "learning_rate": 4.715998812855304e-07, "logits/chosen": 0.33380645513534546, "logits/rejected": 0.29338324069976807, "logps/chosen": -71.09315490722656, "logps/ref_chosen": -57.046993255615234, "logps/ref_rejected": -73.32441711425781, "logps/rejected": -95.2735595703125, "loss": 1.0867, "margin_dpo/margin_mean": 7.902973175048828, "margin_dpo/margin_std": 11.675771713256836, "step": 159 }, { "KL/chosen_KL_mean": -15.937501907348633, "KL/mean": -19.319564819335938, "KL/rejected_KL_mean": -22.701629638671875, "KL/std": 10.381500244140625, "epoch": 0.2418745275888133, "fcm_dpo/beta": 0.05758850276470184, "fcm_dpo/delta": 0.011783316731452942, "fcm_dpo/margin": 6.764126777648926, "fcm_dpo/q_t": 0.4151209890842438, "grad_norm": 18.15164566040039, "learning_rate": 4.7098470178228755e-07, "logits/chosen": 0.19479069113731384, "logits/rejected": 0.14844146370887756, "logps/chosen": -65.74441528320312, "logps/ref_chosen": -49.806915283203125, "logps/ref_rejected": -68.3370132446289, "logps/rejected": -91.03864288330078, "loss": 1.1425, "margin_dpo/margin_mean": 6.764126300811768, "margin_dpo/margin_std": 11.992984771728516, "step": 160 }, { "KL/chosen_KL_mean": -15.949237823486328, "KL/mean": -19.665210723876953, "KL/rejected_KL_mean": -23.381183624267578, "KL/std": 10.27374267578125, "epoch": 0.24338624338624337, "fcm_dpo/beta": 0.05700618401169777, "fcm_dpo/delta": -0.02708788402378559, "fcm_dpo/margin": 7.431947708129883, "fcm_dpo/q_t": 0.4038432836532593, "grad_norm": 17.391952514648438, "learning_rate": 4.703633408618955e-07, "logits/chosen": 0.34192147850990295, "logits/rejected": 0.3018055856227875, "logps/chosen": -68.44972229003906, "logps/ref_chosen": -52.50048828125, "logps/ref_rejected": -66.04540252685547, "logps/rejected": -89.42658996582031, "loss": 1.109, "margin_dpo/margin_mean": 7.431947708129883, "margin_dpo/margin_std": 11.519304275512695, "step": 161 }, { "KL/chosen_KL_mean": -16.880916595458984, "KL/mean": -21.874000549316406, "KL/rejected_KL_mean": -26.867088317871094, "KL/std": 11.709345817565918, "epoch": 0.24489795918367346, "fcm_dpo/beta": 0.052794016897678375, "fcm_dpo/delta": -0.1518443524837494, "fcm_dpo/margin": 9.98617172241211, "fcm_dpo/q_t": 0.376575231552124, "grad_norm": 18.568201065063477, "learning_rate": 4.697358159051549e-07, "logits/chosen": 0.3624524474143982, "logits/rejected": 0.31142184138298035, "logps/chosen": -86.35011291503906, "logps/ref_chosen": -69.46919250488281, "logps/ref_rejected": -92.00952911376953, "logps/rejected": -118.87661743164062, "loss": 1.0213, "margin_dpo/margin_mean": 9.98617172241211, "margin_dpo/margin_std": 12.183841705322266, "step": 162 }, { "KL/chosen_KL_mean": -14.589275360107422, "KL/mean": -19.729949951171875, "KL/rejected_KL_mean": -24.870624542236328, "KL/std": 11.731868743896484, "epoch": 0.24640967498110355, "fcm_dpo/beta": 0.05017222464084625, "fcm_dpo/delta": -0.13923859596252441, "fcm_dpo/margin": 10.28134822845459, "fcm_dpo/q_t": 0.3796467185020447, "grad_norm": 16.747222900390625, "learning_rate": 4.691021444652876e-07, "logits/chosen": 0.30114084482192993, "logits/rejected": 0.25208836793899536, "logps/chosen": -65.20310974121094, "logps/ref_chosen": -50.613834381103516, "logps/ref_rejected": -74.62033081054688, "logps/rejected": -99.49095153808594, "loss": 1.0379, "margin_dpo/margin_mean": 10.281349182128906, "margin_dpo/margin_std": 12.120843887329102, "step": 163 }, { "KL/chosen_KL_mean": -15.791715621948242, "KL/mean": -20.810779571533203, "KL/rejected_KL_mean": -25.829849243164062, "KL/std": 12.496322631835938, "epoch": 0.24792139077853365, "fcm_dpo/beta": 0.04662519693374634, "fcm_dpo/delta": -0.07952342927455902, "fcm_dpo/margin": 10.038135528564453, "fcm_dpo/q_t": 0.3925841450691223, "grad_norm": 14.853240013122559, "learning_rate": 4.6846234426744624e-07, "logits/chosen": 0.29247328639030457, "logits/rejected": 0.22746919095516205, "logps/chosen": -70.63983154296875, "logps/ref_chosen": -54.848114013671875, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -104.89286041259766, "loss": 1.0734, "margin_dpo/margin_mean": 10.038135528564453, "margin_dpo/margin_std": 13.581424713134766, "step": 164 }, { "KL/chosen_KL_mean": -17.847217559814453, "KL/mean": -22.394338607788086, "KL/rejected_KL_mean": -26.94145965576172, "KL/std": 12.855323791503906, "epoch": 0.2494331065759637, "fcm_dpo/beta": 0.04560963064432144, "fcm_dpo/delta": -0.01796562224626541, "fcm_dpo/margin": 9.09423828125, "fcm_dpo/q_t": 0.404633492231369, "grad_norm": 15.527134895324707, "learning_rate": 4.678164332082175e-07, "logits/chosen": 0.3545019328594208, "logits/rejected": 0.29826444387435913, "logps/chosen": -68.93643188476562, "logps/ref_chosen": -51.089210510253906, "logps/ref_rejected": -71.23370361328125, "logps/rejected": -98.17515563964844, "loss": 1.1163, "margin_dpo/margin_mean": 9.09423828125, "margin_dpo/margin_std": 13.98188591003418, "step": 165 }, { "KL/chosen_KL_mean": -17.026296615600586, "KL/mean": -20.65865707397461, "KL/rejected_KL_mean": -24.29102325439453, "KL/std": 13.004007339477539, "epoch": 0.2509448223733938, "fcm_dpo/beta": 0.04612819850444794, "fcm_dpo/delta": 0.07232268154621124, "fcm_dpo/margin": 7.2647247314453125, "fcm_dpo/q_t": 0.42566025257110596, "grad_norm": 18.30398941040039, "learning_rate": 4.6716442935512214e-07, "logits/chosen": 0.3076387047767639, "logits/rejected": 0.2137855738401413, "logps/chosen": -80.21710205078125, "logps/ref_chosen": -63.19081115722656, "logps/ref_rejected": -93.8402099609375, "logps/rejected": -118.13123321533203, "loss": 1.157, "margin_dpo/margin_mean": 7.264724254608154, "margin_dpo/margin_std": 12.455879211425781, "step": 166 }, { "KL/chosen_KL_mean": -14.038339614868164, "KL/mean": -19.74565887451172, "KL/rejected_KL_mean": -25.452983856201172, "KL/std": 13.267440795898438, "epoch": 0.25245653817082386, "fcm_dpo/beta": 0.04366353154182434, "fcm_dpo/delta": -0.12011955678462982, "fcm_dpo/margin": 11.414639472961426, "fcm_dpo/q_t": 0.3832022547721863, "grad_norm": 15.69991683959961, "learning_rate": 4.6650635094610966e-07, "logits/chosen": 0.30109137296676636, "logits/rejected": 0.2652606666088104, "logps/chosen": -72.96261596679688, "logps/ref_chosen": -58.92427062988281, "logps/ref_rejected": -72.97377014160156, "logps/rejected": -98.4267578125, "loss": 1.0246, "margin_dpo/margin_mean": 11.414640426635742, "margin_dpo/margin_std": 13.03073501586914, "step": 167 }, { "KL/chosen_KL_mean": -16.865724563598633, "KL/mean": -21.457231521606445, "KL/rejected_KL_mean": -26.048744201660156, "KL/std": 14.453012466430664, "epoch": 0.25396825396825395, "fcm_dpo/beta": 0.043425120413303375, "fcm_dpo/delta": 0.0012646140530705452, "fcm_dpo/margin": 9.183013916015625, "fcm_dpo/q_t": 0.41033321619033813, "grad_norm": 16.60943603515625, "learning_rate": 4.6584221638904767e-07, "logits/chosen": 0.310530424118042, "logits/rejected": 0.27260318398475647, "logps/chosen": -82.51710510253906, "logps/ref_chosen": -65.65138244628906, "logps/ref_rejected": -79.71418762207031, "logps/rejected": -105.76293182373047, "loss": 1.1069, "margin_dpo/margin_mean": 9.183013916015625, "margin_dpo/margin_std": 13.43674087524414, "step": 168 }, { "KL/chosen_KL_mean": -13.360231399536133, "KL/mean": -18.598499298095703, "KL/rejected_KL_mean": -23.836761474609375, "KL/std": 14.224285125732422, "epoch": 0.25547996976568405, "fcm_dpo/beta": 0.04259002208709717, "fcm_dpo/delta": -0.0533658042550087, "fcm_dpo/margin": 10.476530075073242, "fcm_dpo/q_t": 0.40058034658432007, "grad_norm": 17.076465606689453, "learning_rate": 4.651720442612075e-07, "logits/chosen": 0.38942497968673706, "logits/rejected": 0.3538346290588379, "logps/chosen": -74.78610229492188, "logps/ref_chosen": -61.425865173339844, "logps/ref_rejected": -76.09590148925781, "logps/rejected": -99.93266296386719, "loss": 1.107, "margin_dpo/margin_mean": 10.476531028747559, "margin_dpo/margin_std": 16.978721618652344, "step": 169 }, { "KL/chosen_KL_mean": -12.65814208984375, "KL/mean": -17.938709259033203, "KL/rejected_KL_mean": -23.219276428222656, "KL/std": 14.04989242553711, "epoch": 0.25699168556311414, "fcm_dpo/beta": 0.041654251515865326, "fcm_dpo/delta": -0.04596564173698425, "fcm_dpo/margin": 10.561136245727539, "fcm_dpo/q_t": 0.40057674050331116, "grad_norm": 14.404908180236816, "learning_rate": 4.6449585330874425e-07, "logits/chosen": 0.3427773118019104, "logits/rejected": 0.34061866998672485, "logps/chosen": -69.31133270263672, "logps/ref_chosen": -56.65319061279297, "logps/ref_rejected": -63.45965576171875, "logps/rejected": -86.6789321899414, "loss": 1.1222, "margin_dpo/margin_mean": 10.561136245727539, "margin_dpo/margin_std": 17.518104553222656, "step": 170 }, { "KL/chosen_KL_mean": -14.985160827636719, "KL/mean": -20.769207000732422, "KL/rejected_KL_mean": -26.55325698852539, "KL/std": 16.237634658813477, "epoch": 0.2585034013605442, "fcm_dpo/beta": 0.038949161767959595, "fcm_dpo/delta": -0.06474211066961288, "fcm_dpo/margin": 11.56810188293457, "fcm_dpo/q_t": 0.39703309535980225, "grad_norm": 14.443869590759277, "learning_rate": 4.6381366244617224e-07, "logits/chosen": 0.3847231864929199, "logits/rejected": 0.3326500654220581, "logps/chosen": -78.71992492675781, "logps/ref_chosen": -63.73476028442383, "logps/ref_rejected": -78.50328063964844, "logps/rejected": -105.05653381347656, "loss": 1.1113, "margin_dpo/margin_mean": 11.56810188293457, "margin_dpo/margin_std": 18.460580825805664, "step": 171 }, { "KL/chosen_KL_mean": -16.810169219970703, "KL/mean": -22.31606674194336, "KL/rejected_KL_mean": -27.82196044921875, "KL/std": 16.634769439697266, "epoch": 0.2600151171579743, "fcm_dpo/beta": 0.0396546795964241, "fcm_dpo/delta": -0.04476618766784668, "fcm_dpo/margin": 11.011789321899414, "fcm_dpo/q_t": 0.40239837765693665, "grad_norm": 16.143247604370117, "learning_rate": 4.631254907558365e-07, "logits/chosen": 0.3681456744670868, "logits/rejected": 0.3081226348876953, "logps/chosen": -69.01193237304688, "logps/ref_chosen": -52.201759338378906, "logps/ref_rejected": -82.85285949707031, "logps/rejected": -110.67481994628906, "loss": 1.1085, "margin_dpo/margin_mean": 11.011789321899414, "margin_dpo/margin_std": 17.142223358154297, "step": 172 }, { "KL/chosen_KL_mean": -15.272012710571289, "KL/mean": -21.283023834228516, "KL/rejected_KL_mean": -27.29404067993164, "KL/std": 17.057445526123047, "epoch": 0.2615268329554044, "fcm_dpo/beta": 0.03659302368760109, "fcm_dpo/delta": -0.052808038890361786, "fcm_dpo/margin": 12.02202320098877, "fcm_dpo/q_t": 0.4022437334060669, "grad_norm": 14.902726173400879, "learning_rate": 4.624313574873786e-07, "logits/chosen": 0.39478111267089844, "logits/rejected": 0.30247700214385986, "logps/chosen": -70.70674133300781, "logps/ref_chosen": -55.434722900390625, "logps/ref_rejected": -77.81967163085938, "logps/rejected": -105.11371612548828, "loss": 1.1372, "margin_dpo/margin_mean": 12.022025108337402, "margin_dpo/margin_std": 20.99781036376953, "step": 173 }, { "KL/chosen_KL_mean": -17.438180923461914, "KL/mean": -24.091327667236328, "KL/rejected_KL_mean": -30.744476318359375, "KL/std": 17.439682006835938, "epoch": 0.26303854875283444, "fcm_dpo/beta": 0.03558962047100067, "fcm_dpo/delta": -0.08698572218418121, "fcm_dpo/margin": 13.306293487548828, "fcm_dpo/q_t": 0.3912256360054016, "grad_norm": 15.202779769897461, "learning_rate": 4.61731282057198e-07, "logits/chosen": 0.3748180866241455, "logits/rejected": 0.305186927318573, "logps/chosen": -74.61013793945312, "logps/ref_chosen": -57.17195129394531, "logps/ref_rejected": -85.47578430175781, "logps/rejected": -116.22026062011719, "loss": 1.0715, "margin_dpo/margin_mean": 13.306294441223145, "margin_dpo/margin_std": 18.859642028808594, "step": 174 }, { "KL/chosen_KL_mean": -14.990503311157227, "KL/mean": -22.452695846557617, "KL/rejected_KL_mean": -29.914894104003906, "KL/std": 17.86532211303711, "epoch": 0.26455026455026454, "fcm_dpo/beta": 0.03374076634645462, "fcm_dpo/delta": -0.12154931575059891, "fcm_dpo/margin": 14.924393653869629, "fcm_dpo/q_t": 0.38714203238487244, "grad_norm": 14.585382461547852, "learning_rate": 4.6102528404790965e-07, "logits/chosen": 0.4304041266441345, "logits/rejected": 0.3987458050251007, "logps/chosen": -82.6561279296875, "logps/ref_chosen": -67.6656265258789, "logps/ref_rejected": -84.36766815185547, "logps/rejected": -114.28256225585938, "loss": 1.0598, "margin_dpo/margin_mean": 14.924394607543945, "margin_dpo/margin_std": 20.989219665527344, "step": 175 }, { "KL/chosen_KL_mean": -19.427631378173828, "KL/mean": -24.25865936279297, "KL/rejected_KL_mean": -29.08969497680664, "KL/std": 19.192419052124023, "epoch": 0.2660619803476946, "fcm_dpo/beta": 0.033753879368305206, "fcm_dpo/delta": 0.08193635195493698, "fcm_dpo/margin": 9.66206169128418, "fcm_dpo/q_t": 0.4294675588607788, "grad_norm": 17.79058265686035, "learning_rate": 4.603133832077953e-07, "logits/chosen": 0.3631625771522522, "logits/rejected": 0.33612459897994995, "logps/chosen": -97.28639221191406, "logps/ref_chosen": -77.8587646484375, "logps/ref_rejected": -81.08732604980469, "logps/rejected": -110.17701721191406, "loss": 1.2166, "margin_dpo/margin_mean": 9.66206169128418, "margin_dpo/margin_std": 21.81412696838379, "step": 176 }, { "KL/chosen_KL_mean": -14.337238311767578, "KL/mean": -23.852760314941406, "KL/rejected_KL_mean": -33.3682861328125, "KL/std": 19.595916748046875, "epoch": 0.2675736961451247, "fcm_dpo/beta": 0.03083290532231331, "fcm_dpo/delta": -0.2311885952949524, "fcm_dpo/margin": 19.031044006347656, "fcm_dpo/q_t": 0.360975980758667, "grad_norm": 17.56546974182129, "learning_rate": 4.5959559945025183e-07, "logits/chosen": 0.50090491771698, "logits/rejected": 0.4008740186691284, "logps/chosen": -69.55763244628906, "logps/ref_chosen": -55.22039794921875, "logps/ref_rejected": -92.54973602294922, "logps/rejected": -125.91802215576172, "loss": 0.9693, "margin_dpo/margin_mean": 19.031042098999023, "margin_dpo/margin_std": 20.025856018066406, "step": 177 }, { "KL/chosen_KL_mean": -17.92155647277832, "KL/mean": -23.787391662597656, "KL/rejected_KL_mean": -29.653221130371094, "KL/std": 18.697711944580078, "epoch": 0.2690854119425548, "fcm_dpo/beta": 0.02928192727267742, "fcm_dpo/delta": -0.04489610344171524, "fcm_dpo/margin": 11.731661796569824, "fcm_dpo/q_t": 0.4197039306163788, "grad_norm": 17.016468048095703, "learning_rate": 4.588719528532341e-07, "logits/chosen": 0.3574860692024231, "logits/rejected": 0.3063201308250427, "logps/chosen": -78.73204803466797, "logps/ref_chosen": -60.81049346923828, "logps/ref_rejected": -81.12973022460938, "logps/rejected": -110.78294372558594, "loss": 1.1482, "margin_dpo/margin_mean": 11.731661796569824, "margin_dpo/margin_std": 18.86032485961914, "step": 178 }, { "KL/chosen_KL_mean": -18.39486312866211, "KL/mean": -25.84204864501953, "KL/rejected_KL_mean": -33.28923797607422, "KL/std": 20.058395385742188, "epoch": 0.2705971277399849, "fcm_dpo/beta": 0.02877836301922798, "fcm_dpo/delta": -0.03301222249865532, "fcm_dpo/margin": 14.894369125366211, "fcm_dpo/q_t": 0.4027270972728729, "grad_norm": 14.578384399414062, "learning_rate": 4.581424636586928e-07, "logits/chosen": 0.39680802822113037, "logits/rejected": 0.38149362802505493, "logps/chosen": -84.06657409667969, "logps/ref_chosen": -65.67171478271484, "logps/ref_rejected": -75.32586669921875, "logps/rejected": -108.61510467529297, "loss": 1.1094, "margin_dpo/margin_mean": 14.894369125366211, "margin_dpo/margin_std": 23.633628845214844, "step": 179 }, { "KL/chosen_KL_mean": -14.09145736694336, "KL/mean": -20.09218978881836, "KL/rejected_KL_mean": -26.092912673950195, "KL/std": 19.371274948120117, "epoch": 0.272108843537415, "fcm_dpo/beta": 0.029514048248529434, "fcm_dpo/delta": 0.050767071545124054, "fcm_dpo/margin": 12.001461029052734, "fcm_dpo/q_t": 0.4218587279319763, "grad_norm": 13.016547203063965, "learning_rate": 4.5740715227200897e-07, "logits/chosen": 0.24703602492809296, "logits/rejected": 0.228146493434906, "logps/chosen": -70.77426147460938, "logps/ref_chosen": -56.68280792236328, "logps/ref_rejected": -64.94414520263672, "logps/rejected": -91.03706359863281, "loss": 1.1642, "margin_dpo/margin_mean": 12.001461029052734, "margin_dpo/margin_std": 22.415281295776367, "step": 180 }, { "KL/chosen_KL_mean": -12.105340957641602, "KL/mean": -20.832984924316406, "KL/rejected_KL_mean": -29.560623168945312, "KL/std": 20.69134521484375, "epoch": 0.273620559334845, "fcm_dpo/beta": 0.028023969382047653, "fcm_dpo/delta": -0.10585962235927582, "fcm_dpo/margin": 17.45528793334961, "fcm_dpo/q_t": 0.38700011372566223, "grad_norm": 15.768712997436523, "learning_rate": 4.566660392614228e-07, "logits/chosen": 0.42835646867752075, "logits/rejected": 0.38607752323150635, "logps/chosen": -72.88139343261719, "logps/ref_chosen": -60.77604675292969, "logps/ref_rejected": -83.98361206054688, "logps/rejected": -113.54423522949219, "loss": 1.031, "margin_dpo/margin_mean": 17.45528793334961, "margin_dpo/margin_std": 20.554719924926758, "step": 181 }, { "KL/chosen_KL_mean": -19.097064971923828, "KL/mean": -28.098819732666016, "KL/rejected_KL_mean": -37.1005744934082, "KL/std": 22.797513961791992, "epoch": 0.2751322751322751, "fcm_dpo/beta": 0.026667404919862747, "fcm_dpo/delta": -0.0935642197728157, "fcm_dpo/margin": 18.003517150878906, "fcm_dpo/q_t": 0.39077216386795044, "grad_norm": 16.841510772705078, "learning_rate": 4.5591914535745817e-07, "logits/chosen": 0.4146467447280884, "logits/rejected": 0.32692649960517883, "logps/chosen": -79.35084533691406, "logps/ref_chosen": -60.2537841796875, "logps/ref_rejected": -89.7706298828125, "logps/rejected": -126.87120056152344, "loss": 1.071, "margin_dpo/margin_mean": 18.003517150878906, "margin_dpo/margin_std": 25.671062469482422, "step": 182 }, { "KL/chosen_KL_mean": -23.710100173950195, "KL/mean": -27.463716506958008, "KL/rejected_KL_mean": -31.217327117919922, "KL/std": 21.911300659179688, "epoch": 0.2766439909297052, "fcm_dpo/beta": 0.027505898848176003, "fcm_dpo/delta": 0.07738558948040009, "fcm_dpo/margin": 7.507230281829834, "fcm_dpo/q_t": 0.4531930983066559, "grad_norm": 14.247156143188477, "learning_rate": 4.551664914523433e-07, "logits/chosen": 0.40092331171035767, "logits/rejected": 0.3776063919067383, "logps/chosen": -85.47151947021484, "logps/ref_chosen": -61.76142120361328, "logps/ref_rejected": -72.54627990722656, "logps/rejected": -103.76361083984375, "loss": 1.2818, "margin_dpo/margin_mean": 7.507230281829834, "margin_dpo/margin_std": 22.548179626464844, "step": 183 }, { "KL/chosen_KL_mean": -13.734045028686523, "KL/mean": -20.633419036865234, "KL/rejected_KL_mean": -27.532791137695312, "KL/std": 19.101581573486328, "epoch": 0.2781557067271353, "fcm_dpo/beta": 0.02745695784687996, "fcm_dpo/delta": 0.02308075502514839, "fcm_dpo/margin": 13.798742294311523, "fcm_dpo/q_t": 0.4148487448692322, "grad_norm": 16.434070587158203, "learning_rate": 4.544080985994258e-07, "logits/chosen": 0.4927828907966614, "logits/rejected": 0.42545321583747864, "logps/chosen": -60.57476806640625, "logps/ref_chosen": -46.840721130371094, "logps/ref_rejected": -69.3609390258789, "logps/rejected": -96.89373016357422, "loss": 1.1172, "margin_dpo/margin_mean": 13.798742294311523, "margin_dpo/margin_std": 20.220142364501953, "step": 184 }, { "KL/chosen_KL_mean": -18.83686065673828, "KL/mean": -27.118553161621094, "KL/rejected_KL_mean": -35.40023422241211, "KL/std": 23.780193328857422, "epoch": 0.2796674225245654, "fcm_dpo/beta": 0.026584237813949585, "fcm_dpo/delta": -0.05228573456406593, "fcm_dpo/margin": 16.563377380371094, "fcm_dpo/q_t": 0.40145543217658997, "grad_norm": 14.18293571472168, "learning_rate": 4.5364398801258394e-07, "logits/chosen": 0.43396925926208496, "logits/rejected": 0.38681963086128235, "logps/chosen": -71.15800476074219, "logps/ref_chosen": -52.32114028930664, "logps/ref_rejected": -68.3885726928711, "logps/rejected": -103.78880310058594, "loss": 1.1334, "margin_dpo/margin_mean": 16.56337547302246, "margin_dpo/margin_std": 28.58876609802246, "step": 185 }, { "KL/chosen_KL_mean": -16.70131492614746, "KL/mean": -25.68690299987793, "KL/rejected_KL_mean": -34.67249298095703, "KL/std": 23.681175231933594, "epoch": 0.2811791383219955, "fcm_dpo/beta": 0.025996234267950058, "fcm_dpo/delta": -0.07862785458564758, "fcm_dpo/margin": 17.971176147460938, "fcm_dpo/q_t": 0.3956528306007385, "grad_norm": 17.511899948120117, "learning_rate": 4.5287418106563354e-07, "logits/chosen": 0.3419583737850189, "logits/rejected": 0.299105703830719, "logps/chosen": -84.12144470214844, "logps/ref_chosen": -67.42012786865234, "logps/ref_rejected": -82.50968933105469, "logps/rejected": -117.18218231201172, "loss": 1.0992, "margin_dpo/margin_mean": 17.971176147460938, "margin_dpo/margin_std": 28.787887573242188, "step": 186 }, { "KL/chosen_KL_mean": -22.477245330810547, "KL/mean": -30.472248077392578, "KL/rejected_KL_mean": -38.467254638671875, "KL/std": 25.28232765197754, "epoch": 0.28269085411942557, "fcm_dpo/beta": 0.02547437697649002, "fcm_dpo/delta": -0.008900739252567291, "fcm_dpo/margin": 15.990015983581543, "fcm_dpo/q_t": 0.4118683338165283, "grad_norm": 16.354949951171875, "learning_rate": 4.520986992917297e-07, "logits/chosen": 0.4215141534805298, "logits/rejected": 0.3655751347541809, "logps/chosen": -98.00273895263672, "logps/ref_chosen": -75.52549743652344, "logps/ref_rejected": -94.76289367675781, "logps/rejected": -133.2301483154297, "loss": 1.1527, "margin_dpo/margin_mean": 15.990015983581543, "margin_dpo/margin_std": 29.899837493896484, "step": 187 }, { "KL/chosen_KL_mean": -20.62126922607422, "KL/mean": -30.150714874267578, "KL/rejected_KL_mean": -39.68015670776367, "KL/std": 26.517009735107422, "epoch": 0.2842025699168556, "fcm_dpo/beta": 0.024722332134842873, "fcm_dpo/delta": -0.08292864263057709, "fcm_dpo/margin": 19.058883666992188, "fcm_dpo/q_t": 0.3959978222846985, "grad_norm": 13.136139869689941, "learning_rate": 4.5131756438276466e-07, "logits/chosen": 0.42119163274765015, "logits/rejected": 0.37826281785964966, "logps/chosen": -92.14459991455078, "logps/ref_chosen": -71.52333068847656, "logps/ref_rejected": -78.29949951171875, "logps/rejected": -117.97966003417969, "loss": 1.0819, "margin_dpo/margin_mean": 19.058883666992188, "margin_dpo/margin_std": 28.6160831451416, "step": 188 }, { "KL/chosen_KL_mean": -20.660493850708008, "KL/mean": -29.015668869018555, "KL/rejected_KL_mean": -37.37084197998047, "KL/std": 26.431987762451172, "epoch": 0.2857142857142857, "fcm_dpo/beta": 0.023861508816480637, "fcm_dpo/delta": -0.0019575804471969604, "fcm_dpo/margin": 16.710351943969727, "fcm_dpo/q_t": 0.41077572107315063, "grad_norm": 15.193591117858887, "learning_rate": 4.5053079818876096e-07, "logits/chosen": 0.47629261016845703, "logits/rejected": 0.492626428604126, "logps/chosen": -92.83676147460938, "logps/ref_chosen": -72.17626953125, "logps/ref_rejected": -75.26313781738281, "logps/rejected": -112.63397216796875, "loss": 1.1338, "margin_dpo/margin_mean": 16.71035385131836, "margin_dpo/margin_std": 27.63460350036621, "step": 189 }, { "KL/chosen_KL_mean": -19.903602600097656, "KL/mean": -32.08153533935547, "KL/rejected_KL_mean": -44.25946807861328, "KL/std": 27.69138526916504, "epoch": 0.2872260015117158, "fcm_dpo/beta": 0.02243766561150551, "fcm_dpo/delta": -0.17649930715560913, "fcm_dpo/margin": 24.355867385864258, "fcm_dpo/q_t": 0.3752169609069824, "grad_norm": 19.759334564208984, "learning_rate": 4.4973842271726024e-07, "logits/chosen": 0.5364293456077576, "logits/rejected": 0.3797008693218231, "logps/chosen": -74.52787780761719, "logps/ref_chosen": -54.624271392822266, "logps/ref_rejected": -101.47068786621094, "logps/rejected": -145.73016357421875, "loss": 1.0183, "margin_dpo/margin_mean": 24.355865478515625, "margin_dpo/margin_std": 30.53769302368164, "step": 190 }, { "KL/chosen_KL_mean": -29.63574981689453, "KL/mean": -39.05760192871094, "KL/rejected_KL_mean": -48.479454040527344, "KL/std": 29.330245971679688, "epoch": 0.2887377173091459, "fcm_dpo/beta": 0.021781757473945618, "fcm_dpo/delta": -0.01194111630320549, "fcm_dpo/margin": 18.843711853027344, "fcm_dpo/q_t": 0.4067850112915039, "grad_norm": 15.226569175720215, "learning_rate": 4.48940460132708e-07, "logits/chosen": 0.5249963998794556, "logits/rejected": 0.49379801750183105, "logps/chosen": -102.56826782226562, "logps/ref_chosen": -72.93251037597656, "logps/ref_rejected": -89.95103454589844, "logps/rejected": -138.43048095703125, "loss": 1.1252, "margin_dpo/margin_mean": 18.843713760375977, "margin_dpo/margin_std": 31.241641998291016, "step": 191 }, { "KL/chosen_KL_mean": -33.76609802246094, "KL/mean": -39.7779426574707, "KL/rejected_KL_mean": -45.78978729248047, "KL/std": 30.88973617553711, "epoch": 0.29024943310657597, "fcm_dpo/beta": 0.02324771136045456, "fcm_dpo/delta": 0.1299854815006256, "fcm_dpo/margin": 12.023693084716797, "fcm_dpo/q_t": 0.43884414434432983, "grad_norm": 15.96609878540039, "learning_rate": 4.481369327558329e-07, "logits/chosen": 0.5415316820144653, "logits/rejected": 0.5124877691268921, "logps/chosen": -87.76721954345703, "logps/ref_chosen": -54.001121520996094, "logps/ref_rejected": -63.531551361083984, "logps/rejected": -109.32133483886719, "loss": 1.2218, "margin_dpo/margin_mean": 12.02369213104248, "margin_dpo/margin_std": 27.488590240478516, "step": 192 }, { "KL/chosen_KL_mean": -26.880212783813477, "KL/mean": -38.29176330566406, "KL/rejected_KL_mean": -49.70331573486328, "KL/std": 28.740842819213867, "epoch": 0.29176114890400606, "fcm_dpo/beta": 0.02194758504629135, "fcm_dpo/delta": -0.12258221954107285, "fcm_dpo/margin": 22.823104858398438, "fcm_dpo/q_t": 0.38504743576049805, "grad_norm": 13.015534400939941, "learning_rate": 4.47327863063023e-07, "logits/chosen": 0.41978591680526733, "logits/rejected": 0.39997708797454834, "logps/chosen": -83.62948608398438, "logps/ref_chosen": -56.74927520751953, "logps/ref_rejected": -58.80629348754883, "logps/rejected": -108.50961303710938, "loss": 1.0454, "margin_dpo/margin_mean": 22.82310676574707, "margin_dpo/margin_std": 29.807849884033203, "step": 193 }, { "KL/chosen_KL_mean": -30.15066909790039, "KL/mean": -39.23329162597656, "KL/rejected_KL_mean": -48.315921783447266, "KL/std": 30.352039337158203, "epoch": 0.29327286470143615, "fcm_dpo/beta": 0.02143619954586029, "fcm_dpo/delta": 0.00822427123785019, "fcm_dpo/margin": 18.165246963500977, "fcm_dpo/q_t": 0.4135337471961975, "grad_norm": 17.924869537353516, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 0.526753842830658, "logits/rejected": 0.4955962300300598, "logps/chosen": -86.80010986328125, "logps/ref_chosen": -56.64944076538086, "logps/ref_rejected": -69.98954772949219, "logps/rejected": -118.30546569824219, "loss": 1.1742, "margin_dpo/margin_mean": 18.165246963500977, "margin_dpo/margin_std": 35.39740753173828, "step": 194 }, { "KL/chosen_KL_mean": -35.72456741333008, "KL/mean": -48.061038970947266, "KL/rejected_KL_mean": -60.39750671386719, "KL/std": 33.23961639404297, "epoch": 0.2947845804988662, "fcm_dpo/beta": 0.020506273955106735, "fcm_dpo/delta": -0.1278218924999237, "fcm_dpo/margin": 24.672945022583008, "fcm_dpo/q_t": 0.38505733013153076, "grad_norm": 16.904264450073242, "learning_rate": 4.4569318740967043e-07, "logits/chosen": 0.43730270862579346, "logits/rejected": 0.44750988483428955, "logps/chosen": -106.13434600830078, "logps/ref_chosen": -70.40977478027344, "logps/ref_rejected": -74.39448547363281, "logps/rejected": -134.7919921875, "loss": 1.0634, "margin_dpo/margin_mean": 24.67294692993164, "margin_dpo/margin_std": 35.26850128173828, "step": 195 }, { "KL/chosen_KL_mean": -36.346405029296875, "KL/mean": -45.410797119140625, "KL/rejected_KL_mean": -54.47519302368164, "KL/std": 34.48020935058594, "epoch": 0.2962962962962963, "fcm_dpo/beta": 0.020830024033784866, "fcm_dpo/delta": 0.02376365289092064, "fcm_dpo/margin": 18.128787994384766, "fcm_dpo/q_t": 0.414675235748291, "grad_norm": 18.091453552246094, "learning_rate": 4.448676271745197e-07, "logits/chosen": 0.5153293013572693, "logits/rejected": 0.46972033381462097, "logps/chosen": -95.573974609375, "logps/ref_chosen": -59.227577209472656, "logps/ref_rejected": -83.54757690429688, "logps/rejected": -138.02276611328125, "loss": 1.1365, "margin_dpo/margin_mean": 18.128787994384766, "margin_dpo/margin_std": 29.96463966369629, "step": 196 }, { "KL/chosen_KL_mean": -32.234886169433594, "KL/mean": -43.992149353027344, "KL/rejected_KL_mean": -55.74940872192383, "KL/std": 32.462039947509766, "epoch": 0.29780801209372637, "fcm_dpo/beta": 0.020198438316583633, "fcm_dpo/delta": -0.08790335804224014, "fcm_dpo/margin": 23.51451873779297, "fcm_dpo/q_t": 0.3938829302787781, "grad_norm": 17.229829788208008, "learning_rate": 4.440366160729392e-07, "logits/chosen": 0.593604564666748, "logits/rejected": 0.5391672849655151, "logps/chosen": -83.76402282714844, "logps/ref_chosen": -51.52912902832031, "logps/ref_rejected": -73.70631408691406, "logps/rejected": -129.45571899414062, "loss": 1.1271, "margin_dpo/margin_mean": 23.51451873779297, "margin_dpo/margin_std": 40.65414047241211, "step": 197 }, { "KL/chosen_KL_mean": -35.11063003540039, "KL/mean": -47.747779846191406, "KL/rejected_KL_mean": -60.38494110107422, "KL/std": 33.210731506347656, "epoch": 0.29931972789115646, "fcm_dpo/beta": 0.018990688025951385, "fcm_dpo/delta": -0.09321387112140656, "fcm_dpo/margin": 25.274303436279297, "fcm_dpo/q_t": 0.3890335261821747, "grad_norm": 15.211456298828125, "learning_rate": 4.432001773500957e-07, "logits/chosen": 0.5949510931968689, "logits/rejected": 0.5511294603347778, "logps/chosen": -94.893310546875, "logps/ref_chosen": -59.78268051147461, "logps/ref_rejected": -72.24533081054688, "logps/rejected": -132.63026428222656, "loss": 1.0603, "margin_dpo/margin_mean": 25.274303436279297, "margin_dpo/margin_std": 33.733585357666016, "step": 198 }, { "KL/chosen_KL_mean": -42.495819091796875, "KL/mean": -52.347877502441406, "KL/rejected_KL_mean": -62.19993209838867, "KL/std": 33.97826385498047, "epoch": 0.30083144368858655, "fcm_dpo/beta": 0.0181651059538126, "fcm_dpo/delta": -0.06439946591854095, "fcm_dpo/margin": 19.704111099243164, "fcm_dpo/q_t": 0.41771239042282104, "grad_norm": 13.64396858215332, "learning_rate": 4.4235833440297856e-07, "logits/chosen": 0.5503775477409363, "logits/rejected": 0.4570651054382324, "logps/chosen": -98.8825912475586, "logps/ref_chosen": -56.38677215576172, "logps/ref_rejected": -74.56779479980469, "logps/rejected": -136.76773071289062, "loss": 1.1815, "margin_dpo/margin_mean": 19.704111099243164, "margin_dpo/margin_std": 38.327781677246094, "step": 199 }, { "KL/chosen_KL_mean": -34.580101013183594, "KL/mean": -48.731632232666016, "KL/rejected_KL_mean": -62.88316345214844, "KL/std": 35.259788513183594, "epoch": 0.30234315948601664, "fcm_dpo/beta": 0.016897382214665413, "fcm_dpo/delta": -0.09347662329673767, "fcm_dpo/margin": 28.303054809570312, "fcm_dpo/q_t": 0.39045828580856323, "grad_norm": 12.926334381103516, "learning_rate": 4.415111107797445e-07, "logits/chosen": 0.5941851139068604, "logits/rejected": 0.512370228767395, "logps/chosen": -92.40442657470703, "logps/ref_chosen": -57.82432556152344, "logps/ref_rejected": -89.28246307373047, "logps/rejected": -152.16561889648438, "loss": 1.0802, "margin_dpo/margin_mean": 28.303054809570312, "margin_dpo/margin_std": 41.966915130615234, "step": 200 }, { "KL/chosen_KL_mean": -47.82463836669922, "KL/mean": -58.91985321044922, "KL/rejected_KL_mean": -70.01507568359375, "KL/std": 37.099159240722656, "epoch": 0.30385487528344673, "fcm_dpo/beta": 0.016795823350548744, "fcm_dpo/delta": 0.029392898082733154, "fcm_dpo/margin": 22.190425872802734, "fcm_dpo/q_t": 0.41658544540405273, "grad_norm": 15.126965522766113, "learning_rate": 4.4065853017905953e-07, "logits/chosen": 0.6271176338195801, "logits/rejected": 0.5756611824035645, "logps/chosen": -106.82440185546875, "logps/ref_chosen": -58.999759674072266, "logps/ref_rejected": -84.67575073242188, "logps/rejected": -154.69082641601562, "loss": 1.1508, "margin_dpo/margin_mean": 22.190425872802734, "margin_dpo/margin_std": 38.777530670166016, "step": 201 }, { "KL/chosen_KL_mean": -40.08701705932617, "KL/mean": -54.63224792480469, "KL/rejected_KL_mean": -69.17748260498047, "KL/std": 35.3343505859375, "epoch": 0.30536659108087677, "fcm_dpo/beta": 0.01654578559100628, "fcm_dpo/delta": -0.09474054723978043, "fcm_dpo/margin": 29.090471267700195, "fcm_dpo/q_t": 0.38745981454849243, "grad_norm": 13.597728729248047, "learning_rate": 4.3980061644943575e-07, "logits/chosen": 0.5242820978164673, "logits/rejected": 0.444463849067688, "logps/chosen": -87.74766540527344, "logps/ref_chosen": -47.660648345947266, "logps/ref_rejected": -73.63249969482422, "logps/rejected": -142.8099822998047, "loss": 1.0619, "margin_dpo/margin_mean": 29.090471267700195, "margin_dpo/margin_std": 39.005897521972656, "step": 202 }, { "KL/chosen_KL_mean": -50.41577911376953, "KL/mean": -63.223907470703125, "KL/rejected_KL_mean": -76.03202819824219, "KL/std": 36.80195999145508, "epoch": 0.30687830687830686, "fcm_dpo/beta": 0.016147522255778313, "fcm_dpo/delta": -0.015619541518390179, "fcm_dpo/margin": 25.616247177124023, "fcm_dpo/q_t": 0.4067385792732239, "grad_norm": 16.166540145874023, "learning_rate": 4.3893739358856455e-07, "logits/chosen": 0.5744930505752563, "logits/rejected": 0.4879806637763977, "logps/chosen": -112.74131774902344, "logps/ref_chosen": -62.32553482055664, "logps/ref_rejected": -99.37226104736328, "logps/rejected": -175.404296875, "loss": 1.115, "margin_dpo/margin_mean": 25.616249084472656, "margin_dpo/margin_std": 40.74905014038086, "step": 203 }, { "KL/chosen_KL_mean": -45.187042236328125, "KL/mean": -58.972225189208984, "KL/rejected_KL_mean": -72.75740051269531, "KL/std": 37.236968994140625, "epoch": 0.30839002267573695, "fcm_dpo/beta": 0.015411125496029854, "fcm_dpo/delta": -0.0340307280421257, "fcm_dpo/margin": 27.570358276367188, "fcm_dpo/q_t": 0.40242862701416016, "grad_norm": 14.934419631958008, "learning_rate": 4.380688857426449e-07, "logits/chosen": 0.562065839767456, "logits/rejected": 0.48749643564224243, "logps/chosen": -95.81636047363281, "logps/ref_chosen": -50.62931823730469, "logps/ref_rejected": -66.60475158691406, "logps/rejected": -139.36215209960938, "loss": 1.0957, "margin_dpo/margin_mean": 27.570356369018555, "margin_dpo/margin_std": 38.67369079589844, "step": 204 }, { "KL/chosen_KL_mean": -52.95893859863281, "KL/mean": -66.04808044433594, "KL/rejected_KL_mean": -79.13722229003906, "KL/std": 39.503875732421875, "epoch": 0.30990173847316704, "fcm_dpo/beta": 0.015607166104018688, "fcm_dpo/delta": -0.009851815178990364, "fcm_dpo/margin": 26.17829704284668, "fcm_dpo/q_t": 0.40857863426208496, "grad_norm": 16.72960090637207, "learning_rate": 4.3719511720570814e-07, "logits/chosen": 0.5846419334411621, "logits/rejected": 0.5103527903556824, "logps/chosen": -123.31511688232422, "logps/ref_chosen": -70.3561782836914, "logps/ref_rejected": -93.39848327636719, "logps/rejected": -172.53570556640625, "loss": 1.1293, "margin_dpo/margin_mean": 26.178298950195312, "margin_dpo/margin_std": 44.114990234375, "step": 205 }, { "KL/chosen_KL_mean": -55.539981842041016, "KL/mean": -64.34222412109375, "KL/rejected_KL_mean": -73.14447021484375, "KL/std": 38.425743103027344, "epoch": 0.31141345427059713, "fcm_dpo/beta": 0.015734657645225525, "fcm_dpo/delta": 0.022806979715824127, "fcm_dpo/margin": 17.60448455810547, "fcm_dpo/q_t": 0.4400056302547455, "grad_norm": 14.786349296569824, "learning_rate": 4.363161124189387e-07, "logits/chosen": 0.6014150381088257, "logits/rejected": 0.5844457149505615, "logps/chosen": -123.18546295166016, "logps/ref_chosen": -67.64547729492188, "logps/ref_rejected": -79.89584350585938, "logps/rejected": -153.04031372070312, "loss": 1.2576, "margin_dpo/margin_mean": 17.6044864654541, "margin_dpo/margin_std": 47.01768493652344, "step": 206 }, { "KL/chosen_KL_mean": -63.995445251464844, "KL/mean": -76.38787841796875, "KL/rejected_KL_mean": -88.78030395507812, "KL/std": 39.89057159423828, "epoch": 0.3129251700680272, "fcm_dpo/beta": 0.015928978100419044, "fcm_dpo/delta": 0.005825323052704334, "fcm_dpo/margin": 24.784862518310547, "fcm_dpo/q_t": 0.41365471482276917, "grad_norm": 13.180200576782227, "learning_rate": 4.3543189596998986e-07, "logits/chosen": 0.5777188539505005, "logits/rejected": 0.5047867894172668, "logps/chosen": -131.65963745117188, "logps/ref_chosen": -67.66419219970703, "logps/ref_rejected": -85.10249328613281, "logps/rejected": -173.88279724121094, "loss": 1.1359, "margin_dpo/margin_mean": 24.784862518310547, "margin_dpo/margin_std": 42.67274475097656, "step": 207 }, { "KL/chosen_KL_mean": -54.84553527832031, "KL/mean": -61.52077865600586, "KL/rejected_KL_mean": -68.1960220336914, "KL/std": 38.0513916015625, "epoch": 0.3144368858654573, "fcm_dpo/beta": 0.01662435196340084, "fcm_dpo/delta": 0.0744815543293953, "fcm_dpo/margin": 13.350489616394043, "fcm_dpo/q_t": 0.4523673355579376, "grad_norm": 21.40291976928711, "learning_rate": 4.3454249259229664e-07, "logits/chosen": 0.5781328678131104, "logits/rejected": 0.5493608713150024, "logps/chosen": -112.5772476196289, "logps/ref_chosen": -57.731712341308594, "logps/ref_rejected": -74.19276428222656, "logps/rejected": -142.3887939453125, "loss": 1.3069, "margin_dpo/margin_mean": 13.350488662719727, "margin_dpo/margin_std": 44.465492248535156, "step": 208 }, { "KL/chosen_KL_mean": -54.61763000488281, "KL/mean": -70.42623138427734, "KL/rejected_KL_mean": -86.23484802246094, "KL/std": 42.82319641113281, "epoch": 0.31594860166288735, "fcm_dpo/beta": 0.01570359617471695, "fcm_dpo/delta": -0.1145251914858818, "fcm_dpo/margin": 31.617210388183594, "fcm_dpo/q_t": 0.3877779245376587, "grad_norm": 15.687070846557617, "learning_rate": 4.336479271643833e-07, "logits/chosen": 0.5210272073745728, "logits/rejected": 0.46536776423454285, "logps/chosen": -123.16770935058594, "logps/ref_chosen": -68.55007934570312, "logps/ref_rejected": -87.90541076660156, "logps/rejected": -174.1402587890625, "loss": 1.0937, "margin_dpo/margin_mean": 31.617210388183594, "margin_dpo/margin_std": 50.869163513183594, "step": 209 }, { "KL/chosen_KL_mean": -53.08406066894531, "KL/mean": -69.13988494873047, "KL/rejected_KL_mean": -85.19570922851562, "KL/std": 43.93097686767578, "epoch": 0.31746031746031744, "fcm_dpo/beta": 0.01513850037008524, "fcm_dpo/delta": -0.10149059444665909, "fcm_dpo/margin": 32.111637115478516, "fcm_dpo/q_t": 0.39089614152908325, "grad_norm": 15.86282730102539, "learning_rate": 4.327482247091679e-07, "logits/chosen": 0.6385220289230347, "logits/rejected": 0.5306141376495361, "logps/chosen": -110.35234069824219, "logps/ref_chosen": -57.268272399902344, "logps/ref_rejected": -85.72807312011719, "logps/rejected": -170.9237823486328, "loss": 1.0724, "margin_dpo/margin_mean": 32.111637115478516, "margin_dpo/margin_std": 45.491607666015625, "step": 210 }, { "KL/chosen_KL_mean": -48.97825622558594, "KL/mean": -64.92655944824219, "KL/rejected_KL_mean": -80.8748779296875, "KL/std": 40.996124267578125, "epoch": 0.31897203325774753, "fcm_dpo/beta": 0.014405088499188423, "fcm_dpo/delta": -0.06979866325855255, "fcm_dpo/margin": 31.896615982055664, "fcm_dpo/q_t": 0.39662906527519226, "grad_norm": 17.855039596557617, "learning_rate": 4.3184341039326217e-07, "logits/chosen": 0.6159626245498657, "logits/rejected": 0.5108903050422668, "logps/chosen": -102.61896514892578, "logps/ref_chosen": -53.640708923339844, "logps/ref_rejected": -93.0387954711914, "logps/rejected": -173.91366577148438, "loss": 1.0677, "margin_dpo/margin_mean": 31.896615982055664, "margin_dpo/margin_std": 42.558563232421875, "step": 211 }, { "KL/chosen_KL_mean": -55.17749786376953, "KL/mean": -70.99906158447266, "KL/rejected_KL_mean": -86.82063293457031, "KL/std": 43.388309478759766, "epoch": 0.3204837490551776, "fcm_dpo/beta": 0.013755107298493385, "fcm_dpo/delta": -0.040556252002716064, "fcm_dpo/margin": 31.643131256103516, "fcm_dpo/q_t": 0.3995632529258728, "grad_norm": 10.621484756469727, "learning_rate": 4.309335095262675e-07, "logits/chosen": 0.6260430812835693, "logits/rejected": 0.5469928979873657, "logps/chosen": -112.54423522949219, "logps/ref_chosen": -57.36674499511719, "logps/ref_rejected": -79.89643096923828, "logps/rejected": -166.71707153320312, "loss": 1.096, "margin_dpo/margin_mean": 31.643129348754883, "margin_dpo/margin_std": 46.78819274902344, "step": 212 }, { "KL/chosen_KL_mean": -44.155555725097656, "KL/mean": -62.20250701904297, "KL/rejected_KL_mean": -80.24946594238281, "KL/std": 43.83998107910156, "epoch": 0.3219954648526077, "fcm_dpo/beta": 0.013000023551285267, "fcm_dpo/delta": -0.08304151147603989, "fcm_dpo/margin": 36.09390640258789, "fcm_dpo/q_t": 0.3916971981525421, "grad_norm": 11.67632007598877, "learning_rate": 4.3001854756006724e-07, "logits/chosen": 0.5951361656188965, "logits/rejected": 0.5706344842910767, "logps/chosen": -109.37667083740234, "logps/ref_chosen": -65.22111511230469, "logps/ref_rejected": -80.1810302734375, "logps/rejected": -160.43048095703125, "loss": 1.0649, "margin_dpo/margin_mean": 36.09390640258789, "margin_dpo/margin_std": 48.79711151123047, "step": 213 }, { "KL/chosen_KL_mean": -52.43152618408203, "KL/mean": -69.8787841796875, "KL/rejected_KL_mean": -87.3260269165039, "KL/std": 44.159698486328125, "epoch": 0.3235071806500378, "fcm_dpo/beta": 0.012891286984086037, "fcm_dpo/delta": -0.058638766407966614, "fcm_dpo/margin": 34.89451217651367, "fcm_dpo/q_t": 0.39819642901420593, "grad_norm": 13.59187126159668, "learning_rate": 4.290985500881143e-07, "logits/chosen": 0.47652411460876465, "logits/rejected": 0.46216946840286255, "logps/chosen": -113.7238540649414, "logps/ref_chosen": -61.292327880859375, "logps/ref_rejected": -67.69841003417969, "logps/rejected": -155.02444458007812, "loss": 1.0924, "margin_dpo/margin_mean": 34.89451599121094, "margin_dpo/margin_std": 51.43964385986328, "step": 214 }, { "KL/chosen_KL_mean": -57.65421676635742, "KL/mean": -75.66905212402344, "KL/rejected_KL_mean": -93.68387603759766, "KL/std": 45.626731872558594, "epoch": 0.3250188964474679, "fcm_dpo/beta": 0.012519944459199905, "fcm_dpo/delta": -0.061683304607868195, "fcm_dpo/margin": 36.02965545654297, "fcm_dpo/q_t": 0.3975563645362854, "grad_norm": 12.812163352966309, "learning_rate": 4.281735428447157e-07, "logits/chosen": 0.49123620986938477, "logits/rejected": 0.3736293911933899, "logps/chosen": -121.52335357666016, "logps/ref_chosen": -63.869136810302734, "logps/ref_rejected": -98.7657241821289, "logps/rejected": -192.44960021972656, "loss": 1.099, "margin_dpo/margin_mean": 36.02965545654297, "margin_dpo/margin_std": 53.505271911621094, "step": 215 }, { "KL/chosen_KL_mean": -48.55633544921875, "KL/mean": -68.02696228027344, "KL/rejected_KL_mean": -87.49758911132812, "KL/std": 44.254615783691406, "epoch": 0.32653061224489793, "fcm_dpo/beta": 0.011713217943906784, "fcm_dpo/delta": -0.06535829603672028, "fcm_dpo/margin": 38.941246032714844, "fcm_dpo/q_t": 0.3921303153038025, "grad_norm": 22.394311904907227, "learning_rate": 4.2724355170431247e-07, "logits/chosen": 0.5988740921020508, "logits/rejected": 0.5082088708877563, "logps/chosen": -116.38128662109375, "logps/ref_chosen": -67.824951171875, "logps/ref_rejected": -96.40231323242188, "logps/rejected": -183.89990234375, "loss": 1.0456, "margin_dpo/margin_mean": 38.941246032714844, "margin_dpo/margin_std": 44.615882873535156, "step": 216 }, { "KL/chosen_KL_mean": -52.889949798583984, "KL/mean": -73.1640625, "KL/rejected_KL_mean": -93.43817901611328, "KL/std": 45.83582305908203, "epoch": 0.328042328042328, "fcm_dpo/beta": 0.01119733415544033, "fcm_dpo/delta": -0.06392455846071243, "fcm_dpo/margin": 40.548240661621094, "fcm_dpo/q_t": 0.39370107650756836, "grad_norm": 10.61658000946045, "learning_rate": 4.26308602680756e-07, "logits/chosen": 0.5777255892753601, "logits/rejected": 0.4716952443122864, "logps/chosen": -113.39494323730469, "logps/ref_chosen": -60.5049934387207, "logps/ref_rejected": -84.26618194580078, "logps/rejected": -177.70436096191406, "loss": 1.0503, "margin_dpo/margin_mean": 40.548240661621094, "margin_dpo/margin_std": 47.78947448730469, "step": 217 }, { "KL/chosen_KL_mean": -58.71846008300781, "KL/mean": -70.38279724121094, "KL/rejected_KL_mean": -82.04712677001953, "KL/std": 45.208160400390625, "epoch": 0.3295540438397581, "fcm_dpo/beta": 0.011268625035881996, "fcm_dpo/delta": 0.02180871181190014, "fcm_dpo/margin": 23.328662872314453, "fcm_dpo/q_t": 0.4390827715396881, "grad_norm": 11.695416450500488, "learning_rate": 4.253687219265803e-07, "logits/chosen": 0.4690364599227905, "logits/rejected": 0.472156822681427, "logps/chosen": -129.31277465820312, "logps/ref_chosen": -70.59431457519531, "logps/ref_rejected": -73.89038848876953, "logps/rejected": -155.93751525878906, "loss": 1.234, "margin_dpo/margin_mean": 23.32866096496582, "margin_dpo/margin_std": 54.49486541748047, "step": 218 }, { "KL/chosen_KL_mean": -54.63685989379883, "KL/mean": -67.47407531738281, "KL/rejected_KL_mean": -80.31128692626953, "KL/std": 43.43921661376953, "epoch": 0.3310657596371882, "fcm_dpo/beta": 0.011822700500488281, "fcm_dpo/delta": 0.10650709271430969, "fcm_dpo/margin": 25.674427032470703, "fcm_dpo/q_t": 0.4315260052680969, "grad_norm": 11.611766815185547, "learning_rate": 4.2442393573227043e-07, "logits/chosen": 0.5588555335998535, "logits/rejected": 0.5147054195404053, "logps/chosen": -115.1278076171875, "logps/ref_chosen": -60.490943908691406, "logps/ref_rejected": -75.85001373291016, "logps/rejected": -156.1613006591797, "loss": 1.1746, "margin_dpo/margin_mean": 25.674427032470703, "margin_dpo/margin_std": 45.024803161621094, "step": 219 }, { "KL/chosen_KL_mean": -46.359901428222656, "KL/mean": -61.93693923950195, "KL/rejected_KL_mean": -77.51397705078125, "KL/std": 46.67603302001953, "epoch": 0.3325774754346183, "fcm_dpo/beta": 0.012025123462080956, "fcm_dpo/delta": 0.02687015011906624, "fcm_dpo/margin": 31.15407943725586, "fcm_dpo/q_t": 0.4164389967918396, "grad_norm": 13.288846969604492, "learning_rate": 4.234742705255272e-07, "logits/chosen": 0.6493104100227356, "logits/rejected": 0.5782603025436401, "logps/chosen": -91.37329864501953, "logps/ref_chosen": -45.013397216796875, "logps/ref_rejected": -70.49369812011719, "logps/rejected": -148.0076904296875, "loss": 1.1525, "margin_dpo/margin_mean": 31.154081344604492, "margin_dpo/margin_std": 54.201744079589844, "step": 220 }, { "KL/chosen_KL_mean": -45.98303985595703, "KL/mean": -63.76835632324219, "KL/rejected_KL_mean": -81.55365753173828, "KL/std": 46.344482421875, "epoch": 0.3340891912320484, "fcm_dpo/beta": 0.01212066039443016, "fcm_dpo/delta": -0.03572237864136696, "fcm_dpo/margin": 35.570621490478516, "fcm_dpo/q_t": 0.4028802216053009, "grad_norm": 13.347899436950684, "learning_rate": 4.22519752870528e-07, "logits/chosen": 0.577314019203186, "logits/rejected": 0.494738906621933, "logps/chosen": -105.0788803100586, "logps/ref_chosen": -59.09584045410156, "logps/ref_rejected": -88.64388275146484, "logps/rejected": -170.19754028320312, "loss": 1.0937, "margin_dpo/margin_mean": 35.570621490478516, "margin_dpo/margin_std": 52.52630615234375, "step": 221 }, { "KL/chosen_KL_mean": -46.059627532958984, "KL/mean": -69.68561553955078, "KL/rejected_KL_mean": -93.31161499023438, "KL/std": 52.56859588623047, "epoch": 0.3356009070294785, "fcm_dpo/beta": 0.011211092583835125, "fcm_dpo/delta": -0.15475650131702423, "fcm_dpo/margin": 47.251983642578125, "fcm_dpo/q_t": 0.37726062536239624, "grad_norm": 11.579093933105469, "learning_rate": 4.2156040946718343e-07, "logits/chosen": 0.6488215327262878, "logits/rejected": 0.5492867827415466, "logps/chosen": -102.05731964111328, "logps/ref_chosen": -55.9976921081543, "logps/ref_rejected": -111.94727325439453, "logps/rejected": -205.25888061523438, "loss": 1.0177, "margin_dpo/margin_mean": 47.251983642578125, "margin_dpo/margin_std": 56.59059143066406, "step": 222 }, { "KL/chosen_KL_mean": -55.13484191894531, "KL/mean": -73.27153015136719, "KL/rejected_KL_mean": -91.40821838378906, "KL/std": 49.953399658203125, "epoch": 0.3371126228269085, "fcm_dpo/beta": 0.01093536801636219, "fcm_dpo/delta": 0.003693356178700924, "fcm_dpo/margin": 36.27337646484375, "fcm_dpo/q_t": 0.4097467362880707, "grad_norm": 13.193449974060059, "learning_rate": 4.2059626715039065e-07, "logits/chosen": 0.6461388468742371, "logits/rejected": 0.5862003564834595, "logps/chosen": -115.02626037597656, "logps/ref_chosen": -59.891422271728516, "logps/ref_rejected": -86.28954315185547, "logps/rejected": -177.69775390625, "loss": 1.1034, "margin_dpo/margin_mean": 36.27337646484375, "margin_dpo/margin_std": 51.730384826660156, "step": 223 }, { "KL/chosen_KL_mean": -64.09883117675781, "KL/mean": -74.00580596923828, "KL/rejected_KL_mean": -83.91278076171875, "KL/std": 49.710262298583984, "epoch": 0.3386243386243386, "fcm_dpo/beta": 0.011504280380904675, "fcm_dpo/delta": 0.07703431695699692, "fcm_dpo/margin": 19.813947677612305, "fcm_dpo/q_t": 0.449110746383667, "grad_norm": 16.505056381225586, "learning_rate": 4.1962735288928304e-07, "logits/chosen": 0.6655477285385132, "logits/rejected": 0.6418548822402954, "logps/chosen": -128.14346313476562, "logps/ref_chosen": -64.04463195800781, "logps/ref_rejected": -75.05450439453125, "logps/rejected": -158.96728515625, "loss": 1.2611, "margin_dpo/margin_mean": 19.813949584960938, "margin_dpo/margin_std": 53.515785217285156, "step": 224 }, { "KL/chosen_KL_mean": -58.37687683105469, "KL/mean": -79.15052795410156, "KL/rejected_KL_mean": -99.92417907714844, "KL/std": 54.812477111816406, "epoch": 0.3401360544217687, "fcm_dpo/beta": 0.010968768037855625, "fcm_dpo/delta": -0.06804075092077255, "fcm_dpo/margin": 41.54730987548828, "fcm_dpo/q_t": 0.3937387466430664, "grad_norm": 11.77187728881836, "learning_rate": 4.186536937864752e-07, "logits/chosen": 0.606415867805481, "logits/rejected": 0.4917227327823639, "logps/chosen": -124.47270202636719, "logps/ref_chosen": -66.0958251953125, "logps/ref_rejected": -97.68675231933594, "logps/rejected": -197.61093139648438, "loss": 1.0544, "margin_dpo/margin_mean": 41.54731369018555, "margin_dpo/margin_std": 49.95249938964844, "step": 225 }, { "KL/chosen_KL_mean": -56.503387451171875, "KL/mean": -72.67691802978516, "KL/rejected_KL_mean": -88.85043334960938, "KL/std": 49.93806076049805, "epoch": 0.3416477702191988, "fcm_dpo/beta": 0.011389894410967827, "fcm_dpo/delta": 0.03310755267739296, "fcm_dpo/margin": 32.347049713134766, "fcm_dpo/q_t": 0.41660165786743164, "grad_norm": 13.261937141418457, "learning_rate": 4.176753170773052e-07, "logits/chosen": 0.6644654273986816, "logits/rejected": 0.6175069808959961, "logps/chosen": -107.92025756835938, "logps/ref_chosen": -51.4168701171875, "logps/ref_rejected": -66.30068969726562, "logps/rejected": -155.151123046875, "loss": 1.1674, "margin_dpo/margin_mean": 32.3470458984375, "margin_dpo/margin_std": 60.95067596435547, "step": 226 }, { "KL/chosen_KL_mean": -60.69549560546875, "KL/mean": -77.11978149414062, "KL/rejected_KL_mean": -93.5440673828125, "KL/std": 54.52500915527344, "epoch": 0.3431594860166289, "fcm_dpo/beta": 0.01134055107831955, "fcm_dpo/delta": 0.03080243244767189, "fcm_dpo/margin": 32.84856414794922, "fcm_dpo/q_t": 0.41694962978363037, "grad_norm": 13.040262222290039, "learning_rate": 4.166922501290729e-07, "logits/chosen": 0.6646544933319092, "logits/rejected": 0.6282267570495605, "logps/chosen": -118.68527221679688, "logps/ref_chosen": -57.989776611328125, "logps/ref_rejected": -75.05464172363281, "logps/rejected": -168.5987091064453, "loss": 1.1623, "margin_dpo/margin_mean": 32.84856414794922, "margin_dpo/margin_std": 61.44432830810547, "step": 227 }, { "KL/chosen_KL_mean": -64.08232116699219, "KL/mean": -83.22323608398438, "KL/rejected_KL_mean": -102.36415100097656, "KL/std": 51.782081604003906, "epoch": 0.34467120181405897, "fcm_dpo/beta": 0.011403821408748627, "fcm_dpo/delta": -0.04262223467230797, "fcm_dpo/margin": 38.281822204589844, "fcm_dpo/q_t": 0.3999175429344177, "grad_norm": 15.122424125671387, "learning_rate": 4.1570452044027405e-07, "logits/chosen": 0.684807538986206, "logits/rejected": 0.6012556552886963, "logps/chosen": -119.64169311523438, "logps/ref_chosen": -55.55936813354492, "logps/ref_rejected": -77.02364349365234, "logps/rejected": -179.38778686523438, "loss": 1.0908, "margin_dpo/margin_mean": 38.281822204589844, "margin_dpo/margin_std": 54.99129867553711, "step": 228 }, { "KL/chosen_KL_mean": -54.697181701660156, "KL/mean": -71.32321166992188, "KL/rejected_KL_mean": -87.9492416381836, "KL/std": 50.878578186035156, "epoch": 0.34618291761148906, "fcm_dpo/beta": 0.011315664276480675, "fcm_dpo/delta": 0.02675747312605381, "fcm_dpo/margin": 33.25206756591797, "fcm_dpo/q_t": 0.4166818857192993, "grad_norm": 16.81686019897461, "learning_rate": 4.147121556398312e-07, "logits/chosen": 0.7582691311836243, "logits/rejected": 0.6779066324234009, "logps/chosen": -105.49183654785156, "logps/ref_chosen": -50.79466247558594, "logps/ref_rejected": -78.4474105834961, "logps/rejected": -166.3966522216797, "loss": 1.1589, "margin_dpo/margin_mean": 33.25206756591797, "margin_dpo/margin_std": 61.93717956542969, "step": 229 }, { "KL/chosen_KL_mean": -59.01070022583008, "KL/mean": -77.44501495361328, "KL/rejected_KL_mean": -95.87932586669922, "KL/std": 54.955116271972656, "epoch": 0.3476946334089191, "fcm_dpo/beta": 0.011499742977321148, "fcm_dpo/delta": -0.031111925840377808, "fcm_dpo/margin": 36.868629455566406, "fcm_dpo/q_t": 0.403672993183136, "grad_norm": 13.358758926391602, "learning_rate": 4.137151834863213e-07, "logits/chosen": 0.6462364196777344, "logits/rejected": 0.648872971534729, "logps/chosen": -115.73992919921875, "logps/ref_chosen": -56.729225158691406, "logps/ref_rejected": -62.99180603027344, "logps/rejected": -158.87112426757812, "loss": 1.1114, "margin_dpo/margin_mean": 36.86862564086914, "margin_dpo/margin_std": 56.367958068847656, "step": 230 }, { "KL/chosen_KL_mean": -65.8025131225586, "KL/mean": -93.48104858398438, "KL/rejected_KL_mean": -121.15958404541016, "KL/std": 53.62238693237305, "epoch": 0.3492063492063492, "fcm_dpo/beta": 0.010107467882335186, "fcm_dpo/delta": -0.1956898272037506, "fcm_dpo/margin": 55.35707092285156, "fcm_dpo/q_t": 0.36493778228759766, "grad_norm": 12.845860481262207, "learning_rate": 4.1271363186719835e-07, "logits/chosen": 0.6005295515060425, "logits/rejected": 0.5976369976997375, "logps/chosen": -138.3996124267578, "logps/ref_chosen": -72.59709930419922, "logps/ref_rejected": -86.2322998046875, "logps/rejected": -207.39187622070312, "loss": 0.9845, "margin_dpo/margin_mean": 55.35707092285156, "margin_dpo/margin_std": 58.276145935058594, "step": 231 }, { "KL/chosen_KL_mean": -66.58941650390625, "KL/mean": -85.17105102539062, "KL/rejected_KL_mean": -103.752685546875, "KL/std": 59.081756591796875, "epoch": 0.3507180650037793, "fcm_dpo/beta": 0.010086476802825928, "fcm_dpo/delta": 0.028100861236453056, "fcm_dpo/margin": 37.163265228271484, "fcm_dpo/q_t": 0.4179150462150574, "grad_norm": 12.528980255126953, "learning_rate": 4.1170752879801436e-07, "logits/chosen": 0.6124403476715088, "logits/rejected": 0.5794280767440796, "logps/chosen": -134.70794677734375, "logps/ref_chosen": -68.1185302734375, "logps/ref_rejected": -83.79415893554688, "logps/rejected": -187.54684448242188, "loss": 1.1554, "margin_dpo/margin_mean": 37.163265228271484, "margin_dpo/margin_std": 67.85264587402344, "step": 232 }, { "KL/chosen_KL_mean": -76.2654800415039, "KL/mean": -89.84437561035156, "KL/rejected_KL_mean": -103.42329406738281, "KL/std": 53.858177185058594, "epoch": 0.35222978080120937, "fcm_dpo/beta": 0.009955944493412971, "fcm_dpo/delta": -0.02221892587840557, "fcm_dpo/margin": 27.157814025878906, "fcm_dpo/q_t": 0.4383409917354584, "grad_norm": 12.91989803314209, "learning_rate": 4.106969024216348e-07, "logits/chosen": 0.6682592630386353, "logits/rejected": 0.6073845028877258, "logps/chosen": -131.33563232421875, "logps/ref_chosen": -55.070152282714844, "logps/ref_rejected": -66.61845397949219, "logps/rejected": -170.041748046875, "loss": 1.2163, "margin_dpo/margin_mean": 27.157814025878906, "margin_dpo/margin_std": 56.61811828613281, "step": 233 }, { "KL/chosen_KL_mean": -67.56138610839844, "KL/mean": -80.51182556152344, "KL/rejected_KL_mean": -93.46226501464844, "KL/std": 52.28334426879883, "epoch": 0.35374149659863946, "fcm_dpo/beta": 0.009842677041888237, "fcm_dpo/delta": -0.03836081176996231, "fcm_dpo/margin": 25.900876998901367, "fcm_dpo/q_t": 0.44125378131866455, "grad_norm": 18.20611000061035, "learning_rate": 4.09681781007452e-07, "logits/chosen": 0.621496319770813, "logits/rejected": 0.6191729307174683, "logps/chosen": -123.48728942871094, "logps/ref_chosen": -55.92589569091797, "logps/ref_rejected": -51.11608123779297, "logps/rejected": -144.57833862304688, "loss": 1.2379, "margin_dpo/margin_mean": 25.90087890625, "margin_dpo/margin_std": 57.52040100097656, "step": 234 }, { "KL/chosen_KL_mean": -62.01856231689453, "KL/mean": -85.31228637695312, "KL/rejected_KL_mean": -108.60600280761719, "KL/std": 57.6728630065918, "epoch": 0.35525321239606955, "fcm_dpo/beta": 0.009540073573589325, "fcm_dpo/delta": -0.051198702305555344, "fcm_dpo/margin": 46.587440490722656, "fcm_dpo/q_t": 0.39602601528167725, "grad_norm": 14.494193077087402, "learning_rate": 4.08662192950594e-07, "logits/chosen": 0.6710847616195679, "logits/rejected": 0.6589834690093994, "logps/chosen": -126.55828857421875, "logps/ref_chosen": -64.53972625732422, "logps/ref_rejected": -77.69151306152344, "logps/rejected": -186.29751586914062, "loss": 1.0562, "margin_dpo/margin_mean": 46.58743667602539, "margin_dpo/margin_std": 55.38169860839844, "step": 235 }, { "KL/chosen_KL_mean": -89.08969116210938, "KL/mean": -106.5599365234375, "KL/rejected_KL_mean": -124.0301742553711, "KL/std": 55.50977325439453, "epoch": 0.35676492819349964, "fcm_dpo/beta": 0.009686114266514778, "fcm_dpo/delta": 0.06868328154087067, "fcm_dpo/margin": 34.94047546386719, "fcm_dpo/q_t": 0.42309021949768066, "grad_norm": 11.22431468963623, "learning_rate": 4.076381667711306e-07, "logits/chosen": 0.6049457788467407, "logits/rejected": 0.5927116870880127, "logps/chosen": -160.2444305419922, "logps/ref_chosen": -71.15473937988281, "logps/ref_rejected": -84.88541412353516, "logps/rejected": -208.91558837890625, "loss": 1.1686, "margin_dpo/margin_mean": 34.94047546386719, "margin_dpo/margin_std": 64.02879333496094, "step": 236 }, { "KL/chosen_KL_mean": -74.93379211425781, "KL/mean": -96.43376159667969, "KL/rejected_KL_mean": -117.93374633789062, "KL/std": 55.61524963378906, "epoch": 0.35827664399092973, "fcm_dpo/beta": 0.009799846448004246, "fcm_dpo/delta": -0.024707447737455368, "fcm_dpo/margin": 42.99995422363281, "fcm_dpo/q_t": 0.402961790561676, "grad_norm": 15.107486724853516, "learning_rate": 4.066097311132753e-07, "logits/chosen": 0.6670963168144226, "logits/rejected": 0.6589772701263428, "logps/chosen": -151.0758056640625, "logps/ref_chosen": -76.14201354980469, "logps/ref_rejected": -80.88479614257812, "logps/rejected": -198.81854248046875, "loss": 1.1068, "margin_dpo/margin_mean": 42.99995422363281, "margin_dpo/margin_std": 64.99576568603516, "step": 237 }, { "KL/chosen_KL_mean": -66.69758605957031, "KL/mean": -87.274658203125, "KL/rejected_KL_mean": -107.85171508789062, "KL/std": 57.005157470703125, "epoch": 0.35978835978835977, "fcm_dpo/beta": 0.009592464193701744, "fcm_dpo/delta": 0.004950430244207382, "fcm_dpo/margin": 41.15412902832031, "fcm_dpo/q_t": 0.4092939496040344, "grad_norm": 15.823814392089844, "learning_rate": 4.0557691474458414e-07, "logits/chosen": 0.6299471855163574, "logits/rejected": 0.6191462278366089, "logps/chosen": -135.5824432373047, "logps/ref_chosen": -68.88484954833984, "logps/ref_rejected": -75.8946304321289, "logps/rejected": -183.746337890625, "loss": 1.1119, "margin_dpo/margin_mean": 41.15412139892578, "margin_dpo/margin_std": 60.12267303466797, "step": 238 }, { "KL/chosen_KL_mean": -80.4689712524414, "KL/mean": -102.43445587158203, "KL/rejected_KL_mean": -124.39994049072266, "KL/std": 62.442771911621094, "epoch": 0.36130007558578986, "fcm_dpo/beta": 0.009715170599520206, "fcm_dpo/delta": -0.03162968158721924, "fcm_dpo/margin": 43.93098068237305, "fcm_dpo/q_t": 0.40267249941825867, "grad_norm": 18.650232315063477, "learning_rate": 4.045397465551513e-07, "logits/chosen": 0.7951195240020752, "logits/rejected": 0.6444146633148193, "logps/chosen": -137.2407989501953, "logps/ref_chosen": -56.771827697753906, "logps/ref_rejected": -116.23050689697266, "logps/rejected": -240.6304473876953, "loss": 1.1093, "margin_dpo/margin_mean": 43.93098449707031, "margin_dpo/margin_std": 67.45944213867188, "step": 239 }, { "KL/chosen_KL_mean": -78.93304443359375, "KL/mean": -105.90691375732422, "KL/rejected_KL_mean": -132.8807830810547, "KL/std": 62.66727828979492, "epoch": 0.36281179138321995, "fcm_dpo/beta": 0.009080219082534313, "fcm_dpo/delta": -0.10564997047185898, "fcm_dpo/margin": 53.94773483276367, "fcm_dpo/q_t": 0.38679730892181396, "grad_norm": 12.00704574584961, "learning_rate": 4.0349825555680045e-07, "logits/chosen": 0.68144690990448, "logits/rejected": 0.5815201997756958, "logps/chosen": -132.2871551513672, "logps/ref_chosen": -53.35411071777344, "logps/ref_rejected": -80.12019348144531, "logps/rejected": -213.0009765625, "loss": 1.0426, "margin_dpo/margin_mean": 53.947731018066406, "margin_dpo/margin_std": 67.94525909423828, "step": 240 }, { "KL/chosen_KL_mean": -74.51988983154297, "KL/mean": -92.31866455078125, "KL/rejected_KL_mean": -110.117431640625, "KL/std": 59.715797424316406, "epoch": 0.36432350718065004, "fcm_dpo/beta": 0.009323697537183762, "fcm_dpo/delta": 0.07455047965049744, "fcm_dpo/margin": 35.59754943847656, "fcm_dpo/q_t": 0.42614781856536865, "grad_norm": 17.535839080810547, "learning_rate": 4.0245247088227377e-07, "logits/chosen": 0.6315206289291382, "logits/rejected": 0.5956459641456604, "logps/chosen": -146.41529846191406, "logps/ref_chosen": -71.89541625976562, "logps/ref_rejected": -83.03492736816406, "logps/rejected": -193.15235900878906, "loss": 1.1761, "margin_dpo/margin_mean": 35.5975456237793, "margin_dpo/margin_std": 67.8523941040039, "step": 241 }, { "KL/chosen_KL_mean": -76.13401794433594, "KL/mean": -98.4712905883789, "KL/rejected_KL_mean": -120.80856323242188, "KL/std": 61.277618408203125, "epoch": 0.36583522297808013, "fcm_dpo/beta": 0.009163142181932926, "fcm_dpo/delta": -0.012581542134284973, "fcm_dpo/margin": 44.67455291748047, "fcm_dpo/q_t": 0.40708714723587036, "grad_norm": 11.770535469055176, "learning_rate": 4.0140242178441665e-07, "logits/chosen": 0.6444741487503052, "logits/rejected": 0.624873161315918, "logps/chosen": -134.0614471435547, "logps/ref_chosen": -57.927433013916016, "logps/ref_rejected": -67.838623046875, "logps/rejected": -188.64718627929688, "loss": 1.1051, "margin_dpo/margin_mean": 44.67455291748047, "margin_dpo/margin_std": 65.82492065429688, "step": 242 }, { "KL/chosen_KL_mean": -75.29219055175781, "KL/mean": -95.34475708007812, "KL/rejected_KL_mean": -115.39732360839844, "KL/std": 60.416358947753906, "epoch": 0.3673469387755102, "fcm_dpo/beta": 0.009415372274816036, "fcm_dpo/delta": 0.025139713659882545, "fcm_dpo/margin": 40.10512924194336, "fcm_dpo/q_t": 0.41358304023742676, "grad_norm": 16.480270385742188, "learning_rate": 4.003481376353596e-07, "logits/chosen": 0.6188076734542847, "logits/rejected": 0.6329231262207031, "logps/chosen": -149.56887817382812, "logps/ref_chosen": -74.27667236328125, "logps/ref_rejected": -73.24340057373047, "logps/rejected": -188.64071655273438, "loss": 1.1272, "margin_dpo/margin_mean": 40.105133056640625, "margin_dpo/margin_std": 62.97103500366211, "step": 243 }, { "KL/chosen_KL_mean": -68.44050598144531, "KL/mean": -94.90727996826172, "KL/rejected_KL_mean": -121.3740463256836, "KL/std": 60.30860900878906, "epoch": 0.3688586545729403, "fcm_dpo/beta": 0.009139430709183216, "fcm_dpo/delta": -0.09782852977514267, "fcm_dpo/margin": 52.93354797363281, "fcm_dpo/q_t": 0.38723719120025635, "grad_norm": 11.722265243530273, "learning_rate": 3.9928964792569654e-07, "logits/chosen": 0.6751892566680908, "logits/rejected": 0.586534321308136, "logps/chosen": -121.80440521240234, "logps/ref_chosen": -53.36390686035156, "logps/ref_rejected": -71.10276794433594, "logps/rejected": -192.47682189941406, "loss": 1.0407, "margin_dpo/margin_mean": 52.93354797363281, "margin_dpo/margin_std": 63.211524963378906, "step": 244 }, { "KL/chosen_KL_mean": -77.30024719238281, "KL/mean": -107.91534423828125, "KL/rejected_KL_mean": -138.53045654296875, "KL/std": 62.56925964355469, "epoch": 0.37037037037037035, "fcm_dpo/beta": 0.008342870511114597, "fcm_dpo/delta": -0.13193881511688232, "fcm_dpo/margin": 61.230194091796875, "fcm_dpo/q_t": 0.37750697135925293, "grad_norm": 13.34937572479248, "learning_rate": 3.982269822636601e-07, "logits/chosen": 0.7413665056228638, "logits/rejected": 0.7142044305801392, "logps/chosen": -148.49534606933594, "logps/ref_chosen": -71.19510650634766, "logps/ref_rejected": -80.76235961914062, "logps/rejected": -219.29281616210938, "loss": 1.0067, "margin_dpo/margin_mean": 61.230194091796875, "margin_dpo/margin_std": 64.25137329101562, "step": 245 }, { "KL/chosen_KL_mean": -91.80043029785156, "KL/mean": -113.60960388183594, "KL/rejected_KL_mean": -135.4187774658203, "KL/std": 62.362632751464844, "epoch": 0.37188208616780044, "fcm_dpo/beta": 0.008444206789135933, "fcm_dpo/delta": 0.03405901789665222, "fcm_dpo/margin": 43.61833953857422, "fcm_dpo/q_t": 0.41692936420440674, "grad_norm": 12.155149459838867, "learning_rate": 3.971601703742932e-07, "logits/chosen": 0.7366214990615845, "logits/rejected": 0.6703383922576904, "logps/chosen": -163.42147827148438, "logps/ref_chosen": -71.62104797363281, "logps/ref_rejected": -94.03392028808594, "logps/rejected": -229.45269775390625, "loss": 1.1571, "margin_dpo/margin_mean": 43.61833953857422, "margin_dpo/margin_std": 78.9627914428711, "step": 246 }, { "KL/chosen_KL_mean": -94.80854034423828, "KL/mean": -109.75330352783203, "KL/rejected_KL_mean": -124.69806671142578, "KL/std": 63.64940643310547, "epoch": 0.37339380196523053, "fcm_dpo/beta": 0.008503669872879982, "fcm_dpo/delta": 0.047118451446294785, "fcm_dpo/margin": 29.8895206451416, "fcm_dpo/q_t": 0.4427517056465149, "grad_norm": 15.709403991699219, "learning_rate": 3.960892420986177e-07, "logits/chosen": 0.6784308552742004, "logits/rejected": 0.6688522100448608, "logps/chosen": -174.83108520507812, "logps/ref_chosen": -80.02254486083984, "logps/ref_rejected": -89.22705841064453, "logps/rejected": -213.9251251220703, "loss": 1.2434, "margin_dpo/margin_mean": 29.88951873779297, "margin_dpo/margin_std": 74.12858581542969, "step": 247 }, { "KL/chosen_KL_mean": -80.26335144042969, "KL/mean": -104.7964096069336, "KL/rejected_KL_mean": -129.3294677734375, "KL/std": 65.8143310546875, "epoch": 0.3749055177626606, "fcm_dpo/beta": 0.008583104237914085, "fcm_dpo/delta": -0.024302110075950623, "fcm_dpo/margin": 49.06610107421875, "fcm_dpo/q_t": 0.4045664370059967, "grad_norm": 16.8564395904541, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 0.6709645986557007, "logits/rejected": 0.7285152673721313, "logps/chosen": -145.6413116455078, "logps/ref_chosen": -65.37796020507812, "logps/ref_rejected": -61.365787506103516, "logps/rejected": -190.69525146484375, "loss": 1.1079, "margin_dpo/margin_mean": 49.06610107421875, "margin_dpo/margin_std": 75.50920104980469, "step": 248 }, { "KL/chosen_KL_mean": -101.28555297851562, "KL/mean": -109.88075256347656, "KL/rejected_KL_mean": -118.47596740722656, "KL/std": 61.5130500793457, "epoch": 0.3764172335600907, "fcm_dpo/beta": 0.008645986206829548, "fcm_dpo/delta": 0.05666162446141243, "fcm_dpo/margin": 17.190412521362305, "fcm_dpo/q_t": 0.4651719331741333, "grad_norm": 16.34800910949707, "learning_rate": 3.9393515632731094e-07, "logits/chosen": 0.6691962480545044, "logits/rejected": 0.7120274305343628, "logps/chosen": -175.8870086669922, "logps/ref_chosen": -74.60145568847656, "logps/ref_rejected": -63.79338455200195, "logps/rejected": -182.26934814453125, "loss": 1.3502, "margin_dpo/margin_mean": 17.190414428710938, "margin_dpo/margin_std": 75.83460998535156, "step": 249 }, { "KL/chosen_KL_mean": -88.96881103515625, "KL/mean": -113.87507629394531, "KL/rejected_KL_mean": -138.7813720703125, "KL/std": 69.089599609375, "epoch": 0.3779289493575208, "fcm_dpo/beta": 0.008802786469459534, "fcm_dpo/delta": -0.046542391180992126, "fcm_dpo/margin": 49.81255340576172, "fcm_dpo/q_t": 0.39933088421821594, "grad_norm": 14.993885040283203, "learning_rate": 3.9285205908608934e-07, "logits/chosen": 0.7950924634933472, "logits/rejected": 0.7543875575065613, "logps/chosen": -150.90701293945312, "logps/ref_chosen": -61.938209533691406, "logps/ref_rejected": -72.21602630615234, "logps/rejected": -210.9973907470703, "loss": 1.0931, "margin_dpo/margin_mean": 49.81255340576172, "margin_dpo/margin_std": 71.54025268554688, "step": 250 }, { "KL/chosen_KL_mean": -96.157470703125, "KL/mean": -113.1865234375, "KL/rejected_KL_mean": -130.21556091308594, "KL/std": 60.91024398803711, "epoch": 0.3794406651549509, "fcm_dpo/beta": 0.00891035981476307, "fcm_dpo/delta": 0.10659044981002808, "fcm_dpo/margin": 34.05808639526367, "fcm_dpo/q_t": 0.43218693137168884, "grad_norm": 23.815366744995117, "learning_rate": 3.9176496596569265e-07, "logits/chosen": 0.7540764808654785, "logits/rejected": 0.7111141681671143, "logps/chosen": -163.01441955566406, "logps/ref_chosen": -66.85694885253906, "logps/ref_rejected": -84.83396911621094, "logps/rejected": -215.04953002929688, "loss": 1.1989, "margin_dpo/margin_mean": 34.05808639526367, "margin_dpo/margin_std": 69.84614562988281, "step": 251 }, { "KL/chosen_KL_mean": -90.72431945800781, "KL/mean": -103.80723571777344, "KL/rejected_KL_mean": -116.89015197753906, "KL/std": 63.69288635253906, "epoch": 0.38095238095238093, "fcm_dpo/beta": 0.0090938750654459, "fcm_dpo/delta": -0.007932256907224655, "fcm_dpo/margin": 26.16583824157715, "fcm_dpo/q_t": 0.4449775218963623, "grad_norm": 20.581787109375, "learning_rate": 3.9067390737445254e-07, "logits/chosen": 0.6826932430267334, "logits/rejected": 0.6261128187179565, "logps/chosen": -146.9482421875, "logps/ref_chosen": -56.22393035888672, "logps/ref_rejected": -77.1136245727539, "logps/rejected": -194.0037841796875, "loss": 1.2792, "margin_dpo/margin_mean": 26.16583824157715, "margin_dpo/margin_std": 74.60096740722656, "step": 252 }, { "KL/chosen_KL_mean": -89.88412475585938, "KL/mean": -108.26229095458984, "KL/rejected_KL_mean": -126.64045715332031, "KL/std": 60.83412170410156, "epoch": 0.382464096749811, "fcm_dpo/beta": 0.008921799249947071, "fcm_dpo/delta": -0.027868332341313362, "fcm_dpo/margin": 36.7563362121582, "fcm_dpo/q_t": 0.4237293004989624, "grad_norm": 16.071880340576172, "learning_rate": 3.8957891383162304e-07, "logits/chosen": 0.7912516593933105, "logits/rejected": 0.7534704208374023, "logps/chosen": -142.09414672851562, "logps/ref_chosen": -52.21001434326172, "logps/ref_rejected": -58.75764846801758, "logps/rejected": -185.39810180664062, "loss": 1.1652, "margin_dpo/margin_mean": 36.7563362121582, "margin_dpo/margin_std": 63.523983001708984, "step": 253 }, { "KL/chosen_KL_mean": -98.28326416015625, "KL/mean": -117.90261840820312, "KL/rejected_KL_mean": -137.52197265625, "KL/std": 67.03953552246094, "epoch": 0.3839758125472411, "fcm_dpo/beta": 0.009032230824232101, "fcm_dpo/delta": 0.05078422278165817, "fcm_dpo/margin": 39.23871612548828, "fcm_dpo/q_t": 0.42154332995414734, "grad_norm": 13.182265281677246, "learning_rate": 3.884800159665276e-07, "logits/chosen": 0.70362389087677, "logits/rejected": 0.6483662128448486, "logps/chosen": -163.91958618164062, "logps/ref_chosen": -65.63632202148438, "logps/ref_rejected": -82.34425354003906, "logps/rejected": -219.86622619628906, "loss": 1.1509, "margin_dpo/margin_mean": 39.23871612548828, "margin_dpo/margin_std": 67.09379577636719, "step": 254 }, { "KL/chosen_KL_mean": -87.982666015625, "KL/mean": -110.94923400878906, "KL/rejected_KL_mean": -133.91580200195312, "KL/std": 64.83241271972656, "epoch": 0.3854875283446712, "fcm_dpo/beta": 0.009070008993148804, "fcm_dpo/delta": -0.019309692084789276, "fcm_dpo/margin": 45.93316650390625, "fcm_dpo/q_t": 0.4051594138145447, "grad_norm": 17.457691192626953, "learning_rate": 3.873772445177015e-07, "logits/chosen": 0.6465994119644165, "logits/rejected": 0.6145304441452026, "logps/chosen": -155.8937530517578, "logps/ref_chosen": -67.91108703613281, "logps/ref_rejected": -83.89114379882812, "logps/rejected": -217.8069610595703, "loss": 1.1121, "margin_dpo/margin_mean": 45.933170318603516, "margin_dpo/margin_std": 72.16119384765625, "step": 255 }, { "KL/chosen_KL_mean": -106.05748748779297, "KL/mean": -128.17782592773438, "KL/rejected_KL_mean": -150.2981719970703, "KL/std": 61.828369140625, "epoch": 0.3869992441421013, "fcm_dpo/beta": 0.009000243619084358, "fcm_dpo/delta": 0.0012618564069271088, "fcm_dpo/margin": 44.24068069458008, "fcm_dpo/q_t": 0.4116033911705017, "grad_norm": 15.866179466247559, "learning_rate": 3.862706303320329e-07, "logits/chosen": 0.6873655319213867, "logits/rejected": 0.6170613765716553, "logps/chosen": -169.55746459960938, "logps/ref_chosen": -63.49998474121094, "logps/ref_rejected": -90.77104187011719, "logps/rejected": -241.0692138671875, "loss": 1.1498, "margin_dpo/margin_mean": 44.240684509277344, "margin_dpo/margin_std": 80.80990600585938, "step": 256 }, { "KL/chosen_KL_mean": -99.76148986816406, "KL/mean": -128.57794189453125, "KL/rejected_KL_mean": -157.39439392089844, "KL/std": 69.79254913330078, "epoch": 0.3885109599395314, "fcm_dpo/beta": 0.00856287032365799, "fcm_dpo/delta": -0.11203688383102417, "fcm_dpo/margin": 57.63291931152344, "fcm_dpo/q_t": 0.38817721605300903, "grad_norm": 11.6730318069458, "learning_rate": 3.851602043638994e-07, "logits/chosen": 0.6632556915283203, "logits/rejected": 0.5904806852340698, "logps/chosen": -170.36215209960938, "logps/ref_chosen": -70.60064697265625, "logps/ref_rejected": -108.58313751220703, "logps/rejected": -265.9775390625, "loss": 1.0577, "margin_dpo/margin_mean": 57.63291931152344, "margin_dpo/margin_std": 79.96395111083984, "step": 257 }, { "KL/chosen_KL_mean": -102.39521026611328, "KL/mean": -123.94535827636719, "KL/rejected_KL_mean": -145.49549865722656, "KL/std": 61.29582214355469, "epoch": 0.3900226757369615, "fcm_dpo/beta": 0.008610175922513008, "fcm_dpo/delta": 0.03255663812160492, "fcm_dpo/margin": 43.10028839111328, "fcm_dpo/q_t": 0.4138616621494293, "grad_norm": 14.99991512298584, "learning_rate": 3.840459976743023e-07, "logits/chosen": 0.7334133386611938, "logits/rejected": 0.6779052019119263, "logps/chosen": -161.64938354492188, "logps/ref_chosen": -59.25416564941406, "logps/ref_rejected": -85.58709716796875, "logps/rejected": -231.0825958251953, "loss": 1.12, "margin_dpo/margin_mean": 43.10028839111328, "margin_dpo/margin_std": 63.61687469482422, "step": 258 }, { "KL/chosen_KL_mean": -90.74569702148438, "KL/mean": -121.82440185546875, "KL/rejected_KL_mean": -152.9031219482422, "KL/std": 67.67817687988281, "epoch": 0.3915343915343915, "fcm_dpo/beta": 0.008102967403829098, "fcm_dpo/delta": -0.12510867416858673, "fcm_dpo/margin": 62.157413482666016, "fcm_dpo/q_t": 0.3824307322502136, "grad_norm": 12.970193862915039, "learning_rate": 3.8292804142999796e-07, "logits/chosen": 0.6587529182434082, "logits/rejected": 0.549065113067627, "logps/chosen": -156.18057250976562, "logps/ref_chosen": -65.43487548828125, "logps/ref_rejected": -95.41731262207031, "logps/rejected": -248.3204345703125, "loss": 1.0524, "margin_dpo/margin_mean": 62.157413482666016, "margin_dpo/margin_std": 83.17296600341797, "step": 259 }, { "KL/chosen_KL_mean": -91.60452270507812, "KL/mean": -115.55876922607422, "KL/rejected_KL_mean": -139.5130157470703, "KL/std": 61.73158264160156, "epoch": 0.3930461073318216, "fcm_dpo/beta": 0.008017702028155327, "fcm_dpo/delta": 0.017222033813595772, "fcm_dpo/margin": 47.908504486083984, "fcm_dpo/q_t": 0.41276681423187256, "grad_norm": 18.296205520629883, "learning_rate": 3.818063669026256e-07, "logits/chosen": 0.7184348702430725, "logits/rejected": 0.6148696541786194, "logps/chosen": -140.69410705566406, "logps/ref_chosen": -49.08958435058594, "logps/ref_rejected": -79.01708221435547, "logps/rejected": -218.53009033203125, "loss": 1.1336, "margin_dpo/margin_mean": 47.90850067138672, "margin_dpo/margin_std": 78.28831481933594, "step": 260 }, { "KL/chosen_KL_mean": -91.9481201171875, "KL/mean": -112.35318756103516, "KL/rejected_KL_mean": -132.7582550048828, "KL/std": 60.71597671508789, "epoch": 0.3945578231292517, "fcm_dpo/beta": 0.008456382900476456, "fcm_dpo/delta": 0.05926769599318504, "fcm_dpo/margin": 40.810142517089844, "fcm_dpo/q_t": 0.42231160402297974, "grad_norm": 16.93952178955078, "learning_rate": 3.806810054678331e-07, "logits/chosen": 0.5660909414291382, "logits/rejected": 0.6089234352111816, "logps/chosen": -162.8205108642578, "logps/ref_chosen": -70.87239074707031, "logps/ref_rejected": -65.01522064208984, "logps/rejected": -197.77346801757812, "loss": 1.1557, "margin_dpo/margin_mean": 40.810142517089844, "margin_dpo/margin_std": 70.20431518554688, "step": 261 }, { "KL/chosen_KL_mean": -99.01957702636719, "KL/mean": -119.05585479736328, "KL/rejected_KL_mean": -139.09213256835938, "KL/std": 61.233192443847656, "epoch": 0.3960695389266818, "fcm_dpo/beta": 0.008709060028195381, "fcm_dpo/delta": 0.056011371314525604, "fcm_dpo/margin": 40.072566986083984, "fcm_dpo/q_t": 0.42116984724998474, "grad_norm": 18.52885627746582, "learning_rate": 3.7955198860439887e-07, "logits/chosen": 0.7286216020584106, "logits/rejected": 0.6594671607017517, "logps/chosen": -166.89019775390625, "logps/ref_chosen": -67.8706283569336, "logps/ref_rejected": -88.7205810546875, "logps/rejected": -227.81271362304688, "loss": 1.1519, "margin_dpo/margin_mean": 40.07257080078125, "margin_dpo/margin_std": 68.45884704589844, "step": 262 }, { "KL/chosen_KL_mean": -93.94923400878906, "KL/mean": -115.14928436279297, "KL/rejected_KL_mean": -136.34933471679688, "KL/std": 62.44976806640625, "epoch": 0.3975812547241119, "fcm_dpo/beta": 0.008855888620018959, "fcm_dpo/delta": 0.027387959882616997, "fcm_dpo/margin": 42.40009307861328, "fcm_dpo/q_t": 0.41403600573539734, "grad_norm": 16.228689193725586, "learning_rate": 3.784193478933516e-07, "logits/chosen": 0.6711243391036987, "logits/rejected": 0.5571412444114685, "logps/chosen": -149.14381408691406, "logps/ref_chosen": -55.194583892822266, "logps/ref_rejected": -80.54048156738281, "logps/rejected": -216.88983154296875, "loss": 1.1333, "margin_dpo/margin_mean": 42.40009307861328, "margin_dpo/margin_std": 68.54437255859375, "step": 263 }, { "KL/chosen_KL_mean": -95.62120819091797, "KL/mean": -118.658203125, "KL/rejected_KL_mean": -141.6951904296875, "KL/std": 69.25521087646484, "epoch": 0.39909297052154197, "fcm_dpo/beta": 0.00880684144794941, "fcm_dpo/delta": -0.0066429730504751205, "fcm_dpo/margin": 46.07396697998047, "fcm_dpo/q_t": 0.4083606004714966, "grad_norm": 14.93736743927002, "learning_rate": 3.7728311501708674e-07, "logits/chosen": 0.5669023990631104, "logits/rejected": 0.5313173532485962, "logps/chosen": -178.79190063476562, "logps/ref_chosen": -83.17068481445312, "logps/ref_rejected": -88.33625793457031, "logps/rejected": -230.0314483642578, "loss": 1.1267, "margin_dpo/margin_mean": 46.07396697998047, "margin_dpo/margin_std": 75.87529754638672, "step": 264 }, { "KL/chosen_KL_mean": -91.21208190917969, "KL/mean": -114.84857177734375, "KL/rejected_KL_mean": -138.48507690429688, "KL/std": 64.7674560546875, "epoch": 0.40060468631897206, "fcm_dpo/beta": 0.008796760812401772, "fcm_dpo/delta": -0.018197163939476013, "fcm_dpo/margin": 47.27299499511719, "fcm_dpo/q_t": 0.40623968839645386, "grad_norm": 12.336769104003906, "learning_rate": 3.7614332175848027e-07, "logits/chosen": 0.769055962562561, "logits/rejected": 0.7016372680664062, "logps/chosen": -142.87493896484375, "logps/ref_chosen": -51.66284942626953, "logps/ref_rejected": -67.1720962524414, "logps/rejected": -205.65716552734375, "loss": 1.138, "margin_dpo/margin_mean": 47.27299499511719, "margin_dpo/margin_std": 82.93624114990234, "step": 265 }, { "KL/chosen_KL_mean": -86.67529296875, "KL/mean": -110.70747375488281, "KL/rejected_KL_mean": -134.73963928222656, "KL/std": 65.62783813476562, "epoch": 0.4021164021164021, "fcm_dpo/beta": 0.008618027903139591, "fcm_dpo/delta": -0.016524948179721832, "fcm_dpo/margin": 48.064353942871094, "fcm_dpo/q_t": 0.4059688448905945, "grad_norm": 22.678979873657227, "learning_rate": 3.75e-07, "logits/chosen": 0.6709597110748291, "logits/rejected": 0.5944674015045166, "logps/chosen": -144.12579345703125, "logps/ref_chosen": -57.45049285888672, "logps/ref_rejected": -77.60826110839844, "logps/rejected": -212.347900390625, "loss": 1.1195, "margin_dpo/margin_mean": 48.064353942871094, "margin_dpo/margin_std": 77.88729858398438, "step": 266 }, { "KL/chosen_KL_mean": -79.54592895507812, "KL/mean": -98.55870056152344, "KL/rejected_KL_mean": -117.57147216796875, "KL/std": 63.595970153808594, "epoch": 0.4036281179138322, "fcm_dpo/beta": 0.008466208353638649, "fcm_dpo/delta": -0.06876867264509201, "fcm_dpo/margin": 38.025535583496094, "fcm_dpo/q_t": 0.42545759677886963, "grad_norm": 15.666740417480469, "learning_rate": 3.738531817228131e-07, "logits/chosen": 0.7178699970245361, "logits/rejected": 0.6993035078048706, "logps/chosen": -134.58128356933594, "logps/ref_chosen": -55.03535079956055, "logps/ref_rejected": -66.0953369140625, "logps/rejected": -183.66680908203125, "loss": 1.2011, "margin_dpo/margin_mean": 38.02553939819336, "margin_dpo/margin_std": 74.72657775878906, "step": 267 }, { "KL/chosen_KL_mean": -76.53323364257812, "KL/mean": -100.77192687988281, "KL/rejected_KL_mean": -125.0106201171875, "KL/std": 66.77596282958984, "epoch": 0.4051398337112623, "fcm_dpo/beta": 0.008353805169463158, "fcm_dpo/delta": -0.0063354093581438065, "fcm_dpo/margin": 48.47738265991211, "fcm_dpo/q_t": 0.40715789794921875, "grad_norm": 15.87929916381836, "learning_rate": 3.7270289900589204e-07, "logits/chosen": 0.5625072717666626, "logits/rejected": 0.5523948669433594, "logps/chosen": -141.60498046875, "logps/ref_chosen": -65.07174682617188, "logps/ref_rejected": -71.42485809326172, "logps/rejected": -196.43548583984375, "loss": 1.0939, "margin_dpo/margin_mean": 48.477378845214844, "margin_dpo/margin_std": 65.1787109375, "step": 268 }, { "KL/chosen_KL_mean": -91.21495056152344, "KL/mean": -115.9734115600586, "KL/rejected_KL_mean": -140.7318878173828, "KL/std": 69.93099212646484, "epoch": 0.40665154950869237, "fcm_dpo/beta": 0.008321639150381088, "fcm_dpo/delta": -0.014926902949810028, "fcm_dpo/margin": 49.516929626464844, "fcm_dpo/q_t": 0.4059554636478424, "grad_norm": 14.74932861328125, "learning_rate": 3.7154918402511714e-07, "logits/chosen": 0.7675411701202393, "logits/rejected": 0.7207101583480835, "logps/chosen": -158.35116577148438, "logps/ref_chosen": -67.1362075805664, "logps/ref_rejected": -82.55778503417969, "logps/rejected": -223.2896728515625, "loss": 1.0967, "margin_dpo/margin_mean": 49.51692199707031, "margin_dpo/margin_std": 69.16948699951172, "step": 269 }, { "KL/chosen_KL_mean": -89.67121124267578, "KL/mean": -112.38426971435547, "KL/rejected_KL_mean": -135.09732055664062, "KL/std": 68.95745849609375, "epoch": 0.40816326530612246, "fcm_dpo/beta": 0.008279960602521896, "fcm_dpo/delta": 0.026922225952148438, "fcm_dpo/margin": 45.42612075805664, "fcm_dpo/q_t": 0.41440480947494507, "grad_norm": 16.14615821838379, "learning_rate": 3.7039206905237656e-07, "logits/chosen": 0.7438903450965881, "logits/rejected": 0.6575706005096436, "logps/chosen": -156.3599090576172, "logps/ref_chosen": -66.6886978149414, "logps/ref_rejected": -85.16129302978516, "logps/rejected": -220.2586212158203, "loss": 1.1353, "margin_dpo/margin_mean": 45.426116943359375, "margin_dpo/margin_std": 75.07006072998047, "step": 270 }, { "KL/chosen_KL_mean": -91.91019439697266, "KL/mean": -108.07470703125, "KL/rejected_KL_mean": -124.23921966552734, "KL/std": 74.5631103515625, "epoch": 0.40967498110355255, "fcm_dpo/beta": 0.00850964430719614, "fcm_dpo/delta": 0.0376083105802536, "fcm_dpo/margin": 32.32902145385742, "fcm_dpo/q_t": 0.4424757659435272, "grad_norm": 19.207263946533203, "learning_rate": 3.692315864546635e-07, "logits/chosen": 0.7038223743438721, "logits/rejected": 0.6382116079330444, "logps/chosen": -164.3177490234375, "logps/ref_chosen": -72.40754699707031, "logps/ref_rejected": -92.06311798095703, "logps/rejected": -216.30233764648438, "loss": 1.2645, "margin_dpo/margin_mean": 32.32902145385742, "margin_dpo/margin_std": 89.17990112304688, "step": 271 }, { "KL/chosen_KL_mean": -75.82728576660156, "KL/mean": -107.36683654785156, "KL/rejected_KL_mean": -138.9063720703125, "KL/std": 63.622032165527344, "epoch": 0.41118669690098264, "fcm_dpo/beta": 0.008081506937742233, "fcm_dpo/delta": -0.12915097177028656, "fcm_dpo/margin": 63.079078674316406, "fcm_dpo/q_t": 0.3778059780597687, "grad_norm": 15.495028495788574, "learning_rate": 3.6806776869317067e-07, "logits/chosen": 0.7020183801651001, "logits/rejected": 0.7239981293678284, "logps/chosen": -142.42868041992188, "logps/ref_chosen": -66.60140228271484, "logps/ref_rejected": -67.74340057373047, "logps/rejected": -206.64976501464844, "loss": 0.9983, "margin_dpo/margin_mean": 63.079078674316406, "margin_dpo/margin_std": 61.04596710205078, "step": 272 }, { "KL/chosen_KL_mean": -97.05889892578125, "KL/mean": -120.1033706665039, "KL/rejected_KL_mean": -143.1478271484375, "KL/std": 70.52786254882812, "epoch": 0.4126984126984127, "fcm_dpo/beta": 0.007941139861941338, "fcm_dpo/delta": 0.03800379857420921, "fcm_dpo/margin": 46.088951110839844, "fcm_dpo/q_t": 0.4187663197517395, "grad_norm": 19.712482452392578, "learning_rate": 3.669006483223828e-07, "logits/chosen": 0.722759485244751, "logits/rejected": 0.645821750164032, "logps/chosen": -154.41378784179688, "logps/ref_chosen": -57.35487747192383, "logps/ref_rejected": -84.17168426513672, "logps/rejected": -227.31951904296875, "loss": 1.1726, "margin_dpo/margin_mean": 46.088951110839844, "margin_dpo/margin_std": 88.80404663085938, "step": 273 }, { "KL/chosen_KL_mean": -95.72166442871094, "KL/mean": -120.80400085449219, "KL/rejected_KL_mean": -145.88633728027344, "KL/std": 67.72232055664062, "epoch": 0.41421012849584277, "fcm_dpo/beta": 0.008074736222624779, "fcm_dpo/delta": -0.005904029123485088, "fcm_dpo/margin": 50.16468048095703, "fcm_dpo/q_t": 0.4069592356681824, "grad_norm": 13.46408748626709, "learning_rate": 3.657302579891656e-07, "logits/chosen": 0.6265521049499512, "logits/rejected": 0.609045147895813, "logps/chosen": -155.3631591796875, "logps/ref_chosen": -59.64149475097656, "logps/ref_rejected": -68.29348754882812, "logps/rejected": -214.17982482910156, "loss": 1.12, "margin_dpo/margin_mean": 50.16468048095703, "margin_dpo/margin_std": 79.97119140625, "step": 274 }, { "KL/chosen_KL_mean": -93.24505615234375, "KL/mean": -117.6572265625, "KL/rejected_KL_mean": -142.06939697265625, "KL/std": 64.98899841308594, "epoch": 0.41572184429327286, "fcm_dpo/beta": 0.008090265095233917, "fcm_dpo/delta": 0.005442675203084946, "fcm_dpo/margin": 48.82433319091797, "fcm_dpo/q_t": 0.40933775901794434, "grad_norm": 17.434755325317383, "learning_rate": 3.645566304318526e-07, "logits/chosen": 0.692554235458374, "logits/rejected": 0.6077108383178711, "logps/chosen": -146.51170349121094, "logps/ref_chosen": -53.26664352416992, "logps/ref_rejected": -73.84062194824219, "logps/rejected": -215.91000366210938, "loss": 1.0988, "margin_dpo/margin_mean": 48.82433319091797, "margin_dpo/margin_std": 66.89923858642578, "step": 275 }, { "KL/chosen_KL_mean": -86.27903747558594, "KL/mean": -110.57057189941406, "KL/rejected_KL_mean": -134.8621063232422, "KL/std": 60.111572265625, "epoch": 0.41723356009070295, "fcm_dpo/beta": 0.00810546800494194, "fcm_dpo/delta": 0.006946409121155739, "fcm_dpo/margin": 48.58306121826172, "fcm_dpo/q_t": 0.4088294506072998, "grad_norm": 17.899261474609375, "learning_rate": 3.633797984793294e-07, "logits/chosen": 0.6355469226837158, "logits/rejected": 0.6031860113143921, "logps/chosen": -139.29983520507812, "logps/ref_chosen": -53.02079772949219, "logps/ref_rejected": -61.56678771972656, "logps/rejected": -196.42889404296875, "loss": 1.1036, "margin_dpo/margin_mean": 48.58306121826172, "margin_dpo/margin_std": 68.3792724609375, "step": 276 }, { "KL/chosen_KL_mean": -100.11457824707031, "KL/mean": -111.95452880859375, "KL/rejected_KL_mean": -123.79446411132812, "KL/std": 69.6145248413086, "epoch": 0.41874527588813304, "fcm_dpo/beta": 0.008502400480210781, "fcm_dpo/delta": 0.08324266970157623, "fcm_dpo/margin": 23.679885864257812, "fcm_dpo/q_t": 0.4552352726459503, "grad_norm": 21.53533935546875, "learning_rate": 3.6219979505011555e-07, "logits/chosen": 0.743852972984314, "logits/rejected": 0.7702978253364563, "logps/chosen": -171.54757690429688, "logps/ref_chosen": -71.43299102783203, "logps/ref_rejected": -67.65852355957031, "logps/rejected": -191.4530029296875, "loss": 1.3012, "margin_dpo/margin_mean": 23.679885864257812, "margin_dpo/margin_std": 78.18290710449219, "step": 277 }, { "KL/chosen_KL_mean": -106.93940734863281, "KL/mean": -130.02792358398438, "KL/rejected_KL_mean": -153.11642456054688, "KL/std": 70.84649658203125, "epoch": 0.42025699168556313, "fcm_dpo/beta": 0.0086892768740654, "fcm_dpo/delta": -0.005315154790878296, "fcm_dpo/margin": 46.17700958251953, "fcm_dpo/q_t": 0.4069432318210602, "grad_norm": 23.12691307067871, "learning_rate": 3.6101665315144353e-07, "logits/chosen": 0.6467447876930237, "logits/rejected": 0.5822904109954834, "logps/chosen": -174.0501708984375, "logps/ref_chosen": -67.11076354980469, "logps/ref_rejected": -88.74851989746094, "logps/rejected": -241.8649444580078, "loss": 1.1301, "margin_dpo/margin_mean": 46.17700958251953, "margin_dpo/margin_std": 73.78169250488281, "step": 278 }, { "KL/chosen_KL_mean": -83.75779724121094, "KL/mean": -115.92961883544922, "KL/rejected_KL_mean": -148.10145568847656, "KL/std": 68.06074523925781, "epoch": 0.4217687074829932, "fcm_dpo/beta": 0.008041076362133026, "fcm_dpo/delta": -0.1383289098739624, "fcm_dpo/margin": 64.34365844726562, "fcm_dpo/q_t": 0.376503050327301, "grad_norm": 16.200817108154297, "learning_rate": 3.5983040587833563e-07, "logits/chosen": 0.6431260108947754, "logits/rejected": 0.6045100092887878, "logps/chosen": -138.25527954101562, "logps/ref_chosen": -54.49748611450195, "logps/ref_rejected": -70.42373657226562, "logps/rejected": -218.52517700195312, "loss": 1.0068, "margin_dpo/margin_mean": 64.34365844726562, "margin_dpo/margin_std": 67.10409545898438, "step": 279 }, { "KL/chosen_KL_mean": -89.94000244140625, "KL/mean": -123.85331726074219, "KL/rejected_KL_mean": -157.76663208007812, "KL/std": 71.78616333007812, "epoch": 0.42328042328042326, "fcm_dpo/beta": 0.007400077767670155, "fcm_dpo/delta": -0.12002548575401306, "fcm_dpo/margin": 67.82662963867188, "fcm_dpo/q_t": 0.3807293772697449, "grad_norm": 11.438913345336914, "learning_rate": 3.586410864126781e-07, "logits/chosen": 0.7191169261932373, "logits/rejected": 0.6715154647827148, "logps/chosen": -150.37281799316406, "logps/ref_chosen": -60.43281173706055, "logps/ref_rejected": -78.39051818847656, "logps/rejected": -236.15716552734375, "loss": 1.0056, "margin_dpo/margin_mean": 67.82662963867188, "margin_dpo/margin_std": 68.43364715576172, "step": 280 }, { "KL/chosen_KL_mean": -92.57947540283203, "KL/mean": -119.57850646972656, "KL/rejected_KL_mean": -146.57754516601562, "KL/std": 70.05896759033203, "epoch": 0.42479213907785335, "fcm_dpo/beta": 0.007345996797084808, "fcm_dpo/delta": 0.0029397867619991302, "fcm_dpo/margin": 53.998077392578125, "fcm_dpo/q_t": 0.4082247316837311, "grad_norm": 12.51746940612793, "learning_rate": 3.574487280222929e-07, "logits/chosen": 0.6676753163337708, "logits/rejected": 0.6956182718276978, "logps/chosen": -152.861572265625, "logps/ref_chosen": -60.2820930480957, "logps/ref_rejected": -62.04009246826172, "logps/rejected": -208.61764526367188, "loss": 1.1074, "margin_dpo/margin_mean": 53.998077392578125, "margin_dpo/margin_std": 77.513427734375, "step": 281 }, { "KL/chosen_KL_mean": -101.51087951660156, "KL/mean": -125.53376770019531, "KL/rejected_KL_mean": -149.55665588378906, "KL/std": 70.40267944335938, "epoch": 0.42630385487528344, "fcm_dpo/beta": 0.007150539197027683, "fcm_dpo/delta": -0.056217122822999954, "fcm_dpo/margin": 48.0457763671875, "fcm_dpo/q_t": 0.4206792414188385, "grad_norm": 17.20449447631836, "learning_rate": 3.562533640600075e-07, "logits/chosen": 0.5914024114608765, "logits/rejected": 0.549197256565094, "logps/chosen": -162.13479614257812, "logps/ref_chosen": -60.623924255371094, "logps/ref_rejected": -68.67400360107422, "logps/rejected": -218.23065185546875, "loss": 1.1632, "margin_dpo/margin_mean": 48.045772552490234, "margin_dpo/margin_std": 82.70968627929688, "step": 282 }, { "KL/chosen_KL_mean": -98.51730346679688, "KL/mean": -124.37229919433594, "KL/rejected_KL_mean": -150.22727966308594, "KL/std": 74.53094482421875, "epoch": 0.42781557067271353, "fcm_dpo/beta": 0.007138520944863558, "fcm_dpo/delta": 0.034711360931396484, "fcm_dpo/margin": 51.70996856689453, "fcm_dpo/q_t": 0.416533887386322, "grad_norm": 13.802699089050293, "learning_rate": 3.550550279627215e-07, "logits/chosen": 0.6436029672622681, "logits/rejected": 0.5408251881599426, "logps/chosen": -166.16505432128906, "logps/ref_chosen": -67.64775085449219, "logps/ref_rejected": -99.96835327148438, "logps/rejected": -250.19564819335938, "loss": 1.1355, "margin_dpo/margin_mean": 51.70996856689453, "margin_dpo/margin_std": 84.02940368652344, "step": 283 }, { "KL/chosen_KL_mean": -94.60420227050781, "KL/mean": -119.55040740966797, "KL/rejected_KL_mean": -144.49659729003906, "KL/std": 73.55546569824219, "epoch": 0.4293272864701436, "fcm_dpo/beta": 0.007238998077809811, "fcm_dpo/delta": 0.04319431632757187, "fcm_dpo/margin": 49.892398834228516, "fcm_dpo/q_t": 0.41724467277526855, "grad_norm": 12.608574867248535, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 0.7289709448814392, "logits/rejected": 0.6605273485183716, "logps/chosen": -151.57162475585938, "logps/ref_chosen": -56.96742630004883, "logps/ref_rejected": -86.36236572265625, "logps/rejected": -230.85897827148438, "loss": 1.1231, "margin_dpo/margin_mean": 49.892398834228516, "margin_dpo/margin_std": 71.17445373535156, "step": 284 }, { "KL/chosen_KL_mean": -108.02169799804688, "KL/mean": -129.18577575683594, "KL/rejected_KL_mean": -150.34986877441406, "KL/std": 66.75422668457031, "epoch": 0.4308390022675737, "fcm_dpo/beta": 0.007741519715636969, "fcm_dpo/delta": 0.07731801271438599, "fcm_dpo/margin": 42.32817077636719, "fcm_dpo/q_t": 0.4266880750656128, "grad_norm": 18.31220245361328, "learning_rate": 3.5264957352549375e-07, "logits/chosen": 0.6713681221008301, "logits/rejected": 0.6448211669921875, "logps/chosen": -179.67779541015625, "logps/ref_chosen": -71.65611267089844, "logps/ref_rejected": -81.63829803466797, "logps/rejected": -231.9881591796875, "loss": 1.1678, "margin_dpo/margin_mean": 42.32816696166992, "margin_dpo/margin_std": 75.15558624267578, "step": 285 }, { "KL/chosen_KL_mean": -103.962890625, "KL/mean": -138.55560302734375, "KL/rejected_KL_mean": -173.14833068847656, "KL/std": 73.11457824707031, "epoch": 0.4323507180650038, "fcm_dpo/beta": 0.007291554007679224, "fcm_dpo/delta": -0.12348268926143646, "fcm_dpo/margin": 69.1854476928711, "fcm_dpo/q_t": 0.38189804553985596, "grad_norm": 12.251094818115234, "learning_rate": 3.514425224712835e-07, "logits/chosen": 0.6184149384498596, "logits/rejected": 0.5270093083381653, "logps/chosen": -165.04241943359375, "logps/ref_chosen": -61.07952117919922, "logps/ref_rejected": -91.28128051757812, "logps/rejected": -264.42962646484375, "loss": 1.0169, "margin_dpo/margin_mean": 69.1854476928711, "margin_dpo/margin_std": 77.75883483886719, "step": 286 }, { "KL/chosen_KL_mean": -86.13887023925781, "KL/mean": -118.71305847167969, "KL/rejected_KL_mean": -151.28726196289062, "KL/std": 72.01513671875, "epoch": 0.43386243386243384, "fcm_dpo/beta": 0.006978640798479319, "fcm_dpo/delta": -0.06320708990097046, "fcm_dpo/margin": 65.14839172363281, "fcm_dpo/q_t": 0.3935144543647766, "grad_norm": 11.138228416442871, "learning_rate": 3.502326338516534e-07, "logits/chosen": 0.7138686180114746, "logits/rejected": 0.6729492545127869, "logps/chosen": -132.17465209960938, "logps/ref_chosen": -46.035789489746094, "logps/ref_rejected": -59.95293426513672, "logps/rejected": -211.24020385742188, "loss": 1.0475, "margin_dpo/margin_mean": 65.14839172363281, "margin_dpo/margin_std": 75.49003601074219, "step": 287 }, { "KL/chosen_KL_mean": -108.60914611816406, "KL/mean": -133.51132202148438, "KL/rejected_KL_mean": -158.41348266601562, "KL/std": 69.94869232177734, "epoch": 0.43537414965986393, "fcm_dpo/beta": 0.007091499399393797, "fcm_dpo/delta": 0.05196710675954819, "fcm_dpo/margin": 49.80432891845703, "fcm_dpo/q_t": 0.4193596839904785, "grad_norm": 13.565356254577637, "learning_rate": 3.490199415097892e-07, "logits/chosen": 0.5789628028869629, "logits/rejected": 0.5190857648849487, "logps/chosen": -174.0, "logps/ref_chosen": -65.3908462524414, "logps/ref_rejected": -88.53607940673828, "logps/rejected": -246.94955444335938, "loss": 1.1339, "margin_dpo/margin_mean": 49.80432891845703, "margin_dpo/margin_std": 76.44319915771484, "step": 288 }, { "KL/chosen_KL_mean": -109.88246154785156, "KL/mean": -133.86720275878906, "KL/rejected_KL_mean": -157.85194396972656, "KL/std": 71.7451171875, "epoch": 0.436885865457294, "fcm_dpo/beta": 0.0073424819856882095, "fcm_dpo/delta": 0.05256052315235138, "fcm_dpo/margin": 47.9694938659668, "fcm_dpo/q_t": 0.42131608724594116, "grad_norm": 11.81661319732666, "learning_rate": 3.4780447936730247e-07, "logits/chosen": 0.7805777788162231, "logits/rejected": 0.7439557313919067, "logps/chosen": -164.47608947753906, "logps/ref_chosen": -54.5936279296875, "logps/ref_rejected": -67.20855712890625, "logps/rejected": -225.0605010986328, "loss": 1.1489, "margin_dpo/margin_mean": 47.9694938659668, "margin_dpo/margin_std": 80.595703125, "step": 289 }, { "KL/chosen_KL_mean": -122.74056243896484, "KL/mean": -149.30039978027344, "KL/rejected_KL_mean": -175.86026000976562, "KL/std": 70.5626220703125, "epoch": 0.4383975812547241, "fcm_dpo/beta": 0.007285839878022671, "fcm_dpo/delta": 0.01345156878232956, "fcm_dpo/margin": 53.11968994140625, "fcm_dpo/q_t": 0.4115050137042999, "grad_norm": 14.872110366821289, "learning_rate": 3.465862814232821e-07, "logits/chosen": 0.7720860242843628, "logits/rejected": 0.694909930229187, "logps/chosen": -184.12513732910156, "logps/ref_chosen": -61.38457489013672, "logps/ref_rejected": -91.92778015136719, "logps/rejected": -267.78802490234375, "loss": 1.1188, "margin_dpo/margin_mean": 53.11968994140625, "margin_dpo/margin_std": 80.01852416992188, "step": 290 }, { "KL/chosen_KL_mean": -111.4900894165039, "KL/mean": -144.89549255371094, "KL/rejected_KL_mean": -178.30091857910156, "KL/std": 73.21499633789062, "epoch": 0.4399092970521542, "fcm_dpo/beta": 0.007250026799738407, "fcm_dpo/delta": -0.10002140700817108, "fcm_dpo/margin": 66.81082153320312, "fcm_dpo/q_t": 0.3905301094055176, "grad_norm": 14.139022827148438, "learning_rate": 3.4536538175334343e-07, "logits/chosen": 0.802875280380249, "logits/rejected": 0.7305212020874023, "logps/chosen": -162.35311889648438, "logps/ref_chosen": -50.863037109375, "logps/ref_rejected": -82.20868682861328, "logps/rejected": -260.50958251953125, "loss": 1.0554, "margin_dpo/margin_mean": 66.8108139038086, "margin_dpo/margin_std": 86.83169555664062, "step": 291 }, { "KL/chosen_KL_mean": -114.57568359375, "KL/mean": -136.93133544921875, "KL/rejected_KL_mean": -159.2869873046875, "KL/std": 76.05390930175781, "epoch": 0.4414210128495843, "fcm_dpo/beta": 0.007219928316771984, "fcm_dpo/delta": 0.08566851168870926, "fcm_dpo/margin": 44.71131134033203, "fcm_dpo/q_t": 0.4291321933269501, "grad_norm": 16.09522247314453, "learning_rate": 3.4414181450867465e-07, "logits/chosen": 0.7329747676849365, "logits/rejected": 0.6827974319458008, "logps/chosen": -178.924560546875, "logps/ref_chosen": -64.34888458251953, "logps/ref_rejected": -72.86434173583984, "logps/rejected": -232.15133666992188, "loss": 1.195, "margin_dpo/margin_mean": 44.71131134033203, "margin_dpo/margin_std": 93.11201477050781, "step": 292 }, { "KL/chosen_KL_mean": -108.29093933105469, "KL/mean": -143.0504608154297, "KL/rejected_KL_mean": -177.80999755859375, "KL/std": 77.0187759399414, "epoch": 0.4429327286470144, "fcm_dpo/beta": 0.007083355449140072, "fcm_dpo/delta": -0.10814375430345535, "fcm_dpo/margin": 69.51905822753906, "fcm_dpo/q_t": 0.387387752532959, "grad_norm": 10.112757682800293, "learning_rate": 3.4291561391508185e-07, "logits/chosen": 0.827242374420166, "logits/rejected": 0.7373911142349243, "logps/chosen": -163.160400390625, "logps/ref_chosen": -54.869468688964844, "logps/ref_rejected": -81.858642578125, "logps/rejected": -259.66864013671875, "loss": 1.0591, "margin_dpo/margin_mean": 69.51905822753906, "margin_dpo/margin_std": 93.7623291015625, "step": 293 }, { "KL/chosen_KL_mean": -117.05465698242188, "KL/mean": -140.6841583251953, "KL/rejected_KL_mean": -164.31365966796875, "KL/std": 78.87942504882812, "epoch": 0.4444444444444444, "fcm_dpo/beta": 0.007031150162220001, "fcm_dpo/delta": 0.07512365281581879, "fcm_dpo/margin": 47.259010314941406, "fcm_dpo/q_t": 0.4250057339668274, "grad_norm": 11.902660369873047, "learning_rate": 3.4168681427203153e-07, "logits/chosen": 0.7192553877830505, "logits/rejected": 0.6725665330886841, "logps/chosen": -173.72555541992188, "logps/ref_chosen": -56.670902252197266, "logps/ref_rejected": -70.32819366455078, "logps/rejected": -234.64186096191406, "loss": 1.1411, "margin_dpo/margin_mean": 47.259010314941406, "margin_dpo/margin_std": 70.30636596679688, "step": 294 }, { "KL/chosen_KL_mean": -119.32664489746094, "KL/mean": -141.36685180664062, "KL/rejected_KL_mean": -163.40704345703125, "KL/std": 73.06025695800781, "epoch": 0.4459561602418745, "fcm_dpo/beta": 0.007442053407430649, "fcm_dpo/delta": 0.07993526011705399, "fcm_dpo/margin": 44.08039855957031, "fcm_dpo/q_t": 0.42684632539749146, "grad_norm": 16.47260856628418, "learning_rate": 3.4045544995169125e-07, "logits/chosen": 0.712509036064148, "logits/rejected": 0.6072986721992493, "logps/chosen": -169.7275390625, "logps/ref_chosen": -50.40088653564453, "logps/ref_rejected": -83.43521881103516, "logps/rejected": -246.84225463867188, "loss": 1.1656, "margin_dpo/margin_mean": 44.08039855957031, "margin_dpo/margin_std": 78.5947265625, "step": 295 }, { "KL/chosen_KL_mean": -124.35566711425781, "KL/mean": -153.10382080078125, "KL/rejected_KL_mean": -181.85195922851562, "KL/std": 80.76585388183594, "epoch": 0.4474678760393046, "fcm_dpo/beta": 0.00737424660474062, "fcm_dpo/delta": -0.02867615595459938, "fcm_dpo/margin": 57.496299743652344, "fcm_dpo/q_t": 0.4039427638053894, "grad_norm": 12.168935775756836, "learning_rate": 3.392215553979679e-07, "logits/chosen": 0.6344171762466431, "logits/rejected": 0.5863820910453796, "logps/chosen": -193.50601196289062, "logps/ref_chosen": -69.15034484863281, "logps/ref_rejected": -89.60166931152344, "logps/rejected": -271.45361328125, "loss": 1.1178, "margin_dpo/margin_mean": 57.496299743652344, "margin_dpo/margin_std": 93.3548355102539, "step": 296 }, { "KL/chosen_KL_mean": -126.96737670898438, "KL/mean": -155.42323303222656, "KL/rejected_KL_mean": -183.87908935546875, "KL/std": 70.00838470458984, "epoch": 0.4489795918367347, "fcm_dpo/beta": 0.0074002379551529884, "fcm_dpo/delta": -0.024462923407554626, "fcm_dpo/margin": 56.91170883178711, "fcm_dpo/q_t": 0.4019927382469177, "grad_norm": 12.931771278381348, "learning_rate": 3.3798516512554485e-07, "logits/chosen": 0.6887728571891785, "logits/rejected": 0.6372237801551819, "logps/chosen": -184.98367309570312, "logps/ref_chosen": -58.01630401611328, "logps/ref_rejected": -69.95780944824219, "logps/rejected": -253.8369140625, "loss": 1.0781, "margin_dpo/margin_mean": 56.911705017089844, "margin_dpo/margin_std": 72.6642074584961, "step": 297 }, { "KL/chosen_KL_mean": -124.369384765625, "KL/mean": -145.10752868652344, "KL/rejected_KL_mean": -165.845703125, "KL/std": 74.36654663085938, "epoch": 0.4504913076341648, "fcm_dpo/beta": 0.007596236653625965, "fcm_dpo/delta": 0.09406104683876038, "fcm_dpo/margin": 41.47632598876953, "fcm_dpo/q_t": 0.43052124977111816, "grad_norm": 13.027968406677246, "learning_rate": 3.367463137189156e-07, "logits/chosen": 0.8152356147766113, "logits/rejected": 0.7590856552124023, "logps/chosen": -180.5386962890625, "logps/ref_chosen": -56.1693115234375, "logps/ref_rejected": -68.55052185058594, "logps/rejected": -234.39620971679688, "loss": 1.1905, "margin_dpo/margin_mean": 41.47632598876953, "margin_dpo/margin_std": 82.53489685058594, "step": 298 }, { "KL/chosen_KL_mean": -128.62130737304688, "KL/mean": -147.03543090820312, "KL/rejected_KL_mean": -165.44956970214844, "KL/std": 71.73980712890625, "epoch": 0.4520030234315949, "fcm_dpo/beta": 0.00786143820732832, "fcm_dpo/delta": 0.026317991316318512, "fcm_dpo/margin": 36.82827377319336, "fcm_dpo/q_t": 0.43662887811660767, "grad_norm": 21.02098274230957, "learning_rate": 3.355050358314172e-07, "logits/chosen": 0.631700873374939, "logits/rejected": 0.5993541479110718, "logps/chosen": -190.93911743164062, "logps/ref_chosen": -62.31780242919922, "logps/ref_rejected": -72.60028839111328, "logps/rejected": -238.04986572265625, "loss": 1.2412, "margin_dpo/margin_mean": 36.828269958496094, "margin_dpo/margin_std": 91.29080963134766, "step": 299 }, { "KL/chosen_KL_mean": -127.76425170898438, "KL/mean": -151.45286560058594, "KL/rejected_KL_mean": -175.1414794921875, "KL/std": 74.65655517578125, "epoch": 0.45351473922902497, "fcm_dpo/beta": 0.007995443418622017, "fcm_dpo/delta": 0.023457201197743416, "fcm_dpo/margin": 47.377227783203125, "fcm_dpo/q_t": 0.4160943329334259, "grad_norm": 15.658246040344238, "learning_rate": 3.3426136618426043e-07, "logits/chosen": 0.7396022081375122, "logits/rejected": 0.6713778972625732, "logps/chosen": -188.1458282470703, "logps/ref_chosen": -60.38157653808594, "logps/ref_rejected": -75.45442199707031, "logps/rejected": -250.59591674804688, "loss": 1.152, "margin_dpo/margin_mean": 47.377227783203125, "margin_dpo/margin_std": 84.77381896972656, "step": 300 }, { "KL/chosen_KL_mean": -127.62648010253906, "KL/mean": -148.23974609375, "KL/rejected_KL_mean": -168.85302734375, "KL/std": 72.34912109375, "epoch": 0.455026455026455, "fcm_dpo/beta": 0.00814204104244709, "fcm_dpo/delta": 0.07148119807243347, "fcm_dpo/margin": 41.22654724121094, "fcm_dpo/q_t": 0.426349401473999, "grad_norm": 13.319229125976562, "learning_rate": 3.3301533956555885e-07, "logits/chosen": 0.7542002201080322, "logits/rejected": 0.7245285511016846, "logps/chosen": -180.47738647460938, "logps/ref_chosen": -52.85089111328125, "logps/ref_rejected": -69.97584533691406, "logps/rejected": -238.82888793945312, "loss": 1.1898, "margin_dpo/margin_mean": 41.22654724121094, "margin_dpo/margin_std": 84.22346496582031, "step": 301 }, { "KL/chosen_KL_mean": -134.59283447265625, "KL/mean": -150.45022583007812, "KL/rejected_KL_mean": -166.30764770507812, "KL/std": 76.41629791259766, "epoch": 0.4565381708238851, "fcm_dpo/beta": 0.00883575901389122, "fcm_dpo/delta": 0.1305047869682312, "fcm_dpo/margin": 31.714813232421875, "fcm_dpo/q_t": 0.4405342936515808, "grad_norm": 17.557220458984375, "learning_rate": 3.317669908293554e-07, "logits/chosen": 0.5931792259216309, "logits/rejected": 0.5361766815185547, "logps/chosen": -201.55934143066406, "logps/ref_chosen": -66.96650695800781, "logps/ref_rejected": -88.09510803222656, "logps/rejected": -254.4027557373047, "loss": 1.2561, "margin_dpo/margin_mean": 31.714813232421875, "margin_dpo/margin_std": 85.427001953125, "step": 302 }, { "KL/chosen_KL_mean": -116.82827758789062, "KL/mean": -148.2735137939453, "KL/rejected_KL_mean": -179.71875, "KL/std": 78.29273223876953, "epoch": 0.4580498866213152, "fcm_dpo/beta": 0.00839284434914589, "fcm_dpo/delta": -0.15248456597328186, "fcm_dpo/margin": 62.89048767089844, "fcm_dpo/q_t": 0.38311582803726196, "grad_norm": 13.024530410766602, "learning_rate": 3.3051635489464793e-07, "logits/chosen": 0.6601454019546509, "logits/rejected": 0.5881760120391846, "logps/chosen": -178.94979858398438, "logps/ref_chosen": -62.12152862548828, "logps/ref_rejected": -90.31204223632812, "logps/rejected": -270.0307922363281, "loss": 1.0848, "margin_dpo/margin_mean": 62.89048767089844, "margin_dpo/margin_std": 98.73794555664062, "step": 303 }, { "KL/chosen_KL_mean": -109.92771911621094, "KL/mean": -142.41702270507812, "KL/rejected_KL_mean": -174.90631103515625, "KL/std": 81.21237182617188, "epoch": 0.4595616024187453, "fcm_dpo/beta": 0.007727502845227718, "fcm_dpo/delta": -0.12161770462989807, "fcm_dpo/margin": 64.97858428955078, "fcm_dpo/q_t": 0.38245490193367004, "grad_norm": 12.52614688873291, "learning_rate": 3.292634667444117e-07, "logits/chosen": 0.6955829858779907, "logits/rejected": 0.6446535587310791, "logps/chosen": -170.622802734375, "logps/ref_chosen": -60.695091247558594, "logps/ref_rejected": -78.2525405883789, "logps/rejected": -253.1588592529297, "loss": 1.0246, "margin_dpo/margin_mean": 64.97859191894531, "margin_dpo/margin_std": 75.37732696533203, "step": 304 }, { "KL/chosen_KL_mean": -125.00523376464844, "KL/mean": -144.45281982421875, "KL/rejected_KL_mean": -163.90040588378906, "KL/std": 72.6624755859375, "epoch": 0.46107331821617537, "fcm_dpo/beta": 0.0076552326790988445, "fcm_dpo/delta": 0.004358620848506689, "fcm_dpo/margin": 38.89517593383789, "fcm_dpo/q_t": 0.43163472414016724, "grad_norm": 17.9072208404541, "learning_rate": 3.280083614246217e-07, "logits/chosen": 0.6535234451293945, "logits/rejected": 0.6887332797050476, "logps/chosen": -197.70437622070312, "logps/ref_chosen": -72.69914245605469, "logps/ref_rejected": -65.65670776367188, "logps/rejected": -229.55712890625, "loss": 1.2238, "margin_dpo/margin_mean": 38.89517593383789, "margin_dpo/margin_std": 89.01888275146484, "step": 305 }, { "KL/chosen_KL_mean": -114.75829315185547, "KL/mean": -139.0925750732422, "KL/rejected_KL_mean": -163.42686462402344, "KL/std": 70.30106353759766, "epoch": 0.46258503401360546, "fcm_dpo/beta": 0.007610576692968607, "fcm_dpo/delta": 0.031120628118515015, "fcm_dpo/margin": 48.66856384277344, "fcm_dpo/q_t": 0.4150772988796234, "grad_norm": 11.311169624328613, "learning_rate": 3.267510740432719e-07, "logits/chosen": 0.7671083211898804, "logits/rejected": 0.663329005241394, "logps/chosen": -168.72882080078125, "logps/ref_chosen": -53.97052764892578, "logps/ref_rejected": -71.02423095703125, "logps/rejected": -234.4510955810547, "loss": 1.123, "margin_dpo/margin_mean": 48.66856384277344, "margin_dpo/margin_std": 69.87991333007812, "step": 306 }, { "KL/chosen_KL_mean": -110.68402099609375, "KL/mean": -122.84514617919922, "KL/rejected_KL_mean": -135.00628662109375, "KL/std": 71.01148986816406, "epoch": 0.46409674981103555, "fcm_dpo/beta": 0.007799516897648573, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 24.322265625, "fcm_dpo/q_t": 0.45799070596694946, "grad_norm": 18.40856170654297, "learning_rate": 3.2549163976939285e-07, "logits/chosen": 0.7283176183700562, "logits/rejected": 0.6819779872894287, "logps/chosen": -168.09713745117188, "logps/ref_chosen": -57.413108825683594, "logps/ref_rejected": -68.68010711669922, "logps/rejected": -203.68638610839844, "loss": 1.327, "margin_dpo/margin_mean": 24.322263717651367, "margin_dpo/margin_std": 91.626708984375, "step": 307 }, { "KL/chosen_KL_mean": -111.70747375488281, "KL/mean": -135.7213592529297, "KL/rejected_KL_mean": -159.7352294921875, "KL/std": 73.46157836914062, "epoch": 0.4656084656084656, "fcm_dpo/beta": 0.007773532532155514, "fcm_dpo/delta": 0.02862391620874405, "fcm_dpo/margin": 48.02775573730469, "fcm_dpo/q_t": 0.41647428274154663, "grad_norm": 11.686368942260742, "learning_rate": 3.2423009383206874e-07, "logits/chosen": 0.688902735710144, "logits/rejected": 0.6754565834999084, "logps/chosen": -178.3062744140625, "logps/ref_chosen": -66.59879302978516, "logps/ref_rejected": -74.337158203125, "logps/rejected": -234.0723876953125, "loss": 1.1581, "margin_dpo/margin_mean": 48.02775573730469, "margin_dpo/margin_std": 87.01692199707031, "step": 308 }, { "KL/chosen_KL_mean": -125.97828674316406, "KL/mean": -151.03265380859375, "KL/rejected_KL_mean": -176.0870361328125, "KL/std": 71.87471771240234, "epoch": 0.4671201814058957, "fcm_dpo/beta": 0.007821241393685341, "fcm_dpo/delta": 0.007066238671541214, "fcm_dpo/margin": 50.10874557495117, "fcm_dpo/q_t": 0.4118044972419739, "grad_norm": 13.115415573120117, "learning_rate": 3.229664715194511e-07, "logits/chosen": 0.7336107492446899, "logits/rejected": 0.6807036399841309, "logps/chosen": -191.37303161621094, "logps/ref_chosen": -65.39474487304688, "logps/ref_rejected": -75.70930480957031, "logps/rejected": -251.7963409423828, "loss": 1.1404, "margin_dpo/margin_mean": 50.108741760253906, "margin_dpo/margin_std": 84.24324035644531, "step": 309 }, { "KL/chosen_KL_mean": -134.229248046875, "KL/mean": -147.745849609375, "KL/rejected_KL_mean": -161.26243591308594, "KL/std": 66.22117614746094, "epoch": 0.46863189720332576, "fcm_dpo/beta": 0.008301754482090473, "fcm_dpo/delta": 0.06831841170787811, "fcm_dpo/margin": 27.033180236816406, "fcm_dpo/q_t": 0.4524414539337158, "grad_norm": 15.16258716583252, "learning_rate": 3.2170080817777257e-07, "logits/chosen": 0.7237935066223145, "logits/rejected": 0.7082376480102539, "logps/chosen": -208.89752197265625, "logps/ref_chosen": -74.66827392578125, "logps/ref_rejected": -80.5689697265625, "logps/rejected": -241.83140563964844, "loss": 1.2791, "margin_dpo/margin_mean": 27.033180236816406, "margin_dpo/margin_std": 80.27223205566406, "step": 310 }, { "KL/chosen_KL_mean": -104.70181274414062, "KL/mean": -132.61630249023438, "KL/rejected_KL_mean": -160.53082275390625, "KL/std": 85.0822982788086, "epoch": 0.47014361300075586, "fcm_dpo/beta": 0.007963906042277813, "fcm_dpo/delta": -0.054659221321344376, "fcm_dpo/margin": 55.829002380371094, "fcm_dpo/q_t": 0.40370649099349976, "grad_norm": 13.94991683959961, "learning_rate": 3.204331392103574e-07, "logits/chosen": 0.6307729482650757, "logits/rejected": 0.4885410666465759, "logps/chosen": -164.43984985351562, "logps/ref_chosen": -59.738033294677734, "logps/ref_rejected": -93.60757446289062, "logps/rejected": -254.13839721679688, "loss": 1.1068, "margin_dpo/margin_mean": 55.829002380371094, "margin_dpo/margin_std": 91.33370971679688, "step": 311 }, { "KL/chosen_KL_mean": -108.8864974975586, "KL/mean": -138.23374938964844, "KL/rejected_KL_mean": -167.58099365234375, "KL/std": 72.34786987304688, "epoch": 0.47165532879818595, "fcm_dpo/beta": 0.007836501114070415, "fcm_dpo/delta": -0.06939233839511871, "fcm_dpo/margin": 58.694496154785156, "fcm_dpo/q_t": 0.3929941952228546, "grad_norm": 11.712054252624512, "learning_rate": 3.1916350007663176e-07, "logits/chosen": 0.7151386737823486, "logits/rejected": 0.6277899742126465, "logps/chosen": -162.70294189453125, "logps/ref_chosen": -53.816436767578125, "logps/ref_rejected": -68.6575698852539, "logps/rejected": -236.23855590820312, "loss": 1.0515, "margin_dpo/margin_mean": 58.694496154785156, "margin_dpo/margin_std": 71.33744812011719, "step": 312 }, { "KL/chosen_KL_mean": -114.5650634765625, "KL/mean": -130.61492919921875, "KL/rejected_KL_mean": -146.664794921875, "KL/std": 71.21220397949219, "epoch": 0.47316704459561604, "fcm_dpo/beta": 0.007850449532270432, "fcm_dpo/delta": 0.06122079864144325, "fcm_dpo/margin": 32.0997314453125, "fcm_dpo/q_t": 0.4438740611076355, "grad_norm": 13.880863189697266, "learning_rate": 3.178919262911314e-07, "logits/chosen": 0.7805558443069458, "logits/rejected": 0.7599529027938843, "logps/chosen": -174.52243041992188, "logps/ref_chosen": -59.957359313964844, "logps/ref_rejected": -69.31729888916016, "logps/rejected": -215.98208618164062, "loss": 1.2399, "margin_dpo/margin_mean": 32.0997314453125, "margin_dpo/margin_std": 78.52413177490234, "step": 313 }, { "KL/chosen_KL_mean": -107.243408203125, "KL/mean": -136.32138061523438, "KL/rejected_KL_mean": -165.3993682861328, "KL/std": 67.65750885009766, "epoch": 0.47467876039304613, "fcm_dpo/beta": 0.007725432515144348, "fcm_dpo/delta": -0.05801774933934212, "fcm_dpo/margin": 58.155967712402344, "fcm_dpo/q_t": 0.3972868025302887, "grad_norm": 12.526936531066895, "learning_rate": 3.166184534225087e-07, "logits/chosen": 0.6884793043136597, "logits/rejected": 0.7177518606185913, "logps/chosen": -177.51156616210938, "logps/ref_chosen": -70.26815795898438, "logps/ref_rejected": -69.23971557617188, "logps/rejected": -234.6390838623047, "loss": 1.0679, "margin_dpo/margin_mean": 58.15597152709961, "margin_dpo/margin_std": 75.71588897705078, "step": 314 }, { "KL/chosen_KL_mean": -113.2779541015625, "KL/mean": -135.77932739257812, "KL/rejected_KL_mean": -158.28070068359375, "KL/std": 69.8665771484375, "epoch": 0.47619047619047616, "fcm_dpo/beta": 0.007983740419149399, "fcm_dpo/delta": 0.04358711838722229, "fcm_dpo/margin": 45.00275802612305, "fcm_dpo/q_t": 0.41818535327911377, "grad_norm": 12.93481731414795, "learning_rate": 3.1534311709253723e-07, "logits/chosen": 0.6272811889648438, "logits/rejected": 0.5963453054428101, "logps/chosen": -181.07264709472656, "logps/ref_chosen": -67.79469299316406, "logps/ref_rejected": -74.55148315429688, "logps/rejected": -232.83218383789062, "loss": 1.1472, "margin_dpo/margin_mean": 45.00275802612305, "margin_dpo/margin_std": 75.59745025634766, "step": 315 }, { "KL/chosen_KL_mean": -105.72105407714844, "KL/mean": -138.439697265625, "KL/rejected_KL_mean": -171.1583251953125, "KL/std": 74.41236877441406, "epoch": 0.47770219198790626, "fcm_dpo/beta": 0.007744115777313709, "fcm_dpo/delta": -0.12915290892124176, "fcm_dpo/margin": 65.43728637695312, "fcm_dpo/q_t": 0.38332080841064453, "grad_norm": 13.909781455993652, "learning_rate": 3.1406595297511564e-07, "logits/chosen": 0.5902273058891296, "logits/rejected": 0.45409518480300903, "logps/chosen": -161.00953674316406, "logps/ref_chosen": -55.288482666015625, "logps/ref_rejected": -96.15723419189453, "logps/rejected": -267.3155517578125, "loss": 1.0384, "margin_dpo/margin_mean": 65.43728637695312, "margin_dpo/margin_std": 76.73016357421875, "step": 316 }, { "KL/chosen_KL_mean": -95.98619079589844, "KL/mean": -126.41956329345703, "KL/rejected_KL_mean": -156.85293579101562, "KL/std": 66.47119140625, "epoch": 0.47921390778533635, "fcm_dpo/beta": 0.007124145515263081, "fcm_dpo/delta": -0.04006510600447655, "fcm_dpo/margin": 60.866737365722656, "fcm_dpo/q_t": 0.3974044919013977, "grad_norm": 13.468562126159668, "learning_rate": 3.1278699679526975e-07, "logits/chosen": 0.715498685836792, "logits/rejected": 0.6702585816383362, "logps/chosen": -150.56756591796875, "logps/ref_chosen": -54.58137512207031, "logps/ref_rejected": -72.77232360839844, "logps/rejected": -229.62525939941406, "loss": 1.0662, "margin_dpo/margin_mean": 60.866737365722656, "margin_dpo/margin_std": 74.26116180419922, "step": 317 }, { "KL/chosen_KL_mean": -106.12240600585938, "KL/mean": -128.9796905517578, "KL/rejected_KL_mean": -151.8369598388672, "KL/std": 72.66458129882812, "epoch": 0.48072562358276644, "fcm_dpo/beta": 0.007349280174821615, "fcm_dpo/delta": 0.07138577103614807, "fcm_dpo/margin": 45.71455764770508, "fcm_dpo/q_t": 0.42840999364852905, "grad_norm": 13.374271392822266, "learning_rate": 3.1150628432815336e-07, "logits/chosen": 0.6954464912414551, "logits/rejected": 0.6276102662086487, "logps/chosen": -159.01065063476562, "logps/ref_chosen": -52.88822937011719, "logps/ref_rejected": -80.63988494873047, "logps/rejected": -232.4768524169922, "loss": 1.2033, "margin_dpo/margin_mean": 45.71455764770508, "margin_dpo/margin_std": 100.32778930664062, "step": 318 }, { "KL/chosen_KL_mean": -105.37348937988281, "KL/mean": -137.54539489746094, "KL/rejected_KL_mean": -169.71728515625, "KL/std": 77.01676177978516, "epoch": 0.48223733938019653, "fcm_dpo/beta": 0.007210618816316128, "fcm_dpo/delta": -0.07460720092058182, "fcm_dpo/margin": 64.34378814697266, "fcm_dpo/q_t": 0.3947829604148865, "grad_norm": 10.253146171569824, "learning_rate": 3.1022385139804707e-07, "logits/chosen": 0.6536087989807129, "logits/rejected": 0.6382144689559937, "logps/chosen": -169.73681640625, "logps/ref_chosen": -64.36333465576172, "logps/ref_rejected": -79.47296142578125, "logps/rejected": -249.19024658203125, "loss": 1.0683, "margin_dpo/margin_mean": 64.34378814697266, "margin_dpo/margin_std": 89.17861938476562, "step": 319 }, { "KL/chosen_KL_mean": -96.51083374023438, "KL/mean": -121.93170166015625, "KL/rejected_KL_mean": -147.35256958007812, "KL/std": 78.21331787109375, "epoch": 0.4837490551776266, "fcm_dpo/beta": 0.006960249040275812, "fcm_dpo/delta": -0.08929912000894547, "fcm_dpo/margin": 50.84172821044922, "fcm_dpo/q_t": 0.41984930634498596, "grad_norm": 11.877212524414062, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 0.6033248901367188, "logits/rejected": 0.5647040605545044, "logps/chosen": -146.069580078125, "logps/ref_chosen": -49.558746337890625, "logps/ref_rejected": -71.23444366455078, "logps/rejected": -218.58700561523438, "loss": 1.1531, "margin_dpo/margin_mean": 50.84172439575195, "margin_dpo/margin_std": 83.79927062988281, "step": 320 }, { "KL/chosen_KL_mean": -100.57034301757812, "KL/mean": -128.36460876464844, "KL/rejected_KL_mean": -156.15890502929688, "KL/std": 66.17784881591797, "epoch": 0.4852607709750567, "fcm_dpo/beta": 0.006733224727213383, "fcm_dpo/delta": 0.02710144966840744, "fcm_dpo/margin": 55.58854675292969, "fcm_dpo/q_t": 0.41282814741134644, "grad_norm": 15.027644157409668, "learning_rate": 3.0765396768561004e-07, "logits/chosen": 0.6975396275520325, "logits/rejected": 0.6782538890838623, "logps/chosen": -152.65560913085938, "logps/ref_chosen": -52.08526611328125, "logps/ref_rejected": -55.58674621582031, "logps/rejected": -211.74563598632812, "loss": 1.1195, "margin_dpo/margin_mean": 55.58854675292969, "margin_dpo/margin_std": 78.52020263671875, "step": 321 }, { "KL/chosen_KL_mean": -112.34437561035156, "KL/mean": -147.16134643554688, "KL/rejected_KL_mean": -181.97833251953125, "KL/std": 75.60289001464844, "epoch": 0.48677248677248675, "fcm_dpo/beta": 0.006711442023515701, "fcm_dpo/delta": -0.07818324863910675, "fcm_dpo/margin": 69.63396453857422, "fcm_dpo/q_t": 0.3887701630592346, "grad_norm": 10.670733451843262, "learning_rate": 3.063665887884511e-07, "logits/chosen": 0.7195281982421875, "logits/rejected": 0.6374897956848145, "logps/chosen": -159.74847412109375, "logps/ref_chosen": -47.404109954833984, "logps/ref_rejected": -73.4260025024414, "logps/rejected": -255.40432739257812, "loss": 1.031, "margin_dpo/margin_mean": 69.63395690917969, "margin_dpo/margin_std": 72.7001953125, "step": 322 }, { "KL/chosen_KL_mean": -116.77687072753906, "KL/mean": -138.69090270996094, "KL/rejected_KL_mean": -160.60494995117188, "KL/std": 72.23223876953125, "epoch": 0.48828420256991684, "fcm_dpo/beta": 0.006896693259477615, "fcm_dpo/delta": 0.10736851394176483, "fcm_dpo/margin": 43.82807159423828, "fcm_dpo/q_t": 0.4345787763595581, "grad_norm": 12.62841510772705, "learning_rate": 3.0507763319663517e-07, "logits/chosen": 0.6039708852767944, "logits/rejected": 0.5336369276046753, "logps/chosen": -186.78317260742188, "logps/ref_chosen": -70.00630187988281, "logps/ref_rejected": -86.96690368652344, "logps/rejected": -247.57183837890625, "loss": 1.2031, "margin_dpo/margin_mean": 43.82807159423828, "margin_dpo/margin_std": 92.93321228027344, "step": 323 }, { "KL/chosen_KL_mean": -102.77244567871094, "KL/mean": -132.74996948242188, "KL/rejected_KL_mean": -162.7274932861328, "KL/std": 84.26118469238281, "epoch": 0.4897959183673469, "fcm_dpo/beta": 0.006869921460747719, "fcm_dpo/delta": -0.014991983771324158, "fcm_dpo/margin": 59.95502853393555, "fcm_dpo/q_t": 0.4040718674659729, "grad_norm": 13.32012939453125, "learning_rate": 3.0378713696502097e-07, "logits/chosen": 0.7294822931289673, "logits/rejected": 0.6708847284317017, "logps/chosen": -158.66127014160156, "logps/ref_chosen": -55.88882064819336, "logps/ref_rejected": -75.23088073730469, "logps/rejected": -237.9583740234375, "loss": 1.0831, "margin_dpo/margin_mean": 59.95503234863281, "margin_dpo/margin_std": 75.76171875, "step": 324 }, { "KL/chosen_KL_mean": -124.10344696044922, "KL/mean": -154.2080078125, "KL/rejected_KL_mean": -184.31256103515625, "KL/std": 82.18153381347656, "epoch": 0.491307634164777, "fcm_dpo/beta": 0.006824733689427376, "fcm_dpo/delta": -0.013523155823349953, "fcm_dpo/margin": 60.20912170410156, "fcm_dpo/q_t": 0.4077095687389374, "grad_norm": 11.118515014648438, "learning_rate": 3.0249513619156206e-07, "logits/chosen": 0.6534192562103271, "logits/rejected": 0.5929680466651917, "logps/chosen": -188.25045776367188, "logps/ref_chosen": -64.14701843261719, "logps/ref_rejected": -79.91143798828125, "logps/rejected": -264.2239990234375, "loss": 1.1226, "margin_dpo/margin_mean": 60.20912170410156, "margin_dpo/margin_std": 100.43673706054688, "step": 325 }, { "KL/chosen_KL_mean": -145.35369873046875, "KL/mean": -156.6713409423828, "KL/rejected_KL_mean": -167.98898315429688, "KL/std": 74.04638671875, "epoch": 0.4928193499622071, "fcm_dpo/beta": 0.0068876477889716625, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.635299682617188, "fcm_dpo/q_t": 0.46425288915634155, "grad_norm": 12.687813758850098, "learning_rate": 3.012016670162977e-07, "logits/chosen": 0.6178878545761108, "logits/rejected": 0.6271042823791504, "logps/chosen": -220.885009765625, "logps/ref_chosen": -75.53131103515625, "logps/ref_rejected": -76.5898666381836, "logps/rejected": -244.578857421875, "loss": 1.3161, "margin_dpo/margin_mean": 22.635299682617188, "margin_dpo/margin_std": 83.47059631347656, "step": 326 }, { "KL/chosen_KL_mean": -128.65380859375, "KL/mean": -151.80908203125, "KL/rejected_KL_mean": -174.96434020996094, "KL/std": 78.66702270507812, "epoch": 0.4943310657596372, "fcm_dpo/beta": 0.007172505371272564, "fcm_dpo/delta": 0.07436676323413849, "fcm_dpo/margin": 46.310516357421875, "fcm_dpo/q_t": 0.425973117351532, "grad_norm": 12.866050720214844, "learning_rate": 2.99906765620341e-07, "logits/chosen": 0.5928927063941956, "logits/rejected": 0.5698734521865845, "logps/chosen": -197.990966796875, "logps/ref_chosen": -69.33717346191406, "logps/ref_rejected": -73.37751770019531, "logps/rejected": -248.34185791015625, "loss": 1.1804, "margin_dpo/margin_mean": 46.310516357421875, "margin_dpo/margin_std": 90.25209045410156, "step": 327 }, { "KL/chosen_KL_mean": -113.55990600585938, "KL/mean": -142.6298828125, "KL/rejected_KL_mean": -171.69985961914062, "KL/std": 80.41529846191406, "epoch": 0.4958427815570673, "fcm_dpo/beta": 0.007189217023551464, "fcm_dpo/delta": -0.02078922465443611, "fcm_dpo/margin": 58.13993453979492, "fcm_dpo/q_t": 0.4041920304298401, "grad_norm": 13.29574966430664, "learning_rate": 2.9861046822486766e-07, "logits/chosen": 0.5948680639266968, "logits/rejected": 0.5532065033912659, "logps/chosen": -175.26614379882812, "logps/ref_chosen": -61.70623016357422, "logps/ref_rejected": -83.73808288574219, "logps/rejected": -255.43792724609375, "loss": 1.0833, "margin_dpo/margin_mean": 58.13993835449219, "margin_dpo/margin_std": 76.74394226074219, "step": 328 }, { "KL/chosen_KL_mean": -127.44103240966797, "KL/mean": -156.460205078125, "KL/rejected_KL_mean": -185.47938537597656, "KL/std": 85.77304077148438, "epoch": 0.4973544973544973, "fcm_dpo/beta": 0.007048935629427433, "fcm_dpo/delta": -0.010564148426055908, "fcm_dpo/margin": 58.038368225097656, "fcm_dpo/q_t": 0.40908321738243103, "grad_norm": 14.72163200378418, "learning_rate": 2.9731281109010253e-07, "logits/chosen": 0.7163276672363281, "logits/rejected": 0.6566041111946106, "logps/chosen": -191.939453125, "logps/ref_chosen": -64.4984130859375, "logps/ref_rejected": -83.6591796875, "logps/rejected": -269.1385498046875, "loss": 1.1121, "margin_dpo/margin_mean": 58.038368225097656, "margin_dpo/margin_std": 90.64898681640625, "step": 329 }, { "KL/chosen_KL_mean": -111.15478515625, "KL/mean": -139.06466674804688, "KL/rejected_KL_mean": -166.97454833984375, "KL/std": 77.32144165039062, "epoch": 0.4988662131519274, "fcm_dpo/beta": 0.0071370261721313, "fcm_dpo/delta": 0.001220574602484703, "fcm_dpo/margin": 55.819766998291016, "fcm_dpo/q_t": 0.41049909591674805, "grad_norm": 12.301667213439941, "learning_rate": 2.9601383051430505e-07, "logits/chosen": 0.6728509664535522, "logits/rejected": 0.6075490713119507, "logps/chosen": -165.9594268798828, "logps/ref_chosen": -54.80464172363281, "logps/ref_rejected": -75.3194351196289, "logps/rejected": -242.2939910888672, "loss": 1.1461, "margin_dpo/margin_mean": 55.81976318359375, "margin_dpo/margin_std": 99.13479614257812, "step": 330 }, { "KL/chosen_KL_mean": -118.90031433105469, "KL/mean": -155.21522521972656, "KL/rejected_KL_mean": -191.53013610839844, "KL/std": 81.10183715820312, "epoch": 0.5003779289493575, "fcm_dpo/beta": 0.006745649501681328, "fcm_dpo/delta": -0.105572909116745, "fcm_dpo/margin": 72.62981414794922, "fcm_dpo/q_t": 0.38496410846710205, "grad_norm": 12.032770156860352, "learning_rate": 2.947135628327544e-07, "logits/chosen": 0.7782831192016602, "logits/rejected": 0.7540770769119263, "logps/chosen": -178.14288330078125, "logps/ref_chosen": -59.242584228515625, "logps/ref_rejected": -69.87483215332031, "logps/rejected": -261.40496826171875, "loss": 1.0371, "margin_dpo/margin_mean": 72.62981414794922, "margin_dpo/margin_std": 88.2421875, "step": 331 }, { "KL/chosen_KL_mean": -124.0113754272461, "KL/mean": -153.82049560546875, "KL/rejected_KL_mean": -183.62960815429688, "KL/std": 78.55375671386719, "epoch": 0.5018896447467877, "fcm_dpo/beta": 0.006745035294443369, "fcm_dpo/delta": -0.00457579642534256, "fcm_dpo/margin": 59.618255615234375, "fcm_dpo/q_t": 0.4085049033164978, "grad_norm": 11.804621696472168, "learning_rate": 2.934120444167326e-07, "logits/chosen": 0.6241295337677002, "logits/rejected": 0.5831087827682495, "logps/chosen": -191.12112426757812, "logps/ref_chosen": -67.10975646972656, "logps/ref_rejected": -77.11839294433594, "logps/rejected": -260.74798583984375, "loss": 1.1011, "margin_dpo/margin_mean": 59.618255615234375, "margin_dpo/margin_std": 83.41954803466797, "step": 332 }, { "KL/chosen_KL_mean": -129.91983032226562, "KL/mean": -164.92306518554688, "KL/rejected_KL_mean": -199.92630004882812, "KL/std": 81.36688232421875, "epoch": 0.5034013605442177, "fcm_dpo/beta": 0.0064663151279091835, "fcm_dpo/delta": -0.060843661427497864, "fcm_dpo/margin": 70.00645446777344, "fcm_dpo/q_t": 0.39520421624183655, "grad_norm": 10.76503849029541, "learning_rate": 2.921093116725076e-07, "logits/chosen": 0.6815329790115356, "logits/rejected": 0.604261040687561, "logps/chosen": -188.30096435546875, "logps/ref_chosen": -58.381134033203125, "logps/ref_rejected": -85.02839660644531, "logps/rejected": -284.9547119140625, "loss": 1.0549, "margin_dpo/margin_mean": 70.00645446777344, "margin_dpo/margin_std": 85.70564270019531, "step": 333 }, { "KL/chosen_KL_mean": -125.50909423828125, "KL/mean": -150.45120239257812, "KL/rejected_KL_mean": -175.393310546875, "KL/std": 80.46173858642578, "epoch": 0.5049130763416477, "fcm_dpo/beta": 0.006630011834204197, "fcm_dpo/delta": 0.07646586000919342, "fcm_dpo/margin": 49.88423538208008, "fcm_dpo/q_t": 0.42625001072883606, "grad_norm": 12.834829330444336, "learning_rate": 2.9080540104031484e-07, "logits/chosen": 0.685641348361969, "logits/rejected": 0.6379861831665039, "logps/chosen": -192.40109252929688, "logps/ref_chosen": -66.89199829101562, "logps/ref_rejected": -91.83695220947266, "logps/rejected": -267.23028564453125, "loss": 1.1679, "margin_dpo/margin_mean": 49.88423156738281, "margin_dpo/margin_std": 90.79962158203125, "step": 334 }, { "KL/chosen_KL_mean": -126.83473205566406, "KL/mean": -153.43295288085938, "KL/rejected_KL_mean": -180.03115844726562, "KL/std": 80.2386474609375, "epoch": 0.5064247921390779, "fcm_dpo/beta": 0.00688770180568099, "fcm_dpo/delta": 0.03577427193522453, "fcm_dpo/margin": 53.196434020996094, "fcm_dpo/q_t": 0.41868066787719727, "grad_norm": 14.178523063659668, "learning_rate": 2.895003489933375e-07, "logits/chosen": 0.7071677446365356, "logits/rejected": 0.6692987084388733, "logps/chosen": -188.34918212890625, "logps/ref_chosen": -61.51445770263672, "logps/ref_rejected": -75.68916320800781, "logps/rejected": -255.7203369140625, "loss": 1.1555, "margin_dpo/margin_mean": 53.196434020996094, "margin_dpo/margin_std": 95.25994110107422, "step": 335 }, { "KL/chosen_KL_mean": -140.4132537841797, "KL/mean": -169.2587890625, "KL/rejected_KL_mean": -198.10433959960938, "KL/std": 85.74678039550781, "epoch": 0.5079365079365079, "fcm_dpo/beta": 0.006724413949996233, "fcm_dpo/delta": 0.010933250188827515, "fcm_dpo/margin": 57.69108200073242, "fcm_dpo/q_t": 0.4159916043281555, "grad_norm": 13.194255828857422, "learning_rate": 2.8819419203668675e-07, "logits/chosen": 0.6335151195526123, "logits/rejected": 0.6091711521148682, "logps/chosen": -209.26332092285156, "logps/ref_chosen": -68.85006713867188, "logps/ref_rejected": -92.99603271484375, "logps/rejected": -291.100341796875, "loss": 1.1379, "margin_dpo/margin_mean": 57.69108581542969, "margin_dpo/margin_std": 97.63800811767578, "step": 336 }, { "KL/chosen_KL_mean": -148.26010131835938, "KL/mean": -171.10504150390625, "KL/rejected_KL_mean": -193.94998168945312, "KL/std": 85.65023803710938, "epoch": 0.509448223733938, "fcm_dpo/beta": 0.007140236906707287, "fcm_dpo/delta": 0.08157092332839966, "fcm_dpo/margin": 45.68986511230469, "fcm_dpo/q_t": 0.4279956817626953, "grad_norm": 11.6563138961792, "learning_rate": 2.8688696670638053e-07, "logits/chosen": 0.5941582918167114, "logits/rejected": 0.5655093193054199, "logps/chosen": -221.44793701171875, "logps/ref_chosen": -73.18783569335938, "logps/ref_rejected": -86.89118957519531, "logps/rejected": -280.8411865234375, "loss": 1.1877, "margin_dpo/margin_mean": 45.68986511230469, "margin_dpo/margin_std": 92.1810302734375, "step": 337 }, { "KL/chosen_KL_mean": -145.2806854248047, "KL/mean": -168.95672607421875, "KL/rejected_KL_mean": -192.6327667236328, "KL/std": 78.41549682617188, "epoch": 0.5109599395313681, "fcm_dpo/beta": 0.007375883869826794, "fcm_dpo/delta": 0.05678681656718254, "fcm_dpo/margin": 47.35206604003906, "fcm_dpo/q_t": 0.4219357669353485, "grad_norm": 11.649815559387207, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 0.6714822053909302, "logits/rejected": 0.6458303928375244, "logps/chosen": -209.22030639648438, "logps/ref_chosen": -63.939613342285156, "logps/ref_rejected": -75.34243774414062, "logps/rejected": -267.9752197265625, "loss": 1.1663, "margin_dpo/margin_mean": 47.35206604003906, "margin_dpo/margin_std": 87.88858032226562, "step": 338 }, { "KL/chosen_KL_mean": -118.4639892578125, "KL/mean": -147.70608520507812, "KL/rejected_KL_mean": -176.9481964111328, "KL/std": 79.511962890625, "epoch": 0.5124716553287982, "fcm_dpo/beta": 0.007303786464035511, "fcm_dpo/delta": -0.031887464225292206, "fcm_dpo/margin": 58.484214782714844, "fcm_dpo/q_t": 0.4025455117225647, "grad_norm": 14.439207077026367, "learning_rate": 2.842694572172736e-07, "logits/chosen": 0.8139640688896179, "logits/rejected": 0.7292660474777222, "logps/chosen": -164.01312255859375, "logps/ref_chosen": -45.54913330078125, "logps/ref_rejected": -67.0482177734375, "logps/rejected": -243.9964141845703, "loss": 1.0965, "margin_dpo/margin_mean": 58.484214782714844, "margin_dpo/margin_std": 85.29948425292969, "step": 339 }, { "KL/chosen_KL_mean": -133.04364013671875, "KL/mean": -160.40573120117188, "KL/rejected_KL_mean": -187.767822265625, "KL/std": 85.34622192382812, "epoch": 0.5139833711262283, "fcm_dpo/beta": 0.007357806898653507, "fcm_dpo/delta": -0.0032272376120090485, "fcm_dpo/margin": 54.72416687011719, "fcm_dpo/q_t": 0.41151127219200134, "grad_norm": 12.648419380187988, "learning_rate": 2.8295924627584004e-07, "logits/chosen": 0.6942380666732788, "logits/rejected": 0.6745313405990601, "logps/chosen": -187.04928588867188, "logps/ref_chosen": -54.00564956665039, "logps/ref_rejected": -61.314430236816406, "logps/rejected": -249.08224487304688, "loss": 1.152, "margin_dpo/margin_mean": 54.72416687011719, "margin_dpo/margin_std": 100.8072738647461, "step": 340 }, { "KL/chosen_KL_mean": -135.74366760253906, "KL/mean": -167.9922637939453, "KL/rejected_KL_mean": -200.2408447265625, "KL/std": 85.87152099609375, "epoch": 0.5154950869236583, "fcm_dpo/beta": 0.006678248755633831, "fcm_dpo/delta": -0.15111427009105682, "fcm_dpo/margin": 64.4971923828125, "fcm_dpo/q_t": 0.39761853218078613, "grad_norm": 13.96065902709961, "learning_rate": 2.816481133934373e-07, "logits/chosen": 0.7185596823692322, "logits/rejected": 0.6717268824577332, "logps/chosen": -199.13876342773438, "logps/ref_chosen": -63.39509582519531, "logps/ref_rejected": -76.20973205566406, "logps/rejected": -276.4505920410156, "loss": 1.0987, "margin_dpo/margin_mean": 64.4971923828125, "margin_dpo/margin_std": 91.53335571289062, "step": 341 }, { "KL/chosen_KL_mean": -133.71408081054688, "KL/mean": -164.31256103515625, "KL/rejected_KL_mean": -194.91104125976562, "KL/std": 91.55609130859375, "epoch": 0.5170068027210885, "fcm_dpo/beta": 0.006609394680708647, "fcm_dpo/delta": -0.00558522529900074, "fcm_dpo/margin": 61.19696807861328, "fcm_dpo/q_t": 0.4092737138271332, "grad_norm": 10.6194486618042, "learning_rate": 2.8033609524527046e-07, "logits/chosen": 0.7518157362937927, "logits/rejected": 0.7110329270362854, "logps/chosen": -186.76190185546875, "logps/ref_chosen": -53.047813415527344, "logps/ref_rejected": -68.2854232788086, "logps/rejected": -263.19647216796875, "loss": 1.1123, "margin_dpo/margin_mean": 61.19697189331055, "margin_dpo/margin_std": 93.84374237060547, "step": 342 }, { "KL/chosen_KL_mean": -129.63284301757812, "KL/mean": -152.19546508789062, "KL/rejected_KL_mean": -174.75808715820312, "KL/std": 84.81193542480469, "epoch": 0.5185185185185185, "fcm_dpo/beta": 0.006589554250240326, "fcm_dpo/delta": -0.016698164865374565, "fcm_dpo/margin": 45.125267028808594, "fcm_dpo/q_t": 0.43187424540519714, "grad_norm": 10.550444602966309, "learning_rate": 2.7902322853130753e-07, "logits/chosen": 0.5749995708465576, "logits/rejected": 0.5654809474945068, "logps/chosen": -200.21136474609375, "logps/ref_chosen": -70.57852935791016, "logps/ref_rejected": -84.73873901367188, "logps/rejected": -259.496826171875, "loss": 1.194, "margin_dpo/margin_mean": 45.125267028808594, "margin_dpo/margin_std": 88.6053466796875, "step": 343 }, { "KL/chosen_KL_mean": -140.68621826171875, "KL/mean": -173.6033172607422, "KL/rejected_KL_mean": -206.52041625976562, "KL/std": 85.86953735351562, "epoch": 0.5200302343159486, "fcm_dpo/beta": 0.006495362147688866, "fcm_dpo/delta": -0.031697697937488556, "fcm_dpo/margin": 65.8342056274414, "fcm_dpo/q_t": 0.403338760137558, "grad_norm": 12.14207935333252, "learning_rate": 2.7770954997525274e-07, "logits/chosen": 0.7135556936264038, "logits/rejected": 0.6433905959129333, "logps/chosen": -196.49722290039062, "logps/ref_chosen": -55.811004638671875, "logps/ref_rejected": -84.77637481689453, "logps/rejected": -291.29681396484375, "loss": 1.1007, "margin_dpo/margin_mean": 65.8342056274414, "margin_dpo/margin_std": 99.62645721435547, "step": 344 }, { "KL/chosen_KL_mean": -116.20820617675781, "KL/mean": -145.49966430664062, "KL/rejected_KL_mean": -174.7911376953125, "KL/std": 78.99835205078125, "epoch": 0.5215419501133787, "fcm_dpo/beta": 0.006551923230290413, "fcm_dpo/delta": 0.01797226071357727, "fcm_dpo/margin": 58.582942962646484, "fcm_dpo/q_t": 0.41312161087989807, "grad_norm": 11.81797981262207, "learning_rate": 2.7639509632351927e-07, "logits/chosen": 0.7470645308494568, "logits/rejected": 0.7046425342559814, "logps/chosen": -173.99429321289062, "logps/ref_chosen": -57.78609848022461, "logps/ref_rejected": -78.91847229003906, "logps/rejected": -253.70960998535156, "loss": 1.1272, "margin_dpo/margin_mean": 58.58293914794922, "margin_dpo/margin_std": 93.53245544433594, "step": 345 }, { "KL/chosen_KL_mean": -122.06107330322266, "KL/mean": -158.17715454101562, "KL/rejected_KL_mean": -194.29324340820312, "KL/std": 93.81214904785156, "epoch": 0.5230536659108088, "fcm_dpo/beta": 0.0063993064686656, "fcm_dpo/delta": -0.0722820833325386, "fcm_dpo/margin": 72.23216247558594, "fcm_dpo/q_t": 0.3927100896835327, "grad_norm": 13.295743942260742, "learning_rate": 2.7507990434420123e-07, "logits/chosen": 0.7267513275146484, "logits/rejected": 0.6444227695465088, "logps/chosen": -178.34619140625, "logps/ref_chosen": -56.285125732421875, "logps/ref_rejected": -91.15303039550781, "logps/rejected": -285.4462890625, "loss": 1.0698, "margin_dpo/margin_mean": 72.23216247558594, "margin_dpo/margin_std": 97.0244140625, "step": 346 }, { "KL/chosen_KL_mean": -135.72100830078125, "KL/mean": -162.5164794921875, "KL/rejected_KL_mean": -189.3119354248047, "KL/std": 86.25690460205078, "epoch": 0.5245653817082389, "fcm_dpo/beta": 0.006409412249922752, "fcm_dpo/delta": 0.06314625591039658, "fcm_dpo/margin": 53.5909309387207, "fcm_dpo/q_t": 0.4225504994392395, "grad_norm": 12.886883735656738, "learning_rate": 2.737640108260456e-07, "logits/chosen": 0.8315824270248413, "logits/rejected": 0.784061074256897, "logps/chosen": -189.22055053710938, "logps/ref_chosen": -53.499542236328125, "logps/ref_rejected": -72.52565002441406, "logps/rejected": -261.83758544921875, "loss": 1.1462, "margin_dpo/margin_mean": 53.5909309387207, "margin_dpo/margin_std": 87.14447784423828, "step": 347 }, { "KL/chosen_KL_mean": -122.0595932006836, "KL/mean": -153.3162841796875, "KL/rejected_KL_mean": -184.57296752929688, "KL/std": 81.57878112792969, "epoch": 0.5260770975056689, "fcm_dpo/beta": 0.006419610697776079, "fcm_dpo/delta": -0.002443864941596985, "fcm_dpo/margin": 62.51338195800781, "fcm_dpo/q_t": 0.41006332635879517, "grad_norm": 12.771003723144531, "learning_rate": 2.724474525774229e-07, "logits/chosen": 0.803961992263794, "logits/rejected": 0.7731932401657104, "logps/chosen": -172.846435546875, "logps/ref_chosen": -50.78684997558594, "logps/ref_rejected": -68.63732147216797, "logps/rejected": -253.2102813720703, "loss": 1.114, "margin_dpo/margin_mean": 62.51338195800781, "margin_dpo/margin_std": 96.11529541015625, "step": 348 }, { "KL/chosen_KL_mean": -124.03947448730469, "KL/mean": -155.91339111328125, "KL/rejected_KL_mean": -187.78729248046875, "KL/std": 91.26483154296875, "epoch": 0.527588813303099, "fcm_dpo/beta": 0.006526275537908077, "fcm_dpo/delta": -0.019439280033111572, "fcm_dpo/margin": 63.74781036376953, "fcm_dpo/q_t": 0.4069031774997711, "grad_norm": 11.912677764892578, "learning_rate": 2.711302664252973e-07, "logits/chosen": 0.7378636598587036, "logits/rejected": 0.6440693736076355, "logps/chosen": -177.36448669433594, "logps/ref_chosen": -53.325008392333984, "logps/ref_rejected": -83.21236419677734, "logps/rejected": -270.9996643066406, "loss": 1.1076, "margin_dpo/margin_mean": 63.7478141784668, "margin_dpo/margin_std": 96.80409240722656, "step": 349 }, { "KL/chosen_KL_mean": -126.06622314453125, "KL/mean": -168.70594787597656, "KL/rejected_KL_mean": -211.34567260742188, "KL/std": 97.71580505371094, "epoch": 0.5291005291005291, "fcm_dpo/beta": 0.006007745862007141, "fcm_dpo/delta": -0.13404223322868347, "fcm_dpo/margin": 85.27944946289062, "fcm_dpo/q_t": 0.3789331316947937, "grad_norm": 13.077025413513184, "learning_rate": 2.698124892141971e-07, "logits/chosen": 0.7190489768981934, "logits/rejected": 0.6495992541313171, "logps/chosen": -187.69198608398438, "logps/ref_chosen": -61.625770568847656, "logps/ref_rejected": -87.63627624511719, "logps/rejected": -298.98193359375, "loss": 1.0091, "margin_dpo/margin_mean": 85.27944946289062, "margin_dpo/margin_std": 92.19660186767578, "step": 350 }, { "KL/chosen_KL_mean": -122.51304626464844, "KL/mean": -152.106201171875, "KL/rejected_KL_mean": -181.69935607910156, "KL/std": 81.26626586914062, "epoch": 0.5306122448979592, "fcm_dpo/beta": 0.006009809672832489, "fcm_dpo/delta": 0.0496024414896965, "fcm_dpo/margin": 59.186309814453125, "fcm_dpo/q_t": 0.4185904562473297, "grad_norm": 11.393095970153809, "learning_rate": 2.6849415780518357e-07, "logits/chosen": 0.6613746881484985, "logits/rejected": 0.5912742018699646, "logps/chosen": -178.76937866210938, "logps/ref_chosen": -56.2563362121582, "logps/ref_rejected": -79.11589813232422, "logps/rejected": -260.81524658203125, "loss": 1.1556, "margin_dpo/margin_mean": 59.186309814453125, "margin_dpo/margin_std": 103.63017272949219, "step": 351 }, { "KL/chosen_KL_mean": -124.99972534179688, "KL/mean": -158.94674682617188, "KL/rejected_KL_mean": -192.89376831054688, "KL/std": 92.21725463867188, "epoch": 0.5321239606953893, "fcm_dpo/beta": 0.006002703681588173, "fcm_dpo/delta": -0.009663868695497513, "fcm_dpo/margin": 67.89405822753906, "fcm_dpo/q_t": 0.40806788206100464, "grad_norm": 11.126773834228516, "learning_rate": 2.6717530907482024e-07, "logits/chosen": 0.7351091504096985, "logits/rejected": 0.6848005056381226, "logps/chosen": -188.05166625976562, "logps/ref_chosen": -63.05195236206055, "logps/ref_rejected": -85.52035522460938, "logps/rejected": -278.41412353515625, "loss": 1.1028, "margin_dpo/margin_mean": 67.89405822753906, "margin_dpo/margin_std": 99.7062759399414, "step": 352 }, { "KL/chosen_KL_mean": -121.88029479980469, "KL/mean": -152.44581604003906, "KL/rejected_KL_mean": -183.01132202148438, "KL/std": 83.91139221191406, "epoch": 0.5336356764928194, "fcm_dpo/beta": 0.006154675967991352, "fcm_dpo/delta": 0.026616372168064117, "fcm_dpo/margin": 61.13102340698242, "fcm_dpo/q_t": 0.4135306775569916, "grad_norm": 10.312121391296387, "learning_rate": 2.658559799141411e-07, "logits/chosen": 0.7169567942619324, "logits/rejected": 0.7259862422943115, "logps/chosen": -190.88949584960938, "logps/ref_chosen": -69.00918579101562, "logps/ref_rejected": -72.65840148925781, "logps/rejected": -255.6697235107422, "loss": 1.1138, "margin_dpo/margin_mean": 61.131019592285156, "margin_dpo/margin_std": 87.21917724609375, "step": 353 }, { "KL/chosen_KL_mean": -121.11980438232422, "KL/mean": -157.54824829101562, "KL/rejected_KL_mean": -193.97665405273438, "KL/std": 78.74341583251953, "epoch": 0.5351473922902494, "fcm_dpo/beta": 0.005997260566800833, "fcm_dpo/delta": -0.04329075664281845, "fcm_dpo/margin": 72.85685729980469, "fcm_dpo/q_t": 0.39849966764450073, "grad_norm": 12.67549991607666, "learning_rate": 2.6453620722761895e-07, "logits/chosen": 0.8099727630615234, "logits/rejected": 0.6850665807723999, "logps/chosen": -160.90814208984375, "logps/ref_chosen": -39.78833770751953, "logps/ref_rejected": -69.56885528564453, "logps/rejected": -263.5455322265625, "loss": 1.0772, "margin_dpo/margin_mean": 72.85685729980469, "margin_dpo/margin_std": 97.290283203125, "step": 354 }, { "KL/chosen_KL_mean": -125.32882690429688, "KL/mean": -166.8375244140625, "KL/rejected_KL_mean": -208.34622192382812, "KL/std": 94.46980285644531, "epoch": 0.5366591080876795, "fcm_dpo/beta": 0.005797875113785267, "fcm_dpo/delta": -0.09473682194948196, "fcm_dpo/margin": 83.01737213134766, "fcm_dpo/q_t": 0.38854479789733887, "grad_norm": 13.447663307189941, "learning_rate": 2.632160279321328e-07, "logits/chosen": 0.7900065183639526, "logits/rejected": 0.6645567417144775, "logps/chosen": -171.58421325683594, "logps/ref_chosen": -46.25537872314453, "logps/ref_rejected": -78.20236206054688, "logps/rejected": -286.548583984375, "loss": 1.0509, "margin_dpo/margin_mean": 83.01736450195312, "margin_dpo/margin_std": 104.88021850585938, "step": 355 }, { "KL/chosen_KL_mean": -119.88313293457031, "KL/mean": -152.29454040527344, "KL/rejected_KL_mean": -184.70596313476562, "KL/std": 87.65028381347656, "epoch": 0.5381708238851096, "fcm_dpo/beta": 0.005688229110091925, "fcm_dpo/delta": 0.03451262786984444, "fcm_dpo/margin": 64.82282257080078, "fcm_dpo/q_t": 0.41692763566970825, "grad_norm": 11.188521385192871, "learning_rate": 2.618954789559356e-07, "logits/chosen": 0.7430191040039062, "logits/rejected": 0.6651791334152222, "logps/chosen": -167.78929138183594, "logps/ref_chosen": -47.906158447265625, "logps/ref_rejected": -74.29397583007812, "logps/rejected": -258.99993896484375, "loss": 1.1454, "margin_dpo/margin_mean": 64.82282257080078, "margin_dpo/margin_std": 109.8090591430664, "step": 356 }, { "KL/chosen_KL_mean": -143.23452758789062, "KL/mean": -169.093017578125, "KL/rejected_KL_mean": -194.9515380859375, "KL/std": 87.9165267944336, "epoch": 0.5396825396825397, "fcm_dpo/beta": 0.005629734601825476, "fcm_dpo/delta": -0.04662557691335678, "fcm_dpo/margin": 51.71701431274414, "fcm_dpo/q_t": 0.4305192530155182, "grad_norm": 11.694966316223145, "learning_rate": 2.6057459723762076e-07, "logits/chosen": 0.752831220626831, "logits/rejected": 0.7355064153671265, "logps/chosen": -205.86952209472656, "logps/ref_chosen": -62.63500213623047, "logps/ref_rejected": -65.11399841308594, "logps/rejected": -260.0655517578125, "loss": 1.1955, "margin_dpo/margin_mean": 51.717018127441406, "margin_dpo/margin_std": 97.05239868164062, "step": 357 }, { "KL/chosen_KL_mean": -142.22438049316406, "KL/mean": -176.40380859375, "KL/rejected_KL_mean": -210.58322143554688, "KL/std": 94.13438415527344, "epoch": 0.5411942554799698, "fcm_dpo/beta": 0.00548651535063982, "fcm_dpo/delta": -0.08703271299600601, "fcm_dpo/margin": 68.35882568359375, "fcm_dpo/q_t": 0.41342803835868835, "grad_norm": 15.619354248046875, "learning_rate": 2.5925341972508954e-07, "logits/chosen": 0.6815810203552246, "logits/rejected": 0.6956534385681152, "logps/chosen": -209.43399047851562, "logps/ref_chosen": -67.20960998535156, "logps/ref_rejected": -69.34715270996094, "logps/rejected": -279.93035888671875, "loss": 1.134, "margin_dpo/margin_mean": 68.35882568359375, "margin_dpo/margin_std": 105.12598419189453, "step": 358 }, { "KL/chosen_KL_mean": -158.29672241210938, "KL/mean": -176.74636840820312, "KL/rejected_KL_mean": -195.1959991455078, "KL/std": 92.31513977050781, "epoch": 0.5427059712773998, "fcm_dpo/beta": 0.005430396646261215, "fcm_dpo/delta": 0.05346956476569176, "fcm_dpo/margin": 36.89925765991211, "fcm_dpo/q_t": 0.45445123314857483, "grad_norm": 13.617073059082031, "learning_rate": 2.579319833745169e-07, "logits/chosen": 0.7321456074714661, "logits/rejected": 0.6983045339584351, "logps/chosen": -220.822509765625, "logps/ref_chosen": -62.52578353881836, "logps/ref_rejected": -76.63114929199219, "logps/rejected": -271.8271484375, "loss": 1.2704, "margin_dpo/margin_mean": 36.89925765991211, "margin_dpo/margin_std": 100.25183868408203, "step": 359 }, { "KL/chosen_KL_mean": -151.61859130859375, "KL/mean": -184.96478271484375, "KL/rejected_KL_mean": -218.31097412109375, "KL/std": 98.37626647949219, "epoch": 0.54421768707483, "fcm_dpo/beta": 0.005576558876782656, "fcm_dpo/delta": 0.031639836728572845, "fcm_dpo/margin": 66.6923599243164, "fcm_dpo/q_t": 0.415699303150177, "grad_norm": 11.561031341552734, "learning_rate": 2.5661032514931834e-07, "logits/chosen": 0.6751697063446045, "logits/rejected": 0.5904595255851746, "logps/chosen": -215.1063232421875, "logps/ref_chosen": -63.48772048950195, "logps/ref_rejected": -90.6891098022461, "logps/rejected": -309.00006103515625, "loss": 1.1178, "margin_dpo/margin_mean": 66.6923599243164, "margin_dpo/margin_std": 97.05094909667969, "step": 360 }, { "KL/chosen_KL_mean": -143.3883056640625, "KL/mean": -180.1334228515625, "KL/rejected_KL_mean": -216.8785400390625, "KL/std": 92.39273071289062, "epoch": 0.54572940287226, "fcm_dpo/beta": 0.005615275353193283, "fcm_dpo/delta": -0.014561126939952374, "fcm_dpo/margin": 73.49020385742188, "fcm_dpo/q_t": 0.40417200326919556, "grad_norm": 10.36706256866455, "learning_rate": 2.552884820191154e-07, "logits/chosen": 0.8067169785499573, "logits/rejected": 0.7659634351730347, "logps/chosen": -201.30545043945312, "logps/ref_chosen": -57.917144775390625, "logps/ref_rejected": -72.39089965820312, "logps/rejected": -289.2694396972656, "loss": 1.0856, "margin_dpo/margin_mean": 73.49020385742188, "margin_dpo/margin_std": 96.46673583984375, "step": 361 }, { "KL/chosen_KL_mean": -144.8900146484375, "KL/mean": -188.41073608398438, "KL/rejected_KL_mean": -231.93145751953125, "KL/std": 106.30439758300781, "epoch": 0.54724111866969, "fcm_dpo/beta": 0.005439079366624355, "fcm_dpo/delta": -0.08564796298742294, "fcm_dpo/margin": 87.04143524169922, "fcm_dpo/q_t": 0.39451199769973755, "grad_norm": 11.566519737243652, "learning_rate": 2.53966490958702e-07, "logits/chosen": 0.8194698095321655, "logits/rejected": 0.7212804555892944, "logps/chosen": -208.33348083496094, "logps/ref_chosen": -63.4434700012207, "logps/ref_rejected": -103.45516967773438, "logps/rejected": -335.3866271972656, "loss": 1.0693, "margin_dpo/margin_mean": 87.04142761230469, "margin_dpo/margin_std": 124.4408187866211, "step": 362 }, { "KL/chosen_KL_mean": -147.78335571289062, "KL/mean": -186.64004516601562, "KL/rejected_KL_mean": -225.4967041015625, "KL/std": 92.55686950683594, "epoch": 0.5487528344671202, "fcm_dpo/beta": 0.005266488064080477, "fcm_dpo/delta": -0.010586944408714771, "fcm_dpo/margin": 77.71334075927734, "fcm_dpo/q_t": 0.40681716799736023, "grad_norm": 13.70695686340332, "learning_rate": 2.526443889470099e-07, "logits/chosen": 0.7966769933700562, "logits/rejected": 0.6717593669891357, "logps/chosen": -196.43519592285156, "logps/ref_chosen": -48.65182876586914, "logps/ref_rejected": -88.65904235839844, "logps/rejected": -314.15576171875, "loss": 1.0935, "margin_dpo/margin_mean": 77.71334075927734, "margin_dpo/margin_std": 108.69964599609375, "step": 363 }, { "KL/chosen_KL_mean": -131.58168029785156, "KL/mean": -174.65626525878906, "KL/rejected_KL_mean": -217.7308349609375, "KL/std": 102.02590942382812, "epoch": 0.5502645502645502, "fcm_dpo/beta": 0.0051279449835419655, "fcm_dpo/delta": -0.04850374162197113, "fcm_dpo/margin": 86.14913940429688, "fcm_dpo/q_t": 0.4003210961818695, "grad_norm": 9.81286334991455, "learning_rate": 2.513222129660744e-07, "logits/chosen": 0.6942867040634155, "logits/rejected": 0.6157143115997314, "logps/chosen": -189.4527587890625, "logps/ref_chosen": -57.87107467651367, "logps/ref_rejected": -80.95503234863281, "logps/rejected": -298.68585205078125, "loss": 1.0928, "margin_dpo/margin_mean": 86.14913940429688, "margin_dpo/margin_std": 129.07220458984375, "step": 364 }, { "KL/chosen_KL_mean": -125.02583312988281, "KL/mean": -164.640380859375, "KL/rejected_KL_mean": -204.25491333007812, "KL/std": 108.65450286865234, "epoch": 0.5517762660619804, "fcm_dpo/beta": 0.0050535742193460464, "fcm_dpo/delta": -0.0012763142585754395, "fcm_dpo/margin": 79.22908782958984, "fcm_dpo/q_t": 0.4074895977973938, "grad_norm": 10.674773216247559, "learning_rate": 2.5e-07, "logits/chosen": 0.7654601335525513, "logits/rejected": 0.7636349201202393, "logps/chosen": -189.968017578125, "logps/ref_chosen": -64.94217681884766, "logps/ref_rejected": -74.8599853515625, "logps/rejected": -279.1148986816406, "loss": 1.0785, "margin_dpo/margin_mean": 79.22908782958984, "margin_dpo/margin_std": 95.32018280029297, "step": 365 }, { "KL/chosen_KL_mean": -124.21640014648438, "KL/mean": -156.70164489746094, "KL/rejected_KL_mean": -189.18690490722656, "KL/std": 95.22203063964844, "epoch": 0.5532879818594104, "fcm_dpo/beta": 0.005297908093780279, "fcm_dpo/delta": 0.06093678995966911, "fcm_dpo/margin": 64.97052001953125, "fcm_dpo/q_t": 0.4231628477573395, "grad_norm": 13.27842903137207, "learning_rate": 2.486777870339255e-07, "logits/chosen": 0.7186771631240845, "logits/rejected": 0.7142035365104675, "logps/chosen": -179.38238525390625, "logps/ref_chosen": -55.16598129272461, "logps/ref_rejected": -65.26121520996094, "logps/rejected": -254.4481201171875, "loss": 1.1684, "margin_dpo/margin_mean": 64.97052001953125, "margin_dpo/margin_std": 120.9197769165039, "step": 366 }, { "KL/chosen_KL_mean": -142.8965606689453, "KL/mean": -175.44778442382812, "KL/rejected_KL_mean": -207.9990234375, "KL/std": 94.457275390625, "epoch": 0.5547996976568406, "fcm_dpo/beta": 0.005413766950368881, "fcm_dpo/delta": 0.05324382334947586, "fcm_dpo/margin": 65.10247802734375, "fcm_dpo/q_t": 0.4184330105781555, "grad_norm": 11.966232299804688, "learning_rate": 2.4735561105299014e-07, "logits/chosen": 0.754651665687561, "logits/rejected": 0.658037543296814, "logps/chosen": -198.90701293945312, "logps/ref_chosen": -56.01046371459961, "logps/ref_rejected": -77.31010437011719, "logps/rejected": -285.30914306640625, "loss": 1.1341, "margin_dpo/margin_mean": 65.10247802734375, "margin_dpo/margin_std": 99.43478393554688, "step": 367 }, { "KL/chosen_KL_mean": -154.50941467285156, "KL/mean": -183.74929809570312, "KL/rejected_KL_mean": -212.98915100097656, "KL/std": 93.31791687011719, "epoch": 0.5563114134542706, "fcm_dpo/beta": 0.0056096892803907394, "fcm_dpo/delta": 0.07988474518060684, "fcm_dpo/margin": 58.47974395751953, "fcm_dpo/q_t": 0.42577698826789856, "grad_norm": 13.028302192687988, "learning_rate": 2.46033509041298e-07, "logits/chosen": 0.5896161794662476, "logits/rejected": 0.592335045337677, "logps/chosen": -229.3386993408203, "logps/ref_chosen": -74.82927703857422, "logps/ref_rejected": -76.11680603027344, "logps/rejected": -289.10595703125, "loss": 1.1633, "margin_dpo/margin_mean": 58.47974395751953, "margin_dpo/margin_std": 100.59825134277344, "step": 368 }, { "KL/chosen_KL_mean": -141.26634216308594, "KL/mean": -168.32205200195312, "KL/rejected_KL_mean": -195.3777618408203, "KL/std": 93.77578735351562, "epoch": 0.5578231292517006, "fcm_dpo/beta": 0.0060209897346794605, "fcm_dpo/delta": 0.0800437182188034, "fcm_dpo/margin": 54.111427307128906, "fcm_dpo/q_t": 0.4274851679801941, "grad_norm": 13.085291862487793, "learning_rate": 2.447115179808846e-07, "logits/chosen": 0.7499760389328003, "logits/rejected": 0.704288125038147, "logps/chosen": -199.59255981445312, "logps/ref_chosen": -58.32621765136719, "logps/ref_rejected": -80.92183685302734, "logps/rejected": -276.29962158203125, "loss": 1.1727, "margin_dpo/margin_mean": 54.111427307128906, "margin_dpo/margin_std": 99.9386978149414, "step": 369 }, { "KL/chosen_KL_mean": -133.82888793945312, "KL/mean": -173.8616943359375, "KL/rejected_KL_mean": -213.89450073242188, "KL/std": 97.72712707519531, "epoch": 0.5593348450491308, "fcm_dpo/beta": 0.005765399895608425, "fcm_dpo/delta": -0.07319603860378265, "fcm_dpo/margin": 80.06562805175781, "fcm_dpo/q_t": 0.39234304428100586, "grad_norm": 12.59056568145752, "learning_rate": 2.4338967485068164e-07, "logits/chosen": 0.8812678456306458, "logits/rejected": 0.8152058124542236, "logps/chosen": -186.71258544921875, "logps/ref_chosen": -52.88372039794922, "logps/ref_rejected": -79.43692016601562, "logps/rejected": -293.3314208984375, "loss": 1.0779, "margin_dpo/margin_mean": 80.06562805175781, "margin_dpo/margin_std": 113.96977233886719, "step": 370 }, { "KL/chosen_KL_mean": -131.85989379882812, "KL/mean": -166.550048828125, "KL/rejected_KL_mean": -201.240234375, "KL/std": 91.61387634277344, "epoch": 0.5608465608465608, "fcm_dpo/beta": 0.005842489190399647, "fcm_dpo/delta": -0.008122660219669342, "fcm_dpo/margin": 69.38032531738281, "fcm_dpo/q_t": 0.4071889817714691, "grad_norm": 15.300116539001465, "learning_rate": 2.420680166254831e-07, "logits/chosen": 0.8666425347328186, "logits/rejected": 0.8389255404472351, "logps/chosen": -181.0841064453125, "logps/ref_chosen": -49.224212646484375, "logps/ref_rejected": -63.348472595214844, "logps/rejected": -264.58868408203125, "loss": 1.1081, "margin_dpo/margin_mean": 69.38032531738281, "margin_dpo/margin_std": 101.50696563720703, "step": 371 }, { "KL/chosen_KL_mean": -139.83580017089844, "KL/mean": -160.21075439453125, "KL/rejected_KL_mean": -180.585693359375, "KL/std": 92.19517517089844, "epoch": 0.562358276643991, "fcm_dpo/beta": 0.005775434896349907, "fcm_dpo/delta": 0.02449907548725605, "fcm_dpo/margin": 40.7498779296875, "fcm_dpo/q_t": 0.448069304227829, "grad_norm": 13.797256469726562, "learning_rate": 2.4074658027491044e-07, "logits/chosen": 0.7692815065383911, "logits/rejected": 0.6794453859329224, "logps/chosen": -192.1053466796875, "logps/ref_chosen": -52.269554138183594, "logps/ref_rejected": -72.99522399902344, "logps/rejected": -253.58090209960938, "loss": 1.2809, "margin_dpo/margin_mean": 40.749881744384766, "margin_dpo/margin_std": 119.3675537109375, "step": 372 }, { "KL/chosen_KL_mean": -158.91616821289062, "KL/mean": -186.7388916015625, "KL/rejected_KL_mean": -214.56161499023438, "KL/std": 99.78253173828125, "epoch": 0.563869992441421, "fcm_dpo/beta": 0.006023020949214697, "fcm_dpo/delta": 0.07183150947093964, "fcm_dpo/margin": 55.64544677734375, "fcm_dpo/q_t": 0.42674756050109863, "grad_norm": 13.5969820022583, "learning_rate": 2.394254027623792e-07, "logits/chosen": 0.787378191947937, "logits/rejected": 0.7218998670578003, "logps/chosen": -220.0291748046875, "logps/ref_chosen": -61.112998962402344, "logps/ref_rejected": -76.24851989746094, "logps/rejected": -290.81011962890625, "loss": 1.2072, "margin_dpo/margin_mean": 55.64544677734375, "margin_dpo/margin_std": 123.12017822265625, "step": 373 }, { "KL/chosen_KL_mean": -134.72891235351562, "KL/mean": -178.64892578125, "KL/rejected_KL_mean": -222.56893920898438, "KL/std": 102.30409240722656, "epoch": 0.5653817082388511, "fcm_dpo/beta": 0.005733366124331951, "fcm_dpo/delta": -0.12269183248281479, "fcm_dpo/margin": 87.84001159667969, "fcm_dpo/q_t": 0.38412588834762573, "grad_norm": 14.229789733886719, "learning_rate": 2.381045210440644e-07, "logits/chosen": 0.6749398708343506, "logits/rejected": 0.6806223392486572, "logps/chosen": -207.3981170654297, "logps/ref_chosen": -72.66920471191406, "logps/ref_rejected": -76.83158874511719, "logps/rejected": -299.4005126953125, "loss": 1.0639, "margin_dpo/margin_mean": 87.84001159667969, "margin_dpo/margin_std": 126.37348175048828, "step": 374 }, { "KL/chosen_KL_mean": -133.24624633789062, "KL/mean": -164.95437622070312, "KL/rejected_KL_mean": -196.66249084472656, "KL/std": 110.367431640625, "epoch": 0.5668934240362812, "fcm_dpo/beta": 0.005722872447222471, "fcm_dpo/delta": 0.04165318235754967, "fcm_dpo/margin": 63.41625213623047, "fcm_dpo/q_t": 0.4203999638557434, "grad_norm": 15.683096885681152, "learning_rate": 2.3678397206786715e-07, "logits/chosen": 0.7528005838394165, "logits/rejected": 0.7014273405075073, "logps/chosen": -190.92953491210938, "logps/ref_chosen": -57.68330383300781, "logps/ref_rejected": -79.34097290039062, "logps/rejected": -276.00347900390625, "loss": 1.1684, "margin_dpo/margin_mean": 63.4162483215332, "margin_dpo/margin_std": 121.38080596923828, "step": 375 }, { "KL/chosen_KL_mean": -141.19493103027344, "KL/mean": -181.21041870117188, "KL/rejected_KL_mean": -221.2259521484375, "KL/std": 112.07803344726562, "epoch": 0.5684051398337112, "fcm_dpo/beta": 0.005588100291788578, "fcm_dpo/delta": -0.05586222559213638, "fcm_dpo/margin": 80.03101348876953, "fcm_dpo/q_t": 0.4027999937534332, "grad_norm": 12.794271469116211, "learning_rate": 2.3546379277238103e-07, "logits/chosen": 0.8139768838882446, "logits/rejected": 0.7517552971839905, "logps/chosen": -192.86898803710938, "logps/ref_chosen": -51.674072265625, "logps/ref_rejected": -75.69713592529297, "logps/rejected": -296.923095703125, "loss": 1.112, "margin_dpo/margin_mean": 80.03101348876953, "margin_dpo/margin_std": 133.2711639404297, "step": 376 }, { "KL/chosen_KL_mean": -141.94168090820312, "KL/mean": -169.37771606445312, "KL/rejected_KL_mean": -196.8137664794922, "KL/std": 87.98646545410156, "epoch": 0.5699168556311414, "fcm_dpo/beta": 0.005861810874193907, "fcm_dpo/delta": 0.08542998135089874, "fcm_dpo/margin": 54.872093200683594, "fcm_dpo/q_t": 0.42690926790237427, "grad_norm": 14.541651725769043, "learning_rate": 2.3414402008585886e-07, "logits/chosen": 0.7738527059555054, "logits/rejected": 0.7508142590522766, "logps/chosen": -188.12020874023438, "logps/ref_chosen": -46.17853546142578, "logps/ref_rejected": -57.756500244140625, "logps/rejected": -254.5702667236328, "loss": 1.1706, "margin_dpo/margin_mean": 54.872093200683594, "margin_dpo/margin_std": 98.96908569335938, "step": 377 }, { "KL/chosen_KL_mean": -139.3751678466797, "KL/mean": -166.8651123046875, "KL/rejected_KL_mean": -194.35507202148438, "KL/std": 103.14115142822266, "epoch": 0.5714285714285714, "fcm_dpo/beta": 0.006156137678772211, "fcm_dpo/delta": 0.06586823612451553, "fcm_dpo/margin": 54.979896545410156, "fcm_dpo/q_t": 0.4245694875717163, "grad_norm": 11.343628883361816, "learning_rate": 2.3282469092517977e-07, "logits/chosen": 0.8040895462036133, "logits/rejected": 0.7606953382492065, "logps/chosen": -198.59405517578125, "logps/ref_chosen": -59.21887969970703, "logps/ref_rejected": -71.24818420410156, "logps/rejected": -265.603271484375, "loss": 1.1734, "margin_dpo/margin_mean": 54.97990036010742, "margin_dpo/margin_std": 101.69361877441406, "step": 378 }, { "KL/chosen_KL_mean": -141.87744140625, "KL/mean": -181.7213897705078, "KL/rejected_KL_mean": -221.56533813476562, "KL/std": 107.65896606445312, "epoch": 0.5729402872260015, "fcm_dpo/beta": 0.005909925326704979, "fcm_dpo/delta": -0.08261381089687347, "fcm_dpo/margin": 79.68791961669922, "fcm_dpo/q_t": 0.39472633600234985, "grad_norm": 14.828629493713379, "learning_rate": 2.3150584219481643e-07, "logits/chosen": 0.7641682624816895, "logits/rejected": 0.6950868368148804, "logps/chosen": -218.19403076171875, "logps/ref_chosen": -76.31658935546875, "logps/ref_rejected": -104.26200103759766, "logps/rejected": -325.82733154296875, "loss": 1.0766, "margin_dpo/margin_mean": 79.68791961669922, "margin_dpo/margin_std": 117.0067138671875, "step": 379 }, { "KL/chosen_KL_mean": -118.77130889892578, "KL/mean": -161.55316162109375, "KL/rejected_KL_mean": -204.33502197265625, "KL/std": 95.21235656738281, "epoch": 0.5744520030234316, "fcm_dpo/beta": 0.005602892488241196, "fcm_dpo/delta": -0.09268143028020859, "fcm_dpo/margin": 85.5636978149414, "fcm_dpo/q_t": 0.388763964176178, "grad_norm": 13.451189041137695, "learning_rate": 2.3018751078580283e-07, "logits/chosen": 0.7501556873321533, "logits/rejected": 0.7167741060256958, "logps/chosen": -180.05447387695312, "logps/ref_chosen": -61.283164978027344, "logps/ref_rejected": -72.38892364501953, "logps/rejected": -276.72393798828125, "loss": 1.0568, "margin_dpo/margin_mean": 85.5636978149414, "margin_dpo/margin_std": 113.08406066894531, "step": 380 }, { "KL/chosen_KL_mean": -145.89193725585938, "KL/mean": -162.467041015625, "KL/rejected_KL_mean": -179.04214477539062, "KL/std": 100.26214599609375, "epoch": 0.5759637188208617, "fcm_dpo/beta": 0.005632633343338966, "fcm_dpo/delta": 0.06954426318407059, "fcm_dpo/margin": 33.150203704833984, "fcm_dpo/q_t": 0.45884740352630615, "grad_norm": 11.799565315246582, "learning_rate": 2.288697335747027e-07, "logits/chosen": 0.725066065788269, "logits/rejected": 0.7106317281723022, "logps/chosen": -204.10592651367188, "logps/ref_chosen": -58.2139892578125, "logps/ref_rejected": -60.78669357299805, "logps/rejected": -239.82882690429688, "loss": 1.3176, "margin_dpo/margin_mean": 33.15020751953125, "margin_dpo/margin_std": 121.16805267333984, "step": 381 }, { "KL/chosen_KL_mean": -146.5373077392578, "KL/mean": -177.61614990234375, "KL/rejected_KL_mean": -208.69497680664062, "KL/std": 100.65556335449219, "epoch": 0.5774754346182918, "fcm_dpo/beta": 0.005918251350522041, "fcm_dpo/delta": 0.0343366302549839, "fcm_dpo/margin": 62.157691955566406, "fcm_dpo/q_t": 0.4175347685813904, "grad_norm": 14.061190605163574, "learning_rate": 2.2755254742257706e-07, "logits/chosen": 0.7362748384475708, "logits/rejected": 0.6886606216430664, "logps/chosen": -208.3626251220703, "logps/ref_chosen": -61.82532501220703, "logps/ref_rejected": -83.0452880859375, "logps/rejected": -291.7402648925781, "loss": 1.1333, "margin_dpo/margin_mean": 62.157691955566406, "margin_dpo/margin_std": 98.62821960449219, "step": 382 }, { "KL/chosen_KL_mean": -151.32742309570312, "KL/mean": -177.73983764648438, "KL/rejected_KL_mean": -204.15228271484375, "KL/std": 99.92002868652344, "epoch": 0.5789871504157218, "fcm_dpo/beta": 0.006043318659067154, "fcm_dpo/delta": 0.08946429938077927, "fcm_dpo/margin": 52.82486343383789, "fcm_dpo/q_t": 0.43128010630607605, "grad_norm": 16.85250473022461, "learning_rate": 2.2623598917395436e-07, "logits/chosen": 0.6481820344924927, "logits/rejected": 0.6832894086837769, "logps/chosen": -231.89068603515625, "logps/ref_chosen": -80.56326293945312, "logps/ref_rejected": -74.62922668457031, "logps/rejected": -278.781494140625, "loss": 1.2263, "margin_dpo/margin_mean": 52.824859619140625, "margin_dpo/margin_std": 125.01660919189453, "step": 383 }, { "KL/chosen_KL_mean": -137.10409545898438, "KL/mean": -170.72140502929688, "KL/rejected_KL_mean": -204.33868408203125, "KL/std": 95.64372253417969, "epoch": 0.5804988662131519, "fcm_dpo/beta": 0.006174423731863499, "fcm_dpo/delta": -0.01735561341047287, "fcm_dpo/margin": 67.234619140625, "fcm_dpo/q_t": 0.403439462184906, "grad_norm": 15.312804222106934, "learning_rate": 2.2492009565579875e-07, "logits/chosen": 0.7630481123924255, "logits/rejected": 0.7314999103546143, "logps/chosen": -202.57925415039062, "logps/ref_chosen": -65.47514343261719, "logps/ref_rejected": -79.67378234863281, "logps/rejected": -284.0124816894531, "loss": 1.1058, "margin_dpo/margin_mean": 67.234619140625, "margin_dpo/margin_std": 101.94595336914062, "step": 384 }, { "KL/chosen_KL_mean": -140.1160888671875, "KL/mean": -182.65090942382812, "KL/rejected_KL_mean": -225.18572998046875, "KL/std": 100.97268676757812, "epoch": 0.582010582010582, "fcm_dpo/beta": 0.005884962156414986, "fcm_dpo/delta": -0.11797355115413666, "fcm_dpo/margin": 85.06967163085938, "fcm_dpo/q_t": 0.3843558132648468, "grad_norm": 11.957921028137207, "learning_rate": 2.2360490367648084e-07, "logits/chosen": 0.701429009437561, "logits/rejected": 0.6622118949890137, "logps/chosen": -206.17259216308594, "logps/ref_chosen": -66.0565185546875, "logps/ref_rejected": -86.68023681640625, "logps/rejected": -311.865966796875, "loss": 1.0349, "margin_dpo/margin_mean": 85.06966400146484, "margin_dpo/margin_std": 103.32591247558594, "step": 385 }, { "KL/chosen_KL_mean": -161.50335693359375, "KL/mean": -189.95013427734375, "KL/rejected_KL_mean": -218.39691162109375, "KL/std": 107.01244354248047, "epoch": 0.5835222978080121, "fcm_dpo/beta": 0.005871026776731014, "fcm_dpo/delta": 0.07351464033126831, "fcm_dpo/margin": 56.8935661315918, "fcm_dpo/q_t": 0.42427319288253784, "grad_norm": 13.5010347366333, "learning_rate": 2.2229045002474724e-07, "logits/chosen": 0.6531039476394653, "logits/rejected": 0.6064084768295288, "logps/chosen": -237.12701416015625, "logps/ref_chosen": -75.6236572265625, "logps/ref_rejected": -92.62330627441406, "logps/rejected": -311.02020263671875, "loss": 1.1754, "margin_dpo/margin_mean": 56.89356994628906, "margin_dpo/margin_std": 107.92727661132812, "step": 386 }, { "KL/chosen_KL_mean": -141.63909912109375, "KL/mean": -181.66258239746094, "KL/rejected_KL_mean": -221.68606567382812, "KL/std": 96.4151382446289, "epoch": 0.5850340136054422, "fcm_dpo/beta": 0.005757711827754974, "fcm_dpo/delta": -0.07114600390195847, "fcm_dpo/margin": 80.04695129394531, "fcm_dpo/q_t": 0.3940368890762329, "grad_norm": 13.86131763458252, "learning_rate": 2.209767714686924e-07, "logits/chosen": 0.7648110389709473, "logits/rejected": 0.6623188257217407, "logps/chosen": -188.86080932617188, "logps/ref_chosen": -47.22170639038086, "logps/ref_rejected": -87.338134765625, "logps/rejected": -309.0242004394531, "loss": 1.0566, "margin_dpo/margin_mean": 80.04695129394531, "margin_dpo/margin_std": 101.74493408203125, "step": 387 }, { "KL/chosen_KL_mean": -148.16046142578125, "KL/mean": -172.68748474121094, "KL/rejected_KL_mean": -197.21450805664062, "KL/std": 103.10355377197266, "epoch": 0.5865457294028723, "fcm_dpo/beta": 0.0057308124378323555, "fcm_dpo/delta": 0.003550410969182849, "fcm_dpo/margin": 49.054039001464844, "fcm_dpo/q_t": 0.4384087324142456, "grad_norm": 12.839620590209961, "learning_rate": 2.1966390475472954e-07, "logits/chosen": 0.7350201606750488, "logits/rejected": 0.7327404022216797, "logps/chosen": -222.73992919921875, "logps/ref_chosen": -74.5794677734375, "logps/ref_rejected": -79.92558288574219, "logps/rejected": -277.14007568359375, "loss": 1.2403, "margin_dpo/margin_mean": 49.054039001464844, "margin_dpo/margin_std": 121.98270416259766, "step": 388 }, { "KL/chosen_KL_mean": -138.47047424316406, "KL/mean": -178.14364624023438, "KL/rejected_KL_mean": -217.81683349609375, "KL/std": 97.10962677001953, "epoch": 0.5880574452003023, "fcm_dpo/beta": 0.005619604140520096, "fcm_dpo/delta": -0.05288499593734741, "fcm_dpo/margin": 79.34635925292969, "fcm_dpo/q_t": 0.39804351329803467, "grad_norm": 21.324504852294922, "learning_rate": 2.1835188660656265e-07, "logits/chosen": 0.7512508630752563, "logits/rejected": 0.7208691835403442, "logps/chosen": -200.0948486328125, "logps/ref_chosen": -61.624366760253906, "logps/ref_rejected": -76.50978088378906, "logps/rejected": -294.32659912109375, "loss": 1.0743, "margin_dpo/margin_mean": 79.34635925292969, "margin_dpo/margin_std": 107.70132446289062, "step": 389 }, { "KL/chosen_KL_mean": -123.1337661743164, "KL/mean": -153.21640014648438, "KL/rejected_KL_mean": -183.29904174804688, "KL/std": 92.85203552246094, "epoch": 0.5895691609977324, "fcm_dpo/beta": 0.00567449489608407, "fcm_dpo/delta": 0.06538356095552444, "fcm_dpo/margin": 60.165260314941406, "fcm_dpo/q_t": 0.4235107898712158, "grad_norm": 11.731263160705566, "learning_rate": 2.170407537241599e-07, "logits/chosen": 0.8297513723373413, "logits/rejected": 0.7623568773269653, "logps/chosen": -169.00563049316406, "logps/ref_chosen": -45.871864318847656, "logps/ref_rejected": -61.305999755859375, "logps/rejected": -244.60504150390625, "loss": 1.15, "margin_dpo/margin_mean": 60.16526412963867, "margin_dpo/margin_std": 99.79165649414062, "step": 390 }, { "KL/chosen_KL_mean": -138.01197814941406, "KL/mean": -175.88485717773438, "KL/rejected_KL_mean": -213.7576904296875, "KL/std": 97.07452392578125, "epoch": 0.5910808767951625, "fcm_dpo/beta": 0.005652905907481909, "fcm_dpo/delta": -0.03288257122039795, "fcm_dpo/margin": 75.74574279785156, "fcm_dpo/q_t": 0.40263625979423523, "grad_norm": 12.69221305847168, "learning_rate": 2.1573054278272636e-07, "logits/chosen": 0.7633911967277527, "logits/rejected": 0.7019753456115723, "logps/chosen": -196.198974609375, "logps/ref_chosen": -58.18701171875, "logps/ref_rejected": -83.63442993164062, "logps/rejected": -297.39215087890625, "loss": 1.1052, "margin_dpo/margin_mean": 75.74574279785156, "margin_dpo/margin_std": 116.3956298828125, "step": 391 }, { "KL/chosen_KL_mean": -127.93291473388672, "KL/mean": -166.52969360351562, "KL/rejected_KL_mean": -205.12648010253906, "KL/std": 105.89525604248047, "epoch": 0.5925925925925926, "fcm_dpo/beta": 0.005653767846524715, "fcm_dpo/delta": -0.043347496539354324, "fcm_dpo/margin": 77.19355773925781, "fcm_dpo/q_t": 0.4022483229637146, "grad_norm": 11.86667537689209, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 0.7835770845413208, "logits/rejected": 0.7315959930419922, "logps/chosen": -197.6774444580078, "logps/ref_chosen": -69.7445297241211, "logps/ref_rejected": -94.05877685546875, "logps/rejected": -299.18524169921875, "loss": 1.1083, "margin_dpo/margin_mean": 77.19355773925781, "margin_dpo/margin_std": 123.01547241210938, "step": 392 }, { "KL/chosen_KL_mean": -142.97479248046875, "KL/mean": -184.1305694580078, "KL/rejected_KL_mean": -225.28634643554688, "KL/std": 102.22776794433594, "epoch": 0.5941043083900227, "fcm_dpo/beta": 0.005305946804583073, "fcm_dpo/delta": -0.04566541686654091, "fcm_dpo/margin": 82.31155395507812, "fcm_dpo/q_t": 0.3981964588165283, "grad_norm": 10.274748802185059, "learning_rate": 2.131130332936195e-07, "logits/chosen": 0.7935898303985596, "logits/rejected": 0.7587308883666992, "logps/chosen": -195.3096923828125, "logps/ref_chosen": -52.33489990234375, "logps/ref_rejected": -74.33809661865234, "logps/rejected": -299.62445068359375, "loss": 1.0668, "margin_dpo/margin_mean": 82.3115463256836, "margin_dpo/margin_std": 100.75532531738281, "step": 393 }, { "KL/chosen_KL_mean": -136.1352996826172, "KL/mean": -171.53225708007812, "KL/rejected_KL_mean": -206.92921447753906, "KL/std": 95.84933471679688, "epoch": 0.5956160241874527, "fcm_dpo/beta": 0.005464646033942699, "fcm_dpo/delta": 0.013672813773155212, "fcm_dpo/margin": 70.79390716552734, "fcm_dpo/q_t": 0.41006964445114136, "grad_norm": 12.026927947998047, "learning_rate": 2.1180580796331323e-07, "logits/chosen": 0.7781930565834045, "logits/rejected": 0.751598596572876, "logps/chosen": -196.81143188476562, "logps/ref_chosen": -60.6761360168457, "logps/ref_rejected": -71.36074829101562, "logps/rejected": -278.28997802734375, "loss": 1.1026, "margin_dpo/margin_mean": 70.79390716552734, "margin_dpo/margin_std": 94.3880615234375, "step": 394 }, { "KL/chosen_KL_mean": -137.5908660888672, "KL/mean": -171.83412170410156, "KL/rejected_KL_mean": -206.07737731933594, "KL/std": 98.70809173583984, "epoch": 0.5971277399848829, "fcm_dpo/beta": 0.005507326684892178, "fcm_dpo/delta": 0.02526915818452835, "fcm_dpo/margin": 68.48651885986328, "fcm_dpo/q_t": 0.41558602452278137, "grad_norm": 11.91160774230957, "learning_rate": 2.104996510066625e-07, "logits/chosen": 0.8012561798095703, "logits/rejected": 0.7142231464385986, "logps/chosen": -188.1951904296875, "logps/ref_chosen": -50.60432434082031, "logps/ref_rejected": -77.08731079101562, "logps/rejected": -283.1646728515625, "loss": 1.1236, "margin_dpo/margin_mean": 68.48651885986328, "margin_dpo/margin_std": 106.47164916992188, "step": 395 }, { "KL/chosen_KL_mean": -133.67884826660156, "KL/mean": -168.15289306640625, "KL/rejected_KL_mean": -202.62692260742188, "KL/std": 106.60101318359375, "epoch": 0.5986394557823129, "fcm_dpo/beta": 0.0052660866640508175, "fcm_dpo/delta": -0.07049454748630524, "fcm_dpo/margin": 68.94806671142578, "fcm_dpo/q_t": 0.4145070016384125, "grad_norm": 10.632168769836426, "learning_rate": 2.0919459895968517e-07, "logits/chosen": 0.7917169332504272, "logits/rejected": 0.7078319787979126, "logps/chosen": -185.0384521484375, "logps/ref_chosen": -51.35961151123047, "logps/ref_rejected": -79.89360046386719, "logps/rejected": -282.5205078125, "loss": 1.1199, "margin_dpo/margin_mean": 68.94806671142578, "margin_dpo/margin_std": 95.077392578125, "step": 396 }, { "KL/chosen_KL_mean": -154.8973388671875, "KL/mean": -174.40435791015625, "KL/rejected_KL_mean": -193.911376953125, "KL/std": 99.12435150146484, "epoch": 0.600151171579743, "fcm_dpo/beta": 0.0052660866640508175, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.0140266418457, "fcm_dpo/q_t": 0.45285657048225403, "grad_norm": 11.291228294372559, "learning_rate": 2.078906883274924e-07, "logits/chosen": 0.7051912546157837, "logits/rejected": 0.6572903990745544, "logps/chosen": -221.35357666015625, "logps/ref_chosen": -66.45622253417969, "logps/ref_rejected": -85.74736785888672, "logps/rejected": -279.65875244140625, "loss": 1.2758, "margin_dpo/margin_mean": 39.01403045654297, "margin_dpo/margin_std": 111.57695770263672, "step": 397 }, { "KL/chosen_KL_mean": -129.05348205566406, "KL/mean": -174.99749755859375, "KL/rejected_KL_mean": -220.94149780273438, "KL/std": 103.12016296386719, "epoch": 0.6016628873771731, "fcm_dpo/beta": 0.0050222016870975494, "fcm_dpo/delta": -0.0740644708275795, "fcm_dpo/margin": 91.88800048828125, "fcm_dpo/q_t": 0.39427825808525085, "grad_norm": 9.868289947509766, "learning_rate": 2.065879555832674e-07, "logits/chosen": 0.7551954984664917, "logits/rejected": 0.6983195543289185, "logps/chosen": -178.2977294921875, "logps/ref_chosen": -49.244239807128906, "logps/ref_rejected": -75.18949127197266, "logps/rejected": -296.1309814453125, "loss": 1.0491, "margin_dpo/margin_mean": 91.88800048828125, "margin_dpo/margin_std": 112.2702865600586, "step": 398 }, { "KL/chosen_KL_mean": -154.25454711914062, "KL/mean": -207.419921875, "KL/rejected_KL_mean": -260.58526611328125, "KL/std": 118.88225555419922, "epoch": 0.6031746031746031, "fcm_dpo/beta": 0.004701526835560799, "fcm_dpo/delta": -0.12128760665655136, "fcm_dpo/margin": 106.33074188232422, "fcm_dpo/q_t": 0.3860124349594116, "grad_norm": 10.134847640991211, "learning_rate": 2.052864371672457e-07, "logits/chosen": 0.7207819223403931, "logits/rejected": 0.591735303401947, "logps/chosen": -222.56134033203125, "logps/ref_chosen": -68.30679321289062, "logps/ref_rejected": -113.2708511352539, "logps/rejected": -373.85614013671875, "loss": 1.0369, "margin_dpo/margin_mean": 106.33074188232422, "margin_dpo/margin_std": 135.85537719726562, "step": 399 }, { "KL/chosen_KL_mean": -171.22833251953125, "KL/mean": -200.75521850585938, "KL/rejected_KL_mean": -230.28211975097656, "KL/std": 106.50505065917969, "epoch": 0.6046863189720333, "fcm_dpo/beta": 0.004656361881643534, "fcm_dpo/delta": -0.00973660871386528, "fcm_dpo/margin": 59.05378341674805, "fcm_dpo/q_t": 0.43434858322143555, "grad_norm": 13.978921890258789, "learning_rate": 2.0398616948569493e-07, "logits/chosen": 0.7625016570091248, "logits/rejected": 0.7079190611839294, "logps/chosen": -242.85482788085938, "logps/ref_chosen": -71.62649536132812, "logps/ref_rejected": -90.98765563964844, "logps/rejected": -321.269775390625, "loss": 1.188, "margin_dpo/margin_mean": 59.05378341674805, "margin_dpo/margin_std": 100.4428939819336, "step": 400 }, { "KL/chosen_KL_mean": -128.79498291015625, "KL/mean": -171.2772674560547, "KL/rejected_KL_mean": -213.75953674316406, "KL/std": 112.90093994140625, "epoch": 0.6061980347694633, "fcm_dpo/beta": 0.004625148139894009, "fcm_dpo/delta": 0.007325233891606331, "fcm_dpo/margin": 84.96454620361328, "fcm_dpo/q_t": 0.4098255932331085, "grad_norm": 7.507087230682373, "learning_rate": 2.0268718890989752e-07, "logits/chosen": 0.8112884759902954, "logits/rejected": 0.7324414253234863, "logps/chosen": -182.51995849609375, "logps/ref_chosen": -53.72495651245117, "logps/ref_rejected": -75.06304931640625, "logps/rejected": -288.82257080078125, "loss": 1.0941, "margin_dpo/margin_mean": 84.96454620361328, "margin_dpo/margin_std": 110.92296600341797, "step": 401 }, { "KL/chosen_KL_mean": -143.20652770996094, "KL/mean": -174.81996154785156, "KL/rejected_KL_mean": -206.43341064453125, "KL/std": 94.28620910644531, "epoch": 0.6077097505668935, "fcm_dpo/beta": 0.004691399168223143, "fcm_dpo/delta": 0.005168864969164133, "fcm_dpo/margin": 63.226871490478516, "fcm_dpo/q_t": 0.43117454648017883, "grad_norm": 11.448514938354492, "learning_rate": 2.013895317751323e-07, "logits/chosen": 0.7634487152099609, "logits/rejected": 0.7546102404594421, "logps/chosen": -205.0804443359375, "logps/ref_chosen": -61.873931884765625, "logps/ref_rejected": -66.15198516845703, "logps/rejected": -272.58538818359375, "loss": 1.1843, "margin_dpo/margin_mean": 63.22686767578125, "margin_dpo/margin_std": 115.47822570800781, "step": 402 }, { "KL/chosen_KL_mean": -149.43191528320312, "KL/mean": -191.5021209716797, "KL/rejected_KL_mean": -233.57232666015625, "KL/std": 111.39325714111328, "epoch": 0.6092214663643235, "fcm_dpo/beta": 0.004674067720770836, "fcm_dpo/delta": 0.007327292114496231, "fcm_dpo/margin": 84.14042663574219, "fcm_dpo/q_t": 0.4105861485004425, "grad_norm": 8.619983673095703, "learning_rate": 2.0009323437965898e-07, "logits/chosen": 0.8772906064987183, "logits/rejected": 0.7967815399169922, "logps/chosen": -200.75341796875, "logps/ref_chosen": -51.321502685546875, "logps/ref_rejected": -86.54010772705078, "logps/rejected": -320.1124267578125, "loss": 1.1015, "margin_dpo/margin_mean": 84.14042663574219, "margin_dpo/margin_std": 116.0869140625, "step": 403 }, { "KL/chosen_KL_mean": -143.56373596191406, "KL/mean": -187.47364807128906, "KL/rejected_KL_mean": -231.38356018066406, "KL/std": 115.00575256347656, "epoch": 0.6107331821617535, "fcm_dpo/beta": 0.0046137175522744656, "fcm_dpo/delta": -0.00824352353811264, "fcm_dpo/margin": 87.81982421875, "fcm_dpo/q_t": 0.40820419788360596, "grad_norm": 12.866887092590332, "learning_rate": 1.9879833298370237e-07, "logits/chosen": 0.7568644285202026, "logits/rejected": 0.6709632873535156, "logps/chosen": -205.82662963867188, "logps/ref_chosen": -62.26288604736328, "logps/ref_rejected": -95.19029998779297, "logps/rejected": -326.5738525390625, "loss": 1.1079, "margin_dpo/margin_mean": 87.81982421875, "margin_dpo/margin_std": 128.9308624267578, "step": 404 }, { "KL/chosen_KL_mean": -144.8743896484375, "KL/mean": -178.5138397216797, "KL/rejected_KL_mean": -212.15328979492188, "KL/std": 99.92108154296875, "epoch": 0.6122448979591837, "fcm_dpo/beta": 0.004627191927284002, "fcm_dpo/delta": -0.022051792591810226, "fcm_dpo/margin": 67.27889251708984, "fcm_dpo/q_t": 0.4273110032081604, "grad_norm": 9.795719146728516, "learning_rate": 1.975048638084379e-07, "logits/chosen": 0.8541995286941528, "logits/rejected": 0.8144976496696472, "logps/chosen": -195.458740234375, "logps/ref_chosen": -50.5843391418457, "logps/ref_rejected": -65.43156433105469, "logps/rejected": -277.5848388671875, "loss": 1.1557, "margin_dpo/margin_mean": 67.27889251708984, "margin_dpo/margin_std": 100.67938232421875, "step": 405 }, { "KL/chosen_KL_mean": -136.11376953125, "KL/mean": -183.8330078125, "KL/rejected_KL_mean": -231.55224609375, "KL/std": 109.36830139160156, "epoch": 0.6137566137566137, "fcm_dpo/beta": 0.004584764130413532, "fcm_dpo/delta": -0.04349678382277489, "fcm_dpo/margin": 95.4384536743164, "fcm_dpo/q_t": 0.398017942905426, "grad_norm": 9.676344871520996, "learning_rate": 1.9621286303497914e-07, "logits/chosen": 0.8280668258666992, "logits/rejected": 0.685680627822876, "logps/chosen": -185.109375, "logps/ref_chosen": -48.99560546875, "logps/ref_rejected": -92.47774505615234, "logps/rejected": -324.02996826171875, "loss": 1.0703, "margin_dpo/margin_mean": 95.43846130371094, "margin_dpo/margin_std": 120.8671875, "step": 406 }, { "KL/chosen_KL_mean": -180.22271728515625, "KL/mean": -216.6717529296875, "KL/rejected_KL_mean": -253.1208038330078, "KL/std": 127.8733139038086, "epoch": 0.6152683295540439, "fcm_dpo/beta": 0.004674847237765789, "fcm_dpo/delta": 0.06505510210990906, "fcm_dpo/margin": 72.8980941772461, "fcm_dpo/q_t": 0.4223581552505493, "grad_norm": 11.925191879272461, "learning_rate": 1.9492236680336483e-07, "logits/chosen": 0.6953166723251343, "logits/rejected": 0.6430000066757202, "logps/chosen": -269.623291015625, "logps/ref_chosen": -89.40056610107422, "logps/ref_rejected": -99.28775024414062, "logps/rejected": -352.4085693359375, "loss": 1.1603, "margin_dpo/margin_mean": 72.89810180664062, "margin_dpo/margin_std": 128.1045684814453, "step": 407 }, { "KL/chosen_KL_mean": -133.22731018066406, "KL/mean": -183.61326599121094, "KL/rejected_KL_mean": -233.99923706054688, "KL/std": 101.46388244628906, "epoch": 0.6167800453514739, "fcm_dpo/beta": 0.004601156804710627, "fcm_dpo/delta": -0.07428466528654099, "fcm_dpo/margin": 100.77191162109375, "fcm_dpo/q_t": 0.3899993896484375, "grad_norm": 9.173323631286621, "learning_rate": 1.9363341121154895e-07, "logits/chosen": 0.7866462469100952, "logits/rejected": 0.7229700088500977, "logps/chosen": -187.9312286376953, "logps/ref_chosen": -54.70391845703125, "logps/ref_rejected": -73.98648834228516, "logps/rejected": -307.9857177734375, "loss": 1.0333, "margin_dpo/margin_mean": 100.77191925048828, "margin_dpo/margin_std": 104.93372344970703, "step": 408 }, { "KL/chosen_KL_mean": -165.34664916992188, "KL/mean": -187.67367553710938, "KL/rejected_KL_mean": -210.000732421875, "KL/std": 87.25897216796875, "epoch": 0.618291761148904, "fcm_dpo/beta": 0.004743535537272692, "fcm_dpo/delta": 0.09411386400461197, "fcm_dpo/margin": 44.654083251953125, "fcm_dpo/q_t": 0.45126640796661377, "grad_norm": 11.334348678588867, "learning_rate": 1.9234603231438994e-07, "logits/chosen": 0.798064649105072, "logits/rejected": 0.807654857635498, "logps/chosen": -227.46487426757812, "logps/ref_chosen": -62.11822509765625, "logps/ref_rejected": -61.933509826660156, "logps/rejected": -271.9342346191406, "loss": 1.2371, "margin_dpo/margin_mean": 44.654083251953125, "margin_dpo/margin_std": 96.73648071289062, "step": 409 }, { "KL/chosen_KL_mean": -156.24102783203125, "KL/mean": -196.73635864257812, "KL/rejected_KL_mean": -237.231689453125, "KL/std": 102.72321319580078, "epoch": 0.6198034769463341, "fcm_dpo/beta": 0.0046883877366781235, "fcm_dpo/delta": 0.020214948803186417, "fcm_dpo/margin": 80.99066162109375, "fcm_dpo/q_t": 0.4119493365287781, "grad_norm": 10.344640731811523, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 0.7926585674285889, "logits/rejected": 0.7750309705734253, "logps/chosen": -218.043701171875, "logps/ref_chosen": -61.80266189575195, "logps/ref_rejected": -76.60002136230469, "logps/rejected": -313.83172607421875, "loss": 1.1039, "margin_dpo/margin_mean": 80.99066162109375, "margin_dpo/margin_std": 103.93037414550781, "step": 410 }, { "KL/chosen_KL_mean": -162.68948364257812, "KL/mean": -203.23043823242188, "KL/rejected_KL_mean": -243.77142333984375, "KL/std": 108.2699966430664, "epoch": 0.6213151927437641, "fcm_dpo/beta": 0.004831541329622269, "fcm_dpo/delta": 0.009294688701629639, "fcm_dpo/margin": 81.08193969726562, "fcm_dpo/q_t": 0.4101915955543518, "grad_norm": 9.14470386505127, "learning_rate": 1.8977614860195296e-07, "logits/chosen": 0.8241767287254333, "logits/rejected": 0.779667317867279, "logps/chosen": -217.13487243652344, "logps/ref_chosen": -54.44539260864258, "logps/ref_rejected": -74.5650863647461, "logps/rejected": -318.3365173339844, "loss": 1.1031, "margin_dpo/margin_mean": 81.08193969726562, "margin_dpo/margin_std": 114.11431884765625, "step": 411 }, { "KL/chosen_KL_mean": -166.61082458496094, "KL/mean": -202.292236328125, "KL/rejected_KL_mean": -237.9736328125, "KL/std": 91.02214813232422, "epoch": 0.6228269085411943, "fcm_dpo/beta": 0.004908284172415733, "fcm_dpo/delta": 0.05554809793829918, "fcm_dpo/margin": 71.36279296875, "fcm_dpo/q_t": 0.41911643743515015, "grad_norm": 10.918268203735352, "learning_rate": 1.8849371567184662e-07, "logits/chosen": 0.8318126797676086, "logits/rejected": 0.7821172475814819, "logps/chosen": -221.85891723632812, "logps/ref_chosen": -55.248085021972656, "logps/ref_rejected": -68.96623229980469, "logps/rejected": -306.9398498535156, "loss": 1.1214, "margin_dpo/margin_mean": 71.36279296875, "margin_dpo/margin_std": 96.66009521484375, "step": 412 }, { "KL/chosen_KL_mean": -187.35760498046875, "KL/mean": -221.8975830078125, "KL/rejected_KL_mean": -256.4375305175781, "KL/std": 111.54348754882812, "epoch": 0.6243386243386243, "fcm_dpo/beta": 0.00513922143727541, "fcm_dpo/delta": 0.04962531104683876, "fcm_dpo/margin": 69.0799560546875, "fcm_dpo/q_t": 0.42187249660491943, "grad_norm": 13.688892364501953, "learning_rate": 1.872130032047302e-07, "logits/chosen": 0.6731942296028137, "logits/rejected": 0.6520799398422241, "logps/chosen": -256.0783386230469, "logps/ref_chosen": -68.72074890136719, "logps/ref_rejected": -78.76539611816406, "logps/rejected": -335.20294189453125, "loss": 1.1735, "margin_dpo/margin_mean": 69.0799560546875, "margin_dpo/margin_std": 134.26339721679688, "step": 413 }, { "KL/chosen_KL_mean": -164.39401245117188, "KL/mean": -205.43978881835938, "KL/rejected_KL_mean": -246.48556518554688, "KL/std": 119.57435607910156, "epoch": 0.6258503401360545, "fcm_dpo/beta": 0.005096019245684147, "fcm_dpo/delta": -0.020970303565263748, "fcm_dpo/margin": 82.09153747558594, "fcm_dpo/q_t": 0.40284112095832825, "grad_norm": 10.116288185119629, "learning_rate": 1.8593404702488436e-07, "logits/chosen": 0.8319721221923828, "logits/rejected": 0.7853256464004517, "logps/chosen": -218.5322265625, "logps/ref_chosen": -54.138214111328125, "logps/ref_rejected": -74.65741729736328, "logps/rejected": -321.1429748535156, "loss": 1.0866, "margin_dpo/margin_mean": 82.09153747558594, "margin_dpo/margin_std": 109.47962951660156, "step": 414 }, { "KL/chosen_KL_mean": -164.3896484375, "KL/mean": -195.80303955078125, "KL/rejected_KL_mean": -227.21646118164062, "KL/std": 107.44061279296875, "epoch": 0.6273620559334845, "fcm_dpo/beta": 0.0052430275827646255, "fcm_dpo/delta": 0.07850854098796844, "fcm_dpo/margin": 62.826805114746094, "fcm_dpo/q_t": 0.4264346957206726, "grad_norm": 11.805853843688965, "learning_rate": 1.846568829074628e-07, "logits/chosen": 0.812616765499115, "logits/rejected": 0.801384449005127, "logps/chosen": -220.30819702148438, "logps/ref_chosen": -55.91856002807617, "logps/ref_rejected": -61.747703552246094, "logps/rejected": -288.96417236328125, "loss": 1.1706, "margin_dpo/margin_mean": 62.826805114746094, "margin_dpo/margin_std": 115.88273620605469, "step": 415 }, { "KL/chosen_KL_mean": -174.27407836914062, "KL/mean": -204.4618682861328, "KL/rejected_KL_mean": -234.649658203125, "KL/std": 114.86009216308594, "epoch": 0.6288737717309146, "fcm_dpo/beta": 0.005179405678063631, "fcm_dpo/delta": -0.09270616620779037, "fcm_dpo/margin": 60.37559509277344, "fcm_dpo/q_t": 0.4285816550254822, "grad_norm": 12.966651916503906, "learning_rate": 1.8338154657749128e-07, "logits/chosen": 0.8164198398590088, "logits/rejected": 0.7761438488960266, "logps/chosen": -228.99716186523438, "logps/ref_chosen": -54.72308349609375, "logps/ref_rejected": -69.17388916015625, "logps/rejected": -303.82354736328125, "loss": 1.1915, "margin_dpo/margin_mean": 60.37559509277344, "margin_dpo/margin_std": 108.85138702392578, "step": 416 }, { "KL/chosen_KL_mean": -181.67933654785156, "KL/mean": -218.7706298828125, "KL/rejected_KL_mean": -255.86190795898438, "KL/std": 106.55653381347656, "epoch": 0.6303854875283447, "fcm_dpo/beta": 0.005155562423169613, "fcm_dpo/delta": 0.017802808433771133, "fcm_dpo/margin": 74.18257904052734, "fcm_dpo/q_t": 0.4107144773006439, "grad_norm": 12.372243881225586, "learning_rate": 1.8210807370886849e-07, "logits/chosen": 0.9078543782234192, "logits/rejected": 0.8568928837776184, "logps/chosen": -238.47059631347656, "logps/ref_chosen": -56.791259765625, "logps/ref_rejected": -68.7791748046875, "logps/rejected": -324.64111328125, "loss": 1.1455, "margin_dpo/margin_mean": 74.18257904052734, "margin_dpo/margin_std": 129.5694122314453, "step": 417 }, { "KL/chosen_KL_mean": -196.171875, "KL/mean": -226.3531036376953, "KL/rejected_KL_mean": -256.53436279296875, "KL/std": 122.66242980957031, "epoch": 0.6318972033257747, "fcm_dpo/beta": 0.004903099965304136, "fcm_dpo/delta": -0.06217623129487038, "fcm_dpo/margin": 60.362483978271484, "fcm_dpo/q_t": 0.4314631521701813, "grad_norm": 12.916627883911133, "learning_rate": 1.8083649992336825e-07, "logits/chosen": 0.8412420153617859, "logits/rejected": 0.8448699712753296, "logps/chosen": -265.27984619140625, "logps/ref_chosen": -69.10798645019531, "logps/ref_rejected": -75.09132385253906, "logps/rejected": -331.62567138671875, "loss": 1.2087, "margin_dpo/margin_mean": 60.362483978271484, "margin_dpo/margin_std": 124.63722229003906, "step": 418 }, { "KL/chosen_KL_mean": -157.86094665527344, "KL/mean": -203.56805419921875, "KL/rejected_KL_mean": -249.27517700195312, "KL/std": 108.47840881347656, "epoch": 0.6334089191232048, "fcm_dpo/beta": 0.004795808810740709, "fcm_dpo/delta": -0.04438748210668564, "fcm_dpo/margin": 91.41424560546875, "fcm_dpo/q_t": 0.40094202756881714, "grad_norm": 11.021547317504883, "learning_rate": 1.7956686078964255e-07, "logits/chosen": 0.7512973546981812, "logits/rejected": 0.717224657535553, "logps/chosen": -216.03271484375, "logps/ref_chosen": -58.1717643737793, "logps/ref_rejected": -71.67066955566406, "logps/rejected": -320.94586181640625, "loss": 1.0782, "margin_dpo/margin_mean": 91.41425323486328, "margin_dpo/margin_std": 126.16435241699219, "step": 419 }, { "KL/chosen_KL_mean": -189.92071533203125, "KL/mean": -213.03028869628906, "KL/rejected_KL_mean": -236.13986206054688, "KL/std": 111.73857116699219, "epoch": 0.6349206349206349, "fcm_dpo/beta": 0.004948785062879324, "fcm_dpo/delta": 0.05984485149383545, "fcm_dpo/margin": 46.219139099121094, "fcm_dpo/q_t": 0.4502519965171814, "grad_norm": 11.667963981628418, "learning_rate": 1.782991918222275e-07, "logits/chosen": 0.8240588903427124, "logits/rejected": 0.7886266708374023, "logps/chosen": -246.97422790527344, "logps/ref_chosen": -57.05351257324219, "logps/ref_rejected": -62.670982360839844, "logps/rejected": -298.81085205078125, "loss": 1.2697, "margin_dpo/margin_mean": 46.219139099121094, "margin_dpo/margin_std": 130.39532470703125, "step": 420 }, { "KL/chosen_KL_mean": -178.27980041503906, "KL/mean": -212.07435607910156, "KL/rejected_KL_mean": -245.86886596679688, "KL/std": 111.47808837890625, "epoch": 0.636432350718065, "fcm_dpo/beta": 0.004891910124570131, "fcm_dpo/delta": -0.01926536299288273, "fcm_dpo/margin": 67.58908081054688, "fcm_dpo/q_t": 0.42542341351509094, "grad_norm": 12.780967712402344, "learning_rate": 1.7703352848054887e-07, "logits/chosen": 0.7950316071510315, "logits/rejected": 0.7450087070465088, "logps/chosen": -235.60305786132812, "logps/ref_chosen": -57.32324981689453, "logps/ref_rejected": -75.33782958984375, "logps/rejected": -321.20672607421875, "loss": 1.2105, "margin_dpo/margin_mean": 67.5890884399414, "margin_dpo/margin_std": 150.59071350097656, "step": 421 }, { "KL/chosen_KL_mean": -158.81130981445312, "KL/mean": -200.82315063476562, "KL/rejected_KL_mean": -242.8350067138672, "KL/std": 111.04067993164062, "epoch": 0.6379440665154951, "fcm_dpo/beta": 0.004911776166409254, "fcm_dpo/delta": -0.015146823599934578, "fcm_dpo/margin": 84.02371215820312, "fcm_dpo/q_t": 0.40485405921936035, "grad_norm": 12.647407531738281, "learning_rate": 1.7576990616793137e-07, "logits/chosen": 0.7745881080627441, "logits/rejected": 0.775098443031311, "logps/chosen": -225.86888122558594, "logps/ref_chosen": -67.05757141113281, "logps/ref_rejected": -72.12803649902344, "logps/rejected": -314.9630432128906, "loss": 1.0853, "margin_dpo/margin_mean": 84.0237045288086, "margin_dpo/margin_std": 109.76282501220703, "step": 422 }, { "KL/chosen_KL_mean": -155.72247314453125, "KL/mean": -203.5447998046875, "KL/rejected_KL_mean": -251.36709594726562, "KL/std": 113.12423706054688, "epoch": 0.6394557823129252, "fcm_dpo/beta": 0.004685261286795139, "fcm_dpo/delta": -0.05671604722738266, "fcm_dpo/margin": 95.64463806152344, "fcm_dpo/q_t": 0.39703303575515747, "grad_norm": 11.059937477111816, "learning_rate": 1.745083602306071e-07, "logits/chosen": 0.8790537118911743, "logits/rejected": 0.8249520063400269, "logps/chosen": -209.78414916992188, "logps/ref_chosen": -54.06167221069336, "logps/ref_rejected": -76.64092254638672, "logps/rejected": -328.0080261230469, "loss": 1.0619, "margin_dpo/margin_mean": 95.6446304321289, "margin_dpo/margin_std": 121.1661605834961, "step": 423 }, { "KL/chosen_KL_mean": -170.83416748046875, "KL/mean": -215.72669982910156, "KL/rejected_KL_mean": -260.6192321777344, "KL/std": 103.92689514160156, "epoch": 0.6409674981103552, "fcm_dpo/beta": 0.004581287503242493, "fcm_dpo/delta": -0.014987409114837646, "fcm_dpo/margin": 89.7850570678711, "fcm_dpo/q_t": 0.40572795271873474, "grad_norm": 14.250228881835938, "learning_rate": 1.7324892595672804e-07, "logits/chosen": 0.7274860143661499, "logits/rejected": 0.6937150359153748, "logps/chosen": -224.4430389404297, "logps/ref_chosen": -53.60887145996094, "logps/ref_rejected": -79.2139892578125, "logps/rejected": -339.8332214355469, "loss": 1.0895, "margin_dpo/margin_mean": 89.7850570678711, "margin_dpo/margin_std": 120.62132263183594, "step": 424 }, { "KL/chosen_KL_mean": -167.8055419921875, "KL/mean": -202.077392578125, "KL/rejected_KL_mean": -236.3492431640625, "KL/std": 100.29512023925781, "epoch": 0.6424792139077853, "fcm_dpo/beta": 0.004820922389626503, "fcm_dpo/delta": 0.07685823738574982, "fcm_dpo/margin": 68.54373168945312, "fcm_dpo/q_t": 0.4257993698120117, "grad_norm": 11.25399398803711, "learning_rate": 1.7199163857537824e-07, "logits/chosen": 0.8642111420631409, "logits/rejected": 0.843815803527832, "logps/chosen": -226.22021484375, "logps/ref_chosen": -58.41468048095703, "logps/ref_rejected": -66.59054565429688, "logps/rejected": -302.9397888183594, "loss": 1.157, "margin_dpo/margin_mean": 68.54371643066406, "margin_dpo/margin_std": 115.27527618408203, "step": 425 }, { "KL/chosen_KL_mean": -201.40029907226562, "KL/mean": -221.83224487304688, "KL/rejected_KL_mean": -242.26416015625, "KL/std": 111.92045593261719, "epoch": 0.6439909297052154, "fcm_dpo/beta": 0.004862668924033642, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 40.86387634277344, "fcm_dpo/q_t": 0.4547138810157776, "grad_norm": 15.746712684631348, "learning_rate": 1.7073653325558828e-07, "logits/chosen": 0.7816729545593262, "logits/rejected": 0.7827064990997314, "logps/chosen": -273.1085205078125, "logps/ref_chosen": -71.70822143554688, "logps/ref_rejected": -73.57725524902344, "logps/rejected": -315.8414306640625, "loss": 1.3073, "margin_dpo/margin_mean": 40.86387634277344, "margin_dpo/margin_std": 142.01010131835938, "step": 426 }, { "KL/chosen_KL_mean": -189.36972045898438, "KL/mean": -227.52943420410156, "KL/rejected_KL_mean": -265.6891784667969, "KL/std": 122.76222229003906, "epoch": 0.6455026455026455, "fcm_dpo/beta": 0.004909333307296038, "fcm_dpo/delta": 0.028550054877996445, "fcm_dpo/margin": 76.31942749023438, "fcm_dpo/q_t": 0.417738676071167, "grad_norm": 12.455328941345215, "learning_rate": 1.6948364510535218e-07, "logits/chosen": 0.8194973468780518, "logits/rejected": 0.7630200982093811, "logps/chosen": -248.0124969482422, "logps/ref_chosen": -58.64276885986328, "logps/ref_rejected": -86.25437927246094, "logps/rejected": -351.94354248046875, "loss": 1.1438, "margin_dpo/margin_mean": 76.31942749023438, "margin_dpo/margin_std": 133.41810607910156, "step": 427 }, { "KL/chosen_KL_mean": -184.95486450195312, "KL/mean": -222.78594970703125, "KL/rejected_KL_mean": -260.6170349121094, "KL/std": 124.32109832763672, "epoch": 0.6470143613000756, "fcm_dpo/beta": 0.005052429623901844, "fcm_dpo/delta": 0.018595866858959198, "fcm_dpo/margin": 75.66215515136719, "fcm_dpo/q_t": 0.4149612486362457, "grad_norm": 11.38692569732666, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 0.8025504946708679, "logits/rejected": 0.7721965909004211, "logps/chosen": -251.55091857910156, "logps/ref_chosen": -66.5960464477539, "logps/ref_rejected": -82.3941650390625, "logps/rejected": -343.01116943359375, "loss": 1.1482, "margin_dpo/margin_mean": 75.66214752197266, "margin_dpo/margin_std": 133.01632690429688, "step": 428 }, { "KL/chosen_KL_mean": -187.84649658203125, "KL/mean": -216.41619873046875, "KL/rejected_KL_mean": -244.9859161376953, "KL/std": 100.31537628173828, "epoch": 0.6485260770975056, "fcm_dpo/beta": 0.005307496525347233, "fcm_dpo/delta": 0.10380184650421143, "fcm_dpo/margin": 57.139427185058594, "fcm_dpo/q_t": 0.43025392293930054, "grad_norm": 12.65349006652832, "learning_rate": 1.669846604344412e-07, "logits/chosen": 0.7885690927505493, "logits/rejected": 0.8087594509124756, "logps/chosen": -244.856201171875, "logps/ref_chosen": -57.00970458984375, "logps/ref_rejected": -59.86549377441406, "logps/rejected": -304.8514099121094, "loss": 1.1977, "margin_dpo/margin_mean": 57.139427185058594, "margin_dpo/margin_std": 115.14334106445312, "step": 429 }, { "KL/chosen_KL_mean": -165.86953735351562, "KL/mean": -214.21212768554688, "KL/rejected_KL_mean": -262.5546875, "KL/std": 101.71453094482422, "epoch": 0.6500377928949358, "fcm_dpo/beta": 0.005062808282673359, "fcm_dpo/delta": -0.1054583340883255, "fcm_dpo/margin": 96.68514251708984, "fcm_dpo/q_t": 0.3835219144821167, "grad_norm": 13.401054382324219, "learning_rate": 1.6573863381573954e-07, "logits/chosen": 0.7247724533081055, "logits/rejected": 0.7243069410324097, "logps/chosen": -225.4327392578125, "logps/ref_chosen": -59.563194274902344, "logps/ref_rejected": -70.52289581298828, "logps/rejected": -333.0776062011719, "loss": 1.0434, "margin_dpo/margin_mean": 96.68515014648438, "margin_dpo/margin_std": 119.87529754638672, "step": 430 }, { "KL/chosen_KL_mean": -164.5930633544922, "KL/mean": -201.393310546875, "KL/rejected_KL_mean": -238.1935577392578, "KL/std": 106.98062896728516, "epoch": 0.6515495086923658, "fcm_dpo/beta": 0.0050440579652786255, "fcm_dpo/delta": 0.032343633472919464, "fcm_dpo/margin": 73.6004867553711, "fcm_dpo/q_t": 0.41668662428855896, "grad_norm": 12.450831413269043, "learning_rate": 1.6449496416858282e-07, "logits/chosen": 0.790619969367981, "logits/rejected": 0.7420604825019836, "logps/chosen": -214.79339599609375, "logps/ref_chosen": -50.20032501220703, "logps/ref_rejected": -77.81680297851562, "logps/rejected": -316.0103759765625, "loss": 1.1385, "margin_dpo/margin_mean": 73.60047912597656, "margin_dpo/margin_std": 121.77957916259766, "step": 431 }, { "KL/chosen_KL_mean": -174.41119384765625, "KL/mean": -212.10067749023438, "KL/rejected_KL_mean": -249.7901611328125, "KL/std": 106.63932037353516, "epoch": 0.6530612244897959, "fcm_dpo/beta": 0.005112434271723032, "fcm_dpo/delta": 0.016498606652021408, "fcm_dpo/margin": 75.37895202636719, "fcm_dpo/q_t": 0.4119753837585449, "grad_norm": 13.068375587463379, "learning_rate": 1.632536862810844e-07, "logits/chosen": 0.8498221039772034, "logits/rejected": 0.8017172813415527, "logps/chosen": -236.07394409179688, "logps/ref_chosen": -61.662757873535156, "logps/ref_rejected": -83.94496154785156, "logps/rejected": -333.735107421875, "loss": 1.1373, "margin_dpo/margin_mean": 75.37895202636719, "margin_dpo/margin_std": 127.255615234375, "step": 432 }, { "KL/chosen_KL_mean": -171.37161254882812, "KL/mean": -213.88760375976562, "KL/rejected_KL_mean": -256.4035949707031, "KL/std": 99.43716430664062, "epoch": 0.654572940287226, "fcm_dpo/beta": 0.00509435310959816, "fcm_dpo/delta": -0.03820203244686127, "fcm_dpo/margin": 85.03195190429688, "fcm_dpo/q_t": 0.3992968797683716, "grad_norm": 13.277294158935547, "learning_rate": 1.6201483487445515e-07, "logits/chosen": 0.8910690546035767, "logits/rejected": 0.8961876034736633, "logps/chosen": -235.10079956054688, "logps/ref_chosen": -63.72917938232422, "logps/ref_rejected": -65.8391342163086, "logps/rejected": -322.24273681640625, "loss": 1.079, "margin_dpo/margin_mean": 85.03195190429688, "margin_dpo/margin_std": 113.61282348632812, "step": 433 }, { "KL/chosen_KL_mean": -146.40029907226562, "KL/mean": -194.85731506347656, "KL/rejected_KL_mean": -243.3143310546875, "KL/std": 110.77456665039062, "epoch": 0.656084656084656, "fcm_dpo/beta": 0.004744514357298613, "fcm_dpo/delta": -0.07580451667308807, "fcm_dpo/margin": 96.91404724121094, "fcm_dpo/q_t": 0.3922083377838135, "grad_norm": 11.460370063781738, "learning_rate": 1.6077844460203204e-07, "logits/chosen": 0.9136182069778442, "logits/rejected": 0.8583601713180542, "logps/chosen": -194.3736114501953, "logps/ref_chosen": -47.97331619262695, "logps/ref_rejected": -72.51132202148438, "logps/rejected": -315.8256530761719, "loss": 1.0742, "margin_dpo/margin_mean": 96.91405487060547, "margin_dpo/margin_std": 131.41989135742188, "step": 434 }, { "KL/chosen_KL_mean": -177.25384521484375, "KL/mean": -212.98779296875, "KL/rejected_KL_mean": -248.72177124023438, "KL/std": 106.70513916015625, "epoch": 0.6575963718820862, "fcm_dpo/beta": 0.004973133094608784, "fcm_dpo/delta": 0.04807582497596741, "fcm_dpo/margin": 71.46792602539062, "fcm_dpo/q_t": 0.4196769893169403, "grad_norm": 13.298686027526855, "learning_rate": 1.5954455004830878e-07, "logits/chosen": 0.9066946506500244, "logits/rejected": 0.8741989135742188, "logps/chosen": -234.3140869140625, "logps/ref_chosen": -57.06024932861328, "logps/ref_rejected": -71.69146728515625, "logps/rejected": -320.4132385253906, "loss": 1.1529, "margin_dpo/margin_mean": 71.46792602539062, "margin_dpo/margin_std": 123.43423461914062, "step": 435 }, { "KL/chosen_KL_mean": -178.68710327148438, "KL/mean": -208.48751831054688, "KL/rejected_KL_mean": -238.28793334960938, "KL/std": 103.73971557617188, "epoch": 0.6591080876795162, "fcm_dpo/beta": 0.005142658948898315, "fcm_dpo/delta": 0.1033397912979126, "fcm_dpo/margin": 59.600830078125, "fcm_dpo/q_t": 0.4327392578125, "grad_norm": 13.91799545288086, "learning_rate": 1.5831318572796847e-07, "logits/chosen": 0.8306369781494141, "logits/rejected": 0.7831582427024841, "logps/chosen": -234.84515380859375, "logps/ref_chosen": -56.158050537109375, "logps/ref_rejected": -67.63787841796875, "logps/rejected": -305.9258117675781, "loss": 1.206, "margin_dpo/margin_mean": 59.600830078125, "margin_dpo/margin_std": 127.89569091796875, "step": 436 }, { "KL/chosen_KL_mean": -183.9375457763672, "KL/mean": -221.9730224609375, "KL/rejected_KL_mean": -260.00848388671875, "KL/std": 115.573486328125, "epoch": 0.6606198034769464, "fcm_dpo/beta": 0.005009764805436134, "fcm_dpo/delta": -0.082071453332901, "fcm_dpo/margin": 76.07093811035156, "fcm_dpo/q_t": 0.41367313265800476, "grad_norm": 15.81583023071289, "learning_rate": 1.5708438608491815e-07, "logits/chosen": 0.8169673085212708, "logits/rejected": 0.7099220752716064, "logps/chosen": -240.92333984375, "logps/ref_chosen": -56.98578643798828, "logps/ref_rejected": -85.61524963378906, "logps/rejected": -345.62371826171875, "loss": 1.1729, "margin_dpo/margin_mean": 76.07093811035156, "margin_dpo/margin_std": 148.16189575195312, "step": 437 }, { "KL/chosen_KL_mean": -159.0219268798828, "KL/mean": -206.44488525390625, "KL/rejected_KL_mean": -253.86782836914062, "KL/std": 118.10974884033203, "epoch": 0.6621315192743764, "fcm_dpo/beta": 0.004896242171525955, "fcm_dpo/delta": -0.07475844770669937, "fcm_dpo/margin": 94.84590148925781, "fcm_dpo/q_t": 0.3970157504081726, "grad_norm": 12.760135650634766, "learning_rate": 1.558581854913253e-07, "logits/chosen": 0.8870106935501099, "logits/rejected": 0.8304316997528076, "logps/chosen": -200.29971313476562, "logps/ref_chosen": -41.27777862548828, "logps/ref_rejected": -65.33840942382812, "logps/rejected": -319.20623779296875, "loss": 1.0676, "margin_dpo/margin_mean": 94.84590148925781, "margin_dpo/margin_std": 133.10060119628906, "step": 438 }, { "KL/chosen_KL_mean": -186.03314208984375, "KL/mean": -229.74700927734375, "KL/rejected_KL_mean": -273.4609069824219, "KL/std": 121.18110656738281, "epoch": 0.6636432350718064, "fcm_dpo/beta": 0.004845023155212402, "fcm_dpo/delta": -0.03054000437259674, "fcm_dpo/margin": 87.42778015136719, "fcm_dpo/q_t": 0.40231576561927795, "grad_norm": 11.937276840209961, "learning_rate": 1.5463461824665658e-07, "logits/chosen": 0.726641833782196, "logits/rejected": 0.7027250528335571, "logps/chosen": -267.4508056640625, "logps/ref_chosen": -81.41764831542969, "logps/ref_rejected": -94.72309875488281, "logps/rejected": -368.1839904785156, "loss": 1.0944, "margin_dpo/margin_mean": 87.42778015136719, "margin_dpo/margin_std": 121.04769897460938, "step": 439 }, { "KL/chosen_KL_mean": -155.7847900390625, "KL/mean": -199.2237548828125, "KL/rejected_KL_mean": -242.66273498535156, "KL/std": 106.59927368164062, "epoch": 0.6651549508692366, "fcm_dpo/beta": 0.00472570164129138, "fcm_dpo/delta": -0.012669751420617104, "fcm_dpo/margin": 86.87796020507812, "fcm_dpo/q_t": 0.40606674551963806, "grad_norm": 15.137947082519531, "learning_rate": 1.534137185767178e-07, "logits/chosen": 0.8190924525260925, "logits/rejected": 0.7351869344711304, "logps/chosen": -198.32296752929688, "logps/ref_chosen": -42.538185119628906, "logps/ref_rejected": -69.78813934326172, "logps/rejected": -312.45086669921875, "loss": 1.0963, "margin_dpo/margin_mean": 86.87796783447266, "margin_dpo/margin_std": 120.90702819824219, "step": 440 }, { "KL/chosen_KL_mean": -163.71417236328125, "KL/mean": -209.40249633789062, "KL/rejected_KL_mean": -255.09083557128906, "KL/std": 117.68836212158203, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004568706266582012, "fcm_dpo/delta": -0.021269341930747032, "fcm_dpo/margin": 91.37667083740234, "fcm_dpo/q_t": 0.4020621180534363, "grad_norm": 12.57343864440918, "learning_rate": 1.521955206326976e-07, "logits/chosen": 0.7851335406303406, "logits/rejected": 0.7083240747451782, "logps/chosen": -221.30738830566406, "logps/ref_chosen": -57.593223571777344, "logps/ref_rejected": -84.82878875732422, "logps/rejected": -339.91961669921875, "loss": 1.0718, "margin_dpo/margin_mean": 91.37667083740234, "margin_dpo/margin_std": 108.32664489746094, "step": 441 }, { "KL/chosen_KL_mean": -194.39462280273438, "KL/mean": -241.25390625, "KL/rejected_KL_mean": -288.11322021484375, "KL/std": 117.97537231445312, "epoch": 0.6681783824640968, "fcm_dpo/beta": 0.004501115530729294, "fcm_dpo/delta": -0.026340365409851074, "fcm_dpo/margin": 93.71859741210938, "fcm_dpo/q_t": 0.40267208218574524, "grad_norm": 14.261401176452637, "learning_rate": 1.5098005849021078e-07, "logits/chosen": 0.790300726890564, "logits/rejected": 0.7574805021286011, "logps/chosen": -261.8558349609375, "logps/ref_chosen": -67.46121978759766, "logps/ref_rejected": -89.0693588256836, "logps/rejected": -377.18255615234375, "loss": 1.0786, "margin_dpo/margin_mean": 93.7186050415039, "margin_dpo/margin_std": 120.49788665771484, "step": 442 }, { "KL/chosen_KL_mean": -160.43263244628906, "KL/mean": -216.71011352539062, "KL/rejected_KL_mean": -272.98760986328125, "KL/std": 124.23033905029297, "epoch": 0.6696900982615268, "fcm_dpo/beta": 0.004319561645388603, "fcm_dpo/delta": -0.10150419175624847, "fcm_dpo/margin": 112.55494689941406, "fcm_dpo/q_t": 0.3878478407859802, "grad_norm": 11.641592025756836, "learning_rate": 1.4976736614834662e-07, "logits/chosen": 0.8116977214813232, "logits/rejected": 0.7602815628051758, "logps/chosen": -215.22872924804688, "logps/ref_chosen": -54.79610061645508, "logps/ref_rejected": -77.80781555175781, "logps/rejected": -350.79541015625, "loss": 1.0382, "margin_dpo/margin_mean": 112.5549545288086, "margin_dpo/margin_std": 139.6777801513672, "step": 443 }, { "KL/chosen_KL_mean": -191.93292236328125, "KL/mean": -221.1522674560547, "KL/rejected_KL_mean": -250.37164306640625, "KL/std": 127.5732421875, "epoch": 0.671201814058957, "fcm_dpo/beta": 0.00436423160135746, "fcm_dpo/delta": 0.03717103973031044, "fcm_dpo/margin": 58.438720703125, "fcm_dpo/q_t": 0.4444841146469116, "grad_norm": 12.10918140411377, "learning_rate": 1.4855747752871654e-07, "logits/chosen": 0.8365426063537598, "logits/rejected": 0.7632496356964111, "logps/chosen": -250.68197631835938, "logps/ref_chosen": -58.749061584472656, "logps/ref_rejected": -86.87396240234375, "logps/rejected": -337.24560546875, "loss": 1.2394, "margin_dpo/margin_mean": 58.438720703125, "margin_dpo/margin_std": 142.1253662109375, "step": 444 }, { "KL/chosen_KL_mean": -183.2960968017578, "KL/mean": -231.47491455078125, "KL/rejected_KL_mean": -279.6537170410156, "KL/std": 110.73534393310547, "epoch": 0.672713529856387, "fcm_dpo/beta": 0.004318728111684322, "fcm_dpo/delta": -0.01852697879076004, "fcm_dpo/margin": 96.35762023925781, "fcm_dpo/q_t": 0.4018581509590149, "grad_norm": 10.044200897216797, "learning_rate": 1.473504264745062e-07, "logits/chosen": 0.7895313501358032, "logits/rejected": 0.7898776531219482, "logps/chosen": -244.21353149414062, "logps/ref_chosen": -60.91743850708008, "logps/ref_rejected": -71.5637435913086, "logps/rejected": -351.21746826171875, "loss": 1.0725, "margin_dpo/margin_mean": 96.35762023925781, "margin_dpo/margin_std": 115.74655151367188, "step": 445 }, { "KL/chosen_KL_mean": -164.505126953125, "KL/mean": -212.84637451171875, "KL/rejected_KL_mean": -261.1876525878906, "KL/std": 113.10897827148438, "epoch": 0.674225245653817, "fcm_dpo/beta": 0.004369608126580715, "fcm_dpo/delta": -0.0294049009680748, "fcm_dpo/margin": 96.6825180053711, "fcm_dpo/q_t": 0.4011649489402771, "grad_norm": 11.11204719543457, "learning_rate": 1.461462467495284e-07, "logits/chosen": 0.8308959007263184, "logits/rejected": 0.7669998407363892, "logps/chosen": -213.3043670654297, "logps/ref_chosen": -48.79924774169922, "logps/ref_rejected": -71.8719482421875, "logps/rejected": -333.0596008300781, "loss": 1.0718, "margin_dpo/margin_mean": 96.6825180053711, "margin_dpo/margin_std": 111.4892349243164, "step": 446 }, { "KL/chosen_KL_mean": -160.591796875, "KL/mean": -217.41326904296875, "KL/rejected_KL_mean": -274.2347412109375, "KL/std": 113.17485046386719, "epoch": 0.6757369614512472, "fcm_dpo/beta": 0.004061791114509106, "fcm_dpo/delta": -0.07315800338983536, "fcm_dpo/margin": 113.64291381835938, "fcm_dpo/q_t": 0.3898429274559021, "grad_norm": 9.85269546508789, "learning_rate": 1.4494497203727843e-07, "logits/chosen": 0.8119666576385498, "logits/rejected": 0.7234156131744385, "logps/chosen": -214.27452087402344, "logps/ref_chosen": -53.682716369628906, "logps/ref_rejected": -88.17315673828125, "logps/rejected": -362.40789794921875, "loss": 1.0415, "margin_dpo/margin_mean": 113.6429214477539, "margin_dpo/margin_std": 126.60255432128906, "step": 447 }, { "KL/chosen_KL_mean": -177.05801391601562, "KL/mean": -221.44229125976562, "KL/rejected_KL_mean": -265.82659912109375, "KL/std": 117.35690307617188, "epoch": 0.6772486772486772, "fcm_dpo/beta": 0.0041312845423817635, "fcm_dpo/delta": 0.03732322156429291, "fcm_dpo/margin": 88.76856231689453, "fcm_dpo/q_t": 0.41569164395332336, "grad_norm": 8.617594718933105, "learning_rate": 1.4374663593999256e-07, "logits/chosen": 0.8559887409210205, "logits/rejected": 0.8163323402404785, "logps/chosen": -230.80926513671875, "logps/ref_chosen": -53.75125503540039, "logps/ref_rejected": -77.17623901367188, "logps/rejected": -343.0028076171875, "loss": 1.1175, "margin_dpo/margin_mean": 88.76856231689453, "margin_dpo/margin_std": 127.51617431640625, "step": 448 }, { "KL/chosen_KL_mean": -211.01304626464844, "KL/mean": -233.7032470703125, "KL/rejected_KL_mean": -256.3934326171875, "KL/std": 123.69744873046875, "epoch": 0.6787603930461074, "fcm_dpo/beta": 0.004320129752159119, "fcm_dpo/delta": 0.06683868914842606, "fcm_dpo/margin": 45.38038635253906, "fcm_dpo/q_t": 0.45501431822776794, "grad_norm": 17.40116310119629, "learning_rate": 1.4255127197770707e-07, "logits/chosen": 0.6914898157119751, "logits/rejected": 0.6926765441894531, "logps/chosen": -286.8404235839844, "logps/ref_chosen": -75.82737731933594, "logps/ref_rejected": -82.20687866210938, "logps/rejected": -338.6003112792969, "loss": 1.2687, "margin_dpo/margin_mean": 45.38037872314453, "margin_dpo/margin_std": 122.3283920288086, "step": 449 }, { "KL/chosen_KL_mean": -170.02328491210938, "KL/mean": -208.6387176513672, "KL/rejected_KL_mean": -247.25418090820312, "KL/std": 121.925537109375, "epoch": 0.6802721088435374, "fcm_dpo/beta": 0.0044110435992479324, "fcm_dpo/delta": 0.06613278388977051, "fcm_dpo/margin": 77.23091888427734, "fcm_dpo/q_t": 0.4259797930717468, "grad_norm": 12.112007141113281, "learning_rate": 1.4135891358732205e-07, "logits/chosen": 0.9353618621826172, "logits/rejected": 0.8390960097312927, "logps/chosen": -217.1389923095703, "logps/ref_chosen": -47.11572265625, "logps/ref_rejected": -78.7546615600586, "logps/rejected": -326.00885009765625, "loss": 1.1643, "margin_dpo/margin_mean": 77.23091888427734, "margin_dpo/margin_std": 139.64755249023438, "step": 450 }, { "KL/chosen_KL_mean": -172.9991912841797, "KL/mean": -207.0963134765625, "KL/rejected_KL_mean": -241.19345092773438, "KL/std": 114.673095703125, "epoch": 0.6817838246409675, "fcm_dpo/beta": 0.004698569420725107, "fcm_dpo/delta": 0.08733482658863068, "fcm_dpo/margin": 68.19424438476562, "fcm_dpo/q_t": 0.4278087615966797, "grad_norm": 12.366849899291992, "learning_rate": 1.4016959412166437e-07, "logits/chosen": 0.7637462615966797, "logits/rejected": 0.7285028696060181, "logps/chosen": -236.34963989257812, "logps/ref_chosen": -63.350440979003906, "logps/ref_rejected": -76.28530883789062, "logps/rejected": -317.478759765625, "loss": 1.1642, "margin_dpo/margin_mean": 68.19424438476562, "margin_dpo/margin_std": 118.42993927001953, "step": 451 }, { "KL/chosen_KL_mean": -173.6109161376953, "KL/mean": -212.7416229248047, "KL/rejected_KL_mean": -251.87232971191406, "KL/std": 110.82177734375, "epoch": 0.6832955404383976, "fcm_dpo/beta": 0.004780272487550974, "fcm_dpo/delta": 0.02917136438190937, "fcm_dpo/margin": 78.26138305664062, "fcm_dpo/q_t": 0.41561344265937805, "grad_norm": 13.4918851852417, "learning_rate": 1.3898334684855645e-07, "logits/chosen": 0.7915092706680298, "logits/rejected": 0.723894476890564, "logps/chosen": -229.19674682617188, "logps/ref_chosen": -55.58583450317383, "logps/ref_rejected": -77.68738555908203, "logps/rejected": -329.5596923828125, "loss": 1.1423, "margin_dpo/margin_mean": 78.26138305664062, "margin_dpo/margin_std": 133.20782470703125, "step": 452 }, { "KL/chosen_KL_mean": -175.6853485107422, "KL/mean": -211.41357421875, "KL/rejected_KL_mean": -247.1417999267578, "KL/std": 114.35888671875, "epoch": 0.6848072562358276, "fcm_dpo/beta": 0.004915791098028421, "fcm_dpo/delta": 0.05457682907581329, "fcm_dpo/margin": 71.45646667480469, "fcm_dpo/q_t": 0.42137593030929565, "grad_norm": 15.09415340423584, "learning_rate": 1.3780020494988445e-07, "logits/chosen": 0.7863320708274841, "logits/rejected": 0.772057294845581, "logps/chosen": -237.4635467529297, "logps/ref_chosen": -61.778202056884766, "logps/ref_rejected": -71.51403045654297, "logps/rejected": -318.65582275390625, "loss": 1.1581, "margin_dpo/margin_mean": 71.45646667480469, "margin_dpo/margin_std": 126.16264343261719, "step": 453 }, { "KL/chosen_KL_mean": -158.27056884765625, "KL/mean": -200.40567016601562, "KL/rejected_KL_mean": -242.54075622558594, "KL/std": 111.83387756347656, "epoch": 0.6863189720332578, "fcm_dpo/beta": 0.0049323514103889465, "fcm_dpo/delta": -0.01793980970978737, "fcm_dpo/margin": 84.27018737792969, "fcm_dpo/q_t": 0.4046657383441925, "grad_norm": 11.837895393371582, "learning_rate": 1.366202015206706e-07, "logits/chosen": 0.8560349345207214, "logits/rejected": 0.8259134292602539, "logps/chosen": -209.86572265625, "logps/ref_chosen": -51.59515380859375, "logps/ref_rejected": -63.96732711791992, "logps/rejected": -306.5080871582031, "loss": 1.1011, "margin_dpo/margin_mean": 84.27018737792969, "margin_dpo/margin_std": 126.40806579589844, "step": 454 }, { "KL/chosen_KL_mean": -186.71917724609375, "KL/mean": -226.54803466796875, "KL/rejected_KL_mean": -266.37689208984375, "KL/std": 115.4573745727539, "epoch": 0.6878306878306878, "fcm_dpo/beta": 0.004945638123899698, "fcm_dpo/delta": 0.006844331510365009, "fcm_dpo/margin": 79.65769958496094, "fcm_dpo/q_t": 0.41296789050102234, "grad_norm": 13.294573783874512, "learning_rate": 1.354433695681474e-07, "logits/chosen": 0.7359009385108948, "logits/rejected": 0.717836320400238, "logps/chosen": -257.3708801269531, "logps/ref_chosen": -70.65170288085938, "logps/ref_rejected": -77.44276428222656, "logps/rejected": -343.81964111328125, "loss": 1.1178, "margin_dpo/margin_mean": 79.65769958496094, "margin_dpo/margin_std": 125.02285766601562, "step": 455 }, { "KL/chosen_KL_mean": -177.96502685546875, "KL/mean": -217.13229370117188, "KL/rejected_KL_mean": -256.299560546875, "KL/std": 114.41992950439453, "epoch": 0.6893424036281179, "fcm_dpo/beta": 0.0049400972202420235, "fcm_dpo/delta": 0.014393320307135582, "fcm_dpo/margin": 78.33451843261719, "fcm_dpo/q_t": 0.41560494899749756, "grad_norm": 17.193586349487305, "learning_rate": 1.3426974201083439e-07, "logits/chosen": 0.7663091421127319, "logits/rejected": 0.7156355381011963, "logps/chosen": -234.36331176757812, "logps/ref_chosen": -56.398284912109375, "logps/ref_rejected": -82.61642456054688, "logps/rejected": -338.91595458984375, "loss": 1.1366, "margin_dpo/margin_mean": 78.33451843261719, "margin_dpo/margin_std": 135.641357421875, "step": 456 }, { "KL/chosen_KL_mean": -173.41329956054688, "KL/mean": -213.361572265625, "KL/rejected_KL_mean": -253.3098602294922, "KL/std": 117.31248474121094, "epoch": 0.690854119425548, "fcm_dpo/beta": 0.004963235929608345, "fcm_dpo/delta": 0.003583187237381935, "fcm_dpo/margin": 79.89659118652344, "fcm_dpo/q_t": 0.4083296060562134, "grad_norm": 14.671182632446289, "learning_rate": 1.3309935167761717e-07, "logits/chosen": 0.9071280360221863, "logits/rejected": 0.8378027677536011, "logps/chosen": -218.1338653564453, "logps/ref_chosen": -44.72057342529297, "logps/ref_rejected": -68.1158676147461, "logps/rejected": -321.42572021484375, "loss": 1.1027, "margin_dpo/margin_mean": 79.8965835571289, "margin_dpo/margin_std": 113.3616943359375, "step": 457 }, { "KL/chosen_KL_mean": -168.78573608398438, "KL/mean": -213.8314208984375, "KL/rejected_KL_mean": -258.87713623046875, "KL/std": 119.34392547607422, "epoch": 0.6923658352229781, "fcm_dpo/beta": 0.004918850027024746, "fcm_dpo/delta": -0.049693845212459564, "fcm_dpo/margin": 90.09138488769531, "fcm_dpo/q_t": 0.3987731337547302, "grad_norm": 12.683366775512695, "learning_rate": 1.3193223130682936e-07, "logits/chosen": 0.8713840246200562, "logits/rejected": 0.7701141834259033, "logps/chosen": -218.7914276123047, "logps/ref_chosen": -50.00569152832031, "logps/ref_rejected": -87.50015258789062, "logps/rejected": -346.37725830078125, "loss": 1.0915, "margin_dpo/margin_mean": 90.09138488769531, "margin_dpo/margin_std": 134.1453399658203, "step": 458 }, { "KL/chosen_KL_mean": -167.32614135742188, "KL/mean": -218.5425262451172, "KL/rejected_KL_mean": -269.7589111328125, "KL/std": 142.56143188476562, "epoch": 0.6938775510204082, "fcm_dpo/beta": 0.004762580618262291, "fcm_dpo/delta": -0.10504220426082611, "fcm_dpo/margin": 102.4327392578125, "fcm_dpo/q_t": 0.3850099444389343, "grad_norm": 11.739134788513184, "learning_rate": 1.3076841354533658e-07, "logits/chosen": 0.868898868560791, "logits/rejected": 0.8451403975486755, "logps/chosen": -232.7041015625, "logps/ref_chosen": -65.37794494628906, "logps/ref_rejected": -88.19244384765625, "logps/rejected": -357.95135498046875, "loss": 1.0621, "margin_dpo/margin_mean": 102.43274688720703, "margin_dpo/margin_std": 134.36404418945312, "step": 459 }, { "KL/chosen_KL_mean": -177.44586181640625, "KL/mean": -228.05621337890625, "KL/rejected_KL_mean": -278.66656494140625, "KL/std": 122.83148193359375, "epoch": 0.6953892668178382, "fcm_dpo/beta": 0.0044008782133460045, "fcm_dpo/delta": -0.05401448905467987, "fcm_dpo/margin": 101.220703125, "fcm_dpo/q_t": 0.3985062539577484, "grad_norm": 10.761886596679688, "learning_rate": 1.2960793094762345e-07, "logits/chosen": 0.8326910734176636, "logits/rejected": 0.7360147833824158, "logps/chosen": -242.00753784179688, "logps/ref_chosen": -64.5616683959961, "logps/ref_rejected": -88.67890167236328, "logps/rejected": -367.345458984375, "loss": 1.0705, "margin_dpo/margin_mean": 101.220703125, "margin_dpo/margin_std": 134.64776611328125, "step": 460 }, { "KL/chosen_KL_mean": -146.41845703125, "KL/mean": -197.54617309570312, "KL/rejected_KL_mean": -248.6739044189453, "KL/std": 113.24662780761719, "epoch": 0.6969009826152683, "fcm_dpo/beta": 0.004217734094709158, "fcm_dpo/delta": -0.04221995174884796, "fcm_dpo/margin": 102.25545501708984, "fcm_dpo/q_t": 0.3992024064064026, "grad_norm": 10.860950469970703, "learning_rate": 1.2845081597488286e-07, "logits/chosen": 0.897905707359314, "logits/rejected": 0.8329803943634033, "logps/chosen": -195.89637756347656, "logps/ref_chosen": -49.4779167175293, "logps/ref_rejected": -72.65262603759766, "logps/rejected": -321.3265380859375, "loss": 1.0704, "margin_dpo/margin_mean": 102.25545501708984, "margin_dpo/margin_std": 122.31402587890625, "step": 461 }, { "KL/chosen_KL_mean": -162.05267333984375, "KL/mean": -212.8834228515625, "KL/rejected_KL_mean": -263.71417236328125, "KL/std": 105.76100158691406, "epoch": 0.6984126984126984, "fcm_dpo/beta": 0.004214158747345209, "fcm_dpo/delta": -0.0330844447016716, "fcm_dpo/margin": 101.66149139404297, "fcm_dpo/q_t": 0.39864546060562134, "grad_norm": 10.956618309020996, "learning_rate": 1.27297100994108e-07, "logits/chosen": 0.8604691028594971, "logits/rejected": 0.8234196901321411, "logps/chosen": -222.54779052734375, "logps/ref_chosen": -60.4951171875, "logps/ref_rejected": -74.82136535644531, "logps/rejected": -338.5355224609375, "loss": 1.062, "margin_dpo/margin_mean": 101.66149139404297, "margin_dpo/margin_std": 117.94256591796875, "step": 462 }, { "KL/chosen_KL_mean": -197.11163330078125, "KL/mean": -225.55178833007812, "KL/rejected_KL_mean": -253.991943359375, "KL/std": 109.48463439941406, "epoch": 0.6999244142101285, "fcm_dpo/beta": 0.004320027772337198, "fcm_dpo/delta": 0.039919111877679825, "fcm_dpo/margin": 56.880313873291016, "fcm_dpo/q_t": 0.44253993034362793, "grad_norm": 17.169588088989258, "learning_rate": 1.2614681827718695e-07, "logits/chosen": 0.8233051896095276, "logits/rejected": 0.8192650675773621, "logps/chosen": -264.7967529296875, "logps/ref_chosen": -67.68511962890625, "logps/ref_rejected": -71.32196044921875, "logps/rejected": -325.31390380859375, "loss": 1.2257, "margin_dpo/margin_mean": 56.88031005859375, "margin_dpo/margin_std": 123.54747009277344, "step": 463 }, { "KL/chosen_KL_mean": -176.97918701171875, "KL/mean": -224.4793701171875, "KL/rejected_KL_mean": -271.97955322265625, "KL/std": 127.88569641113281, "epoch": 0.7014361300075586, "fcm_dpo/beta": 0.0043390789069235325, "fcm_dpo/delta": -0.014604151248931885, "fcm_dpo/margin": 95.00037384033203, "fcm_dpo/q_t": 0.40515702962875366, "grad_norm": 10.686640739440918, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 0.8410533666610718, "logits/rejected": 0.8190803527832031, "logps/chosen": -236.14483642578125, "logps/ref_chosen": -59.16564178466797, "logps/ref_rejected": -69.56146240234375, "logps/rejected": -341.541015625, "loss": 1.108, "margin_dpo/margin_mean": 95.0003662109375, "margin_dpo/margin_std": 141.3034210205078, "step": 464 }, { "KL/chosen_KL_mean": -184.62344360351562, "KL/mean": -226.74331665039062, "KL/rejected_KL_mean": -268.8631896972656, "KL/std": 118.66059875488281, "epoch": 0.7029478458049887, "fcm_dpo/beta": 0.004308546893298626, "fcm_dpo/delta": 0.04102543368935585, "fcm_dpo/margin": 84.23976135253906, "fcm_dpo/q_t": 0.4176803529262543, "grad_norm": 11.88809871673584, "learning_rate": 1.238566782415197e-07, "logits/chosen": 0.8855515718460083, "logits/rejected": 0.836800217628479, "logps/chosen": -243.13711547851562, "logps/ref_chosen": -58.513671875, "logps/ref_rejected": -84.31745910644531, "logps/rejected": -353.1806640625, "loss": 1.1367, "margin_dpo/margin_mean": 84.23976135253906, "margin_dpo/margin_std": 132.88246154785156, "step": 465 }, { "KL/chosen_KL_mean": -203.14230346679688, "KL/mean": -222.7254638671875, "KL/rejected_KL_mean": -242.3086395263672, "KL/std": 127.04100799560547, "epoch": 0.7044595616024187, "fcm_dpo/beta": 0.004587305244058371, "fcm_dpo/delta": 0.07363003492355347, "fcm_dpo/margin": 39.16636276245117, "fcm_dpo/q_t": 0.45767632126808167, "grad_norm": 16.998245239257812, "learning_rate": 1.2271688498291334e-07, "logits/chosen": 0.8211065530776978, "logits/rejected": 0.8255276083946228, "logps/chosen": -276.4081115722656, "logps/ref_chosen": -73.26580810546875, "logps/ref_rejected": -74.83621215820312, "logps/rejected": -317.14483642578125, "loss": 1.3027, "margin_dpo/margin_mean": 39.166358947753906, "margin_dpo/margin_std": 131.44540405273438, "step": 466 }, { "KL/chosen_KL_mean": -176.02005004882812, "KL/mean": -217.30421447753906, "KL/rejected_KL_mean": -258.58837890625, "KL/std": 126.46763610839844, "epoch": 0.7059712773998488, "fcm_dpo/beta": 0.004604835994541645, "fcm_dpo/delta": 0.022210007533431053, "fcm_dpo/margin": 82.56832885742188, "fcm_dpo/q_t": 0.41688525676727295, "grad_norm": 10.904779434204102, "learning_rate": 1.2158065210664848e-07, "logits/chosen": 0.8689050674438477, "logits/rejected": 0.7562901973724365, "logps/chosen": -223.59951782226562, "logps/ref_chosen": -47.57947540283203, "logps/ref_rejected": -78.68522644042969, "logps/rejected": -337.27362060546875, "loss": 1.1317, "margin_dpo/margin_mean": 82.56832885742188, "margin_dpo/margin_std": 138.4664306640625, "step": 467 }, { "KL/chosen_KL_mean": -171.49169921875, "KL/mean": -218.56295776367188, "KL/rejected_KL_mean": -265.6342468261719, "KL/std": 121.87532806396484, "epoch": 0.7074829931972789, "fcm_dpo/beta": 0.004571876488626003, "fcm_dpo/delta": -0.03497558832168579, "fcm_dpo/margin": 94.14254760742188, "fcm_dpo/q_t": 0.39979416131973267, "grad_norm": 13.04074764251709, "learning_rate": 1.204480113956011e-07, "logits/chosen": 0.798638105392456, "logits/rejected": 0.7876749038696289, "logps/chosen": -235.4194793701172, "logps/ref_chosen": -63.92778778076172, "logps/ref_rejected": -76.51626586914062, "logps/rejected": -342.1505126953125, "loss": 1.101, "margin_dpo/margin_mean": 94.14254760742188, "margin_dpo/margin_std": 142.54898071289062, "step": 468 }, { "KL/chosen_KL_mean": -172.53428649902344, "KL/mean": -215.07339477539062, "KL/rejected_KL_mean": -257.6125183105469, "KL/std": 118.53016662597656, "epoch": 0.708994708994709, "fcm_dpo/beta": 0.0045113954693078995, "fcm_dpo/delta": 0.016616512089967728, "fcm_dpo/margin": 85.07823181152344, "fcm_dpo/q_t": 0.4123454689979553, "grad_norm": 13.645792961120605, "learning_rate": 1.1931899453216697e-07, "logits/chosen": 0.8709956407546997, "logits/rejected": 0.855722188949585, "logps/chosen": -231.59246826171875, "logps/ref_chosen": -59.05818176269531, "logps/ref_rejected": -75.67672729492188, "logps/rejected": -333.28924560546875, "loss": 1.1028, "margin_dpo/margin_mean": 85.0782241821289, "margin_dpo/margin_std": 114.00239562988281, "step": 469 }, { "KL/chosen_KL_mean": -160.6546630859375, "KL/mean": -205.504150390625, "KL/rejected_KL_mean": -250.3536376953125, "KL/std": 114.10494995117188, "epoch": 0.7105064247921391, "fcm_dpo/beta": 0.004605274647474289, "fcm_dpo/delta": -0.015301994979381561, "fcm_dpo/margin": 89.69898986816406, "fcm_dpo/q_t": 0.4052748680114746, "grad_norm": 11.65384578704834, "learning_rate": 1.1819363309737438e-07, "logits/chosen": 0.8430336713790894, "logits/rejected": 0.7940839529037476, "logps/chosen": -208.5220947265625, "logps/ref_chosen": -47.86743927001953, "logps/ref_rejected": -65.96859741210938, "logps/rejected": -316.32220458984375, "loss": 1.0995, "margin_dpo/margin_mean": 89.69898986816406, "margin_dpo/margin_std": 129.3060760498047, "step": 470 }, { "KL/chosen_KL_mean": -158.75082397460938, "KL/mean": -207.02197265625, "KL/rejected_KL_mean": -255.29310607910156, "KL/std": 111.86181640625, "epoch": 0.7120181405895691, "fcm_dpo/beta": 0.004474523477256298, "fcm_dpo/delta": -0.03683408349752426, "fcm_dpo/margin": 96.54228210449219, "fcm_dpo/q_t": 0.3983868360519409, "grad_norm": 12.488258361816406, "learning_rate": 1.1707195857000215e-07, "logits/chosen": 0.8283665180206299, "logits/rejected": 0.7850099802017212, "logps/chosen": -216.52867126464844, "logps/ref_chosen": -57.777854919433594, "logps/ref_rejected": -73.81172180175781, "logps/rejected": -329.1048278808594, "loss": 1.0719, "margin_dpo/margin_mean": 96.54228210449219, "margin_dpo/margin_std": 121.92526245117188, "step": 471 }, { "KL/chosen_KL_mean": -159.306884765625, "KL/mean": -197.11712646484375, "KL/rejected_KL_mean": -234.9273681640625, "KL/std": 117.94495391845703, "epoch": 0.7135298563869993, "fcm_dpo/beta": 0.004620303399860859, "fcm_dpo/delta": 0.054885368794202805, "fcm_dpo/margin": 75.62045288085938, "fcm_dpo/q_t": 0.4230712652206421, "grad_norm": 12.124162673950195, "learning_rate": 1.1595400232569768e-07, "logits/chosen": 0.8315516710281372, "logits/rejected": 0.7926985025405884, "logps/chosen": -215.21556091308594, "logps/ref_chosen": -55.908668518066406, "logps/ref_rejected": -74.70294189453125, "logps/rejected": -309.63031005859375, "loss": 1.1772, "margin_dpo/margin_mean": 75.62045288085938, "margin_dpo/margin_std": 149.9696044921875, "step": 472 }, { "KL/chosen_KL_mean": -163.22857666015625, "KL/mean": -215.5937042236328, "KL/rejected_KL_mean": -267.95880126953125, "KL/std": 143.45526123046875, "epoch": 0.7150415721844293, "fcm_dpo/beta": 0.004440706688910723, "fcm_dpo/delta": -0.0759398564696312, "fcm_dpo/margin": 104.73027038574219, "fcm_dpo/q_t": 0.3982590436935425, "grad_norm": 13.992444038391113, "learning_rate": 1.1483979563610069e-07, "logits/chosen": 0.9021942615509033, "logits/rejected": 0.809353232383728, "logps/chosen": -217.3894500732422, "logps/ref_chosen": -54.16088104248047, "logps/ref_rejected": -92.76789855957031, "logps/rejected": -360.7267150878906, "loss": 1.096, "margin_dpo/margin_mean": 104.73027038574219, "margin_dpo/margin_std": 166.75250244140625, "step": 473 }, { "KL/chosen_KL_mean": -165.42330932617188, "KL/mean": -207.98435974121094, "KL/rejected_KL_mean": -250.54539489746094, "KL/std": 122.40956115722656, "epoch": 0.7165532879818595, "fcm_dpo/beta": 0.004457107279449701, "fcm_dpo/delta": 0.02313932217657566, "fcm_dpo/margin": 85.1220703125, "fcm_dpo/q_t": 0.41587191820144653, "grad_norm": 17.439899444580078, "learning_rate": 1.1372936966796709e-07, "logits/chosen": 0.9221115112304688, "logits/rejected": 0.8645721673965454, "logps/chosen": -212.10902404785156, "logps/ref_chosen": -46.685707092285156, "logps/ref_rejected": -71.44731903076172, "logps/rejected": -321.9927062988281, "loss": 1.1342, "margin_dpo/margin_mean": 85.1220703125, "margin_dpo/margin_std": 141.8304443359375, "step": 474 }, { "KL/chosen_KL_mean": -164.73910522460938, "KL/mean": -222.14610290527344, "KL/rejected_KL_mean": -279.5531005859375, "KL/std": 123.50313568115234, "epoch": 0.7180650037792895, "fcm_dpo/beta": 0.004225746262818575, "fcm_dpo/delta": -0.10140831023454666, "fcm_dpo/margin": 114.81401062011719, "fcm_dpo/q_t": 0.3880518078804016, "grad_norm": 8.140756607055664, "learning_rate": 1.126227554822985e-07, "logits/chosen": 0.8333104848861694, "logits/rejected": 0.7912643551826477, "logps/chosen": -223.22640991210938, "logps/ref_chosen": -58.4873046875, "logps/ref_rejected": -87.00187683105469, "logps/rejected": -366.55499267578125, "loss": 1.0399, "margin_dpo/margin_mean": 114.81401062011719, "margin_dpo/margin_std": 140.57183837890625, "step": 475 }, { "KL/chosen_KL_mean": -198.56451416015625, "KL/mean": -232.54180908203125, "KL/rejected_KL_mean": -266.51910400390625, "KL/std": 132.65438842773438, "epoch": 0.7195767195767195, "fcm_dpo/beta": 0.004227438010275364, "fcm_dpo/delta": 0.02281678467988968, "fcm_dpo/margin": 67.95459747314453, "fcm_dpo/q_t": 0.43554818630218506, "grad_norm": 13.2657470703125, "learning_rate": 1.1151998403347243e-07, "logits/chosen": 0.7603304386138916, "logits/rejected": 0.764335036277771, "logps/chosen": -273.9461364746094, "logps/ref_chosen": -75.38162231445312, "logps/ref_rejected": -76.99822235107422, "logps/rejected": -343.517333984375, "loss": 1.2024, "margin_dpo/margin_mean": 67.95459747314453, "margin_dpo/margin_std": 140.07485961914062, "step": 476 }, { "KL/chosen_KL_mean": -194.00865173339844, "KL/mean": -236.6685791015625, "KL/rejected_KL_mean": -279.3285217285156, "KL/std": 133.24874877929688, "epoch": 0.7210884353741497, "fcm_dpo/beta": 0.004316464997828007, "fcm_dpo/delta": 0.035700492560863495, "fcm_dpo/margin": 85.31986999511719, "fcm_dpo/q_t": 0.4191203713417053, "grad_norm": 12.513143539428711, "learning_rate": 1.1042108616837692e-07, "logits/chosen": 0.8567191362380981, "logits/rejected": 0.8166630268096924, "logps/chosen": -255.08203125, "logps/ref_chosen": -61.073387145996094, "logps/ref_rejected": -81.34375, "logps/rejected": -360.6722717285156, "loss": 1.1779, "margin_dpo/margin_mean": 85.31986236572266, "margin_dpo/margin_std": 170.67071533203125, "step": 477 }, { "KL/chosen_KL_mean": -171.2495574951172, "KL/mean": -202.14273071289062, "KL/rejected_KL_mean": -233.03590393066406, "KL/std": 106.54711151123047, "epoch": 0.7226001511715797, "fcm_dpo/beta": 0.004573522135615349, "fcm_dpo/delta": 0.1288546621799469, "fcm_dpo/margin": 61.78632736206055, "fcm_dpo/q_t": 0.4368034899234772, "grad_norm": 14.673408508300781, "learning_rate": 1.0932609262554746e-07, "logits/chosen": 0.8044695854187012, "logits/rejected": 0.8161039352416992, "logps/chosen": -228.4168701171875, "logps/ref_chosen": -57.16731643676758, "logps/ref_rejected": -53.30917739868164, "logps/rejected": -286.3450927734375, "loss": 1.2093, "margin_dpo/margin_mean": 61.78633117675781, "margin_dpo/margin_std": 129.24046325683594, "step": 478 }, { "KL/chosen_KL_mean": -177.13055419921875, "KL/mean": -204.7392578125, "KL/rejected_KL_mean": -232.34796142578125, "KL/std": 110.43617248535156, "epoch": 0.7241118669690099, "fcm_dpo/beta": 0.004763299599289894, "fcm_dpo/delta": 0.022982783615589142, "fcm_dpo/margin": 55.2174072265625, "fcm_dpo/q_t": 0.44107237458229065, "grad_norm": 14.663484573364258, "learning_rate": 1.0823503403430734e-07, "logits/chosen": 0.8042382001876831, "logits/rejected": 0.778433084487915, "logps/chosen": -236.0438690185547, "logps/ref_chosen": -58.91331481933594, "logps/ref_rejected": -63.7403450012207, "logps/rejected": -296.08831787109375, "loss": 1.2346, "margin_dpo/margin_mean": 55.2174072265625, "margin_dpo/margin_std": 131.55545043945312, "step": 479 }, { "KL/chosen_KL_mean": -184.4405975341797, "KL/mean": -227.0792236328125, "KL/rejected_KL_mean": -269.71783447265625, "KL/std": 115.04316711425781, "epoch": 0.7256235827664399, "fcm_dpo/beta": 0.004858708009123802, "fcm_dpo/delta": -0.02168682962656021, "fcm_dpo/margin": 85.27723693847656, "fcm_dpo/q_t": 0.40630629658699036, "grad_norm": 15.03400707244873, "learning_rate": 1.0714794091391072e-07, "logits/chosen": 0.7868234515190125, "logits/rejected": 0.7834052443504333, "logps/chosen": -247.2412109375, "logps/ref_chosen": -62.80061340332031, "logps/ref_rejected": -67.58859252929688, "logps/rejected": -337.30645751953125, "loss": 1.1233, "margin_dpo/margin_mean": 85.27723693847656, "margin_dpo/margin_std": 134.57638549804688, "step": 480 }, { "KL/chosen_KL_mean": -182.68031311035156, "KL/mean": -222.99514770507812, "KL/rejected_KL_mean": -263.3099670410156, "KL/std": 121.83908081054688, "epoch": 0.72713529856387, "fcm_dpo/beta": 0.004709561821073294, "fcm_dpo/delta": 0.022568443790078163, "fcm_dpo/margin": 80.62967681884766, "fcm_dpo/q_t": 0.4156482219696045, "grad_norm": 12.423121452331543, "learning_rate": 1.0606484367268906e-07, "logits/chosen": 0.8025602102279663, "logits/rejected": 0.8090294599533081, "logps/chosen": -247.966796875, "logps/ref_chosen": -65.28649139404297, "logps/ref_rejected": -70.78668212890625, "logps/rejected": -334.0966491699219, "loss": 1.132, "margin_dpo/margin_mean": 80.62968444824219, "margin_dpo/margin_std": 133.42701721191406, "step": 481 }, { "KL/chosen_KL_mean": -201.8643798828125, "KL/mean": -246.51490783691406, "KL/rejected_KL_mean": -291.16546630859375, "KL/std": 135.26222229003906, "epoch": 0.7286470143613001, "fcm_dpo/beta": 0.004750003106892109, "fcm_dpo/delta": -0.02802390605211258, "fcm_dpo/margin": 89.30106353759766, "fcm_dpo/q_t": 0.41019535064697266, "grad_norm": 15.123866081237793, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 0.7807713747024536, "logits/rejected": 0.6487196087837219, "logps/chosen": -262.77056884765625, "logps/ref_chosen": -60.906185150146484, "logps/ref_rejected": -103.44656372070312, "logps/rejected": -394.61199951171875, "loss": 1.1465, "margin_dpo/margin_mean": 89.30105590820312, "margin_dpo/margin_std": 167.93521118164062, "step": 482 }, { "KL/chosen_KL_mean": -169.6586151123047, "KL/mean": -220.8936309814453, "KL/rejected_KL_mean": -272.128662109375, "KL/std": 117.83233642578125, "epoch": 0.7301587301587301, "fcm_dpo/beta": 0.004558105953037739, "fcm_dpo/delta": -0.07779208570718765, "fcm_dpo/margin": 102.47002410888672, "fcm_dpo/q_t": 0.39235061407089233, "grad_norm": 13.63161849975586, "learning_rate": 1.0391075790138232e-07, "logits/chosen": 0.8855106830596924, "logits/rejected": 0.7963603734970093, "logps/chosen": -222.85061645507812, "logps/ref_chosen": -53.192012786865234, "logps/ref_rejected": -81.83927154541016, "logps/rejected": -353.9678955078125, "loss": 1.0606, "margin_dpo/margin_mean": 102.47001647949219, "margin_dpo/margin_std": 134.73834228515625, "step": 483 }, { "KL/chosen_KL_mean": -176.22354125976562, "KL/mean": -208.54843139648438, "KL/rejected_KL_mean": -240.87332153320312, "KL/std": 116.13772583007812, "epoch": 0.7316704459561603, "fcm_dpo/beta": 0.004734124056994915, "fcm_dpo/delta": 0.10121459513902664, "fcm_dpo/margin": 64.64977264404297, "fcm_dpo/q_t": 0.4296380281448364, "grad_norm": 16.20368003845215, "learning_rate": 1.0283982962570681e-07, "logits/chosen": 0.8983777761459351, "logits/rejected": 0.8746373057365417, "logps/chosen": -233.9929962158203, "logps/ref_chosen": -57.76945877075195, "logps/ref_rejected": -71.6829833984375, "logps/rejected": -312.5563049316406, "loss": 1.159, "margin_dpo/margin_mean": 64.64977264404297, "margin_dpo/margin_std": 101.06908416748047, "step": 484 }, { "KL/chosen_KL_mean": -182.181396484375, "KL/mean": -218.22222900390625, "KL/rejected_KL_mean": -254.26309204101562, "KL/std": 114.957763671875, "epoch": 0.7331821617535903, "fcm_dpo/beta": 0.004620219115167856, "fcm_dpo/delta": -0.04711120203137398, "fcm_dpo/margin": 72.08170318603516, "fcm_dpo/q_t": 0.4228460192680359, "grad_norm": 13.281914710998535, "learning_rate": 1.0177301773633992e-07, "logits/chosen": 0.9076398611068726, "logits/rejected": 0.8928012847900391, "logps/chosen": -238.81724548339844, "logps/ref_chosen": -56.63584899902344, "logps/ref_rejected": -70.85614013671875, "logps/rejected": -325.1192321777344, "loss": 1.1514, "margin_dpo/margin_mean": 72.08171081542969, "margin_dpo/margin_std": 115.27439880371094, "step": 485 }, { "KL/chosen_KL_mean": -207.69192504882812, "KL/mean": -243.96141052246094, "KL/rejected_KL_mean": -280.23089599609375, "KL/std": 137.62863159179688, "epoch": 0.7346938775510204, "fcm_dpo/beta": 0.00475216843187809, "fcm_dpo/delta": 0.0615973100066185, "fcm_dpo/margin": 72.53897857666016, "fcm_dpo/q_t": 0.42720019817352295, "grad_norm": 11.114177703857422, "learning_rate": 1.007103520743035e-07, "logits/chosen": 0.8727377653121948, "logits/rejected": 0.7796794176101685, "logps/chosen": -264.0389404296875, "logps/ref_chosen": -56.347023010253906, "logps/ref_rejected": -85.97221374511719, "logps/rejected": -366.203125, "loss": 1.2038, "margin_dpo/margin_mean": 72.53898620605469, "margin_dpo/margin_std": 161.3694610595703, "step": 486 }, { "KL/chosen_KL_mean": -187.3951416015625, "KL/mean": -228.60194396972656, "KL/rejected_KL_mean": -269.80877685546875, "KL/std": 126.41487121582031, "epoch": 0.7362055933484505, "fcm_dpo/beta": 0.004806404933333397, "fcm_dpo/delta": 0.004405636340379715, "fcm_dpo/margin": 82.41361999511719, "fcm_dpo/q_t": 0.4161032438278198, "grad_norm": 13.856200218200684, "learning_rate": 9.965186236464046e-08, "logits/chosen": 0.9156618118286133, "logits/rejected": 0.8678491115570068, "logps/chosen": -248.01235961914062, "logps/ref_chosen": -60.617218017578125, "logps/ref_rejected": -82.50975036621094, "logps/rejected": -352.3184814453125, "loss": 1.1419, "margin_dpo/margin_mean": 82.41361236572266, "margin_dpo/margin_std": 151.87709045410156, "step": 487 }, { "KL/chosen_KL_mean": -175.20208740234375, "KL/mean": -221.89056396484375, "KL/rejected_KL_mean": -268.57904052734375, "KL/std": 118.11080932617188, "epoch": 0.7377173091458806, "fcm_dpo/beta": 0.004773295484483242, "fcm_dpo/delta": -0.05380316078662872, "fcm_dpo/margin": 93.37692260742188, "fcm_dpo/q_t": 0.39976638555526733, "grad_norm": 14.908041000366211, "learning_rate": 9.859757821558337e-08, "logits/chosen": 0.8313355445861816, "logits/rejected": 0.792324423789978, "logps/chosen": -238.31114196777344, "logps/ref_chosen": -63.10905075073242, "logps/ref_rejected": -82.49348449707031, "logps/rejected": -351.072509765625, "loss": 1.0806, "margin_dpo/margin_mean": 93.3769302368164, "margin_dpo/margin_std": 130.49966430664062, "step": 488 }, { "KL/chosen_KL_mean": -204.91387939453125, "KL/mean": -233.1622314453125, "KL/rejected_KL_mean": -261.41058349609375, "KL/std": 130.28018188476562, "epoch": 0.7392290249433107, "fcm_dpo/beta": 0.0049452874809503555, "fcm_dpo/delta": 0.13130411505699158, "fcm_dpo/margin": 56.49671936035156, "fcm_dpo/q_t": 0.4389078915119171, "grad_norm": 13.527437210083008, "learning_rate": 9.754752911772615e-08, "logits/chosen": 0.8165801763534546, "logits/rejected": 0.780007004737854, "logps/chosen": -269.90283203125, "logps/ref_chosen": -64.98896026611328, "logps/ref_rejected": -84.39607238769531, "logps/rejected": -345.806640625, "loss": 1.2361, "margin_dpo/margin_mean": 56.49671936035156, "margin_dpo/margin_std": 137.77053833007812, "step": 489 }, { "KL/chosen_KL_mean": -184.7069091796875, "KL/mean": -218.8941192626953, "KL/rejected_KL_mean": -253.08131408691406, "KL/std": 124.6688232421875, "epoch": 0.7407407407407407, "fcm_dpo/beta": 0.005108346696943045, "fcm_dpo/delta": 0.05644252896308899, "fcm_dpo/margin": 68.3744125366211, "fcm_dpo/q_t": 0.4248546361923218, "grad_norm": 12.403369903564453, "learning_rate": 9.650174444319956e-08, "logits/chosen": 0.8969284892082214, "logits/rejected": 0.8864274024963379, "logps/chosen": -246.61566162109375, "logps/ref_chosen": -61.90874481201172, "logps/ref_rejected": -70.58566284179688, "logps/rejected": -323.6669921875, "loss": 1.2101, "margin_dpo/margin_mean": 68.3744125366211, "margin_dpo/margin_std": 154.39389038085938, "step": 490 }, { "KL/chosen_KL_mean": -178.02613830566406, "KL/mean": -212.85516357421875, "KL/rejected_KL_mean": -247.6842041015625, "KL/std": 112.87677001953125, "epoch": 0.7422524565381708, "fcm_dpo/beta": 0.005029967520385981, "fcm_dpo/delta": -0.05832719802856445, "fcm_dpo/margin": 69.65803527832031, "fcm_dpo/q_t": 0.41990724205970764, "grad_norm": 10.252985000610352, "learning_rate": 9.546025344484868e-08, "logits/chosen": 0.7992293834686279, "logits/rejected": 0.745780348777771, "logps/chosen": -233.50184631347656, "logps/ref_chosen": -55.47570037841797, "logps/ref_rejected": -78.70318603515625, "logps/rejected": -326.38739013671875, "loss": 1.1552, "margin_dpo/margin_mean": 69.65803527832031, "margin_dpo/margin_std": 120.66893005371094, "step": 491 }, { "KL/chosen_KL_mean": -211.10516357421875, "KL/mean": -243.54579162597656, "KL/rejected_KL_mean": -275.98638916015625, "KL/std": 131.36009216308594, "epoch": 0.7437641723356009, "fcm_dpo/beta": 0.005244538653641939, "fcm_dpo/delta": 0.06441338360309601, "fcm_dpo/margin": 64.88124084472656, "fcm_dpo/q_t": 0.4269523024559021, "grad_norm": 15.789043426513672, "learning_rate": 9.442308525541589e-08, "logits/chosen": 0.8516812920570374, "logits/rejected": 0.7955601215362549, "logps/chosen": -278.39154052734375, "logps/ref_chosen": -67.28638458251953, "logps/ref_rejected": -82.78628540039062, "logps/rejected": -358.772705078125, "loss": 1.2185, "margin_dpo/margin_mean": 64.88124084472656, "margin_dpo/margin_std": 148.1044921875, "step": 492 }, { "KL/chosen_KL_mean": -176.27764892578125, "KL/mean": -222.05245971679688, "KL/rejected_KL_mean": -267.8272705078125, "KL/std": 127.64932250976562, "epoch": 0.745275888133031, "fcm_dpo/beta": 0.005175878759473562, "fcm_dpo/delta": -0.08929616212844849, "fcm_dpo/margin": 91.54963684082031, "fcm_dpo/q_t": 0.3954174220561981, "grad_norm": 15.408149719238281, "learning_rate": 9.339026888672468e-08, "logits/chosen": 0.8074703812599182, "logits/rejected": 0.7508785128593445, "logps/chosen": -232.20513916015625, "logps/ref_chosen": -55.92750549316406, "logps/ref_rejected": -79.12149810791016, "logps/rejected": -346.9487609863281, "loss": 1.0922, "margin_dpo/margin_mean": 91.54963684082031, "margin_dpo/margin_std": 139.12684631347656, "step": 493 }, { "KL/chosen_KL_mean": -181.6630096435547, "KL/mean": -223.94744873046875, "KL/rejected_KL_mean": -266.23193359375, "KL/std": 128.83995056152344, "epoch": 0.7467876039304611, "fcm_dpo/beta": 0.004889290779829025, "fcm_dpo/delta": -0.015794701874256134, "fcm_dpo/margin": 84.56891632080078, "fcm_dpo/q_t": 0.4098696708679199, "grad_norm": 14.83998966217041, "learning_rate": 9.236183322886945e-08, "logits/chosen": 0.7640923857688904, "logits/rejected": 0.7272478342056274, "logps/chosen": -249.61712646484375, "logps/ref_chosen": -67.95410919189453, "logps/ref_rejected": -90.50865173339844, "logps/rejected": -356.7405700683594, "loss": 1.1521, "margin_dpo/margin_mean": 84.56890869140625, "margin_dpo/margin_std": 161.0829620361328, "step": 494 }, { "KL/chosen_KL_mean": -177.00015258789062, "KL/mean": -209.45477294921875, "KL/rejected_KL_mean": -241.90940856933594, "KL/std": 125.46236419677734, "epoch": 0.7482993197278912, "fcm_dpo/beta": 0.004900210537016392, "fcm_dpo/delta": -0.005768109112977982, "fcm_dpo/margin": 64.90924835205078, "fcm_dpo/q_t": 0.4349347949028015, "grad_norm": 16.218822479248047, "learning_rate": 9.133780704940594e-08, "logits/chosen": 0.8546461462974548, "logits/rejected": 0.8084453344345093, "logps/chosen": -229.6256103515625, "logps/ref_chosen": -52.62546157836914, "logps/ref_rejected": -72.06781005859375, "logps/rejected": -313.97723388671875, "loss": 1.2211, "margin_dpo/margin_mean": 64.90924835205078, "margin_dpo/margin_std": 156.70840454101562, "step": 495 }, { "KL/chosen_KL_mean": -191.11280822753906, "KL/mean": -238.58676147460938, "KL/rejected_KL_mean": -286.0606994628906, "KL/std": 138.51930236816406, "epoch": 0.7498110355253212, "fcm_dpo/beta": 0.004659120924770832, "fcm_dpo/delta": -0.054637569934129715, "fcm_dpo/margin": 94.94786834716797, "fcm_dpo/q_t": 0.40979450941085815, "grad_norm": 11.95200252532959, "learning_rate": 9.031821899254797e-08, "logits/chosen": 0.8415661454200745, "logits/rejected": 0.7441533803939819, "logps/chosen": -248.71014404296875, "logps/ref_chosen": -57.597320556640625, "logps/ref_rejected": -94.36127471923828, "logps/rejected": -380.4219665527344, "loss": 1.1355, "margin_dpo/margin_mean": 94.94786834716797, "margin_dpo/margin_std": 179.43215942382812, "step": 496 }, { "KL/chosen_KL_mean": -197.67959594726562, "KL/mean": -241.09596252441406, "KL/rejected_KL_mean": -284.5123291015625, "KL/std": 124.52940368652344, "epoch": 0.7513227513227513, "fcm_dpo/beta": 0.004687286913394928, "fcm_dpo/delta": -0.008316351100802422, "fcm_dpo/margin": 86.83271789550781, "fcm_dpo/q_t": 0.406663715839386, "grad_norm": 13.023819923400879, "learning_rate": 8.930309757836516e-08, "logits/chosen": 0.853161096572876, "logits/rejected": 0.8242659568786621, "logps/chosen": -270.46954345703125, "logps/ref_chosen": -72.78994750976562, "logps/ref_rejected": -89.48483276367188, "logps/rejected": -373.99713134765625, "loss": 1.116, "margin_dpo/margin_mean": 86.83271789550781, "margin_dpo/margin_std": 136.6311492919922, "step": 497 }, { "KL/chosen_KL_mean": -175.44488525390625, "KL/mean": -222.20950317382812, "KL/rejected_KL_mean": -268.97412109375, "KL/std": 117.231201171875, "epoch": 0.7528344671201814, "fcm_dpo/beta": 0.00461503816768527, "fcm_dpo/delta": -0.036642152816057205, "fcm_dpo/margin": 93.52923583984375, "fcm_dpo/q_t": 0.39883777499198914, "grad_norm": 13.756954193115234, "learning_rate": 8.829247120198563e-08, "logits/chosen": 0.8330202102661133, "logits/rejected": 0.8214312791824341, "logps/chosen": -243.81060791015625, "logps/ref_chosen": -68.36572265625, "logps/ref_rejected": -71.28846740722656, "logps/rejected": -340.2626037597656, "loss": 1.0839, "margin_dpo/margin_mean": 93.52923583984375, "margin_dpo/margin_std": 129.33668518066406, "step": 498 }, { "KL/chosen_KL_mean": -179.2896728515625, "KL/mean": -229.2539825439453, "KL/rejected_KL_mean": -279.2182922363281, "KL/std": 132.14337158203125, "epoch": 0.7543461829176115, "fcm_dpo/beta": 0.004479332827031612, "fcm_dpo/delta": -0.055337462574243546, "fcm_dpo/margin": 99.9286117553711, "fcm_dpo/q_t": 0.40216517448425293, "grad_norm": 19.308687210083008, "learning_rate": 8.728636813280163e-08, "logits/chosen": 0.8368034362792969, "logits/rejected": 0.7836736440658569, "logps/chosen": -241.19850158691406, "logps/ref_chosen": -61.90882873535156, "logps/ref_rejected": -91.9411392211914, "logps/rejected": -371.159423828125, "loss": 1.1267, "margin_dpo/margin_mean": 99.92861938476562, "margin_dpo/margin_std": 174.06784057617188, "step": 499 }, { "KL/chosen_KL_mean": -188.59982299804688, "KL/mean": -225.63168334960938, "KL/rejected_KL_mean": -262.66351318359375, "KL/std": 115.21099853515625, "epoch": 0.7558578987150416, "fcm_dpo/beta": 0.00457107275724411, "fcm_dpo/delta": 0.06856130063533783, "fcm_dpo/margin": 74.0636978149414, "fcm_dpo/q_t": 0.42126649618148804, "grad_norm": 14.0758695602417, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.8072834014892578, "logits/rejected": 0.8198223114013672, "logps/chosen": -258.8256530761719, "logps/ref_chosen": -70.225830078125, "logps/ref_rejected": -71.72203063964844, "logps/rejected": -334.38555908203125, "loss": 1.1835, "margin_dpo/margin_mean": 74.0636978149414, "margin_dpo/margin_std": 146.86541748046875, "step": 500 }, { "KL/chosen_KL_mean": -187.39126586914062, "KL/mean": -222.43211364746094, "KL/rejected_KL_mean": -257.47296142578125, "KL/std": 122.28995513916016, "epoch": 0.7573696145124716, "fcm_dpo/beta": 0.004816969856619835, "fcm_dpo/delta": 0.06878426671028137, "fcm_dpo/margin": 70.08174133300781, "fcm_dpo/q_t": 0.42239677906036377, "grad_norm": 11.63576889038086, "learning_rate": 8.528784436016878e-08, "logits/chosen": 0.833807110786438, "logits/rejected": 0.8403280973434448, "logps/chosen": -251.9900665283203, "logps/ref_chosen": -64.59880828857422, "logps/ref_rejected": -70.59329223632812, "logps/rejected": -328.0662841796875, "loss": 1.132, "margin_dpo/margin_mean": 70.08173370361328, "margin_dpo/margin_std": 99.08871459960938, "step": 501 }, { "KL/chosen_KL_mean": -186.89634704589844, "KL/mean": -225.32411193847656, "KL/rejected_KL_mean": -263.75189208984375, "KL/std": 133.69595336914062, "epoch": 0.7588813303099018, "fcm_dpo/beta": 0.004946129396557808, "fcm_dpo/delta": 0.021091684699058533, "fcm_dpo/margin": 76.85551452636719, "fcm_dpo/q_t": 0.41427573561668396, "grad_norm": 16.61263656616211, "learning_rate": 8.4295479559726e-08, "logits/chosen": 0.8703242540359497, "logits/rejected": 0.8291156888008118, "logps/chosen": -252.36297607421875, "logps/ref_chosen": -65.46662902832031, "logps/ref_rejected": -90.22233581542969, "logps/rejected": -353.9742126464844, "loss": 1.1302, "margin_dpo/margin_mean": 76.85551452636719, "margin_dpo/margin_std": 123.59288024902344, "step": 502 }, { "KL/chosen_KL_mean": -165.91510009765625, "KL/mean": -204.1113739013672, "KL/rejected_KL_mean": -242.3076171875, "KL/std": 110.12651062011719, "epoch": 0.7603930461073318, "fcm_dpo/beta": 0.004937716759741306, "fcm_dpo/delta": 0.02568972110748291, "fcm_dpo/margin": 76.39253997802734, "fcm_dpo/q_t": 0.41301050782203674, "grad_norm": 12.290018081665039, "learning_rate": 8.330774987092712e-08, "logits/chosen": 0.8349542617797852, "logits/rejected": 0.8423024415969849, "logps/chosen": -217.74986267089844, "logps/ref_chosen": -51.83476257324219, "logps/ref_rejected": -57.62522506713867, "logps/rejected": -299.932861328125, "loss": 1.1346, "margin_dpo/margin_mean": 76.39253997802734, "margin_dpo/margin_std": 124.44419860839844, "step": 503 }, { "KL/chosen_KL_mean": -174.50894165039062, "KL/mean": -224.36419677734375, "KL/rejected_KL_mean": -274.2194519042969, "KL/std": 112.46851348876953, "epoch": 0.7619047619047619, "fcm_dpo/beta": 0.0048708608373999596, "fcm_dpo/delta": -0.10153862833976746, "fcm_dpo/margin": 99.71051025390625, "fcm_dpo/q_t": 0.3862113058567047, "grad_norm": 11.172093391418457, "learning_rate": 8.232468292269479e-08, "logits/chosen": 0.8066504001617432, "logits/rejected": 0.7978000044822693, "logps/chosen": -243.16014099121094, "logps/ref_chosen": -68.65119934082031, "logps/ref_rejected": -77.91394805908203, "logps/rejected": -352.1334228515625, "loss": 1.0413, "margin_dpo/margin_mean": 99.71051025390625, "margin_dpo/margin_std": 116.26593017578125, "step": 504 }, { "KL/chosen_KL_mean": -184.58160400390625, "KL/mean": -219.14266967773438, "KL/rejected_KL_mean": -253.7037353515625, "KL/std": 134.02723693847656, "epoch": 0.763416477702192, "fcm_dpo/beta": 0.004590325523167849, "fcm_dpo/delta": -0.03364703059196472, "fcm_dpo/margin": 69.12216186523438, "fcm_dpo/q_t": 0.4291911721229553, "grad_norm": 13.124581336975098, "learning_rate": 8.134630621352483e-08, "logits/chosen": 0.8404581546783447, "logits/rejected": 0.8074555397033691, "logps/chosen": -244.5804443359375, "logps/ref_chosen": -59.99884796142578, "logps/ref_rejected": -76.88048553466797, "logps/rejected": -330.584228515625, "loss": 1.2016, "margin_dpo/margin_mean": 69.12216186523438, "margin_dpo/margin_std": 145.76710510253906, "step": 505 }, { "KL/chosen_KL_mean": -185.93222045898438, "KL/mean": -222.56521606445312, "KL/rejected_KL_mean": -259.1982116699219, "KL/std": 118.15262603759766, "epoch": 0.764928193499622, "fcm_dpo/beta": 0.00469227135181427, "fcm_dpo/delta": 0.06280513852834702, "fcm_dpo/margin": 73.26598358154297, "fcm_dpo/q_t": 0.42215216159820557, "grad_norm": 12.451192855834961, "learning_rate": 8.037264711071698e-08, "logits/chosen": 0.8122513890266418, "logits/rejected": 0.8038109540939331, "logps/chosen": -256.0035400390625, "logps/ref_chosen": -70.07130432128906, "logps/ref_rejected": -82.03775024414062, "logps/rejected": -341.2359619140625, "loss": 1.1876, "margin_dpo/margin_mean": 73.2659912109375, "margin_dpo/margin_std": 149.51535034179688, "step": 506 }, { "KL/chosen_KL_mean": -195.21424865722656, "KL/mean": -238.96438598632812, "KL/rejected_KL_mean": -282.71453857421875, "KL/std": 134.91099548339844, "epoch": 0.7664399092970522, "fcm_dpo/beta": 0.004667514935135841, "fcm_dpo/delta": -0.011594630777835846, "fcm_dpo/margin": 87.50030517578125, "fcm_dpo/q_t": 0.41374701261520386, "grad_norm": 13.628171920776367, "learning_rate": 7.940373284960933e-08, "logits/chosen": 0.852962076663971, "logits/rejected": 0.8136993646621704, "logps/chosen": -267.2212829589844, "logps/ref_chosen": -72.00703430175781, "logps/ref_rejected": -93.94987487792969, "logps/rejected": -376.6644287109375, "loss": 1.1472, "margin_dpo/margin_mean": 87.50030517578125, "margin_dpo/margin_std": 159.53916931152344, "step": 507 }, { "KL/chosen_KL_mean": -180.30194091796875, "KL/mean": -231.80496215820312, "KL/rejected_KL_mean": -283.3079833984375, "KL/std": 141.48910522460938, "epoch": 0.7679516250944822, "fcm_dpo/beta": 0.004642902873456478, "fcm_dpo/delta": -0.0925033912062645, "fcm_dpo/margin": 103.00605773925781, "fcm_dpo/q_t": 0.3941517472267151, "grad_norm": 15.925118446350098, "learning_rate": 7.843959053281663e-08, "logits/chosen": 0.8314013481140137, "logits/rejected": 0.7309713959693909, "logps/chosen": -240.5218505859375, "logps/ref_chosen": -60.21992492675781, "logps/ref_rejected": -95.9200668334961, "logps/rejected": -379.2280578613281, "loss": 1.0892, "margin_dpo/margin_mean": 103.00605773925781, "margin_dpo/margin_std": 155.87930297851562, "step": 508 }, { "KL/chosen_KL_mean": -190.84176635742188, "KL/mean": -227.35723876953125, "KL/rejected_KL_mean": -263.8727111816406, "KL/std": 123.76275634765625, "epoch": 0.7694633408919124, "fcm_dpo/beta": 0.004593648016452789, "fcm_dpo/delta": 0.07191221415996552, "fcm_dpo/margin": 73.03093719482422, "fcm_dpo/q_t": 0.42444539070129395, "grad_norm": 15.941633224487305, "learning_rate": 7.748024712947204e-08, "logits/chosen": 0.8122048377990723, "logits/rejected": 0.7954943776130676, "logps/chosen": -257.1119384765625, "logps/ref_chosen": -66.27017211914062, "logps/ref_rejected": -71.73065185546875, "logps/rejected": -335.60333251953125, "loss": 1.1895, "margin_dpo/margin_mean": 73.03093719482422, "margin_dpo/margin_std": 148.45220947265625, "step": 509 }, { "KL/chosen_KL_mean": -189.88351440429688, "KL/mean": -238.0792999267578, "KL/rejected_KL_mean": -286.27508544921875, "KL/std": 139.2322540283203, "epoch": 0.7709750566893424, "fcm_dpo/beta": 0.004529876634478569, "fcm_dpo/delta": -0.04380728304386139, "fcm_dpo/margin": 96.39158630371094, "fcm_dpo/q_t": 0.40820297598838806, "grad_norm": 14.016731262207031, "learning_rate": 7.652572947447272e-08, "logits/chosen": 0.9248769283294678, "logits/rejected": 0.8472373485565186, "logps/chosen": -243.4283905029297, "logps/ref_chosen": -53.54487609863281, "logps/ref_rejected": -91.36648559570312, "logps/rejected": -377.6415710449219, "loss": 1.1419, "margin_dpo/margin_mean": 96.39158630371094, "margin_dpo/margin_std": 180.8556365966797, "step": 510 }, { "KL/chosen_KL_mean": -177.66510009765625, "KL/mean": -231.65658569335938, "KL/rejected_KL_mean": -285.6480712890625, "KL/std": 121.31975555419922, "epoch": 0.7724867724867724, "fcm_dpo/beta": 0.004389429464936256, "fcm_dpo/delta": -0.08617927134037018, "fcm_dpo/margin": 107.98297119140625, "fcm_dpo/q_t": 0.38881558179855347, "grad_norm": 14.997942924499512, "learning_rate": 7.557606426772961e-08, "logits/chosen": 0.8680216670036316, "logits/rejected": 0.8205208778381348, "logps/chosen": -233.50949096679688, "logps/ref_chosen": -55.844383239746094, "logps/ref_rejected": -86.49819946289062, "logps/rejected": -372.1462707519531, "loss": 1.0434, "margin_dpo/margin_mean": 107.98297119140625, "margin_dpo/margin_std": 127.46925354003906, "step": 511 }, { "KL/chosen_KL_mean": -185.30795288085938, "KL/mean": -217.4866485595703, "KL/rejected_KL_mean": -249.66534423828125, "KL/std": 113.47346496582031, "epoch": 0.7739984882842026, "fcm_dpo/beta": 0.004385386127978563, "fcm_dpo/delta": 0.023245226591825485, "fcm_dpo/margin": 64.3573989868164, "fcm_dpo/q_t": 0.43406805396080017, "grad_norm": 19.79949378967285, "learning_rate": 7.463127807341966e-08, "logits/chosen": 0.7447441816329956, "logits/rejected": 0.7479703426361084, "logps/chosen": -246.96099853515625, "logps/ref_chosen": -61.653038024902344, "logps/ref_rejected": -72.83148193359375, "logps/rejected": -322.496826171875, "loss": 1.2125, "margin_dpo/margin_mean": 64.3573989868164, "margin_dpo/margin_std": 137.24069213867188, "step": 512 }, { "KL/chosen_KL_mean": -166.38235473632812, "KL/mean": -207.5938720703125, "KL/rejected_KL_mean": -248.80538940429688, "KL/std": 113.1106948852539, "epoch": 0.7755102040816326, "fcm_dpo/beta": 0.00444423221051693, "fcm_dpo/delta": 0.03789909556508064, "fcm_dpo/margin": 82.42303466796875, "fcm_dpo/q_t": 0.4171144366264343, "grad_norm": 10.346599578857422, "learning_rate": 7.369139731924401e-08, "logits/chosen": 0.9161933660507202, "logits/rejected": 0.8713808655738831, "logps/chosen": -217.23492431640625, "logps/ref_chosen": -50.85256576538086, "logps/ref_rejected": -69.21754455566406, "logps/rejected": -318.02294921875, "loss": 1.1225, "margin_dpo/margin_mean": 82.42303466796875, "margin_dpo/margin_std": 122.04438018798828, "step": 513 }, { "KL/chosen_KL_mean": -187.51318359375, "KL/mean": -235.22531127929688, "KL/rejected_KL_mean": -282.93743896484375, "KL/std": 125.52702331542969, "epoch": 0.7770219198790628, "fcm_dpo/beta": 0.004477824550122023, "fcm_dpo/delta": -0.03195594251155853, "fcm_dpo/margin": 95.42424011230469, "fcm_dpo/q_t": 0.40270817279815674, "grad_norm": 15.198481559753418, "learning_rate": 7.275644829568747e-08, "logits/chosen": 0.8606992959976196, "logits/rejected": 0.837975263595581, "logps/chosen": -256.89813232421875, "logps/ref_chosen": -69.38493347167969, "logps/ref_rejected": -83.32447814941406, "logps/rejected": -366.26190185546875, "loss": 1.0998, "margin_dpo/margin_mean": 95.42424774169922, "margin_dpo/margin_std": 142.98733520507812, "step": 514 }, { "KL/chosen_KL_mean": -194.24365234375, "KL/mean": -232.44851684570312, "KL/rejected_KL_mean": -270.65338134765625, "KL/std": 117.28639221191406, "epoch": 0.7785336356764928, "fcm_dpo/beta": 0.004507084377110004, "fcm_dpo/delta": 0.062156371772289276, "fcm_dpo/margin": 76.40972900390625, "fcm_dpo/q_t": 0.4212613105773926, "grad_norm": 12.018317222595215, "learning_rate": 7.182645715528435e-08, "logits/chosen": 0.8571234345436096, "logits/rejected": 0.7873523235321045, "logps/chosen": -247.93069458007812, "logps/ref_chosen": -53.687034606933594, "logps/ref_rejected": -83.59614562988281, "logps/rejected": -354.24951171875, "loss": 1.1574, "margin_dpo/margin_mean": 76.40972900390625, "margin_dpo/margin_std": 133.1450653076172, "step": 515 }, { "KL/chosen_KL_mean": -168.87564086914062, "KL/mean": -206.1563262939453, "KL/rejected_KL_mean": -243.43699645996094, "KL/std": 115.80433654785156, "epoch": 0.780045351473923, "fcm_dpo/beta": 0.004723469726741314, "fcm_dpo/delta": 0.05215982347726822, "fcm_dpo/margin": 74.56133270263672, "fcm_dpo/q_t": 0.41921383142471313, "grad_norm": 15.109283447265625, "learning_rate": 7.090144991188568e-08, "logits/chosen": 0.8263188600540161, "logits/rejected": 0.7975659370422363, "logps/chosen": -225.77737426757812, "logps/ref_chosen": -56.9017219543457, "logps/ref_rejected": -67.83477783203125, "logps/rejected": -311.27178955078125, "loss": 1.1633, "margin_dpo/margin_mean": 74.56134033203125, "margin_dpo/margin_std": 136.02789306640625, "step": 516 }, { "KL/chosen_KL_mean": -201.57992553710938, "KL/mean": -228.9405517578125, "KL/rejected_KL_mean": -256.3011779785156, "KL/std": 123.93075561523438, "epoch": 0.781557067271353, "fcm_dpo/beta": 0.004781331401318312, "fcm_dpo/delta": 0.04728348180651665, "fcm_dpo/margin": 54.72124481201172, "fcm_dpo/q_t": 0.4420250654220581, "grad_norm": 13.745712280273438, "learning_rate": 6.998145243993284e-08, "logits/chosen": 0.8844262361526489, "logits/rejected": 0.8897521495819092, "logps/chosen": -263.3550720214844, "logps/ref_chosen": -61.775142669677734, "logps/ref_rejected": -62.88270950317383, "logps/rejected": -319.18389892578125, "loss": 1.2287, "margin_dpo/margin_mean": 54.72125244140625, "margin_dpo/margin_std": 127.82027435302734, "step": 517 }, { "KL/chosen_KL_mean": -173.06509399414062, "KL/mean": -213.97628784179688, "KL/rejected_KL_mean": -254.88751220703125, "KL/std": 121.97323608398438, "epoch": 0.783068783068783, "fcm_dpo/beta": 0.004860128741711378, "fcm_dpo/delta": 0.0026370445266366005, "fcm_dpo/margin": 81.82240295410156, "fcm_dpo/q_t": 0.41280555725097656, "grad_norm": 13.351316452026367, "learning_rate": 6.906649047373245e-08, "logits/chosen": 0.8314996957778931, "logits/rejected": 0.799196720123291, "logps/chosen": -235.09033203125, "logps/ref_chosen": -62.02523422241211, "logps/ref_rejected": -79.06085205078125, "logps/rejected": -333.9483642578125, "loss": 1.13, "margin_dpo/margin_mean": 81.82240295410156, "margin_dpo/margin_std": 138.0740966796875, "step": 518 }, { "KL/chosen_KL_mean": -204.05413818359375, "KL/mean": -227.598388671875, "KL/rejected_KL_mean": -251.1426239013672, "KL/std": 121.79615020751953, "epoch": 0.7845804988662132, "fcm_dpo/beta": 0.00496459798887372, "fcm_dpo/delta": 0.07236956804990768, "fcm_dpo/margin": 47.088470458984375, "fcm_dpo/q_t": 0.44760170578956604, "grad_norm": 19.102073669433594, "learning_rate": 6.815658960673781e-08, "logits/chosen": 0.8614056706428528, "logits/rejected": 0.8238772749900818, "logps/chosen": -265.6605224609375, "logps/ref_chosen": -61.60636901855469, "logps/ref_rejected": -74.50727844238281, "logps/rejected": -325.64990234375, "loss": 1.3117, "margin_dpo/margin_mean": 47.088478088378906, "margin_dpo/margin_std": 159.3134765625, "step": 519 }, { "KL/chosen_KL_mean": -189.42742919921875, "KL/mean": -226.57359313964844, "KL/rejected_KL_mean": -263.71978759765625, "KL/std": 123.97309112548828, "epoch": 0.7860922146636432, "fcm_dpo/beta": 0.005040885414928198, "fcm_dpo/delta": 0.02684013918042183, "fcm_dpo/margin": 74.29235076904297, "fcm_dpo/q_t": 0.4172334671020508, "grad_norm": 16.04875373840332, "learning_rate": 6.725177529083209e-08, "logits/chosen": 0.9040594100952148, "logits/rejected": 0.8583179116249084, "logps/chosen": -252.30084228515625, "logps/ref_chosen": -62.87343215942383, "logps/ref_rejected": -76.505615234375, "logps/rejected": -340.2253723144531, "loss": 1.1434, "margin_dpo/margin_mean": 74.29235076904297, "margin_dpo/margin_std": 126.32000732421875, "step": 520 }, { "KL/chosen_KL_mean": -187.01785278320312, "KL/mean": -239.00657653808594, "KL/rejected_KL_mean": -290.9953308105469, "KL/std": 124.35540008544922, "epoch": 0.7876039304610734, "fcm_dpo/beta": 0.00489568617194891, "fcm_dpo/delta": -0.1282760053873062, "fcm_dpo/margin": 103.97747802734375, "fcm_dpo/q_t": 0.3860055208206177, "grad_norm": 11.902304649353027, "learning_rate": 6.63520728356167e-08, "logits/chosen": 0.7507075071334839, "logits/rejected": 0.691694974899292, "logps/chosen": -251.22451782226562, "logps/ref_chosen": -64.20668029785156, "logps/ref_rejected": -92.28083038330078, "logps/rejected": -383.2761535644531, "loss": 1.0477, "margin_dpo/margin_mean": 103.97747802734375, "margin_dpo/margin_std": 142.90042114257812, "step": 521 }, { "KL/chosen_KL_mean": -189.99844360351562, "KL/mean": -219.89682006835938, "KL/rejected_KL_mean": -249.79522705078125, "KL/std": 121.7298583984375, "epoch": 0.7891156462585034, "fcm_dpo/beta": 0.004985436797142029, "fcm_dpo/delta": 0.1123264878988266, "fcm_dpo/margin": 59.796783447265625, "fcm_dpo/q_t": 0.43511706590652466, "grad_norm": 14.604763984680176, "learning_rate": 6.545750740770336e-08, "logits/chosen": 0.8380166292190552, "logits/rejected": 0.8317359089851379, "logps/chosen": -248.3681640625, "logps/ref_chosen": -58.369720458984375, "logps/ref_rejected": -68.79248046875, "logps/rejected": -318.58770751953125, "loss": 1.2558, "margin_dpo/margin_mean": 59.796783447265625, "margin_dpo/margin_std": 157.91360473632812, "step": 522 }, { "KL/chosen_KL_mean": -195.73423767089844, "KL/mean": -234.60708618164062, "KL/rejected_KL_mean": -273.47991943359375, "KL/std": 125.57742309570312, "epoch": 0.7906273620559335, "fcm_dpo/beta": 0.005116622429341078, "fcm_dpo/delta": 0.002492782659828663, "fcm_dpo/margin": 77.7457046508789, "fcm_dpo/q_t": 0.4087350368499756, "grad_norm": 18.966442108154297, "learning_rate": 6.456810403001012e-08, "logits/chosen": 0.8302779793739319, "logits/rejected": 0.728390634059906, "logps/chosen": -261.4474792480469, "logps/ref_chosen": -65.71324157714844, "logps/ref_rejected": -91.98896789550781, "logps/rejected": -365.4689025878906, "loss": 1.1459, "margin_dpo/margin_mean": 77.74569702148438, "margin_dpo/margin_std": 139.64659118652344, "step": 523 }, { "KL/chosen_KL_mean": -173.21641540527344, "KL/mean": -211.4966278076172, "KL/rejected_KL_mean": -249.77685546875, "KL/std": 121.24716186523438, "epoch": 0.7921390778533636, "fcm_dpo/beta": 0.005164645612239838, "fcm_dpo/delta": 0.004730384796857834, "fcm_dpo/margin": 76.56043243408203, "fcm_dpo/q_t": 0.4102930724620819, "grad_norm": 15.195032119750977, "learning_rate": 6.368388758106134e-08, "logits/chosen": 0.7700381875038147, "logits/rejected": 0.7530121803283691, "logps/chosen": -249.56765747070312, "logps/ref_chosen": -76.35124969482422, "logps/ref_rejected": -89.96072387695312, "logps/rejected": -339.7375793457031, "loss": 1.1218, "margin_dpo/margin_mean": 76.5604248046875, "margin_dpo/margin_std": 122.09478759765625, "step": 524 }, { "KL/chosen_KL_mean": -193.4317626953125, "KL/mean": -225.12921142578125, "KL/rejected_KL_mean": -256.82666015625, "KL/std": 117.6169204711914, "epoch": 0.7936507936507936, "fcm_dpo/beta": 0.0052588703110814095, "fcm_dpo/delta": 0.07416895776987076, "fcm_dpo/margin": 63.3948974609375, "fcm_dpo/q_t": 0.42921310663223267, "grad_norm": 19.115665435791016, "learning_rate": 6.280488279429185e-08, "logits/chosen": 0.7093310952186584, "logits/rejected": 0.7094483971595764, "logps/chosen": -268.92755126953125, "logps/ref_chosen": -75.49578857421875, "logps/ref_rejected": -84.04852294921875, "logps/rejected": -340.87518310546875, "loss": 1.2039, "margin_dpo/margin_mean": 63.394893646240234, "margin_dpo/margin_std": 140.00247192382812, "step": 525 }, { "KL/chosen_KL_mean": -198.68052673339844, "KL/mean": -229.0758819580078, "KL/rejected_KL_mean": -259.47125244140625, "KL/std": 122.75657653808594, "epoch": 0.7951625094482238, "fcm_dpo/beta": 0.005194986704736948, "fcm_dpo/delta": -0.052471913397312164, "fcm_dpo/margin": 60.79069900512695, "fcm_dpo/q_t": 0.42842093110084534, "grad_norm": 14.487711906433105, "learning_rate": 6.193111425735515e-08, "logits/chosen": 0.8522295951843262, "logits/rejected": 0.7942419648170471, "logps/chosen": -259.9729309082031, "logps/ref_chosen": -61.29241943359375, "logps/ref_rejected": -82.47763061523438, "logps/rejected": -341.9488525390625, "loss": 1.1995, "margin_dpo/margin_mean": 60.79069900512695, "margin_dpo/margin_std": 125.36583709716797, "step": 526 }, { "KL/chosen_KL_mean": -207.10394287109375, "KL/mean": -233.09361267089844, "KL/rejected_KL_mean": -259.0832824707031, "KL/std": 121.70289611816406, "epoch": 0.7966742252456538, "fcm_dpo/beta": 0.005217268131673336, "fcm_dpo/delta": 0.014235904440283775, "fcm_dpo/margin": 51.979347229003906, "fcm_dpo/q_t": 0.44098371267318726, "grad_norm": 15.067662239074707, "learning_rate": 6.106260641143546e-08, "logits/chosen": 0.9322826266288757, "logits/rejected": 0.8576953411102295, "logps/chosen": -268.5765686035156, "logps/ref_chosen": -61.472625732421875, "logps/ref_rejected": -90.52831268310547, "logps/rejected": -349.6116027832031, "loss": 1.2546, "margin_dpo/margin_mean": 51.979347229003906, "margin_dpo/margin_std": 137.49215698242188, "step": 527 }, { "KL/chosen_KL_mean": -183.95230102539062, "KL/mean": -214.3380584716797, "KL/rejected_KL_mean": -244.72381591796875, "KL/std": 114.09736633300781, "epoch": 0.7981859410430839, "fcm_dpo/beta": 0.005392709746956825, "fcm_dpo/delta": 0.08037621527910233, "fcm_dpo/margin": 60.771522521972656, "fcm_dpo/q_t": 0.4303974509239197, "grad_norm": 17.312894821166992, "learning_rate": 6.019938355056422e-08, "logits/chosen": 0.8145349621772766, "logits/rejected": 0.7549326419830322, "logps/chosen": -242.7443084716797, "logps/ref_chosen": -58.792015075683594, "logps/ref_rejected": -71.82516479492188, "logps/rejected": -316.5489807128906, "loss": 1.2365, "margin_dpo/margin_mean": 60.771522521972656, "margin_dpo/margin_std": 148.80494689941406, "step": 528 }, { "KL/chosen_KL_mean": -178.44883728027344, "KL/mean": -234.34661865234375, "KL/rejected_KL_mean": -290.2444152832031, "KL/std": 112.45913696289062, "epoch": 0.799697656840514, "fcm_dpo/beta": 0.005035213194787502, "fcm_dpo/delta": -0.1965881735086441, "fcm_dpo/margin": 111.79557800292969, "fcm_dpo/q_t": 0.3671392798423767, "grad_norm": 15.271794319152832, "learning_rate": 5.934146982094049e-08, "logits/chosen": 0.7759064435958862, "logits/rejected": 0.7353440523147583, "logps/chosen": -233.51980590820312, "logps/ref_chosen": -55.070960998535156, "logps/ref_rejected": -75.44007873535156, "logps/rejected": -365.68450927734375, "loss": 0.9889, "margin_dpo/margin_mean": 111.79557800292969, "margin_dpo/margin_std": 122.36137390136719, "step": 529 }, { "KL/chosen_KL_mean": -176.23171997070312, "KL/mean": -214.2334747314453, "KL/rejected_KL_mean": -252.23526000976562, "KL/std": 122.73760986328125, "epoch": 0.8012093726379441, "fcm_dpo/beta": 0.004925016779452562, "fcm_dpo/delta": 0.028898127377033234, "fcm_dpo/margin": 76.00357055664062, "fcm_dpo/q_t": 0.4183453917503357, "grad_norm": 16.061771392822266, "learning_rate": 5.848888922025552e-08, "logits/chosen": 0.8739749193191528, "logits/rejected": 0.836665153503418, "logps/chosen": -232.97552490234375, "logps/ref_chosen": -56.743812561035156, "logps/ref_rejected": -76.6692123413086, "logps/rejected": -328.90447998046875, "loss": 1.1416, "margin_dpo/margin_mean": 76.00357055664062, "margin_dpo/margin_std": 129.30343627929688, "step": 530 }, { "KL/chosen_KL_mean": -176.384521484375, "KL/mean": -215.5994873046875, "KL/rejected_KL_mean": -254.81443786621094, "KL/std": 120.86744689941406, "epoch": 0.8027210884353742, "fcm_dpo/beta": 0.0049905176274478436, "fcm_dpo/delta": 0.009740887209773064, "fcm_dpo/margin": 78.42990112304688, "fcm_dpo/q_t": 0.41260451078414917, "grad_norm": 13.00410270690918, "learning_rate": 5.7641665597021435e-08, "logits/chosen": 0.8380810618400574, "logits/rejected": 0.7674177885055542, "logps/chosen": -227.5009765625, "logps/ref_chosen": -51.116455078125, "logps/ref_rejected": -79.52884674072266, "logps/rejected": -334.34326171875, "loss": 1.1312, "margin_dpo/margin_mean": 78.42990112304688, "margin_dpo/margin_std": 130.77659606933594, "step": 531 }, { "KL/chosen_KL_mean": -206.09271240234375, "KL/mean": -245.57955932617188, "KL/rejected_KL_mean": -285.06646728515625, "KL/std": 119.67216491699219, "epoch": 0.8042328042328042, "fcm_dpo/beta": 0.005016631446778774, "fcm_dpo/delta": 0.004318161401897669, "fcm_dpo/margin": 78.97373962402344, "fcm_dpo/q_t": 0.4110661447048187, "grad_norm": 14.44060230255127, "learning_rate": 5.679982264990424e-08, "logits/chosen": 0.8174869418144226, "logits/rejected": 0.774748682975769, "logps/chosen": -264.3726501464844, "logps/ref_chosen": -58.279945373535156, "logps/ref_rejected": -78.05426788330078, "logps/rejected": -363.1207275390625, "loss": 1.1321, "margin_dpo/margin_mean": 78.97373962402344, "margin_dpo/margin_std": 132.41571044921875, "step": 532 }, { "KL/chosen_KL_mean": -148.63015747070312, "KL/mean": -195.14443969726562, "KL/rejected_KL_mean": -241.65870666503906, "KL/std": 127.76698303222656, "epoch": 0.8057445200302343, "fcm_dpo/beta": 0.004902138374745846, "fcm_dpo/delta": -0.06478457897901535, "fcm_dpo/margin": 93.02857208251953, "fcm_dpo/q_t": 0.39741477370262146, "grad_norm": 14.071173667907715, "learning_rate": 5.596338392706076e-08, "logits/chosen": 0.9038737416267395, "logits/rejected": 0.8549951314926147, "logps/chosen": -205.04815673828125, "logps/ref_chosen": -56.41801071166992, "logps/ref_rejected": -73.89324951171875, "logps/rejected": -315.55194091796875, "loss": 1.0814, "margin_dpo/margin_mean": 93.02857971191406, "margin_dpo/margin_std": 134.64431762695312, "step": 533 }, { "KL/chosen_KL_mean": -185.22732543945312, "KL/mean": -220.9058837890625, "KL/rejected_KL_mean": -256.58441162109375, "KL/std": 115.06756591796875, "epoch": 0.8072562358276644, "fcm_dpo/beta": 0.004894358105957508, "fcm_dpo/delta": 0.0565091148018837, "fcm_dpo/margin": 71.35710144042969, "fcm_dpo/q_t": 0.423191636800766, "grad_norm": 13.81146240234375, "learning_rate": 5.513237282548033e-08, "logits/chosen": 0.816210150718689, "logits/rejected": 0.7883972525596619, "logps/chosen": -245.97601318359375, "logps/ref_chosen": -60.748687744140625, "logps/ref_rejected": -73.8623046875, "logps/rejected": -330.44671630859375, "loss": 1.1783, "margin_dpo/margin_mean": 71.35710144042969, "margin_dpo/margin_std": 140.3214111328125, "step": 534 }, { "KL/chosen_KL_mean": -196.07479858398438, "KL/mean": -230.17123413085938, "KL/rejected_KL_mean": -264.26763916015625, "KL/std": 123.80078887939453, "epoch": 0.8087679516250945, "fcm_dpo/beta": 0.005145275965332985, "fcm_dpo/delta": 0.054210808128118515, "fcm_dpo/margin": 68.1928482055664, "fcm_dpo/q_t": 0.4239533543586731, "grad_norm": 15.887540817260742, "learning_rate": 5.430681259032957e-08, "logits/chosen": 0.7688905000686646, "logits/rejected": 0.7201669812202454, "logps/chosen": -257.71221923828125, "logps/ref_chosen": -61.637413024902344, "logps/ref_rejected": -80.93138885498047, "logps/rejected": -345.19903564453125, "loss": 1.1858, "margin_dpo/margin_mean": 68.1928482055664, "margin_dpo/margin_std": 138.06951904296875, "step": 535 }, { "KL/chosen_KL_mean": -169.49693298339844, "KL/mean": -222.1195068359375, "KL/rejected_KL_mean": -274.7420959472656, "KL/std": 123.30923461914062, "epoch": 0.8102796674225246, "fcm_dpo/beta": 0.004823877010494471, "fcm_dpo/delta": -0.12901368737220764, "fcm_dpo/margin": 105.24514770507812, "fcm_dpo/q_t": 0.3823744058609009, "grad_norm": 10.788253784179688, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 0.8770267963409424, "logits/rejected": 0.8094578981399536, "logps/chosen": -221.3859100341797, "logps/ref_chosen": -51.88897705078125, "logps/ref_rejected": -73.34864044189453, "logps/rejected": -348.0907287597656, "loss": 1.0211, "margin_dpo/margin_mean": 105.24514770507812, "margin_dpo/margin_std": 122.67505645751953, "step": 536 }, { "KL/chosen_KL_mean": -192.197509765625, "KL/mean": -238.49530029296875, "KL/rejected_KL_mean": -284.7930603027344, "KL/std": 133.47552490234375, "epoch": 0.8117913832199547, "fcm_dpo/beta": 0.004660390317440033, "fcm_dpo/delta": -0.03684820234775543, "fcm_dpo/margin": 92.59556579589844, "fcm_dpo/q_t": 0.4039597511291504, "grad_norm": 13.52630615234375, "learning_rate": 5.267213693697695e-08, "logits/chosen": 0.9239789843559265, "logits/rejected": 0.8453102707862854, "logps/chosen": -246.44613647460938, "logps/ref_chosen": -54.248619079589844, "logps/ref_rejected": -94.94343566894531, "logps/rejected": -379.73651123046875, "loss": 1.1087, "margin_dpo/margin_mean": 92.59556579589844, "margin_dpo/margin_std": 146.90977478027344, "step": 537 }, { "KL/chosen_KL_mean": -195.64366149902344, "KL/mean": -239.24957275390625, "KL/rejected_KL_mean": -282.8554992675781, "KL/std": 131.49520874023438, "epoch": 0.8133030990173847, "fcm_dpo/beta": 0.004660367965698242, "fcm_dpo/delta": -0.00734717445448041, "fcm_dpo/margin": 87.21183776855469, "fcm_dpo/q_t": 0.407853901386261, "grad_norm": 12.92613697052002, "learning_rate": 5.1863067244167144e-08, "logits/chosen": 0.8298152685165405, "logits/rejected": 0.8131669759750366, "logps/chosen": -265.7371826171875, "logps/ref_chosen": -70.09353637695312, "logps/ref_rejected": -79.49833679199219, "logps/rejected": -362.35382080078125, "loss": 1.1085, "margin_dpo/margin_mean": 87.21183776855469, "margin_dpo/margin_std": 133.052001953125, "step": 538 }, { "KL/chosen_KL_mean": -199.86004638671875, "KL/mean": -234.01394653320312, "KL/rejected_KL_mean": -268.1678466796875, "KL/std": 120.2940902709961, "epoch": 0.8148148148148148, "fcm_dpo/beta": 0.004803318530321121, "fcm_dpo/delta": 0.0798788070678711, "fcm_dpo/margin": 68.30778503417969, "fcm_dpo/q_t": 0.43010759353637695, "grad_norm": 14.349104881286621, "learning_rate": 5.105953986729195e-08, "logits/chosen": 0.8133977651596069, "logits/rejected": 0.7414849400520325, "logps/chosen": -261.791748046875, "logps/ref_chosen": -61.93169403076172, "logps/ref_rejected": -84.08946228027344, "logps/rejected": -352.25732421875, "loss": 1.1762, "margin_dpo/margin_mean": 68.30778503417969, "margin_dpo/margin_std": 132.4424591064453, "step": 539 }, { "KL/chosen_KL_mean": -183.73110961914062, "KL/mean": -237.5785369873047, "KL/rejected_KL_mean": -291.42596435546875, "KL/std": 140.208251953125, "epoch": 0.8163265306122449, "fcm_dpo/beta": 0.004663660190999508, "fcm_dpo/delta": -0.11989345401525497, "fcm_dpo/margin": 107.69487762451172, "fcm_dpo/q_t": 0.3844473659992218, "grad_norm": 12.575929641723633, "learning_rate": 5.026157728273966e-08, "logits/chosen": 0.8718093633651733, "logits/rejected": 0.7981296181678772, "logps/chosen": -246.4353485107422, "logps/ref_chosen": -62.704254150390625, "logps/ref_rejected": -95.63597106933594, "logps/rejected": -387.06195068359375, "loss": 1.0385, "margin_dpo/margin_mean": 107.69487762451172, "margin_dpo/margin_std": 134.9456787109375, "step": 540 }, { "KL/chosen_KL_mean": -179.12994384765625, "KL/mean": -221.66107177734375, "KL/rejected_KL_mean": -264.1922302246094, "KL/std": 119.99224853515625, "epoch": 0.817838246409675, "fcm_dpo/beta": 0.004524096846580505, "fcm_dpo/delta": 0.016618637368083, "fcm_dpo/margin": 85.06229400634766, "fcm_dpo/q_t": 0.41030341386795044, "grad_norm": 11.988428115844727, "learning_rate": 4.9469201811239035e-08, "logits/chosen": 0.8793637752532959, "logits/rejected": 0.9092428684234619, "logps/chosen": -241.61077880859375, "logps/ref_chosen": -62.48084259033203, "logps/ref_rejected": -57.55541229248047, "logps/rejected": -321.74761962890625, "loss": 1.1052, "margin_dpo/margin_mean": 85.06229400634766, "margin_dpo/margin_std": 115.75038146972656, "step": 541 }, { "KL/chosen_KL_mean": -155.74496459960938, "KL/mean": -203.70249938964844, "KL/rejected_KL_mean": -251.66004943847656, "KL/std": 112.92686462402344, "epoch": 0.8193499622071051, "fcm_dpo/beta": 0.004484962671995163, "fcm_dpo/delta": -0.03508245199918747, "fcm_dpo/margin": 95.91510772705078, "fcm_dpo/q_t": 0.40033650398254395, "grad_norm": 11.718820571899414, "learning_rate": 4.868243561723534e-08, "logits/chosen": 0.9004342555999756, "logits/rejected": 0.8623563051223755, "logps/chosen": -205.19984436035156, "logps/ref_chosen": -49.454891204833984, "logps/ref_rejected": -65.33275604248047, "logps/rejected": -316.9927978515625, "loss": 1.0898, "margin_dpo/margin_mean": 95.91511535644531, "margin_dpo/margin_std": 136.8139190673828, "step": 542 }, { "KL/chosen_KL_mean": -170.90618896484375, "KL/mean": -218.6575164794922, "KL/rejected_KL_mean": -266.4088134765625, "KL/std": 120.44759368896484, "epoch": 0.8208616780045351, "fcm_dpo/beta": 0.0044562616385519505, "fcm_dpo/delta": -0.029431413859128952, "fcm_dpo/margin": 95.50263977050781, "fcm_dpo/q_t": 0.40224945545196533, "grad_norm": 10.733229637145996, "learning_rate": 4.790130070827028e-08, "logits/chosen": 0.8563024997711182, "logits/rejected": 0.7859128713607788, "logps/chosen": -222.00704956054688, "logps/ref_chosen": -51.100860595703125, "logps/ref_rejected": -76.06130981445312, "logps/rejected": -342.47015380859375, "loss": 1.0909, "margin_dpo/margin_mean": 95.50263977050781, "margin_dpo/margin_std": 133.79779052734375, "step": 543 }, { "KL/chosen_KL_mean": -183.34523010253906, "KL/mean": -237.28253173828125, "KL/rejected_KL_mean": -291.21978759765625, "KL/std": 133.13616943359375, "epoch": 0.8223733938019653, "fcm_dpo/beta": 0.004260823130607605, "fcm_dpo/delta": -0.06947409361600876, "fcm_dpo/margin": 107.87458038330078, "fcm_dpo/q_t": 0.39583128690719604, "grad_norm": 13.217146873474121, "learning_rate": 4.7125818934366454e-08, "logits/chosen": 0.8630880117416382, "logits/rejected": 0.7988213300704956, "logps/chosen": -243.62246704101562, "logps/ref_chosen": -60.2772331237793, "logps/ref_rejected": -88.40553283691406, "logps/rejected": -379.6253356933594, "loss": 1.0805, "margin_dpo/margin_mean": 107.87457275390625, "margin_dpo/margin_std": 156.6991424560547, "step": 544 }, { "KL/chosen_KL_mean": -197.96246337890625, "KL/mean": -227.96221923828125, "KL/rejected_KL_mean": -257.9619445800781, "KL/std": 122.77239990234375, "epoch": 0.8238851095993953, "fcm_dpo/beta": 0.0045236810110509396, "fcm_dpo/delta": 0.13886934518814087, "fcm_dpo/margin": 59.999481201171875, "fcm_dpo/q_t": 0.4393478035926819, "grad_norm": 13.92799186706543, "learning_rate": 4.635601198741607e-08, "logits/chosen": 0.8469263315200806, "logits/rejected": 0.7915807366371155, "logps/chosen": -259.57769775390625, "logps/ref_chosen": -61.61524963378906, "logps/ref_rejected": -78.71266174316406, "logps/rejected": -336.67462158203125, "loss": 1.2181, "margin_dpo/margin_mean": 59.99948501586914, "margin_dpo/margin_std": 132.14329528808594, "step": 545 }, { "KL/chosen_KL_mean": -189.22254943847656, "KL/mean": -221.83804321289062, "KL/rejected_KL_mean": -254.45355224609375, "KL/std": 116.31465911865234, "epoch": 0.8253968253968254, "fcm_dpo/beta": 0.004792468622326851, "fcm_dpo/delta": 0.09647992998361588, "fcm_dpo/margin": 65.2309799194336, "fcm_dpo/q_t": 0.4293987452983856, "grad_norm": 18.117361068725586, "learning_rate": 4.559190140057428e-08, "logits/chosen": 0.957390308380127, "logits/rejected": 0.95533686876297, "logps/chosen": -248.5358123779297, "logps/ref_chosen": -59.313262939453125, "logps/ref_rejected": -64.73631286621094, "logps/rejected": -319.1898498535156, "loss": 1.209, "margin_dpo/margin_mean": 65.23097229003906, "margin_dpo/margin_std": 142.85653686523438, "step": 546 }, { "KL/chosen_KL_mean": -165.288330078125, "KL/mean": -215.5997772216797, "KL/rejected_KL_mean": -265.9112243652344, "KL/std": 121.85418701171875, "epoch": 0.8269085411942555, "fcm_dpo/beta": 0.004634900949895382, "fcm_dpo/delta": -0.07973036170005798, "fcm_dpo/margin": 100.62291717529297, "fcm_dpo/q_t": 0.3919479250907898, "grad_norm": 12.286870002746582, "learning_rate": 4.483350854765672e-08, "logits/chosen": 0.7986713647842407, "logits/rejected": 0.7496638298034668, "logps/chosen": -220.26507568359375, "logps/ref_chosen": -54.97674560546875, "logps/ref_rejected": -75.35922241210938, "logps/rejected": -341.27044677734375, "loss": 1.0699, "margin_dpo/margin_mean": 100.6229248046875, "margin_dpo/margin_std": 139.28582763671875, "step": 547 }, { "KL/chosen_KL_mean": -196.86660766601562, "KL/mean": -226.04827880859375, "KL/rejected_KL_mean": -255.2299346923828, "KL/std": 120.6140365600586, "epoch": 0.8284202569916855, "fcm_dpo/beta": 0.004941076040267944, "fcm_dpo/delta": 0.12047231942415237, "fcm_dpo/margin": 58.363319396972656, "fcm_dpo/q_t": 0.43659114837646484, "grad_norm": 16.297290802001953, "learning_rate": 4.4080854642541826e-08, "logits/chosen": 0.8177128434181213, "logits/rejected": 0.7706683278083801, "logps/chosen": -260.0772705078125, "logps/ref_chosen": -63.21067428588867, "logps/ref_rejected": -81.23347473144531, "logps/rejected": -336.4634094238281, "loss": 1.2194, "margin_dpo/margin_mean": 58.36331558227539, "margin_dpo/margin_std": 132.25997924804688, "step": 548 }, { "KL/chosen_KL_mean": -194.89645385742188, "KL/mean": -231.360595703125, "KL/rejected_KL_mean": -267.8247375488281, "KL/std": 134.2811737060547, "epoch": 0.8299319727891157, "fcm_dpo/beta": 0.005007403437048197, "fcm_dpo/delta": 0.03835400193929672, "fcm_dpo/margin": 72.92826080322266, "fcm_dpo/q_t": 0.4223543405532837, "grad_norm": 18.85927963256836, "learning_rate": 4.333396073857723e-08, "logits/chosen": 0.9243029952049255, "logits/rejected": 0.8670613765716553, "logps/chosen": -259.16998291015625, "logps/ref_chosen": -64.27351379394531, "logps/ref_rejected": -92.31663513183594, "logps/rejected": -360.141357421875, "loss": 1.199, "margin_dpo/margin_mean": 72.92826080322266, "margin_dpo/margin_std": 159.06471252441406, "step": 549 }, { "KL/chosen_KL_mean": -193.56687927246094, "KL/mean": -220.8571319580078, "KL/rejected_KL_mean": -248.14736938476562, "KL/std": 117.43412017822266, "epoch": 0.8314436885865457, "fcm_dpo/beta": 0.005119511857628822, "fcm_dpo/delta": 0.004839625675231218, "fcm_dpo/margin": 54.58049392700195, "fcm_dpo/q_t": 0.43776458501815796, "grad_norm": 15.661591529846191, "learning_rate": 4.259284772799099e-08, "logits/chosen": 0.8754514455795288, "logits/rejected": 0.8566372990608215, "logps/chosen": -249.7973175048828, "logps/ref_chosen": -56.230438232421875, "logps/ref_rejected": -62.59788513183594, "logps/rejected": -310.7452697753906, "loss": 1.2314, "margin_dpo/margin_mean": 54.58049392700195, "margin_dpo/margin_std": 128.6197509765625, "step": 550 }, { "KL/chosen_KL_mean": -201.9175262451172, "KL/mean": -235.73777770996094, "KL/rejected_KL_mean": -269.55804443359375, "KL/std": 127.38159942626953, "epoch": 0.8329554043839759, "fcm_dpo/beta": 0.0052591776475310326, "fcm_dpo/delta": 0.0491047166287899, "fcm_dpo/margin": 67.64051055908203, "fcm_dpo/q_t": 0.42484039068222046, "grad_norm": 14.79029369354248, "learning_rate": 4.1857536341307176e-08, "logits/chosen": 0.8869715929031372, "logits/rejected": 0.8607189655303955, "logps/chosen": -269.66473388671875, "logps/ref_chosen": -67.74720764160156, "logps/ref_rejected": -87.04285430908203, "logps/rejected": -356.60089111328125, "loss": 1.1578, "margin_dpo/margin_mean": 67.64051818847656, "margin_dpo/margin_std": 126.78274536132812, "step": 551 }, { "KL/chosen_KL_mean": -186.17919921875, "KL/mean": -225.56101989746094, "KL/rejected_KL_mean": -264.94281005859375, "KL/std": 130.92601013183594, "epoch": 0.8344671201814059, "fcm_dpo/beta": 0.005316844675689936, "fcm_dpo/delta": -0.023466050624847412, "fcm_dpo/margin": 78.76361083984375, "fcm_dpo/q_t": 0.4041834771633148, "grad_norm": 18.497468948364258, "learning_rate": 4.112804714676593e-08, "logits/chosen": 0.851292073726654, "logits/rejected": 0.8060963153839111, "logps/chosen": -249.10546875, "logps/ref_chosen": -62.92625427246094, "logps/ref_rejected": -82.98365783691406, "logps/rejected": -347.92645263671875, "loss": 1.1122, "margin_dpo/margin_mean": 78.76361083984375, "margin_dpo/margin_std": 122.3577651977539, "step": 552 }, { "KL/chosen_KL_mean": -201.0130615234375, "KL/mean": -239.95428466796875, "KL/rejected_KL_mean": -278.8955078125, "KL/std": 127.23973083496094, "epoch": 0.8359788359788359, "fcm_dpo/beta": 0.005192439071834087, "fcm_dpo/delta": -0.0050092036835849285, "fcm_dpo/margin": 77.88245391845703, "fcm_dpo/q_t": 0.41371631622314453, "grad_norm": 14.231207847595215, "learning_rate": 4.0404400549748144e-08, "logits/chosen": 0.838943362236023, "logits/rejected": 0.7557328939437866, "logps/chosen": -257.0515441894531, "logps/ref_chosen": -56.038490295410156, "logps/ref_rejected": -84.48454284667969, "logps/rejected": -363.3800354003906, "loss": 1.158, "margin_dpo/margin_mean": 77.8824462890625, "margin_dpo/margin_std": 150.22286987304688, "step": 553 }, { "KL/chosen_KL_mean": -182.27084350585938, "KL/mean": -227.0911865234375, "KL/rejected_KL_mean": -271.91156005859375, "KL/std": 118.93246459960938, "epoch": 0.8374905517762661, "fcm_dpo/beta": 0.0050364104099571705, "fcm_dpo/delta": -0.05972806736826897, "fcm_dpo/margin": 89.64070129394531, "fcm_dpo/q_t": 0.39440518617630005, "grad_norm": 14.413543701171875, "learning_rate": 3.968661679220467e-08, "logits/chosen": 0.8107480406761169, "logits/rejected": 0.8082457780838013, "logps/chosen": -246.80142211914062, "logps/ref_chosen": -64.53059387207031, "logps/ref_rejected": -71.2155990600586, "logps/rejected": -343.12713623046875, "loss": 1.0842, "margin_dpo/margin_mean": 89.64070129394531, "margin_dpo/margin_std": 128.74749755859375, "step": 554 }, { "KL/chosen_KL_mean": -196.64315795898438, "KL/mean": -230.64242553710938, "KL/rejected_KL_mean": -264.6416931152344, "KL/std": 117.89613342285156, "epoch": 0.8390022675736961, "fcm_dpo/beta": 0.005218766164034605, "fcm_dpo/delta": 0.04598980396986008, "fcm_dpo/margin": 67.9985580444336, "fcm_dpo/q_t": 0.4178510904312134, "grad_norm": 15.634132385253906, "learning_rate": 3.89747159520904e-08, "logits/chosen": 0.8837528228759766, "logits/rejected": 0.8687556982040405, "logps/chosen": -263.2950744628906, "logps/ref_chosen": -66.65191650390625, "logps/ref_rejected": -68.6667251586914, "logps/rejected": -333.30841064453125, "loss": 1.1975, "margin_dpo/margin_mean": 67.9985580444336, "margin_dpo/margin_std": 140.62289428710938, "step": 555 }, { "KL/chosen_KL_mean": -193.26910400390625, "KL/mean": -228.2431640625, "KL/rejected_KL_mean": -263.21722412109375, "KL/std": 122.90667724609375, "epoch": 0.8405139833711263, "fcm_dpo/beta": 0.005200220271945, "fcm_dpo/delta": 0.040658943355083466, "fcm_dpo/margin": 69.94815063476562, "fcm_dpo/q_t": 0.42058825492858887, "grad_norm": 14.569718360900879, "learning_rate": 3.826871794280192e-08, "logits/chosen": 0.8939093351364136, "logits/rejected": 0.8580869436264038, "logps/chosen": -246.10147094726562, "logps/ref_chosen": -52.832366943359375, "logps/ref_rejected": -64.49044036865234, "logps/rejected": -327.7076721191406, "loss": 1.173, "margin_dpo/margin_mean": 69.94815826416016, "margin_dpo/margin_std": 135.2369842529297, "step": 556 }, { "KL/chosen_KL_mean": -191.77627563476562, "KL/mean": -241.3262939453125, "KL/rejected_KL_mean": -290.87628173828125, "KL/std": 122.63204193115234, "epoch": 0.8420256991685563, "fcm_dpo/beta": 0.004956225864589214, "fcm_dpo/delta": -0.10954894870519638, "fcm_dpo/margin": 99.10000610351562, "fcm_dpo/q_t": 0.38677504658699036, "grad_norm": 12.53800106048584, "learning_rate": 3.756864251262143e-08, "logits/chosen": 0.9151580929756165, "logits/rejected": 0.8559887409210205, "logps/chosen": -246.81227111816406, "logps/ref_chosen": -55.03598403930664, "logps/ref_rejected": -75.80644989013672, "logps/rejected": -366.6827392578125, "loss": 1.0388, "margin_dpo/margin_mean": 99.10000610351562, "margin_dpo/margin_std": 120.31071472167969, "step": 557 }, { "KL/chosen_KL_mean": -180.4897003173828, "KL/mean": -234.2262420654297, "KL/rejected_KL_mean": -287.9627685546875, "KL/std": 130.8162078857422, "epoch": 0.8435374149659864, "fcm_dpo/beta": 0.00461628008633852, "fcm_dpo/delta": -0.11640501767396927, "fcm_dpo/margin": 107.4730453491211, "fcm_dpo/q_t": 0.38629502058029175, "grad_norm": 10.66162395477295, "learning_rate": 3.687450924416341e-08, "logits/chosen": 0.8702242970466614, "logits/rejected": 0.8275398015975952, "logps/chosen": -243.716064453125, "logps/ref_chosen": -63.226348876953125, "logps/ref_rejected": -91.46881866455078, "logps/rejected": -379.43157958984375, "loss": 1.0436, "margin_dpo/margin_mean": 107.4730453491211, "margin_dpo/margin_std": 138.13763427734375, "step": 558 }, { "KL/chosen_KL_mean": -184.31173706054688, "KL/mean": -231.5626678466797, "KL/rejected_KL_mean": -278.8135986328125, "KL/std": 130.44276428222656, "epoch": 0.8450491307634165, "fcm_dpo/beta": 0.004464943893253803, "fcm_dpo/delta": -0.027830161154270172, "fcm_dpo/margin": 94.5018310546875, "fcm_dpo/q_t": 0.408642053604126, "grad_norm": 11.145119667053223, "learning_rate": 3.6186337553827743e-08, "logits/chosen": 0.8411096334457397, "logits/rejected": 0.7925565242767334, "logps/chosen": -245.83338928222656, "logps/ref_chosen": -61.521644592285156, "logps/ref_rejected": -82.83859252929688, "logps/rejected": -361.65216064453125, "loss": 1.1223, "margin_dpo/margin_mean": 94.5018310546875, "margin_dpo/margin_std": 158.6476287841797, "step": 559 }, { "KL/chosen_KL_mean": -199.8177490234375, "KL/mean": -244.57350158691406, "KL/rejected_KL_mean": -289.3292541503906, "KL/std": 123.44962310791016, "epoch": 0.8465608465608465, "fcm_dpo/beta": 0.004556549713015556, "fcm_dpo/delta": -0.009785711765289307, "fcm_dpo/margin": 89.5114974975586, "fcm_dpo/q_t": 0.4058662950992584, "grad_norm": 16.259973526000977, "learning_rate": 3.550414669125573e-08, "logits/chosen": 0.8841683864593506, "logits/rejected": 0.8540163040161133, "logps/chosen": -260.458984375, "logps/ref_chosen": -60.64122009277344, "logps/ref_rejected": -78.75474548339844, "logps/rejected": -368.083984375, "loss": 1.0881, "margin_dpo/margin_mean": 89.5114974975586, "margin_dpo/margin_std": 116.04913330078125, "step": 560 }, { "KL/chosen_KL_mean": -187.08404541015625, "KL/mean": -225.16224670410156, "KL/rejected_KL_mean": -263.2404479980469, "KL/std": 119.03514099121094, "epoch": 0.8480725623582767, "fcm_dpo/beta": 0.004581731744110584, "fcm_dpo/delta": 0.05710749328136444, "fcm_dpo/margin": 76.15641784667969, "fcm_dpo/q_t": 0.42259570956230164, "grad_norm": 11.56666374206543, "learning_rate": 3.482795573879241e-08, "logits/chosen": 0.8361924886703491, "logits/rejected": 0.8081060647964478, "logps/chosen": -249.5826416015625, "logps/ref_chosen": -62.49859619140625, "logps/ref_rejected": -78.72064208984375, "logps/rejected": -341.9610900878906, "loss": 1.1554, "margin_dpo/margin_mean": 76.15641784667969, "margin_dpo/margin_std": 131.91342163085938, "step": 561 }, { "KL/chosen_KL_mean": -197.9278564453125, "KL/mean": -250.18325805664062, "KL/rejected_KL_mean": -302.4386901855469, "KL/std": 145.4912109375, "epoch": 0.8495842781557067, "fcm_dpo/beta": 0.004399956203997135, "fcm_dpo/delta": -0.07504507899284363, "fcm_dpo/margin": 104.51081848144531, "fcm_dpo/q_t": 0.3958006501197815, "grad_norm": 14.884467124938965, "learning_rate": 3.415778361095226e-08, "logits/chosen": 0.8424015045166016, "logits/rejected": 0.820836067199707, "logps/chosen": -272.7095947265625, "logps/ref_chosen": -74.78173828125, "logps/ref_rejected": -92.63499450683594, "logps/rejected": -395.07366943359375, "loss": 1.0671, "margin_dpo/margin_mean": 104.51081848144531, "margin_dpo/margin_std": 139.0022430419922, "step": 562 }, { "KL/chosen_KL_mean": -164.1915283203125, "KL/mean": -207.3955535888672, "KL/rejected_KL_mean": -250.5995635986328, "KL/std": 106.54350280761719, "epoch": 0.8510959939531368, "fcm_dpo/beta": 0.004485203884541988, "fcm_dpo/delta": 0.01402625348418951, "fcm_dpo/margin": 86.40805053710938, "fcm_dpo/q_t": 0.41050225496292114, "grad_norm": 18.291183471679688, "learning_rate": 3.349364905389032e-08, "logits/chosen": 0.9199409484863281, "logits/rejected": 0.8749498128890991, "logps/chosen": -214.39004516601562, "logps/ref_chosen": -50.19850158691406, "logps/ref_rejected": -66.76687622070312, "logps/rejected": -317.366455078125, "loss": 1.1242, "margin_dpo/margin_mean": 86.40804290771484, "margin_dpo/margin_std": 135.92982482910156, "step": 563 }, { "KL/chosen_KL_mean": -165.2862091064453, "KL/mean": -219.43699645996094, "KL/rejected_KL_mean": -273.5877685546875, "KL/std": 115.89987182617188, "epoch": 0.8526077097505669, "fcm_dpo/beta": 0.004321890417486429, "fcm_dpo/delta": -0.07944278419017792, "fcm_dpo/margin": 108.30156707763672, "fcm_dpo/q_t": 0.3904789388179779, "grad_norm": 11.61489486694336, "learning_rate": 3.283557064487785e-08, "logits/chosen": 0.8549599051475525, "logits/rejected": 0.8323065042495728, "logps/chosen": -221.02706909179688, "logps/ref_chosen": -55.7408447265625, "logps/ref_rejected": -74.82323455810547, "logps/rejected": -348.4110107421875, "loss": 1.0597, "margin_dpo/margin_mean": 108.30156707763672, "margin_dpo/margin_std": 141.85757446289062, "step": 564 }, { "KL/chosen_KL_mean": -198.53341674804688, "KL/mean": -242.3411865234375, "KL/rejected_KL_mean": -286.14898681640625, "KL/std": 123.26646423339844, "epoch": 0.854119425547997, "fcm_dpo/beta": 0.004352842457592487, "fcm_dpo/delta": 0.02035663276910782, "fcm_dpo/margin": 87.61554718017578, "fcm_dpo/q_t": 0.4115191698074341, "grad_norm": 13.795531272888184, "learning_rate": 3.218356679178252e-08, "logits/chosen": 0.886420488357544, "logits/rejected": 0.8450945615768433, "logps/chosen": -256.87078857421875, "logps/ref_chosen": -58.33738327026367, "logps/ref_rejected": -78.31776428222656, "logps/rejected": -364.46673583984375, "loss": 1.1088, "margin_dpo/margin_mean": 87.61554718017578, "margin_dpo/margin_std": 122.73006439208984, "step": 565 }, { "KL/chosen_KL_mean": -189.62576293945312, "KL/mean": -228.21856689453125, "KL/rejected_KL_mean": -266.81134033203125, "KL/std": 130.59353637695312, "epoch": 0.8556311413454271, "fcm_dpo/beta": 0.004506157711148262, "fcm_dpo/delta": 0.055811285972595215, "fcm_dpo/margin": 77.18556213378906, "fcm_dpo/q_t": 0.4241322875022888, "grad_norm": 12.119365692138672, "learning_rate": 3.1537655732553764e-08, "logits/chosen": 0.8291478157043457, "logits/rejected": 0.8274587392807007, "logps/chosen": -260.8495178222656, "logps/ref_chosen": -71.22373962402344, "logps/ref_rejected": -71.11601257324219, "logps/rejected": -337.9273681640625, "loss": 1.1956, "margin_dpo/margin_mean": 77.18555450439453, "margin_dpo/margin_std": 164.42337036132812, "step": 566 }, { "KL/chosen_KL_mean": -179.55332946777344, "KL/mean": -228.03167724609375, "KL/rejected_KL_mean": -276.5100402832031, "KL/std": 120.30104064941406, "epoch": 0.8571428571428571, "fcm_dpo/beta": 0.004341046325862408, "fcm_dpo/delta": -0.02707264944911003, "fcm_dpo/margin": 96.95671844482422, "fcm_dpo/q_t": 0.4018208384513855, "grad_norm": 10.208983421325684, "learning_rate": 3.089785553471233e-08, "logits/chosen": 0.8846025466918945, "logits/rejected": 0.8115559816360474, "logps/chosen": -232.22259521484375, "logps/ref_chosen": -52.669273376464844, "logps/ref_rejected": -74.34785461425781, "logps/rejected": -350.85791015625, "loss": 1.0904, "margin_dpo/margin_mean": 96.95671081542969, "margin_dpo/margin_std": 131.9867401123047, "step": 567 }, { "KL/chosen_KL_mean": -167.61322021484375, "KL/mean": -225.8235626220703, "KL/rejected_KL_mean": -284.0339050292969, "KL/std": 128.29510498046875, "epoch": 0.8586545729402872, "fcm_dpo/beta": 0.0042817965149879456, "fcm_dpo/delta": -0.1166883111000061, "fcm_dpo/margin": 116.42066955566406, "fcm_dpo/q_t": 0.3852314352989197, "grad_norm": 13.811408042907715, "learning_rate": 3.026418409484513e-08, "logits/chosen": 0.8656144738197327, "logits/rejected": 0.7998012900352478, "logps/chosen": -219.79122924804688, "logps/ref_chosen": -52.178001403808594, "logps/ref_rejected": -85.8277587890625, "logps/rejected": -369.8616638183594, "loss": 1.0303, "margin_dpo/margin_mean": 116.42066955566406, "margin_dpo/margin_std": 132.6590576171875, "step": 568 }, { "KL/chosen_KL_mean": -197.28277587890625, "KL/mean": -225.33621215820312, "KL/rejected_KL_mean": -253.38963317871094, "KL/std": 129.16079711914062, "epoch": 0.8601662887377173, "fcm_dpo/beta": 0.004134716931730509, "fcm_dpo/delta": 0.010899978689849377, "fcm_dpo/margin": 56.106849670410156, "fcm_dpo/q_t": 0.4470366835594177, "grad_norm": 13.65986442565918, "learning_rate": 2.963665913810451e-08, "logits/chosen": 0.7979574203491211, "logits/rejected": 0.7730275392532349, "logps/chosen": -259.9320373535156, "logps/ref_chosen": -62.649261474609375, "logps/ref_rejected": -75.4298324584961, "logps/rejected": -328.8194580078125, "loss": 1.2424, "margin_dpo/margin_mean": 56.106842041015625, "margin_dpo/margin_std": 131.40196228027344, "step": 569 }, { "KL/chosen_KL_mean": -172.97483825683594, "KL/mean": -227.10557556152344, "KL/rejected_KL_mean": -281.23626708984375, "KL/std": 119.91664123535156, "epoch": 0.8616780045351474, "fcm_dpo/beta": 0.004098374396562576, "fcm_dpo/delta": -0.05100153386592865, "fcm_dpo/margin": 108.261474609375, "fcm_dpo/q_t": 0.39809536933898926, "grad_norm": 11.868734359741211, "learning_rate": 2.9015298217712453e-08, "logits/chosen": 0.8359570503234863, "logits/rejected": 0.7631780505180359, "logps/chosen": -223.01663208007812, "logps/ref_chosen": -50.04179382324219, "logps/ref_rejected": -78.27146911621094, "logps/rejected": -359.50775146484375, "loss": 1.0673, "margin_dpo/margin_mean": 108.261474609375, "margin_dpo/margin_std": 139.59825134277344, "step": 570 }, { "KL/chosen_KL_mean": -183.54083251953125, "KL/mean": -215.38575744628906, "KL/rejected_KL_mean": -247.2306671142578, "KL/std": 116.32156372070312, "epoch": 0.8631897203325775, "fcm_dpo/beta": 0.004064176231622696, "fcm_dpo/delta": 0.044348590075969696, "fcm_dpo/margin": 63.68983840942383, "fcm_dpo/q_t": 0.44058936834335327, "grad_norm": 11.032413482666016, "learning_rate": 2.840011871446962e-08, "logits/chosen": 0.8487415909767151, "logits/rejected": 0.8350812196731567, "logps/chosen": -237.19764709472656, "logps/ref_chosen": -53.65681457519531, "logps/ref_rejected": -66.13298034667969, "logps/rejected": -313.3636474609375, "loss": 1.2164, "margin_dpo/margin_mean": 63.689842224121094, "margin_dpo/margin_std": 133.0120849609375, "step": 571 }, { "KL/chosen_KL_mean": -191.580078125, "KL/mean": -224.85147094726562, "KL/rejected_KL_mean": -258.12286376953125, "KL/std": 121.44296264648438, "epoch": 0.8647014361300076, "fcm_dpo/beta": 0.004147428087890148, "fcm_dpo/delta": 0.023455314338207245, "fcm_dpo/margin": 66.54278564453125, "fcm_dpo/q_t": 0.4355035126209259, "grad_norm": 13.11365795135498, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 0.8420843482017517, "logits/rejected": 0.8802669048309326, "logps/chosen": -266.39801025390625, "logps/ref_chosen": -74.81792449951172, "logps/ref_rejected": -65.88681030273438, "logps/rejected": -324.0096435546875, "loss": 1.1947, "margin_dpo/margin_mean": 66.54278564453125, "margin_dpo/margin_std": 124.58303833007812, "step": 572 }, { "KL/chosen_KL_mean": -204.505126953125, "KL/mean": -246.52902221679688, "KL/rejected_KL_mean": -288.55291748046875, "KL/std": 137.63925170898438, "epoch": 0.8662131519274376, "fcm_dpo/beta": 0.004270592704415321, "fcm_dpo/delta": 0.04590342938899994, "fcm_dpo/margin": 84.04781341552734, "fcm_dpo/q_t": 0.4227134585380554, "grad_norm": 12.49295425415039, "learning_rate": 2.718837261761528e-08, "logits/chosen": 0.8661466836929321, "logits/rejected": 0.8327881097793579, "logps/chosen": -273.23077392578125, "logps/ref_chosen": -68.72564697265625, "logps/ref_rejected": -88.16201782226562, "logps/rejected": -376.7149658203125, "loss": 1.1742, "margin_dpo/margin_mean": 84.04782104492188, "margin_dpo/margin_std": 166.25271606445312, "step": 573 }, { "KL/chosen_KL_mean": -182.09056091308594, "KL/mean": -235.65748596191406, "KL/rejected_KL_mean": -289.22442626953125, "KL/std": 127.6211166381836, "epoch": 0.8677248677248677, "fcm_dpo/beta": 0.004246646538376808, "fcm_dpo/delta": -0.06463881582021713, "fcm_dpo/margin": 107.13386535644531, "fcm_dpo/q_t": 0.3947995901107788, "grad_norm": 10.736037254333496, "learning_rate": 2.659183991914696e-08, "logits/chosen": 0.8799293041229248, "logits/rejected": 0.8327153921127319, "logps/chosen": -238.40396118164062, "logps/ref_chosen": -56.31340026855469, "logps/ref_rejected": -83.91553497314453, "logps/rejected": -373.13995361328125, "loss": 1.0468, "margin_dpo/margin_mean": 107.13386535644531, "margin_dpo/margin_std": 122.67759704589844, "step": 574 }, { "KL/chosen_KL_mean": -188.4454345703125, "KL/mean": -226.3487091064453, "KL/rejected_KL_mean": -264.2519836425781, "KL/std": 127.46905517578125, "epoch": 0.8692365835222978, "fcm_dpo/beta": 0.004053239710628986, "fcm_dpo/delta": -0.031237700954079628, "fcm_dpo/margin": 75.80654907226562, "fcm_dpo/q_t": 0.43161332607269287, "grad_norm": 11.321064949035645, "learning_rate": 2.600155642716606e-08, "logits/chosen": 0.8765904903411865, "logits/rejected": 0.8120791912078857, "logps/chosen": -253.02957153320312, "logps/ref_chosen": -64.5841293334961, "logps/ref_rejected": -93.47034454345703, "logps/rejected": -357.72235107421875, "loss": 1.1997, "margin_dpo/margin_mean": 75.80654907226562, "margin_dpo/margin_std": 154.2758331298828, "step": 575 }, { "KL/chosen_KL_mean": -165.71563720703125, "KL/mean": -223.3694610595703, "KL/rejected_KL_mean": -281.02325439453125, "KL/std": 128.47235107421875, "epoch": 0.8707482993197279, "fcm_dpo/beta": 0.0038730669766664505, "fcm_dpo/delta": -0.05786684527993202, "fcm_dpo/margin": 115.30767822265625, "fcm_dpo/q_t": 0.3955625295639038, "grad_norm": 10.995619773864746, "learning_rate": 2.5417538653170754e-08, "logits/chosen": 0.8378143310546875, "logits/rejected": 0.7612361907958984, "logps/chosen": -218.99615478515625, "logps/ref_chosen": -53.28052520751953, "logps/ref_rejected": -84.2000503540039, "logps/rejected": -365.22332763671875, "loss": 1.0621, "margin_dpo/margin_mean": 115.30767822265625, "margin_dpo/margin_std": 140.68199157714844, "step": 576 }, { "KL/chosen_KL_mean": -185.73255920410156, "KL/mean": -220.54190063476562, "KL/rejected_KL_mean": -255.35122680664062, "KL/std": 123.33712768554688, "epoch": 0.872260015117158, "fcm_dpo/beta": 0.004132484085857868, "fcm_dpo/delta": 0.12298332154750824, "fcm_dpo/margin": 69.61865234375, "fcm_dpo/q_t": 0.43443581461906433, "grad_norm": 12.665448188781738, "learning_rate": 2.4839802933393607e-08, "logits/chosen": 0.8501470685005188, "logits/rejected": 0.8491848707199097, "logps/chosen": -248.0572509765625, "logps/ref_chosen": -62.32468795776367, "logps/ref_rejected": -67.300537109375, "logps/rejected": -322.6517333984375, "loss": 1.1952, "margin_dpo/margin_mean": 69.61865234375, "margin_dpo/margin_std": 131.3208770751953, "step": 577 }, { "KL/chosen_KL_mean": -177.81094360351562, "KL/mean": -212.05685424804688, "KL/rejected_KL_mean": -246.3027801513672, "KL/std": 130.85694885253906, "epoch": 0.873771730914588, "fcm_dpo/beta": 0.004423568025231361, "fcm_dpo/delta": 0.10660433769226074, "fcm_dpo/margin": 68.4918441772461, "fcm_dpo/q_t": 0.43448588252067566, "grad_norm": 13.477178573608398, "learning_rate": 2.4268365428344733e-08, "logits/chosen": 0.8857970237731934, "logits/rejected": 0.8791966438293457, "logps/chosen": -234.46652221679688, "logps/ref_chosen": -56.65557861328125, "logps/ref_rejected": -68.21835327148438, "logps/rejected": -314.5211181640625, "loss": 1.21, "margin_dpo/margin_mean": 68.49185180664062, "margin_dpo/margin_std": 150.7880096435547, "step": 578 }, { "KL/chosen_KL_mean": -182.95477294921875, "KL/mean": -233.42739868164062, "KL/rejected_KL_mean": -283.9000244140625, "KL/std": 118.61689758300781, "epoch": 0.8752834467120182, "fcm_dpo/beta": 0.004369863774627447, "fcm_dpo/delta": -0.04803343117237091, "fcm_dpo/margin": 100.94525146484375, "fcm_dpo/q_t": 0.3958134353160858, "grad_norm": 10.97381591796875, "learning_rate": 2.3703242122359357e-08, "logits/chosen": 0.805631697177887, "logits/rejected": 0.7936950922012329, "logps/chosen": -239.76443481445312, "logps/ref_chosen": -56.809661865234375, "logps/ref_rejected": -68.09613037109375, "logps/rejected": -351.99615478515625, "loss": 1.0605, "margin_dpo/margin_mean": 100.94525146484375, "margin_dpo/margin_std": 122.35321807861328, "step": 579 }, { "KL/chosen_KL_mean": -186.7950439453125, "KL/mean": -232.87738037109375, "KL/rejected_KL_mean": -278.959716796875, "KL/std": 143.08419799804688, "epoch": 0.8767951625094482, "fcm_dpo/beta": 0.004418280906975269, "fcm_dpo/delta": -0.009930390864610672, "fcm_dpo/margin": 92.1646728515625, "fcm_dpo/q_t": 0.41188204288482666, "grad_norm": 13.299028396606445, "learning_rate": 2.3144448823151392e-08, "logits/chosen": 0.8283756375312805, "logits/rejected": 0.7941699624061584, "logps/chosen": -244.4951629638672, "logps/ref_chosen": -57.70011520385742, "logps/ref_rejected": -77.90664672851562, "logps/rejected": -356.8663635253906, "loss": 1.1349, "margin_dpo/margin_mean": 92.16466522216797, "margin_dpo/margin_std": 160.25291442871094, "step": 580 }, { "KL/chosen_KL_mean": -203.46536254882812, "KL/mean": -243.61395263671875, "KL/rejected_KL_mean": -283.7625732421875, "KL/std": 123.7155532836914, "epoch": 0.8783068783068783, "fcm_dpo/beta": 0.004408898763358593, "fcm_dpo/delta": 0.0514557883143425, "fcm_dpo/margin": 80.29718017578125, "fcm_dpo/q_t": 0.42192864418029785, "grad_norm": 11.928445816040039, "learning_rate": 2.259200116137039e-08, "logits/chosen": 0.8557339906692505, "logits/rejected": 0.8073595762252808, "logps/chosen": -262.7977294921875, "logps/ref_chosen": -59.332359313964844, "logps/ref_rejected": -83.64482116699219, "logps/rejected": -367.4073791503906, "loss": 1.1666, "margin_dpo/margin_mean": 80.29718017578125, "margin_dpo/margin_std": 150.23489379882812, "step": 581 }, { "KL/chosen_KL_mean": -186.45236206054688, "KL/mean": -223.76060485839844, "KL/rejected_KL_mean": -261.06884765625, "KL/std": 116.26499938964844, "epoch": 0.8798185941043084, "fcm_dpo/beta": 0.004633273929357529, "fcm_dpo/delta": 0.05958259850740433, "fcm_dpo/margin": 74.61648559570312, "fcm_dpo/q_t": 0.423893004655838, "grad_norm": 11.573064804077148, "learning_rate": 2.204591459016525e-08, "logits/chosen": 0.8565012812614441, "logits/rejected": 0.8867921829223633, "logps/chosen": -250.61521911621094, "logps/ref_chosen": -64.16285705566406, "logps/ref_rejected": -58.632896423339844, "logps/rejected": -319.70172119140625, "loss": 1.1645, "margin_dpo/margin_mean": 74.61648559570312, "margin_dpo/margin_std": 135.59817504882812, "step": 582 }, { "KL/chosen_KL_mean": -183.05465698242188, "KL/mean": -235.2905731201172, "KL/rejected_KL_mean": -287.5264892578125, "KL/std": 138.89993286132812, "epoch": 0.8813303099017384, "fcm_dpo/beta": 0.004453700035810471, "fcm_dpo/delta": -0.07676573097705841, "fcm_dpo/margin": 104.47183990478516, "fcm_dpo/q_t": 0.39537179470062256, "grad_norm": 13.700339317321777, "learning_rate": 2.1506204384751064e-08, "logits/chosen": 0.9295327067375183, "logits/rejected": 0.8535386323928833, "logps/chosen": -234.9270477294922, "logps/ref_chosen": -51.87239456176758, "logps/ref_rejected": -83.86331176757812, "logps/rejected": -371.3898010253906, "loss": 1.0891, "margin_dpo/margin_mean": 104.47184753417969, "margin_dpo/margin_std": 159.20358276367188, "step": 583 }, { "KL/chosen_KL_mean": -168.95220947265625, "KL/mean": -212.09420776367188, "KL/rejected_KL_mean": -255.2362060546875, "KL/std": 120.96197509765625, "epoch": 0.8828420256991686, "fcm_dpo/beta": 0.00443212129175663, "fcm_dpo/delta": 0.019545778632164, "fcm_dpo/margin": 86.28401184082031, "fcm_dpo/q_t": 0.4162449240684509, "grad_norm": 13.947019577026367, "learning_rate": 2.09728856419826e-08, "logits/chosen": 0.9570223093032837, "logits/rejected": 0.8717998266220093, "logps/chosen": -215.52359008789062, "logps/ref_chosen": -46.571388244628906, "logps/ref_rejected": -80.67969512939453, "logps/rejected": -335.9158935546875, "loss": 1.1478, "margin_dpo/margin_mean": 86.28401184082031, "margin_dpo/margin_std": 154.09945678710938, "step": 584 }, { "KL/chosen_KL_mean": -197.41429138183594, "KL/mean": -228.14865112304688, "KL/rejected_KL_mean": -258.88299560546875, "KL/std": 134.65402221679688, "epoch": 0.8843537414965986, "fcm_dpo/beta": 0.004502854309976101, "fcm_dpo/delta": 0.015462912619113922, "fcm_dpo/margin": 61.46870422363281, "fcm_dpo/q_t": 0.4372748136520386, "grad_norm": 13.030404090881348, "learning_rate": 2.044597327993153e-08, "logits/chosen": 0.8678371906280518, "logits/rejected": 0.8205997943878174, "logps/chosen": -255.538818359375, "logps/ref_chosen": -58.124534606933594, "logps/ref_rejected": -79.00538635253906, "logps/rejected": -337.88836669921875, "loss": 1.212, "margin_dpo/margin_mean": 61.46870422363281, "margin_dpo/margin_std": 131.03814697265625, "step": 585 }, { "KL/chosen_KL_mean": -179.77459716796875, "KL/mean": -218.6102294921875, "KL/rejected_KL_mean": -257.44586181640625, "KL/std": 107.11343383789062, "epoch": 0.8858654572940288, "fcm_dpo/beta": 0.004600149113684893, "fcm_dpo/delta": 0.04789983481168747, "fcm_dpo/margin": 77.6712646484375, "fcm_dpo/q_t": 0.4177996516227722, "grad_norm": 11.622013092041016, "learning_rate": 1.9925482037469187e-08, "logits/chosen": 0.8768935203552246, "logits/rejected": 0.8456597328186035, "logps/chosen": -233.876220703125, "logps/ref_chosen": -54.10163879394531, "logps/ref_rejected": -63.72113037109375, "logps/rejected": -321.1669921875, "loss": 1.1115, "margin_dpo/margin_mean": 77.6712646484375, "margin_dpo/margin_std": 100.46601104736328, "step": 586 }, { "KL/chosen_KL_mean": -193.71388244628906, "KL/mean": -231.7980194091797, "KL/rejected_KL_mean": -269.88214111328125, "KL/std": 131.6758270263672, "epoch": 0.8873771730914588, "fcm_dpo/beta": 0.004727904684841633, "fcm_dpo/delta": 0.04477377235889435, "fcm_dpo/margin": 76.16827392578125, "fcm_dpo/q_t": 0.41351163387298584, "grad_norm": 16.236108779907227, "learning_rate": 1.9411426473854687e-08, "logits/chosen": 0.9039092063903809, "logits/rejected": 0.9038503766059875, "logps/chosen": -257.1310729980469, "logps/ref_chosen": -63.41719436645508, "logps/ref_rejected": -63.47003936767578, "logps/rejected": -333.3522033691406, "loss": 1.2072, "margin_dpo/margin_mean": 76.16828155517578, "margin_dpo/margin_std": 177.19003295898438, "step": 587 }, { "KL/chosen_KL_mean": -197.713134765625, "KL/mean": -240.94564819335938, "KL/rejected_KL_mean": -284.1781921386719, "KL/std": 131.87918090820312, "epoch": 0.8888888888888888, "fcm_dpo/beta": 0.004706019535660744, "fcm_dpo/delta": -0.009034395217895508, "fcm_dpo/margin": 86.46505737304688, "fcm_dpo/q_t": 0.410349041223526, "grad_norm": 16.405550003051758, "learning_rate": 1.890382096832699e-08, "logits/chosen": 0.9087653756141663, "logits/rejected": 0.8741371631622314, "logps/chosen": -259.9141845703125, "logps/ref_chosen": -62.20103454589844, "logps/ref_rejected": -82.10249328613281, "logps/rejected": -366.28070068359375, "loss": 1.1565, "margin_dpo/margin_mean": 86.46505737304688, "margin_dpo/margin_std": 164.84034729003906, "step": 588 }, { "KL/chosen_KL_mean": -183.16661071777344, "KL/mean": -228.384765625, "KL/rejected_KL_mean": -273.6029357910156, "KL/std": 123.86094665527344, "epoch": 0.890400604686319, "fcm_dpo/beta": 0.0047372374683618546, "fcm_dpo/delta": -0.033031828701496124, "fcm_dpo/margin": 90.43630981445312, "fcm_dpo/q_t": 0.4000184237957001, "grad_norm": 11.501230239868164, "learning_rate": 1.840267971970344e-08, "logits/chosen": 0.8425379991531372, "logits/rejected": 0.8174772262573242, "logps/chosen": -239.88021850585938, "logps/ref_chosen": -56.71361541748047, "logps/ref_rejected": -76.7366943359375, "logps/rejected": -350.339599609375, "loss": 1.0785, "margin_dpo/margin_mean": 90.43630981445312, "margin_dpo/margin_std": 118.53308868408203, "step": 589 }, { "KL/chosen_KL_mean": -199.31167602539062, "KL/mean": -239.75074768066406, "KL/rejected_KL_mean": -280.1898193359375, "KL/std": 112.27293395996094, "epoch": 0.891912320483749, "fcm_dpo/beta": 0.00475288974121213, "fcm_dpo/delta": 0.016453173011541367, "fcm_dpo/margin": 80.87815856933594, "fcm_dpo/q_t": 0.41211652755737305, "grad_norm": 16.781558990478516, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 0.8056505918502808, "logits/rejected": 0.7747730016708374, "logps/chosen": -265.82550048828125, "logps/ref_chosen": -66.5138168334961, "logps/ref_rejected": -85.70820617675781, "logps/rejected": -365.89801025390625, "loss": 1.1167, "margin_dpo/margin_mean": 80.87814331054688, "margin_dpo/margin_std": 119.72245788574219, "step": 590 }, { "KL/chosen_KL_mean": -168.68548583984375, "KL/mean": -223.0603485107422, "KL/rejected_KL_mean": -277.43524169921875, "KL/std": 136.9473876953125, "epoch": 0.8934240362811792, "fcm_dpo/beta": 0.004390774294734001, "fcm_dpo/delta": -0.09836189448833466, "fcm_dpo/margin": 108.74974060058594, "fcm_dpo/q_t": 0.3937166929244995, "grad_norm": 13.011321067810059, "learning_rate": 1.7419845883949098e-08, "logits/chosen": 0.9268543124198914, "logits/rejected": 0.8832913637161255, "logps/chosen": -229.38265991210938, "logps/ref_chosen": -60.697181701660156, "logps/ref_rejected": -86.12278747558594, "logps/rejected": -363.5580139160156, "loss": 1.0917, "margin_dpo/margin_mean": 108.7497329711914, "margin_dpo/margin_std": 168.9647216796875, "step": 591 }, { "KL/chosen_KL_mean": -185.19931030273438, "KL/mean": -225.90817260742188, "KL/rejected_KL_mean": -266.6170349121094, "KL/std": 114.66996765136719, "epoch": 0.8949357520786092, "fcm_dpo/beta": 0.004453813657164574, "fcm_dpo/delta": 0.04094696044921875, "fcm_dpo/margin": 81.41773986816406, "fcm_dpo/q_t": 0.4200698137283325, "grad_norm": 14.020329475402832, "learning_rate": 1.6938180788793556e-08, "logits/chosen": 0.9028749465942383, "logits/rejected": 0.8090362548828125, "logps/chosen": -236.4366455078125, "logps/ref_chosen": -51.237327575683594, "logps/ref_rejected": -81.60242462158203, "logps/rejected": -348.219482421875, "loss": 1.137, "margin_dpo/margin_mean": 81.41773986816406, "margin_dpo/margin_std": 130.4501190185547, "step": 592 }, { "KL/chosen_KL_mean": -151.83436584472656, "KL/mean": -198.61888122558594, "KL/rejected_KL_mean": -245.40338134765625, "KL/std": 116.2724380493164, "epoch": 0.8964474678760394, "fcm_dpo/beta": 0.004477858543395996, "fcm_dpo/delta": -0.022135818377137184, "fcm_dpo/margin": 93.56902313232422, "fcm_dpo/q_t": 0.40188807249069214, "grad_norm": 15.795121192932129, "learning_rate": 1.6463034933723336e-08, "logits/chosen": 0.8935588598251343, "logits/rejected": 0.8158241510391235, "logps/chosen": -193.91436767578125, "logps/ref_chosen": -42.08000183105469, "logps/ref_rejected": -68.47499084472656, "logps/rejected": -313.87835693359375, "loss": 1.0948, "margin_dpo/margin_mean": 93.56902313232422, "margin_dpo/margin_std": 133.15293884277344, "step": 593 }, { "KL/chosen_KL_mean": -195.93930053710938, "KL/mean": -232.91555786132812, "KL/rejected_KL_mean": -269.891845703125, "KL/std": 117.66835021972656, "epoch": 0.8979591836734694, "fcm_dpo/beta": 0.004622071050107479, "fcm_dpo/delta": 0.06481346487998962, "fcm_dpo/margin": 73.95257568359375, "fcm_dpo/q_t": 0.4218658208847046, "grad_norm": 12.912012100219727, "learning_rate": 1.5994421609589385e-08, "logits/chosen": 0.7965784668922424, "logits/rejected": 0.7883630990982056, "logps/chosen": -259.59796142578125, "logps/ref_chosen": -63.658668518066406, "logps/ref_rejected": -70.35597229003906, "logps/rejected": -340.2478332519531, "loss": 1.1477, "margin_dpo/margin_mean": 73.95257568359375, "margin_dpo/margin_std": 120.45957946777344, "step": 594 }, { "KL/chosen_KL_mean": -178.886962890625, "KL/mean": -230.2701416015625, "KL/rejected_KL_mean": -281.6533203125, "KL/std": 126.58238220214844, "epoch": 0.8994708994708994, "fcm_dpo/beta": 0.004551096819341183, "fcm_dpo/delta": -0.07861794531345367, "fcm_dpo/margin": 102.76634216308594, "fcm_dpo/q_t": 0.39468061923980713, "grad_norm": 10.457369804382324, "learning_rate": 1.553235392451377e-08, "logits/chosen": 0.8931225538253784, "logits/rejected": 0.8216215372085571, "logps/chosen": -235.105712890625, "logps/ref_chosen": -56.21875762939453, "logps/ref_rejected": -83.95773315429688, "logps/rejected": -365.6110534667969, "loss": 1.0852, "margin_dpo/margin_mean": 102.76634979248047, "margin_dpo/margin_std": 154.8719940185547, "step": 595 }, { "KL/chosen_KL_mean": -209.03253173828125, "KL/mean": -227.578369140625, "KL/rejected_KL_mean": -246.12420654296875, "KL/std": 113.8258056640625, "epoch": 0.9009826152683296, "fcm_dpo/beta": 0.004681180696934462, "fcm_dpo/delta": 0.08331310749053955, "fcm_dpo/margin": 37.091651916503906, "fcm_dpo/q_t": 0.4618571698665619, "grad_norm": 15.247252464294434, "learning_rate": 1.507684480352292e-08, "logits/chosen": 0.7848924398422241, "logits/rejected": 0.8063439726829529, "logps/chosen": -277.513427734375, "logps/ref_chosen": -68.48088073730469, "logps/ref_rejected": -61.732967376708984, "logps/rejected": -307.857177734375, "loss": 1.2958, "margin_dpo/margin_mean": 37.09164810180664, "margin_dpo/margin_std": 118.8961181640625, "step": 596 }, { "KL/chosen_KL_mean": -160.05877685546875, "KL/mean": -198.61622619628906, "KL/rejected_KL_mean": -237.17367553710938, "KL/std": 100.68923950195312, "epoch": 0.9024943310657596, "fcm_dpo/beta": 0.004797500092536211, "fcm_dpo/delta": 0.032740212976932526, "fcm_dpo/margin": 77.11491394042969, "fcm_dpo/q_t": 0.41552621126174927, "grad_norm": 11.356983184814453, "learning_rate": 1.4627906988186111e-08, "logits/chosen": 0.8747761845588684, "logits/rejected": 0.8632344007492065, "logps/chosen": -208.91627502441406, "logps/ref_chosen": -48.85750961303711, "logps/ref_rejected": -55.068084716796875, "logps/rejected": -292.24176025390625, "loss": 1.1215, "margin_dpo/margin_mean": 77.11491394042969, "margin_dpo/margin_std": 113.88622283935547, "step": 597 }, { "KL/chosen_KL_mean": -207.90806579589844, "KL/mean": -234.0879364013672, "KL/rejected_KL_mean": -260.267822265625, "KL/std": 122.74574279785156, "epoch": 0.9040060468631897, "fcm_dpo/beta": 0.004856814630329609, "fcm_dpo/delta": 0.056810446083545685, "fcm_dpo/margin": 52.3597412109375, "fcm_dpo/q_t": 0.44433027505874634, "grad_norm": 12.908487319946289, "learning_rate": 1.4185553036259095e-08, "logits/chosen": 0.896747350692749, "logits/rejected": 0.8310115337371826, "logps/chosen": -266.79522705078125, "logps/ref_chosen": -58.88715362548828, "logps/ref_rejected": -81.43145751953125, "logps/rejected": -341.69927978515625, "loss": 1.2418, "margin_dpo/margin_mean": 52.35974884033203, "margin_dpo/margin_std": 130.03427124023438, "step": 598 }, { "KL/chosen_KL_mean": -208.6796875, "KL/mean": -237.74996948242188, "KL/rejected_KL_mean": -266.8202819824219, "KL/std": 116.68116760253906, "epoch": 0.9055177626606198, "fcm_dpo/beta": 0.004964320454746485, "fcm_dpo/delta": 0.016610831022262573, "fcm_dpo/margin": 58.14057922363281, "fcm_dpo/q_t": 0.43591034412384033, "grad_norm": 16.04824447631836, "learning_rate": 1.3749795321332885e-08, "logits/chosen": 0.910493016242981, "logits/rejected": 0.8786226511001587, "logps/chosen": -266.286865234375, "logps/ref_chosen": -57.60719299316406, "logps/ref_rejected": -71.80469512939453, "logps/rejected": -338.6249694824219, "loss": 1.2245, "margin_dpo/margin_mean": 58.14057922363281, "margin_dpo/margin_std": 135.10581970214844, "step": 599 }, { "KL/chosen_KL_mean": -190.2998046875, "KL/mean": -226.334228515625, "KL/rejected_KL_mean": -262.36865234375, "KL/std": 129.5638885498047, "epoch": 0.9070294784580499, "fcm_dpo/beta": 0.004913763143122196, "fcm_dpo/delta": -0.05108189955353737, "fcm_dpo/margin": 72.0688247680664, "fcm_dpo/q_t": 0.4227983355522156, "grad_norm": 16.685800552368164, "learning_rate": 1.3320646032487393e-08, "logits/chosen": 0.8785399198532104, "logits/rejected": 0.8318706750869751, "logps/chosen": -248.7421112060547, "logps/ref_chosen": -58.44231414794922, "logps/ref_rejected": -83.64639282226562, "logps/rejected": -346.0150146484375, "loss": 1.1762, "margin_dpo/margin_mean": 72.0688247680664, "margin_dpo/margin_std": 140.14089965820312, "step": 600 }, { "KL/chosen_KL_mean": -171.17523193359375, "KL/mean": -220.55459594726562, "KL/rejected_KL_mean": -269.9339599609375, "KL/std": 136.19314575195312, "epoch": 0.90854119425548, "fcm_dpo/beta": 0.0046309540048241615, "fcm_dpo/delta": -0.0687466561794281, "fcm_dpo/margin": 98.75874328613281, "fcm_dpo/q_t": 0.39738285541534424, "grad_norm": 11.404427528381348, "learning_rate": 1.2898117173950868e-08, "logits/chosen": 0.8668086528778076, "logits/rejected": 0.8014013171195984, "logps/chosen": -226.76954650878906, "logps/ref_chosen": -55.59432601928711, "logps/ref_rejected": -83.68630981445312, "logps/rejected": -353.62030029296875, "loss": 1.0981, "margin_dpo/margin_mean": 98.75874328613281, "margin_dpo/margin_std": 154.93240356445312, "step": 601 }, { "KL/chosen_KL_mean": -160.80662536621094, "KL/mean": -204.2389373779297, "KL/rejected_KL_mean": -247.67123413085938, "KL/std": 116.13655853271484, "epoch": 0.91005291005291, "fcm_dpo/beta": 0.004675906151533127, "fcm_dpo/delta": -0.007718130946159363, "fcm_dpo/margin": 86.8646011352539, "fcm_dpo/q_t": 0.4061970114707947, "grad_norm": 13.002616882324219, "learning_rate": 1.2482220564763667e-08, "logits/chosen": 0.8623030185699463, "logits/rejected": 0.8387429118156433, "logps/chosen": -217.15582275390625, "logps/ref_chosen": -56.349185943603516, "logps/ref_rejected": -71.9959716796875, "logps/rejected": -319.6672058105469, "loss": 1.0989, "margin_dpo/margin_mean": 86.8646011352539, "margin_dpo/margin_std": 123.03114318847656, "step": 602 }, { "KL/chosen_KL_mean": -177.90234375, "KL/mean": -222.01742553710938, "KL/rejected_KL_mean": -266.13250732421875, "KL/std": 118.25772094726562, "epoch": 0.9115646258503401, "fcm_dpo/beta": 0.0045766388066112995, "fcm_dpo/delta": -0.004777451977133751, "fcm_dpo/margin": 88.23017883300781, "fcm_dpo/q_t": 0.40897947549819946, "grad_norm": 11.585609436035156, "learning_rate": 1.2072967838448051e-08, "logits/chosen": 0.8506954908370972, "logits/rejected": 0.8117414712905884, "logps/chosen": -231.0707244873047, "logps/ref_chosen": -53.16838836669922, "logps/ref_rejected": -73.8604736328125, "logps/rejected": -339.99298095703125, "loss": 1.1146, "margin_dpo/margin_mean": 88.23017883300781, "margin_dpo/margin_std": 137.55128479003906, "step": 603 }, { "KL/chosen_KL_mean": -174.488037109375, "KL/mean": -210.52664184570312, "KL/rejected_KL_mean": -246.56524658203125, "KL/std": 110.95604705810547, "epoch": 0.9130763416477702, "fcm_dpo/beta": 0.004770040512084961, "fcm_dpo/delta": 0.06184397637844086, "fcm_dpo/margin": 72.07720947265625, "fcm_dpo/q_t": 0.42401158809661865, "grad_norm": 13.443038940429688, "learning_rate": 1.1670370442682459e-08, "logits/chosen": 0.8126691579818726, "logits/rejected": 0.8226346969604492, "logps/chosen": -247.13746643066406, "logps/ref_chosen": -72.64942169189453, "logps/ref_rejected": -69.8792724609375, "logps/rejected": -316.44451904296875, "loss": 1.1842, "margin_dpo/margin_mean": 72.07720947265625, "margin_dpo/margin_std": 146.4234619140625, "step": 604 }, { "KL/chosen_KL_mean": -195.28396606445312, "KL/mean": -235.2552490234375, "KL/rejected_KL_mean": -275.22650146484375, "KL/std": 115.78852844238281, "epoch": 0.9145880574452003, "fcm_dpo/beta": 0.00478768814355135, "fcm_dpo/delta": 0.01924796961247921, "fcm_dpo/margin": 79.94253540039062, "fcm_dpo/q_t": 0.4138326644897461, "grad_norm": 14.64280891418457, "learning_rate": 1.1274439638981532e-08, "logits/chosen": 0.9054049253463745, "logits/rejected": 0.8625191450119019, "logps/chosen": -256.8968200683594, "logps/ref_chosen": -61.61284637451172, "logps/ref_rejected": -79.34398651123047, "logps/rejected": -354.57049560546875, "loss": 1.142, "margin_dpo/margin_mean": 79.94253540039062, "margin_dpo/margin_std": 137.07847595214844, "step": 605 }, { "KL/chosen_KL_mean": -170.9767608642578, "KL/mean": -221.29107666015625, "KL/rejected_KL_mean": -271.6053466796875, "KL/std": 122.00587463378906, "epoch": 0.9160997732426304, "fcm_dpo/beta": 0.004710886627435684, "fcm_dpo/delta": -0.08624030649662018, "fcm_dpo/margin": 100.62860870361328, "fcm_dpo/q_t": 0.3922450542449951, "grad_norm": 14.318273544311523, "learning_rate": 1.0885186502381016e-08, "logits/chosen": 0.8514528274536133, "logits/rejected": 0.7999155521392822, "logps/chosen": -225.4409942626953, "logps/ref_chosen": -54.46424102783203, "logps/ref_rejected": -79.62708282470703, "logps/rejected": -351.2324523925781, "loss": 1.0643, "margin_dpo/margin_mean": 100.62861633300781, "margin_dpo/margin_std": 136.915283203125, "step": 606 }, { "KL/chosen_KL_mean": -189.23269653320312, "KL/mean": -231.669921875, "KL/rejected_KL_mean": -274.10711669921875, "KL/std": 117.58612060546875, "epoch": 0.9176114890400605, "fcm_dpo/beta": 0.004336424637585878, "fcm_dpo/delta": -0.09618955850601196, "fcm_dpo/margin": 84.87444305419922, "fcm_dpo/q_t": 0.4125783443450928, "grad_norm": 11.543998718261719, "learning_rate": 1.0502621921127774e-08, "logits/chosen": 0.8325244188308716, "logits/rejected": 0.8169835805892944, "logps/chosen": -252.09356689453125, "logps/ref_chosen": -62.86086654663086, "logps/ref_rejected": -72.5501937866211, "logps/rejected": -346.6573486328125, "loss": 1.1358, "margin_dpo/margin_mean": 84.87443542480469, "margin_dpo/margin_std": 131.11700439453125, "step": 607 }, { "KL/chosen_KL_mean": -191.9585723876953, "KL/mean": -239.46746826171875, "KL/rejected_KL_mean": -286.976318359375, "KL/std": 132.16444396972656, "epoch": 0.9191232048374905, "fcm_dpo/beta": 0.00433045718818903, "fcm_dpo/delta": -0.013165525160729885, "fcm_dpo/margin": 95.01773834228516, "fcm_dpo/q_t": 0.4065626263618469, "grad_norm": 12.041932106018066, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 0.8287484645843506, "logits/rejected": 0.7658513784408569, "logps/chosen": -255.13929748535156, "logps/ref_chosen": -63.18071746826172, "logps/ref_rejected": -99.15888214111328, "logps/rejected": -386.13519287109375, "loss": 1.0962, "margin_dpo/margin_mean": 95.01774597167969, "margin_dpo/margin_std": 135.38153076171875, "step": 608 }, { "KL/chosen_KL_mean": -171.56765747070312, "KL/mean": -219.2465057373047, "KL/rejected_KL_mean": -266.9253845214844, "KL/std": 113.94578552246094, "epoch": 0.9206349206349206, "fcm_dpo/beta": 0.004231642931699753, "fcm_dpo/delta": -0.005523219704627991, "fcm_dpo/margin": 95.35771942138672, "fcm_dpo/q_t": 0.4046892821788788, "grad_norm": 11.678070068359375, "learning_rate": 9.757601041885694e-09, "logits/chosen": 0.9319273829460144, "logits/rejected": 0.9010778665542603, "logps/chosen": -220.1908721923828, "logps/ref_chosen": -48.62322235107422, "logps/ref_rejected": -68.28271484375, "logps/rejected": -335.2080993652344, "loss": 1.0812, "margin_dpo/margin_mean": 95.35771942138672, "margin_dpo/margin_std": 111.58070373535156, "step": 609 }, { "KL/chosen_KL_mean": -191.14016723632812, "KL/mean": -233.82876586914062, "KL/rejected_KL_mean": -276.51739501953125, "KL/std": 129.33859252929688, "epoch": 0.9221466364323507, "fcm_dpo/beta": 0.004307628609240055, "fcm_dpo/delta": 0.03571552410721779, "fcm_dpo/margin": 85.37721252441406, "fcm_dpo/q_t": 0.41646891832351685, "grad_norm": 13.54400634765625, "learning_rate": 9.395165583732379e-09, "logits/chosen": 0.86832594871521, "logits/rejected": 0.8651347160339355, "logps/chosen": -263.8052978515625, "logps/ref_chosen": -72.66513061523438, "logps/ref_rejected": -87.15310668945312, "logps/rejected": -363.67047119140625, "loss": 1.1555, "margin_dpo/margin_mean": 85.37721252441406, "margin_dpo/margin_std": 151.74803161621094, "step": 610 }, { "KL/chosen_KL_mean": -172.24514770507812, "KL/mean": -208.3623504638672, "KL/rejected_KL_mean": -244.47955322265625, "KL/std": 112.20661926269531, "epoch": 0.9236583522297808, "fcm_dpo/beta": 0.004518900997936726, "fcm_dpo/delta": 0.08180320262908936, "fcm_dpo/margin": 72.23439025878906, "fcm_dpo/q_t": 0.4254700541496277, "grad_norm": 12.482162475585938, "learning_rate": 9.03946036001449e-09, "logits/chosen": 0.8952933549880981, "logits/rejected": 0.8567260503768921, "logps/chosen": -220.55374145507812, "logps/ref_chosen": -48.30857849121094, "logps/ref_rejected": -70.6141128540039, "logps/rejected": -315.0936584472656, "loss": 1.149, "margin_dpo/margin_mean": 72.23438262939453, "margin_dpo/margin_std": 112.78076171875, "step": 611 }, { "KL/chosen_KL_mean": -189.6739044189453, "KL/mean": -245.22344970703125, "KL/rejected_KL_mean": -300.77301025390625, "KL/std": 119.71464538574219, "epoch": 0.9251700680272109, "fcm_dpo/beta": 0.004395745228976011, "fcm_dpo/delta": -0.10357952862977982, "fcm_dpo/margin": 111.09910583496094, "fcm_dpo/q_t": 0.3854334354400635, "grad_norm": 11.041399955749512, "learning_rate": 8.690495320571839e-09, "logits/chosen": 0.7935118675231934, "logits/rejected": 0.735627293586731, "logps/chosen": -250.90545654296875, "logps/ref_chosen": -61.23155975341797, "logps/ref_rejected": -94.37979888916016, "logps/rejected": -395.15283203125, "loss": 1.0329, "margin_dpo/margin_mean": 111.09910583496094, "margin_dpo/margin_std": 131.01829528808594, "step": 612 }, { "KL/chosen_KL_mean": -166.52000427246094, "KL/mean": -213.49661254882812, "KL/rejected_KL_mean": -260.47320556640625, "KL/std": 132.3287811279297, "epoch": 0.926681783824641, "fcm_dpo/beta": 0.004340745974332094, "fcm_dpo/delta": -0.009288674220442772, "fcm_dpo/margin": 93.95318603515625, "fcm_dpo/q_t": 0.4047100841999054, "grad_norm": 10.03150749206543, "learning_rate": 8.348280226706722e-09, "logits/chosen": 0.8340524435043335, "logits/rejected": 0.8326983451843262, "logps/chosen": -220.50311279296875, "logps/ref_chosen": -53.98310852050781, "logps/ref_rejected": -58.32208251953125, "logps/rejected": -318.7952880859375, "loss": 1.0976, "margin_dpo/margin_mean": 93.95318603515625, "margin_dpo/margin_std": 131.71676635742188, "step": 613 }, { "KL/chosen_KL_mean": -189.92941284179688, "KL/mean": -233.43902587890625, "KL/rejected_KL_mean": -276.9486389160156, "KL/std": 110.43850708007812, "epoch": 0.9281934996220711, "fcm_dpo/beta": 0.004416479729115963, "fcm_dpo/delta": 0.014550581574440002, "fcm_dpo/margin": 87.01921844482422, "fcm_dpo/q_t": 0.41044163703918457, "grad_norm": 13.81963062286377, "learning_rate": 8.012824650910937e-09, "logits/chosen": 0.8715497851371765, "logits/rejected": 0.8666037321090698, "logps/chosen": -250.17245483398438, "logps/ref_chosen": -60.24303436279297, "logps/ref_rejected": -72.26258850097656, "logps/rejected": -349.21124267578125, "loss": 1.1157, "margin_dpo/margin_mean": 87.01921844482422, "margin_dpo/margin_std": 125.52960205078125, "step": 614 }, { "KL/chosen_KL_mean": -195.23587036132812, "KL/mean": -247.610107421875, "KL/rejected_KL_mean": -299.9843444824219, "KL/std": 148.3843994140625, "epoch": 0.9297052154195011, "fcm_dpo/beta": 0.004148758016526699, "fcm_dpo/delta": -0.04528382048010826, "fcm_dpo/margin": 104.74848937988281, "fcm_dpo/q_t": 0.40149933099746704, "grad_norm": 11.946541786193848, "learning_rate": 7.684137976598088e-09, "logits/chosen": 0.8352917432785034, "logits/rejected": 0.7855337858200073, "logps/chosen": -267.3305358886719, "logps/ref_chosen": -72.09467315673828, "logps/ref_rejected": -104.02980041503906, "logps/rejected": -404.01416015625, "loss": 1.1023, "margin_dpo/margin_mean": 104.74848937988281, "margin_dpo/margin_std": 158.98904418945312, "step": 615 }, { "KL/chosen_KL_mean": -187.78256225585938, "KL/mean": -227.31333923339844, "KL/rejected_KL_mean": -266.8441162109375, "KL/std": 136.93423461914062, "epoch": 0.9312169312169312, "fcm_dpo/beta": 0.004337611608207226, "fcm_dpo/delta": 0.06366471946239471, "fcm_dpo/margin": 79.06153869628906, "fcm_dpo/q_t": 0.4219726324081421, "grad_norm": 11.41348648071289, "learning_rate": 7.36222939784098e-09, "logits/chosen": 0.8961197137832642, "logits/rejected": 0.8402373194694519, "logps/chosen": -246.31329345703125, "logps/ref_chosen": -58.530723571777344, "logps/ref_rejected": -75.48025512695312, "logps/rejected": -342.3243713378906, "loss": 1.1435, "margin_dpo/margin_mean": 79.06153869628906, "margin_dpo/margin_std": 126.18814086914062, "step": 616 }, { "KL/chosen_KL_mean": -195.61953735351562, "KL/mean": -232.73971557617188, "KL/rejected_KL_mean": -269.85992431640625, "KL/std": 123.03448486328125, "epoch": 0.9327286470143613, "fcm_dpo/beta": 0.004304808564484119, "fcm_dpo/delta": -0.031089257448911667, "fcm_dpo/margin": 74.24037170410156, "fcm_dpo/q_t": 0.4274696111679077, "grad_norm": 15.603584289550781, "learning_rate": 7.047107919114586e-09, "logits/chosen": 0.8908584713935852, "logits/rejected": 0.8490594029426575, "logps/chosen": -253.2282257080078, "logps/ref_chosen": -57.608673095703125, "logps/ref_rejected": -81.22109985351562, "logps/rejected": -351.0810241699219, "loss": 1.1746, "margin_dpo/margin_mean": 74.24037170410156, "margin_dpo/margin_std": 133.52430725097656, "step": 617 }, { "KL/chosen_KL_mean": -182.47540283203125, "KL/mean": -227.52099609375, "KL/rejected_KL_mean": -272.5666198730469, "KL/std": 136.0541229248047, "epoch": 0.9342403628117913, "fcm_dpo/beta": 0.004296266473829746, "fcm_dpo/delta": 0.014270158484578133, "fcm_dpo/margin": 90.0912094116211, "fcm_dpo/q_t": 0.4141734838485718, "grad_norm": 15.708011627197266, "learning_rate": 6.738782355044048e-09, "logits/chosen": 0.8595304489135742, "logits/rejected": 0.7856009006500244, "logps/chosen": -239.17135620117188, "logps/ref_chosen": -56.69594192504883, "logps/ref_rejected": -85.92362976074219, "logps/rejected": -358.490234375, "loss": 1.1172, "margin_dpo/margin_mean": 90.09120178222656, "margin_dpo/margin_std": 138.923583984375, "step": 618 }, { "KL/chosen_KL_mean": -174.67626953125, "KL/mean": -222.02593994140625, "KL/rejected_KL_mean": -269.3756408691406, "KL/std": 125.88426971435547, "epoch": 0.9357520786092215, "fcm_dpo/beta": 0.004293524194508791, "fcm_dpo/delta": -0.007992902770638466, "fcm_dpo/margin": 94.6993637084961, "fcm_dpo/q_t": 0.40730637311935425, "grad_norm": 11.522953987121582, "learning_rate": 6.437261330158206e-09, "logits/chosen": 0.9194344282150269, "logits/rejected": 0.8575088977813721, "logps/chosen": -228.73468017578125, "logps/ref_chosen": -54.05841827392578, "logps/ref_rejected": -83.55493927001953, "logps/rejected": -352.9305725097656, "loss": 1.103, "margin_dpo/margin_mean": 94.6993637084961, "margin_dpo/margin_std": 138.9624481201172, "step": 619 }, { "KL/chosen_KL_mean": -182.78765869140625, "KL/mean": -217.02841186523438, "KL/rejected_KL_mean": -251.26914978027344, "KL/std": 116.54869842529297, "epoch": 0.9372637944066515, "fcm_dpo/beta": 0.004260548390448093, "fcm_dpo/delta": -0.04734001308679581, "fcm_dpo/margin": 68.48147583007812, "fcm_dpo/q_t": 0.43087059259414673, "grad_norm": 12.573121070861816, "learning_rate": 6.142553278648238e-09, "logits/chosen": 0.8781119585037231, "logits/rejected": 0.8820315599441528, "logps/chosen": -246.15737915039062, "logps/ref_chosen": -63.36971664428711, "logps/ref_rejected": -65.68269348144531, "logps/rejected": -316.95184326171875, "loss": 1.1914, "margin_dpo/margin_mean": 68.48147583007812, "margin_dpo/margin_std": 121.66095733642578, "step": 620 }, { "KL/chosen_KL_mean": -175.8046875, "KL/mean": -221.00616455078125, "KL/rejected_KL_mean": -266.2076416015625, "KL/std": 130.67002868652344, "epoch": 0.9387755102040817, "fcm_dpo/beta": 0.004282790701836348, "fcm_dpo/delta": 0.012993309646844864, "fcm_dpo/margin": 90.40296936035156, "fcm_dpo/q_t": 0.415774405002594, "grad_norm": 12.253860473632812, "learning_rate": 5.854666444131934e-09, "logits/chosen": 0.9052057862281799, "logits/rejected": 0.8152543902397156, "logps/chosen": -228.12591552734375, "logps/ref_chosen": -52.321224212646484, "logps/ref_rejected": -88.09001159667969, "logps/rejected": -354.29766845703125, "loss": 1.1415, "margin_dpo/margin_mean": 90.4029769897461, "margin_dpo/margin_std": 156.630859375, "step": 621 }, { "KL/chosen_KL_mean": -193.3104248046875, "KL/mean": -235.54830932617188, "KL/rejected_KL_mean": -277.78619384765625, "KL/std": 133.7432861328125, "epoch": 0.9402872260015117, "fcm_dpo/beta": 0.004352467134594917, "fcm_dpo/delta": 0.034792713820934296, "fcm_dpo/margin": 84.47576904296875, "fcm_dpo/q_t": 0.41774168610572815, "grad_norm": 13.547627449035645, "learning_rate": 5.573608879422875e-09, "logits/chosen": 0.8366628885269165, "logits/rejected": 0.797753095626831, "logps/chosen": -253.17588806152344, "logps/ref_chosen": -59.86545944213867, "logps/ref_rejected": -81.86668395996094, "logps/rejected": -359.65289306640625, "loss": 1.1378, "margin_dpo/margin_mean": 84.47576904296875, "margin_dpo/margin_std": 138.1892852783203, "step": 622 }, { "KL/chosen_KL_mean": -206.13430786132812, "KL/mean": -251.16110229492188, "KL/rejected_KL_mean": -296.18792724609375, "KL/std": 141.82073974609375, "epoch": 0.9417989417989417, "fcm_dpo/beta": 0.004277583211660385, "fcm_dpo/delta": 0.014890506863594055, "fcm_dpo/margin": 90.05360412597656, "fcm_dpo/q_t": 0.4128887951374054, "grad_norm": 12.97014045715332, "learning_rate": 5.299388446305342e-09, "logits/chosen": 0.8766790628433228, "logits/rejected": 0.8372782468795776, "logps/chosen": -273.50274658203125, "logps/ref_chosen": -67.36846160888672, "logps/ref_rejected": -82.02733612060547, "logps/rejected": -378.21527099609375, "loss": 1.1335, "margin_dpo/margin_mean": 90.05360412597656, "margin_dpo/margin_std": 147.62298583984375, "step": 623 }, { "KL/chosen_KL_mean": -175.81561279296875, "KL/mean": -221.06178283691406, "KL/rejected_KL_mean": -266.30792236328125, "KL/std": 129.63758850097656, "epoch": 0.9433106575963719, "fcm_dpo/beta": 0.0043399459682404995, "fcm_dpo/delta": 0.007810791954398155, "fcm_dpo/margin": 90.49232482910156, "fcm_dpo/q_t": 0.41192591190338135, "grad_norm": 14.457840919494629, "learning_rate": 5.03201281531429e-09, "logits/chosen": 0.8671579957008362, "logits/rejected": 0.7905720472335815, "logps/chosen": -226.8421630859375, "logps/ref_chosen": -51.02655029296875, "logps/ref_rejected": -76.49203491210938, "logps/rejected": -342.79998779296875, "loss": 1.117, "margin_dpo/margin_mean": 90.49234008789062, "margin_dpo/margin_std": 139.15968322753906, "step": 624 }, { "KL/chosen_KL_mean": -175.61386108398438, "KL/mean": -212.56109619140625, "KL/rejected_KL_mean": -249.50831604003906, "KL/std": 123.15960693359375, "epoch": 0.9448223733938019, "fcm_dpo/beta": 0.004533303901553154, "fcm_dpo/delta": 0.07208952307701111, "fcm_dpo/margin": 73.89444732666016, "fcm_dpo/q_t": 0.427746057510376, "grad_norm": 12.597397804260254, "learning_rate": 4.7714894655209174e-09, "logits/chosen": 0.9103964567184448, "logits/rejected": 0.836378276348114, "logps/chosen": -229.82147216796875, "logps/ref_chosen": -54.20761489868164, "logps/ref_rejected": -84.93669128417969, "logps/rejected": -334.44500732421875, "loss": 1.1816, "margin_dpo/margin_mean": 73.89444732666016, "margin_dpo/margin_std": 147.33132934570312, "step": 625 }, { "KL/chosen_KL_mean": -165.85633850097656, "KL/mean": -216.77398681640625, "KL/rejected_KL_mean": -267.691650390625, "KL/std": 134.21945190429688, "epoch": 0.9463340891912321, "fcm_dpo/beta": 0.004439115524291992, "fcm_dpo/delta": -0.06053989380598068, "fcm_dpo/margin": 101.8353271484375, "fcm_dpo/q_t": 0.40118056535720825, "grad_norm": 13.96564769744873, "learning_rate": 4.517825684323323e-09, "logits/chosen": 0.9465058445930481, "logits/rejected": 0.8354614973068237, "logps/chosen": -210.91836547851562, "logps/ref_chosen": -45.06201934814453, "logps/ref_rejected": -89.66368103027344, "logps/rejected": -357.3553466796875, "loss": 1.0962, "margin_dpo/margin_mean": 101.8353271484375, "margin_dpo/margin_std": 162.71804809570312, "step": 626 }, { "KL/chosen_KL_mean": -190.51968383789062, "KL/mean": -244.76681518554688, "KL/rejected_KL_mean": -299.0139465332031, "KL/std": 139.9709930419922, "epoch": 0.9478458049886621, "fcm_dpo/beta": 0.00424289982765913, "fcm_dpo/delta": -0.07100832462310791, "fcm_dpo/margin": 108.49424743652344, "fcm_dpo/q_t": 0.39188235998153687, "grad_norm": 12.912845611572266, "learning_rate": 4.271028567242818e-09, "logits/chosen": 0.8388135433197021, "logits/rejected": 0.7388994693756104, "logps/chosen": -249.31072998046875, "logps/ref_chosen": -58.791053771972656, "logps/ref_rejected": -94.90802001953125, "logps/rejected": -393.9219665527344, "loss": 1.0535, "margin_dpo/margin_mean": 108.49424743652344, "margin_dpo/margin_std": 133.7566375732422, "step": 627 }, { "KL/chosen_KL_mean": -177.43270874023438, "KL/mean": -227.32901000976562, "KL/rejected_KL_mean": -277.22528076171875, "KL/std": 129.86306762695312, "epoch": 0.9493575207860923, "fcm_dpo/beta": 0.004253713879734278, "fcm_dpo/delta": -0.030077699571847916, "fcm_dpo/margin": 99.7926025390625, "fcm_dpo/q_t": 0.4006233811378479, "grad_norm": 13.415197372436523, "learning_rate": 4.0311050177251895e-09, "logits/chosen": 0.8698492646217346, "logits/rejected": 0.838459849357605, "logps/chosen": -230.2362823486328, "logps/ref_chosen": -52.80357360839844, "logps/ref_rejected": -76.49468994140625, "logps/rejected": -353.719970703125, "loss": 1.1146, "margin_dpo/margin_mean": 99.7926025390625, "margin_dpo/margin_std": 148.1082763671875, "step": 628 }, { "KL/chosen_KL_mean": -189.38441467285156, "KL/mean": -225.45184326171875, "KL/rejected_KL_mean": -261.51922607421875, "KL/std": 118.73931884765625, "epoch": 0.9508692365835223, "fcm_dpo/beta": 0.004356414079666138, "fcm_dpo/delta": 0.09377999603748322, "fcm_dpo/margin": 72.13484191894531, "fcm_dpo/q_t": 0.43087315559387207, "grad_norm": 11.81552505493164, "learning_rate": 3.798061746947995e-09, "logits/chosen": 0.8537815809249878, "logits/rejected": 0.857978343963623, "logps/chosen": -260.1019287109375, "logps/ref_chosen": -70.71749877929688, "logps/ref_rejected": -78.96273803710938, "logps/rejected": -340.48199462890625, "loss": 1.1703, "margin_dpo/margin_mean": 72.13484191894531, "margin_dpo/margin_std": 128.88232421875, "step": 629 }, { "KL/chosen_KL_mean": -175.94125366210938, "KL/mean": -225.40545654296875, "KL/rejected_KL_mean": -274.8696594238281, "KL/std": 126.93406677246094, "epoch": 0.9523809523809523, "fcm_dpo/beta": 0.004350777715444565, "fcm_dpo/delta": -0.03495955839753151, "fcm_dpo/margin": 98.92840576171875, "fcm_dpo/q_t": 0.4007263779640198, "grad_norm": 10.979743003845215, "learning_rate": 3.5719052736323806e-09, "logits/chosen": 0.8208729028701782, "logits/rejected": 0.7930917739868164, "logps/chosen": -232.14266967773438, "logps/ref_chosen": -56.201412200927734, "logps/ref_rejected": -74.69807434082031, "logps/rejected": -349.5677490234375, "loss": 1.0891, "margin_dpo/margin_mean": 98.92840576171875, "margin_dpo/margin_std": 140.695556640625, "step": 630 }, { "KL/chosen_KL_mean": -166.6142120361328, "KL/mean": -226.4366455078125, "KL/rejected_KL_mean": -286.25909423828125, "KL/std": 137.6940460205078, "epoch": 0.9538926681783825, "fcm_dpo/beta": 0.00401288690045476, "fcm_dpo/delta": -0.10077112913131714, "fcm_dpo/margin": 119.64486694335938, "fcm_dpo/q_t": 0.3887532353401184, "grad_norm": 11.224316596984863, "learning_rate": 3.352641923861144e-09, "logits/chosen": 0.9621773362159729, "logits/rejected": 0.8688410520553589, "logps/chosen": -225.434814453125, "logps/ref_chosen": -58.82059860229492, "logps/ref_rejected": -96.51437377929688, "logps/rejected": -382.7734680175781, "loss": 1.052, "margin_dpo/margin_mean": 119.64486694335938, "margin_dpo/margin_std": 151.0715789794922, "step": 631 }, { "KL/chosen_KL_mean": -169.34906005859375, "KL/mean": -222.4647216796875, "KL/rejected_KL_mean": -275.58038330078125, "KL/std": 118.47867584228516, "epoch": 0.9554043839758125, "fcm_dpo/beta": 0.004019321873784065, "fcm_dpo/delta": -0.031023576855659485, "fcm_dpo/margin": 106.2313232421875, "fcm_dpo/q_t": 0.39956796169281006, "grad_norm": 11.955537796020508, "learning_rate": 3.140277830901428e-09, "logits/chosen": 0.8837977051734924, "logits/rejected": 0.8737013339996338, "logps/chosen": -228.13510131835938, "logps/ref_chosen": -58.786048889160156, "logps/ref_rejected": -67.21923828125, "logps/rejected": -342.79962158203125, "loss": 1.0674, "margin_dpo/margin_mean": 106.2313232421875, "margin_dpo/margin_std": 127.79652404785156, "step": 632 }, { "KL/chosen_KL_mean": -165.90658569335938, "KL/mean": -208.74703979492188, "KL/rejected_KL_mean": -251.58749389648438, "KL/std": 116.66265869140625, "epoch": 0.9569160997732427, "fcm_dpo/beta": 0.004116720985621214, "fcm_dpo/delta": 0.05110543221235275, "fcm_dpo/margin": 85.68087768554688, "fcm_dpo/q_t": 0.42053651809692383, "grad_norm": 12.592107772827148, "learning_rate": 2.9348189350335007e-09, "logits/chosen": 0.8748718500137329, "logits/rejected": 0.8350610733032227, "logps/chosen": -218.0367889404297, "logps/ref_chosen": -52.13019561767578, "logps/ref_rejected": -67.23016357421875, "logps/rejected": -318.817626953125, "loss": 1.1355, "margin_dpo/margin_mean": 85.68087768554688, "margin_dpo/margin_std": 131.61073303222656, "step": 633 }, { "KL/chosen_KL_mean": -198.66600036621094, "KL/mean": -213.45736694335938, "KL/rejected_KL_mean": -228.24871826171875, "KL/std": 122.29270935058594, "epoch": 0.9584278155706727, "fcm_dpo/beta": 0.004098186735063791, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 29.582706451416016, "fcm_dpo/q_t": 0.47323864698410034, "grad_norm": 14.698355674743652, "learning_rate": 2.736270983384276e-09, "logits/chosen": 0.9170514941215515, "logits/rejected": 0.9342811703681946, "logps/chosen": -259.64581298828125, "logps/ref_chosen": -60.97979736328125, "logps/ref_rejected": -58.50825119018555, "logps/rejected": -286.7569580078125, "loss": 1.3477, "margin_dpo/margin_mean": 29.582706451416016, "margin_dpo/margin_std": 139.38327026367188, "step": 634 }, { "KL/chosen_KL_mean": -215.3644256591797, "KL/mean": -250.3736572265625, "KL/rejected_KL_mean": -285.38287353515625, "KL/std": 116.93312072753906, "epoch": 0.9599395313681028, "fcm_dpo/beta": 0.004337867721915245, "fcm_dpo/delta": 0.10398625582456589, "fcm_dpo/margin": 70.0184555053711, "fcm_dpo/q_t": 0.43173298239707947, "grad_norm": 11.96898078918457, "learning_rate": 2.5446395297668287e-09, "logits/chosen": 0.7966681718826294, "logits/rejected": 0.7563410997390747, "logps/chosen": -281.3374328613281, "logps/ref_chosen": -65.9730224609375, "logps/ref_rejected": -85.61317443847656, "logps/rejected": -370.99603271484375, "loss": 1.1911, "margin_dpo/margin_mean": 70.0184555053711, "margin_dpo/margin_std": 137.1169891357422, "step": 635 }, { "KL/chosen_KL_mean": -167.98959350585938, "KL/mean": -215.552490234375, "KL/rejected_KL_mean": -263.1153564453125, "KL/std": 120.42471313476562, "epoch": 0.9614512471655329, "fcm_dpo/beta": 0.004312177188694477, "fcm_dpo/delta": -0.011981801129877567, "fcm_dpo/margin": 95.12577819824219, "fcm_dpo/q_t": 0.4046896696090698, "grad_norm": 9.827645301818848, "learning_rate": 2.359929934524829e-09, "logits/chosen": 0.8474336862564087, "logits/rejected": 0.7673357129096985, "logps/chosen": -217.1297607421875, "logps/ref_chosen": -49.140167236328125, "logps/ref_rejected": -81.26971435546875, "logps/rejected": -344.38507080078125, "loss": 1.0884, "margin_dpo/margin_mean": 95.12577819824219, "margin_dpo/margin_std": 126.56785583496094, "step": 636 }, { "KL/chosen_KL_mean": -226.68218994140625, "KL/mean": -262.2201232910156, "KL/rejected_KL_mean": -297.758056640625, "KL/std": 124.49876403808594, "epoch": 0.9629629629629629, "fcm_dpo/beta": 0.0045396797358989716, "fcm_dpo/delta": 0.08410540223121643, "fcm_dpo/margin": 71.07585144042969, "fcm_dpo/q_t": 0.4289219081401825, "grad_norm": 13.477401733398438, "learning_rate": 2.1821473643827137e-09, "logits/chosen": 0.8291600942611694, "logits/rejected": 0.7868282794952393, "logps/chosen": -300.3787841796875, "logps/ref_chosen": -73.69658660888672, "logps/ref_rejected": -83.01487731933594, "logps/rejected": -380.77294921875, "loss": 1.1968, "margin_dpo/margin_mean": 71.07585144042969, "margin_dpo/margin_std": 150.5892333984375, "step": 637 }, { "KL/chosen_KL_mean": -196.17276000976562, "KL/mean": -238.7854461669922, "KL/rejected_KL_mean": -281.39813232421875, "KL/std": 120.97940063476562, "epoch": 0.9644746787603931, "fcm_dpo/beta": 0.0045569646172225475, "fcm_dpo/delta": 0.013032622635364532, "fcm_dpo/margin": 85.22538757324219, "fcm_dpo/q_t": 0.4132801294326782, "grad_norm": 13.217300415039062, "learning_rate": 2.0112967923011646e-09, "logits/chosen": 0.8621759414672852, "logits/rejected": 0.8292132616043091, "logps/chosen": -258.954345703125, "logps/ref_chosen": -62.78158187866211, "logps/ref_rejected": -85.40478515625, "logps/rejected": -366.80291748046875, "loss": 1.1234, "margin_dpo/margin_mean": 85.22538757324219, "margin_dpo/margin_std": 135.74893188476562, "step": 638 }, { "KL/chosen_KL_mean": -175.19105529785156, "KL/mean": -225.06832885742188, "KL/rejected_KL_mean": -274.94561767578125, "KL/std": 131.4615478515625, "epoch": 0.9659863945578231, "fcm_dpo/beta": 0.004495006985962391, "fcm_dpo/delta": -0.05581279098987579, "fcm_dpo/margin": 99.75454711914062, "fcm_dpo/q_t": 0.39895889163017273, "grad_norm": 15.222760200500488, "learning_rate": 1.847382997337943e-09, "logits/chosen": 0.8577284812927246, "logits/rejected": 0.7885833978652954, "logps/chosen": -228.9576416015625, "logps/ref_chosen": -53.76658630371094, "logps/ref_rejected": -72.30009460449219, "logps/rejected": -347.2456970214844, "loss": 1.0828, "margin_dpo/margin_mean": 99.75454711914062, "margin_dpo/margin_std": 143.67074584960938, "step": 639 }, { "KL/chosen_KL_mean": -182.3665771484375, "KL/mean": -225.64895629882812, "KL/rejected_KL_mean": -268.9313049316406, "KL/std": 123.47915649414062, "epoch": 0.9674981103552532, "fcm_dpo/beta": 0.004439163953065872, "fcm_dpo/delta": 0.017474712803959846, "fcm_dpo/margin": 86.56473541259766, "fcm_dpo/q_t": 0.4128446578979492, "grad_norm": 13.212712287902832, "learning_rate": 1.690410564514244e-09, "logits/chosen": 0.9096243381500244, "logits/rejected": 0.8597659468650818, "logps/chosen": -233.78436279296875, "logps/ref_chosen": -51.41777801513672, "logps/ref_rejected": -77.27879333496094, "logps/rejected": -346.2101135253906, "loss": 1.1238, "margin_dpo/margin_mean": 86.56473541259766, "margin_dpo/margin_std": 135.56790161132812, "step": 640 }, { "KL/chosen_KL_mean": -191.44329833984375, "KL/mean": -231.0288543701172, "KL/rejected_KL_mean": -270.6143798828125, "KL/std": 114.08880615234375, "epoch": 0.9690098261526833, "fcm_dpo/beta": 0.004546813666820526, "fcm_dpo/delta": 0.04486197605729103, "fcm_dpo/margin": 79.17110443115234, "fcm_dpo/q_t": 0.41722530126571655, "grad_norm": 13.189962387084961, "learning_rate": 1.5403838846864692e-09, "logits/chosen": 0.8492909073829651, "logits/rejected": 0.8432836532592773, "logps/chosen": -262.49798583984375, "logps/ref_chosen": -71.0546646118164, "logps/ref_rejected": -82.2440185546875, "logps/rejected": -352.8583984375, "loss": 1.1164, "margin_dpo/margin_mean": 79.17110443115234, "margin_dpo/margin_std": 108.83973693847656, "step": 641 }, { "KL/chosen_KL_mean": -200.26123046875, "KL/mean": -229.521728515625, "KL/rejected_KL_mean": -258.7822265625, "KL/std": 112.85456085205078, "epoch": 0.9705215419501134, "fcm_dpo/beta": 0.004678209777921438, "fcm_dpo/delta": 0.02495141327381134, "fcm_dpo/margin": 58.52101135253906, "fcm_dpo/q_t": 0.43944495916366577, "grad_norm": 15.50776481628418, "learning_rate": 1.3973071544233218e-09, "logits/chosen": 0.8077861070632935, "logits/rejected": 0.8297359943389893, "logps/chosen": -269.19049072265625, "logps/ref_chosen": -68.92927551269531, "logps/ref_rejected": -70.85682678222656, "logps/rejected": -329.6390686035156, "loss": 1.2261, "margin_dpo/margin_mean": 58.521018981933594, "margin_dpo/margin_std": 135.40769958496094, "step": 642 }, { "KL/chosen_KL_mean": -182.4813232421875, "KL/mean": -230.08364868164062, "KL/rejected_KL_mean": -277.68597412109375, "KL/std": 124.2804946899414, "epoch": 0.9720332577475435, "fcm_dpo/beta": 0.004592161625623703, "fcm_dpo/delta": -0.04280085489153862, "fcm_dpo/margin": 95.20464324951172, "fcm_dpo/q_t": 0.401271790266037, "grad_norm": 18.248327255249023, "learning_rate": 1.261184375888541e-09, "logits/chosen": 0.8441503047943115, "logits/rejected": 0.778213381767273, "logps/chosen": -247.79037475585938, "logps/ref_chosen": -65.30903625488281, "logps/ref_rejected": -83.61613464355469, "logps/rejected": -361.3021240234375, "loss": 1.0961, "margin_dpo/margin_mean": 95.20463562011719, "margin_dpo/margin_std": 142.61578369140625, "step": 643 }, { "KL/chosen_KL_mean": -173.4718017578125, "KL/mean": -202.7007293701172, "KL/rejected_KL_mean": -231.92965698242188, "KL/std": 106.54061889648438, "epoch": 0.9735449735449735, "fcm_dpo/beta": 0.004578079096972942, "fcm_dpo/delta": 0.013454647734761238, "fcm_dpo/margin": 58.45787048339844, "fcm_dpo/q_t": 0.43763917684555054, "grad_norm": 11.429241180419922, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 0.939828634262085, "logits/rejected": 0.912724494934082, "logps/chosen": -224.47439575195312, "logps/ref_chosen": -51.002601623535156, "logps/ref_rejected": -64.46372985839844, "logps/rejected": -296.39337158203125, "loss": 1.2201, "margin_dpo/margin_mean": 58.45787048339844, "margin_dpo/margin_std": 126.97639465332031, "step": 644 }, { "KL/chosen_KL_mean": -190.52029418945312, "KL/mean": -227.52386474609375, "KL/rejected_KL_mean": -264.5274353027344, "KL/std": 117.05931091308594, "epoch": 0.9750566893424036, "fcm_dpo/beta": 0.004725134931504726, "fcm_dpo/delta": 0.05598610267043114, "fcm_dpo/margin": 74.00714111328125, "fcm_dpo/q_t": 0.4198160469532013, "grad_norm": 16.172887802124023, "learning_rate": 1.0098157099674987e-09, "logits/chosen": 0.868065357208252, "logits/rejected": 0.8554788827896118, "logps/chosen": -251.48370361328125, "logps/ref_chosen": -60.963409423828125, "logps/ref_rejected": -69.73353576660156, "logps/rejected": -334.260986328125, "loss": 1.1443, "margin_dpo/margin_mean": 74.00714111328125, "margin_dpo/margin_std": 120.89301300048828, "step": 645 }, { "KL/chosen_KL_mean": -205.87545776367188, "KL/mean": -243.1065673828125, "KL/rejected_KL_mean": -280.3376770019531, "KL/std": 141.6141357421875, "epoch": 0.9765684051398337, "fcm_dpo/beta": 0.004856896586716175, "fcm_dpo/delta": 0.04285380616784096, "fcm_dpo/margin": 74.46222686767578, "fcm_dpo/q_t": 0.4203829765319824, "grad_norm": 12.586745262145996, "learning_rate": 8.945768539031783e-10, "logits/chosen": 0.8746490478515625, "logits/rejected": 0.8254890441894531, "logps/chosen": -268.16552734375, "logps/ref_chosen": -62.290069580078125, "logps/ref_rejected": -85.54812622070312, "logps/rejected": -365.88580322265625, "loss": 1.1618, "margin_dpo/margin_mean": 74.46221923828125, "margin_dpo/margin_std": 138.90460205078125, "step": 646 }, { "KL/chosen_KL_mean": -203.10272216796875, "KL/mean": -261.1683654785156, "KL/rejected_KL_mean": -319.2340087890625, "KL/std": 138.07496643066406, "epoch": 0.9780801209372638, "fcm_dpo/beta": 0.004566292278468609, "fcm_dpo/delta": -0.15473519265651703, "fcm_dpo/margin": 116.13129425048828, "fcm_dpo/q_t": 0.3759966492652893, "grad_norm": 14.314106941223145, "learning_rate": 7.863060120144316e-10, "logits/chosen": 0.8773380517959595, "logits/rejected": 0.7988033294677734, "logps/chosen": -270.61859130859375, "logps/ref_chosen": -67.515869140625, "logps/ref_rejected": -101.50871276855469, "logps/rejected": -420.7427062988281, "loss": 1.0135, "margin_dpo/margin_mean": 116.13129425048828, "margin_dpo/margin_std": 136.10638427734375, "step": 647 }, { "KL/chosen_KL_mean": -204.65838623046875, "KL/mean": -243.85040283203125, "KL/rejected_KL_mean": -283.04241943359375, "KL/std": 119.53384399414062, "epoch": 0.9795918367346939, "fcm_dpo/beta": 0.004495234228670597, "fcm_dpo/delta": 0.05285690724849701, "fcm_dpo/margin": 78.38400268554688, "fcm_dpo/q_t": 0.422693133354187, "grad_norm": 12.690834999084473, "learning_rate": 6.850062128694045e-10, "logits/chosen": 0.8503772020339966, "logits/rejected": 0.8010015487670898, "logps/chosen": -269.25433349609375, "logps/ref_chosen": -64.59593963623047, "logps/ref_rejected": -83.384033203125, "logps/rejected": -366.42645263671875, "loss": 1.1861, "margin_dpo/margin_mean": 78.38400268554688, "margin_dpo/margin_std": 156.04074096679688, "step": 648 }, { "KL/chosen_KL_mean": -184.55648803710938, "KL/mean": -225.07196044921875, "KL/rejected_KL_mean": -265.5874328613281, "KL/std": 115.69061279296875, "epoch": 0.981103552532124, "fcm_dpo/beta": 0.0046140593476593494, "fcm_dpo/delta": 0.0292498916387558, "fcm_dpo/margin": 81.03094482421875, "fcm_dpo/q_t": 0.4144395589828491, "grad_norm": 17.35785484313965, "learning_rate": 5.906802900412788e-10, "logits/chosen": 0.9027397632598877, "logits/rejected": 0.850858211517334, "logps/chosen": -233.8661346435547, "logps/ref_chosen": -49.30964660644531, "logps/ref_rejected": -73.73710632324219, "logps/rejected": -339.32452392578125, "loss": 1.1432, "margin_dpo/margin_mean": 81.03094482421875, "margin_dpo/margin_std": 137.96710205078125, "step": 649 }, { "KL/chosen_KL_mean": -178.10488891601562, "KL/mean": -222.58074951171875, "KL/rejected_KL_mean": -267.05657958984375, "KL/std": 120.7571029663086, "epoch": 0.982615268329554, "fcm_dpo/beta": 0.004608334973454475, "fcm_dpo/delta": -0.011857246980071068, "fcm_dpo/margin": 88.95169067382812, "fcm_dpo/q_t": 0.40797871351242065, "grad_norm": 13.712696075439453, "learning_rate": 5.033308820289184e-10, "logits/chosen": 0.9457135200500488, "logits/rejected": 0.896229088306427, "logps/chosen": -233.16815185546875, "logps/ref_chosen": -55.06325912475586, "logps/ref_rejected": -77.39610290527344, "logps/rejected": -344.45269775390625, "loss": 1.1262, "margin_dpo/margin_mean": 88.95169067382812, "margin_dpo/margin_std": 148.16856384277344, "step": 650 }, { "KL/chosen_KL_mean": -176.51058959960938, "KL/mean": -214.7784881591797, "KL/rejected_KL_mean": -253.04638671875, "KL/std": 118.41923522949219, "epoch": 0.9841269841269841, "fcm_dpo/beta": 0.004761428106576204, "fcm_dpo/delta": 0.03895041346549988, "fcm_dpo/margin": 76.53578186035156, "fcm_dpo/q_t": 0.4195203185081482, "grad_norm": 13.958356857299805, "learning_rate": 4.2296043218295606e-10, "logits/chosen": 0.9171409606933594, "logits/rejected": 0.8589807748794556, "logps/chosen": -230.5757598876953, "logps/ref_chosen": -54.065162658691406, "logps/ref_rejected": -77.79080200195312, "logps/rejected": -330.8371887207031, "loss": 1.1358, "margin_dpo/margin_mean": 76.53578186035156, "margin_dpo/margin_std": 123.71870422363281, "step": 651 }, { "KL/chosen_KL_mean": -209.03729248046875, "KL/mean": -242.5619659423828, "KL/rejected_KL_mean": -276.086669921875, "KL/std": 132.26441955566406, "epoch": 0.9856386999244142, "fcm_dpo/beta": 0.004862497560679913, "fcm_dpo/delta": 0.0819351002573967, "fcm_dpo/margin": 67.04937744140625, "fcm_dpo/q_t": 0.4319889545440674, "grad_norm": 16.18764877319336, "learning_rate": 3.4957118863768176e-10, "logits/chosen": 0.8910359144210815, "logits/rejected": 0.8544408082962036, "logps/chosen": -272.6775817871094, "logps/ref_chosen": -63.64030456542969, "logps/ref_rejected": -78.86882019042969, "logps/rejected": -354.95550537109375, "loss": 1.2244, "margin_dpo/margin_mean": 67.04937744140625, "margin_dpo/margin_std": 160.25506591796875, "step": 652 }, { "KL/chosen_KL_mean": -185.3717803955078, "KL/mean": -227.00796508789062, "KL/rejected_KL_mean": -268.6441955566406, "KL/std": 115.03123474121094, "epoch": 0.9871504157218443, "fcm_dpo/beta": 0.0050035035237669945, "fcm_dpo/delta": -0.01987006887793541, "fcm_dpo/margin": 83.27240753173828, "fcm_dpo/q_t": 0.4065983295440674, "grad_norm": 15.927544593811035, "learning_rate": 2.831652042480093e-10, "logits/chosen": 0.8695135116577148, "logits/rejected": 0.8330869674682617, "logps/chosen": -247.04014587402344, "logps/ref_chosen": -61.668373107910156, "logps/ref_rejected": -73.83012390136719, "logps/rejected": -342.47430419921875, "loss": 1.1174, "margin_dpo/margin_mean": 83.27241516113281, "margin_dpo/margin_std": 133.44725036621094, "step": 653 }, { "KL/chosen_KL_mean": -192.90228271484375, "KL/mean": -233.95965576171875, "KL/rejected_KL_mean": -275.01702880859375, "KL/std": 139.22682189941406, "epoch": 0.9886621315192744, "fcm_dpo/beta": 0.0046582408249378204, "fcm_dpo/delta": -0.09315278381109238, "fcm_dpo/margin": 82.11475372314453, "fcm_dpo/q_t": 0.41140812635421753, "grad_norm": 13.904915809631348, "learning_rate": 2.2374433653205016e-10, "logits/chosen": 0.8664817810058594, "logits/rejected": 0.7882392406463623, "logps/chosen": -250.47055053710938, "logps/ref_chosen": -57.568267822265625, "logps/ref_rejected": -87.74789428710938, "logps/rejected": -362.7649230957031, "loss": 1.1324, "margin_dpo/margin_mean": 82.11475372314453, "margin_dpo/margin_std": 129.8606719970703, "step": 654 }, { "KL/chosen_KL_mean": -152.3089599609375, "KL/mean": -211.10946655273438, "KL/rejected_KL_mean": -269.90997314453125, "KL/std": 120.35342407226562, "epoch": 0.9901738473167044, "fcm_dpo/beta": 0.004261736758053303, "fcm_dpo/delta": -0.1305772364139557, "fcm_dpo/margin": 117.60102081298828, "fcm_dpo/q_t": 0.38035333156585693, "grad_norm": 11.05696964263916, "learning_rate": 1.7131024761923852e-10, "logits/chosen": 0.8656521439552307, "logits/rejected": 0.7972902655601501, "logps/chosen": -204.45611572265625, "logps/ref_chosen": -52.14714813232422, "logps/ref_rejected": -80.85014343261719, "logps/rejected": -350.7601318359375, "loss": 1.0168, "margin_dpo/margin_mean": 117.60101318359375, "margin_dpo/margin_std": 123.00428771972656, "step": 655 }, { "KL/chosen_KL_mean": -204.72833251953125, "KL/mean": -246.2228546142578, "KL/rejected_KL_mean": -287.7173767089844, "KL/std": 123.21992492675781, "epoch": 0.9916855631141346, "fcm_dpo/beta": 0.004393207840621471, "fcm_dpo/delta": 0.0396343432366848, "fcm_dpo/margin": 82.98904418945312, "fcm_dpo/q_t": 0.4176909923553467, "grad_norm": 11.27653694152832, "learning_rate": 1.2586440420372934e-10, "logits/chosen": 0.8115413188934326, "logits/rejected": 0.7813726663589478, "logps/chosen": -277.98504638671875, "logps/ref_chosen": -73.25672912597656, "logps/ref_rejected": -85.35127258300781, "logps/rejected": -373.06866455078125, "loss": 1.138, "margin_dpo/margin_mean": 82.98904418945312, "margin_dpo/margin_std": 135.81607055664062, "step": 656 }, { "KL/chosen_KL_mean": -176.87982177734375, "KL/mean": -229.69300842285156, "KL/rejected_KL_mean": -282.5062255859375, "KL/std": 135.0663604736328, "epoch": 0.9931972789115646, "fcm_dpo/beta": 0.0043113818392157555, "fcm_dpo/delta": -0.06402204930782318, "fcm_dpo/margin": 105.62638854980469, "fcm_dpo/q_t": 0.3964206874370575, "grad_norm": 11.030098915100098, "learning_rate": 8.740807750345913e-11, "logits/chosen": 0.9459801912307739, "logits/rejected": 0.8843368291854858, "logps/chosen": -226.60321044921875, "logps/ref_chosen": -49.72339630126953, "logps/ref_rejected": -75.1568603515625, "logps/rejected": -357.6630859375, "loss": 1.0776, "margin_dpo/margin_mean": 105.62638854980469, "margin_dpo/margin_std": 149.3883056640625, "step": 657 }, { "KL/chosen_KL_mean": -190.58401489257812, "KL/mean": -233.33816528320312, "KL/rejected_KL_mean": -276.09234619140625, "KL/std": 143.47869873046875, "epoch": 0.9947089947089947, "fcm_dpo/beta": 0.004380069673061371, "fcm_dpo/delta": 0.025314655154943466, "fcm_dpo/margin": 85.5083236694336, "fcm_dpo/q_t": 0.41635990142822266, "grad_norm": 10.85285758972168, "learning_rate": 5.594234322453539e-11, "logits/chosen": 0.8922824859619141, "logits/rejected": 0.8569316864013672, "logps/chosen": -253.63035583496094, "logps/ref_chosen": -63.04634094238281, "logps/ref_rejected": -83.44963073730469, "logps/rejected": -359.5419616699219, "loss": 1.1715, "margin_dpo/margin_mean": 85.50831604003906, "margin_dpo/margin_std": 166.18760681152344, "step": 658 }, { "KL/chosen_KL_mean": -192.74554443359375, "KL/mean": -224.75161743164062, "KL/rejected_KL_mean": -256.7576904296875, "KL/std": 123.44963073730469, "epoch": 0.9962207105064248, "fcm_dpo/beta": 0.004360191989690065, "fcm_dpo/delta": 0.01944735273718834, "fcm_dpo/margin": 64.01215362548828, "fcm_dpo/q_t": 0.4362792372703552, "grad_norm": 14.053616523742676, "learning_rate": 3.146808153123293e-11, "logits/chosen": 0.9274283051490784, "logits/rejected": 0.8760404586791992, "logps/chosen": -247.82574462890625, "logps/ref_chosen": -55.0802001953125, "logps/ref_rejected": -71.91049194335938, "logps/rejected": -328.6681823730469, "loss": 1.2169, "margin_dpo/margin_mean": 64.01214599609375, "margin_dpo/margin_std": 140.3470458984375, "step": 659 }, { "KL/chosen_KL_mean": -180.63677978515625, "KL/mean": -235.10302734375, "KL/rejected_KL_mean": -289.56927490234375, "KL/std": 133.57896423339844, "epoch": 0.9977324263038548, "fcm_dpo/beta": 0.004226707387715578, "fcm_dpo/delta": -0.07013650238513947, "fcm_dpo/margin": 108.93248748779297, "fcm_dpo/q_t": 0.3952319025993347, "grad_norm": 13.667136192321777, "learning_rate": 1.3985977021235829e-11, "logits/chosen": 0.9608404636383057, "logits/rejected": 0.9127137660980225, "logps/chosen": -235.16268920898438, "logps/ref_chosen": -54.525917053222656, "logps/ref_rejected": -81.23604583740234, "logps/rejected": -370.8052978515625, "loss": 1.0608, "margin_dpo/margin_mean": 108.93248748779297, "margin_dpo/margin_std": 145.68505859375, "step": 660 }, { "KL/chosen_KL_mean": -210.68487548828125, "KL/mean": -243.6689910888672, "KL/rejected_KL_mean": -276.653076171875, "KL/std": 131.54515075683594, "epoch": 0.999244142101285, "fcm_dpo/beta": 0.004441337194293737, "fcm_dpo/delta": 0.1156606376171112, "fcm_dpo/margin": 65.96818542480469, "fcm_dpo/q_t": 0.4353044927120209, "grad_norm": 11.607975959777832, "learning_rate": 3.4965187065971735e-12, "logits/chosen": 0.8547237515449524, "logits/rejected": 0.7953514456748962, "logps/chosen": -271.0575256347656, "logps/ref_chosen": -60.37263870239258, "logps/ref_rejected": -77.42874145507812, "logps/rejected": -354.08184814453125, "loss": 1.2196, "margin_dpo/margin_mean": 65.96818542480469, "margin_dpo/margin_std": 151.36160278320312, "step": 661 }, { "epoch": 0.999244142101285, "step": 661, "total_flos": 0.0, "train_loss": 1.1683954546924078, "train_runtime": 1648.9587, "train_samples_per_second": 25.674, "train_steps_per_second": 0.401 } ], "logging_steps": 1, "max_steps": 661, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }