{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.999244142101285, "eval_steps": 200, "global_step": 661, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.02867889404296875, "KL/mean": 0.029354453086853027, "KL/rejected_KL_mean": 0.030029296875, "KL/std": 0.2071000635623932, "epoch": 0.0015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013532638549804688, "fcm_dpo/q_t": 0.5000336766242981, "grad_norm": 28.22030258178711, "learning_rate": 0.0, "logits/chosen": 0.13337239623069763, "logits/rejected": 0.12492949515581131, "logps/chosen": -64.5841293334961, "logps/ref_chosen": -64.61280822753906, "logps/ref_rejected": -64.17195129394531, "logps/rejected": -64.14192199707031, "loss": 1.3866, "margin_dpo/margin_mean": -0.0013527870178222656, "margin_dpo/margin_std": 0.2561596930027008, "step": 1 }, { "KL/chosen_KL_mean": -0.00289154052734375, "KL/mean": -0.021616414189338684, "KL/rejected_KL_mean": -0.04033660888671875, "KL/std": 0.19624735414981842, "epoch": 0.0030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.037450045347213745, "fcm_dpo/q_t": 0.4990639090538025, "grad_norm": 27.827123641967773, "learning_rate": 7.462686567164179e-09, "logits/chosen": 0.09414851665496826, "logits/rejected": 0.07363267242908478, "logps/chosen": -56.101890563964844, "logps/ref_chosen": -56.0989990234375, "logps/ref_rejected": -66.59971618652344, "logps/rejected": -66.64006042480469, "loss": 1.3828, "margin_dpo/margin_mean": 0.03744968771934509, "margin_dpo/margin_std": 0.27811938524246216, "step": 2 }, { "KL/chosen_KL_mean": 0.02416229248046875, "KL/mean": 0.028771430253982544, "KL/rejected_KL_mean": 0.033382415771484375, "KL/std": 0.23812416195869446, "epoch": 0.0045351473922902496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.009217977523803711, "fcm_dpo/q_t": 0.5002306699752808, "grad_norm": 31.222238540649414, "learning_rate": 1.4925373134328357e-08, "logits/chosen": 0.09942537546157837, "logits/rejected": 0.06147611141204834, "logps/chosen": -65.43309783935547, "logps/ref_chosen": -65.45726013183594, "logps/ref_rejected": -90.82853698730469, "logps/rejected": -90.79515075683594, "loss": 1.3874, "margin_dpo/margin_mean": -0.00921773910522461, "margin_dpo/margin_std": 0.303035706281662, "step": 3 }, { "KL/chosen_KL_mean": 0.028865814208984375, "KL/mean": 0.006366938352584839, "KL/rejected_KL_mean": -0.0161285400390625, "KL/std": 0.23680132627487183, "epoch": 0.006046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04498940706253052, "fcm_dpo/q_t": 0.49887585639953613, "grad_norm": 34.199424743652344, "learning_rate": 2.2388059701492534e-08, "logits/chosen": 0.11272860318422318, "logits/rejected": 0.0964307188987732, "logps/chosen": -76.83131408691406, "logps/ref_chosen": -76.86018371582031, "logps/ref_rejected": -79.91523742675781, "logps/rejected": -79.93136596679688, "loss": 1.3821, "margin_dpo/margin_mean": 0.04498884081840515, "margin_dpo/margin_std": 0.3451710343360901, "step": 4 }, { "KL/chosen_KL_mean": -0.006206512451171875, "KL/mean": -0.012510225176811218, "KL/rejected_KL_mean": -0.018810272216796875, "KL/std": 0.2228856384754181, "epoch": 0.007558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012603014707565308, "fcm_dpo/q_t": 0.4996851086616516, "grad_norm": 29.621192932128906, "learning_rate": 2.9850746268656714e-08, "logits/chosen": 0.08257527649402618, "logits/rejected": 0.043656595051288605, "logps/chosen": -62.97754669189453, "logps/ref_chosen": -62.97134017944336, "logps/ref_rejected": -79.9192123413086, "logps/rejected": -79.93801879882812, "loss": 1.3853, "margin_dpo/margin_mean": 0.012603580951690674, "margin_dpo/margin_std": 0.31694692373275757, "step": 5 }, { "KL/chosen_KL_mean": -0.02547454833984375, "KL/mean": -0.015959739685058594, "KL/rejected_KL_mean": -0.0064449310302734375, "KL/std": 0.24643856287002563, "epoch": 0.009070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.019029557704925537, "fcm_dpo/q_t": 0.5004755854606628, "grad_norm": 29.573598861694336, "learning_rate": 3.731343283582089e-08, "logits/chosen": 0.13114216923713684, "logits/rejected": 0.09240274131298065, "logps/chosen": -51.33283996582031, "logps/ref_chosen": -51.30736541748047, "logps/ref_rejected": -82.77239227294922, "logps/rejected": -82.77883911132812, "loss": 1.3885, "margin_dpo/margin_mean": -0.019030064344406128, "margin_dpo/margin_std": 0.326235294342041, "step": 6 }, { "KL/chosen_KL_mean": -0.0041332244873046875, "KL/mean": 0.01974809169769287, "KL/rejected_KL_mean": 0.043628692626953125, "KL/std": 0.22289568185806274, "epoch": 0.010582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04776144027709961, "fcm_dpo/q_t": 0.5011939406394958, "grad_norm": 27.26719856262207, "learning_rate": 4.477611940298507e-08, "logits/chosen": 0.019673092290759087, "logits/rejected": -0.02439572475850582, "logps/chosen": -51.46354675292969, "logps/ref_chosen": -51.45941162109375, "logps/ref_rejected": -66.3828125, "logps/rejected": -66.33917999267578, "loss": 1.3913, "margin_dpo/margin_mean": -0.047761619091033936, "margin_dpo/margin_std": 0.2788490951061249, "step": 7 }, { "KL/chosen_KL_mean": 0.0001125335693359375, "KL/mean": -0.004021286964416504, "KL/rejected_KL_mean": -0.008153915405273438, "KL/std": 0.22656387090682983, "epoch": 0.012093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.008267223834991455, "fcm_dpo/q_t": 0.4997938871383667, "grad_norm": 28.687152862548828, "learning_rate": 5.223880597014925e-08, "logits/chosen": 0.09223589301109314, "logits/rejected": 0.06963017582893372, "logps/chosen": -62.19743347167969, "logps/ref_chosen": -62.197547912597656, "logps/ref_rejected": -74.66180419921875, "logps/rejected": -74.66996002197266, "loss": 1.3858, "margin_dpo/margin_mean": 0.008266836404800415, "margin_dpo/margin_std": 0.33951717615127563, "step": 8 }, { "KL/chosen_KL_mean": -0.017620086669921875, "KL/mean": -0.03803481161594391, "KL/rejected_KL_mean": -0.05844879150390625, "KL/std": 0.24063098430633545, "epoch": 0.013605442176870748, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04083278775215149, "fcm_dpo/q_t": 0.4989795386791229, "grad_norm": 31.502504348754883, "learning_rate": 5.970149253731343e-08, "logits/chosen": 0.18066352605819702, "logits/rejected": 0.12050867080688477, "logps/chosen": -55.6473388671875, "logps/ref_chosen": -55.629722595214844, "logps/ref_rejected": -86.21221923828125, "logps/rejected": -86.27067565917969, "loss": 1.3825, "margin_dpo/margin_mean": 0.040832966566085815, "margin_dpo/margin_std": 0.31956762075424194, "step": 9 }, { "KL/chosen_KL_mean": 0.003444671630859375, "KL/mean": 0.024548858404159546, "KL/rejected_KL_mean": 0.045650482177734375, "KL/std": 0.27166080474853516, "epoch": 0.015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04221159219741821, "fcm_dpo/q_t": 0.5010551810264587, "grad_norm": 29.839445114135742, "learning_rate": 6.71641791044776e-08, "logits/chosen": 0.14076688885688782, "logits/rejected": 0.10929006338119507, "logps/chosen": -62.687156677246094, "logps/ref_chosen": -62.69060134887695, "logps/ref_rejected": -90.610107421875, "logps/rejected": -90.56446075439453, "loss": 1.3909, "margin_dpo/margin_mean": -0.04221177101135254, "margin_dpo/margin_std": 0.3878580927848816, "step": 10 }, { "KL/chosen_KL_mean": 0.0004329681396484375, "KL/mean": -0.0009850859642028809, "KL/rejected_KL_mean": -0.00240325927734375, "KL/std": 0.224456787109375, "epoch": 0.016628873771730914, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0028373003005981445, "fcm_dpo/q_t": 0.4999290108680725, "grad_norm": 29.369600296020508, "learning_rate": 7.462686567164178e-08, "logits/chosen": 0.13556398451328278, "logits/rejected": 0.12829625606536865, "logps/chosen": -65.76668548583984, "logps/ref_chosen": -65.76712036132812, "logps/ref_rejected": -72.4764633178711, "logps/rejected": -72.47886657714844, "loss": 1.3862, "margin_dpo/margin_mean": 0.002837926149368286, "margin_dpo/margin_std": 0.2853427529335022, "step": 11 }, { "KL/chosen_KL_mean": -0.0237274169921875, "KL/mean": -0.03283458948135376, "KL/rejected_KL_mean": -0.04194450378417969, "KL/std": 0.22529613971710205, "epoch": 0.018140589569160998, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.018218934535980225, "fcm_dpo/q_t": 0.4995444715023041, "grad_norm": 28.207260131835938, "learning_rate": 8.208955223880596e-08, "logits/chosen": 0.017520083114504814, "logits/rejected": 0.0015218239277601242, "logps/chosen": -60.72861862182617, "logps/ref_chosen": -60.704891204833984, "logps/ref_rejected": -69.41564178466797, "logps/rejected": -69.45758819580078, "loss": 1.3848, "margin_dpo/margin_mean": 0.018218576908111572, "margin_dpo/margin_std": 0.3317432403564453, "step": 12 }, { "KL/chosen_KL_mean": 0.013547897338867188, "KL/mean": 0.008224859833717346, "KL/rejected_KL_mean": 0.002910614013671875, "KL/std": 0.22614026069641113, "epoch": 0.019652305366591082, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01064404845237732, "fcm_dpo/q_t": 0.4997338652610779, "grad_norm": 29.149938583374023, "learning_rate": 8.955223880597014e-08, "logits/chosen": 0.1031138002872467, "logits/rejected": 0.041284747421741486, "logps/chosen": -49.89570617675781, "logps/ref_chosen": -49.90925598144531, "logps/ref_rejected": -92.37818145751953, "logps/rejected": -92.37527465820312, "loss": 1.3855, "margin_dpo/margin_mean": 0.010643362998962402, "margin_dpo/margin_std": 0.32827091217041016, "step": 13 }, { "KL/chosen_KL_mean": 0.05055809020996094, "KL/mean": 0.008084923028945923, "KL/rejected_KL_mean": -0.034389495849609375, "KL/std": 0.2188139706850052, "epoch": 0.021164021164021163, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08494538068771362, "fcm_dpo/q_t": 0.4978775382041931, "grad_norm": 29.400705337524414, "learning_rate": 9.701492537313432e-08, "logits/chosen": 0.10135047137737274, "logits/rejected": 0.08315829187631607, "logps/chosen": -60.5682373046875, "logps/ref_chosen": -60.61879348754883, "logps/ref_rejected": -71.79306030273438, "logps/rejected": -71.82745361328125, "loss": 1.3781, "margin_dpo/margin_mean": 0.08494532108306885, "margin_dpo/margin_std": 0.3461207151412964, "step": 14 }, { "KL/chosen_KL_mean": -0.011671066284179688, "KL/mean": -0.011060148477554321, "KL/rejected_KL_mean": -0.010448455810546875, "KL/std": 0.21656280755996704, "epoch": 0.022675736961451247, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0012268424034118652, "fcm_dpo/q_t": 0.5000307559967041, "grad_norm": 33.25237274169922, "learning_rate": 1.044776119402985e-07, "logits/chosen": 0.08315175771713257, "logits/rejected": 0.03908975049853325, "logps/chosen": -63.48120880126953, "logps/ref_chosen": -63.46953582763672, "logps/ref_rejected": -88.88951110839844, "logps/rejected": -88.89996337890625, "loss": 1.3866, "margin_dpo/margin_mean": -0.0012272894382476807, "margin_dpo/margin_std": 0.27439266443252563, "step": 15 }, { "KL/chosen_KL_mean": 0.01654815673828125, "KL/mean": -0.007980212569236755, "KL/rejected_KL_mean": -0.0325164794921875, "KL/std": 0.16992035508155823, "epoch": 0.02418745275888133, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04905608296394348, "fcm_dpo/q_t": 0.4987736940383911, "grad_norm": 26.78843879699707, "learning_rate": 1.1194029850746268e-07, "logits/chosen": 0.11605009436607361, "logits/rejected": 0.07887766510248184, "logps/chosen": -46.515750885009766, "logps/ref_chosen": -46.53229904174805, "logps/ref_rejected": -74.27533721923828, "logps/rejected": -74.30785369873047, "loss": 1.3816, "margin_dpo/margin_mean": 0.049056172370910645, "margin_dpo/margin_std": 0.253105491399765, "step": 16 }, { "KL/chosen_KL_mean": 0.04642677307128906, "KL/mean": -0.011630088090896606, "KL/rejected_KL_mean": -0.06968307495117188, "KL/std": 0.2327060103416443, "epoch": 0.025699168556311415, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11611384153366089, "fcm_dpo/q_t": 0.4970986545085907, "grad_norm": 32.61309814453125, "learning_rate": 1.1940298507462686e-07, "logits/chosen": 0.07137341052293777, "logits/rejected": 0.052080146968364716, "logps/chosen": -64.03140258789062, "logps/ref_chosen": -64.07783508300781, "logps/ref_rejected": -86.40876770019531, "logps/rejected": -86.47845458984375, "loss": 1.375, "margin_dpo/margin_mean": 0.11611288785934448, "margin_dpo/margin_std": 0.324762225151062, "step": 17 }, { "KL/chosen_KL_mean": 0.026746749877929688, "KL/mean": -0.001529976725578308, "KL/rejected_KL_mean": -0.029804229736328125, "KL/std": 0.21335983276367188, "epoch": 0.027210884353741496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05655255913734436, "fcm_dpo/q_t": 0.498586505651474, "grad_norm": 27.79302215576172, "learning_rate": 1.2686567164179106e-07, "logits/chosen": 0.09060671925544739, "logits/rejected": 0.04522589594125748, "logps/chosen": -44.84758758544922, "logps/ref_chosen": -44.87433624267578, "logps/ref_rejected": -70.97604370117188, "logps/rejected": -71.00585174560547, "loss": 1.3808, "margin_dpo/margin_mean": 0.05655243992805481, "margin_dpo/margin_std": 0.26377081871032715, "step": 18 }, { "KL/chosen_KL_mean": 0.028326034545898438, "KL/mean": -0.006911836564540863, "KL/rejected_KL_mean": -0.042148590087890625, "KL/std": 0.2588089108467102, "epoch": 0.02872260015117158, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07047758996486664, "fcm_dpo/q_t": 0.4982374310493469, "grad_norm": 30.838409423828125, "learning_rate": 1.343283582089552e-07, "logits/chosen": 0.07254800200462341, "logits/rejected": 0.05900269374251366, "logps/chosen": -68.13148498535156, "logps/ref_chosen": -68.1598129272461, "logps/ref_rejected": -81.17138671875, "logps/rejected": -81.21353912353516, "loss": 1.3796, "margin_dpo/margin_mean": 0.0704774409532547, "margin_dpo/margin_std": 0.34581679105758667, "step": 19 }, { "KL/chosen_KL_mean": 0.00238800048828125, "KL/mean": -0.0007894262671470642, "KL/rejected_KL_mean": -0.003963470458984375, "KL/std": 0.2168654203414917, "epoch": 0.030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.006352677941322327, "fcm_dpo/q_t": 0.499840646982193, "grad_norm": 29.309476852416992, "learning_rate": 1.4179104477611938e-07, "logits/chosen": 0.13272634148597717, "logits/rejected": 0.10981430113315582, "logps/chosen": -53.67617416381836, "logps/ref_chosen": -53.67856216430664, "logps/ref_rejected": -74.16911315917969, "logps/rejected": -74.17308044433594, "loss": 1.3859, "margin_dpo/margin_mean": 0.0063527971506118774, "margin_dpo/margin_std": 0.3347752094268799, "step": 20 }, { "KL/chosen_KL_mean": -0.01772308349609375, "KL/mean": 0.0006627514958381653, "KL/rejected_KL_mean": 0.01905059814453125, "KL/std": 0.2528798580169678, "epoch": 0.031746031746031744, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03677301108837128, "fcm_dpo/q_t": 0.5009191036224365, "grad_norm": 29.429983139038086, "learning_rate": 1.4925373134328355e-07, "logits/chosen": 0.13152486085891724, "logits/rejected": 0.10482591390609741, "logps/chosen": -64.71927642822266, "logps/ref_chosen": -64.70155334472656, "logps/ref_rejected": -81.02095031738281, "logps/rejected": -81.00189208984375, "loss": 1.3903, "margin_dpo/margin_mean": -0.03677418828010559, "margin_dpo/margin_std": 0.3447566628456116, "step": 21 }, { "KL/chosen_KL_mean": -0.02715301513671875, "KL/mean": -0.05641151964664459, "KL/rejected_KL_mean": -0.08567047119140625, "KL/std": 0.2099946141242981, "epoch": 0.03325774754346183, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05851736664772034, "fcm_dpo/q_t": 0.49853768944740295, "grad_norm": 28.81707763671875, "learning_rate": 1.5671641791044775e-07, "logits/chosen": 0.01210310310125351, "logits/rejected": -0.009254289790987968, "logps/chosen": -58.06314468383789, "logps/ref_chosen": -58.03599166870117, "logps/ref_rejected": -80.72721862792969, "logps/rejected": -80.8128890991211, "loss": 1.3807, "margin_dpo/margin_mean": 0.05851736664772034, "margin_dpo/margin_std": 0.29362723231315613, "step": 22 }, { "KL/chosen_KL_mean": 0.003162384033203125, "KL/mean": -0.0018716305494308472, "KL/rejected_KL_mean": -0.006908416748046875, "KL/std": 0.2396647185087204, "epoch": 0.03476946334089191, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.010072559118270874, "fcm_dpo/q_t": 0.4997485876083374, "grad_norm": 32.89107131958008, "learning_rate": 1.6417910447761193e-07, "logits/chosen": 0.14491821825504303, "logits/rejected": 0.11915301531553268, "logps/chosen": -66.35292053222656, "logps/ref_chosen": -66.35608673095703, "logps/ref_rejected": -93.02769470214844, "logps/rejected": -93.03459930419922, "loss": 1.3856, "margin_dpo/margin_mean": 0.010072082281112671, "margin_dpo/margin_std": 0.3504166007041931, "step": 23 }, { "KL/chosen_KL_mean": -0.030969619750976562, "KL/mean": -0.0021247342228889465, "KL/rejected_KL_mean": 0.026720046997070312, "KL/std": 0.1989734172821045, "epoch": 0.036281179138321996, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.057684049010276794, "fcm_dpo/q_t": 0.5014417767524719, "grad_norm": 26.36972999572754, "learning_rate": 1.716417910447761e-07, "logits/chosen": 0.11135051399469376, "logits/rejected": 0.0792173445224762, "logps/chosen": -54.492210388183594, "logps/ref_chosen": -54.461238861083984, "logps/ref_rejected": -68.33817291259766, "logps/rejected": -68.31145477294922, "loss": 1.3922, "margin_dpo/margin_mean": -0.057683974504470825, "margin_dpo/margin_std": 0.26602524518966675, "step": 24 }, { "KL/chosen_KL_mean": -0.037654876708984375, "KL/mean": -0.04850602149963379, "KL/rejected_KL_mean": -0.059356689453125, "KL/std": 0.23352733254432678, "epoch": 0.03779289493575208, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02170264720916748, "fcm_dpo/q_t": 0.49945753812789917, "grad_norm": 29.69145965576172, "learning_rate": 1.7910447761194027e-07, "logits/chosen": 0.12192443013191223, "logits/rejected": 0.0692099779844284, "logps/chosen": -60.0418586730957, "logps/ref_chosen": -60.00420379638672, "logps/ref_rejected": -90.47376251220703, "logps/rejected": -90.53311920166016, "loss": 1.3844, "margin_dpo/margin_mean": 0.021703064441680908, "margin_dpo/margin_std": 0.30193835496902466, "step": 25 }, { "KL/chosen_KL_mean": -0.04012489318847656, "KL/mean": -0.026201292872428894, "KL/rejected_KL_mean": -0.012279510498046875, "KL/std": 0.23137395083904266, "epoch": 0.039304610733182165, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.027844339609146118, "fcm_dpo/q_t": 0.5006958246231079, "grad_norm": 29.982654571533203, "learning_rate": 1.8656716417910447e-07, "logits/chosen": 0.12360776960849762, "logits/rejected": 0.10471150279045105, "logps/chosen": -56.85927963256836, "logps/ref_chosen": -56.81915283203125, "logps/ref_rejected": -77.84333038330078, "logps/rejected": -77.85560607910156, "loss": 1.3894, "margin_dpo/margin_mean": -0.02784484624862671, "margin_dpo/margin_std": 0.35270577669143677, "step": 26 }, { "KL/chosen_KL_mean": -0.0006389617919921875, "KL/mean": -2.32011079788208e-05, "KL/rejected_KL_mean": 0.000591278076171875, "KL/std": 0.22139030694961548, "epoch": 0.04081632653061224, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0012349188327789307, "fcm_dpo/q_t": 0.5000306963920593, "grad_norm": 28.985897064208984, "learning_rate": 1.9402985074626865e-07, "logits/chosen": 0.11917729675769806, "logits/rejected": 0.09365066885948181, "logps/chosen": -62.877662658691406, "logps/ref_chosen": -62.87702560424805, "logps/ref_rejected": -71.34437561035156, "logps/rejected": -71.34378051757812, "loss": 1.3866, "margin_dpo/margin_mean": -0.0012350082397460938, "margin_dpo/margin_std": 0.27922385931015015, "step": 27 }, { "KL/chosen_KL_mean": -0.015308380126953125, "KL/mean": -0.01000969111919403, "KL/rejected_KL_mean": -0.0047149658203125, "KL/std": 0.21849636733531952, "epoch": 0.042328042328042326, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.010593622922897339, "fcm_dpo/q_t": 0.500264585018158, "grad_norm": 27.6954345703125, "learning_rate": 2.0149253731343282e-07, "logits/chosen": 0.07589665800333023, "logits/rejected": 0.06671995669603348, "logps/chosen": -59.84868621826172, "logps/ref_chosen": -59.8333740234375, "logps/ref_rejected": -70.39804077148438, "logps/rejected": -70.40275573730469, "loss": 1.3876, "margin_dpo/margin_mean": -0.010593622922897339, "margin_dpo/margin_std": 0.30503910779953003, "step": 28 }, { "KL/chosen_KL_mean": -0.04323577880859375, "KL/mean": -0.0652160719037056, "KL/rejected_KL_mean": -0.08718490600585938, "KL/std": 0.2304714471101761, "epoch": 0.04383975812547241, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04395388066768646, "fcm_dpo/q_t": 0.4989010691642761, "grad_norm": 32.57505416870117, "learning_rate": 2.08955223880597e-07, "logits/chosen": 0.15698179602622986, "logits/rejected": 0.13876661658287048, "logps/chosen": -74.16343688964844, "logps/ref_chosen": -74.12020111083984, "logps/ref_rejected": -83.33099365234375, "logps/rejected": -83.41817474365234, "loss": 1.3822, "margin_dpo/margin_mean": 0.04395401477813721, "margin_dpo/margin_std": 0.35602355003356934, "step": 29 }, { "KL/chosen_KL_mean": -0.008335113525390625, "KL/mean": -0.02375747263431549, "KL/rejected_KL_mean": -0.03917694091796875, "KL/std": 0.22164683043956757, "epoch": 0.045351473922902494, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03084227442741394, "fcm_dpo/q_t": 0.4992305040359497, "grad_norm": 30.354887008666992, "learning_rate": 2.1641791044776117e-07, "logits/chosen": 0.11798453330993652, "logits/rejected": 0.06400823593139648, "logps/chosen": -50.75962448120117, "logps/ref_chosen": -50.75128936767578, "logps/ref_rejected": -89.29063415527344, "logps/rejected": -89.3298110961914, "loss": 1.3835, "margin_dpo/margin_mean": 0.030841588973999023, "margin_dpo/margin_std": 0.33075201511383057, "step": 30 }, { "KL/chosen_KL_mean": -0.010515213012695312, "KL/mean": -0.03813314437866211, "KL/rejected_KL_mean": -0.0657501220703125, "KL/std": 0.25133079290390015, "epoch": 0.04686318972033258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05523216724395752, "fcm_dpo/q_t": 0.49862000346183777, "grad_norm": 34.2333869934082, "learning_rate": 2.2388059701492537e-07, "logits/chosen": 0.11453694105148315, "logits/rejected": 0.06778549402952194, "logps/chosen": -65.3472671508789, "logps/ref_chosen": -65.33675384521484, "logps/ref_rejected": -100.76666259765625, "logps/rejected": -100.83241271972656, "loss": 1.3811, "margin_dpo/margin_mean": 0.055232226848602295, "margin_dpo/margin_std": 0.3529159724712372, "step": 31 }, { "KL/chosen_KL_mean": -0.051006317138671875, "KL/mean": -0.03746132552623749, "KL/rejected_KL_mean": -0.02391815185546875, "KL/std": 0.24133098125457764, "epoch": 0.04837490551776266, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.027087301015853882, "fcm_dpo/q_t": 0.5006769895553589, "grad_norm": 30.098434448242188, "learning_rate": 2.3134328358208954e-07, "logits/chosen": 0.08350340276956558, "logits/rejected": 0.07563582062721252, "logps/chosen": -67.23433685302734, "logps/ref_chosen": -67.18333435058594, "logps/ref_rejected": -82.80763244628906, "logps/rejected": -82.83155822753906, "loss": 1.3893, "margin_dpo/margin_mean": -0.027087002992630005, "margin_dpo/margin_std": 0.3504248857498169, "step": 32 }, { "KL/chosen_KL_mean": -0.03555488586425781, "KL/mean": -0.0537736639380455, "KL/rejected_KL_mean": -0.071990966796875, "KL/std": 0.21045207977294922, "epoch": 0.049886621315192746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03643660247325897, "fcm_dpo/q_t": 0.4990893006324768, "grad_norm": 31.021913528442383, "learning_rate": 2.388059701492537e-07, "logits/chosen": 0.042121004313230515, "logits/rejected": 0.01561102643609047, "logps/chosen": -64.07503509521484, "logps/ref_chosen": -64.03948211669922, "logps/ref_rejected": -75.68357849121094, "logps/rejected": -75.75556945800781, "loss": 1.3829, "margin_dpo/margin_mean": 0.036436840891838074, "margin_dpo/margin_std": 0.2771301865577698, "step": 33 }, { "KL/chosen_KL_mean": -0.03723907470703125, "KL/mean": -0.061888277530670166, "KL/rejected_KL_mean": -0.08653831481933594, "KL/std": 0.23711252212524414, "epoch": 0.05139833711262283, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04930323362350464, "fcm_dpo/q_t": 0.498767614364624, "grad_norm": 28.449756622314453, "learning_rate": 2.4626865671641786e-07, "logits/chosen": 0.10028792172670364, "logits/rejected": 0.07043236494064331, "logps/chosen": -53.701534271240234, "logps/ref_chosen": -53.6642951965332, "logps/ref_rejected": -65.77989959716797, "logps/rejected": -65.86643981933594, "loss": 1.3817, "margin_dpo/margin_mean": 0.04930317401885986, "margin_dpo/margin_std": 0.34076127409935, "step": 34 }, { "KL/chosen_KL_mean": -0.09243202209472656, "KL/mean": -0.09407998621463776, "KL/rejected_KL_mean": -0.09572601318359375, "KL/std": 0.2501530349254608, "epoch": 0.05291005291005291, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003294020891189575, "fcm_dpo/q_t": 0.49991774559020996, "grad_norm": 27.86534309387207, "learning_rate": 2.537313432835821e-07, "logits/chosen": 0.05358413979411125, "logits/rejected": 0.031220555305480957, "logps/chosen": -61.10929489135742, "logps/ref_chosen": -61.01686096191406, "logps/ref_rejected": -72.78598022460938, "logps/rejected": -72.8817138671875, "loss": 1.3863, "margin_dpo/margin_mean": 0.0032941102981567383, "margin_dpo/margin_std": 0.36158987879753113, "step": 35 }, { "KL/chosen_KL_mean": -0.07643318176269531, "KL/mean": -0.08448834717273712, "KL/rejected_KL_mean": -0.09254074096679688, "KL/std": 0.2451927661895752, "epoch": 0.05442176870748299, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01610603928565979, "fcm_dpo/q_t": 0.4995979070663452, "grad_norm": 28.73918342590332, "learning_rate": 2.611940298507462e-07, "logits/chosen": 0.09999187290668488, "logits/rejected": 0.04726378619670868, "logps/chosen": -50.613792419433594, "logps/ref_chosen": -50.53736114501953, "logps/ref_rejected": -78.11678314208984, "logps/rejected": -78.20932006835938, "loss": 1.385, "margin_dpo/margin_mean": 0.01610630750656128, "margin_dpo/margin_std": 0.35909271240234375, "step": 36 }, { "KL/chosen_KL_mean": -0.008823394775390625, "KL/mean": -0.0899229645729065, "KL/rejected_KL_mean": -0.17102432250976562, "KL/std": 0.2953256070613861, "epoch": 0.055933484504913075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16220009326934814, "fcm_dpo/q_t": 0.49594852328300476, "grad_norm": 37.488372802734375, "learning_rate": 2.686567164179104e-07, "logits/chosen": 0.09731757640838623, "logits/rejected": 0.0182504802942276, "logps/chosen": -59.56277084350586, "logps/ref_chosen": -59.55394744873047, "logps/ref_rejected": -108.27702331542969, "logps/rejected": -108.44805145263672, "loss": 1.3706, "margin_dpo/margin_mean": 0.1622002124786377, "margin_dpo/margin_std": 0.436340868473053, "step": 37 }, { "KL/chosen_KL_mean": -0.06951332092285156, "KL/mean": -0.07658199965953827, "KL/rejected_KL_mean": -0.08365058898925781, "KL/std": 0.28319302201271057, "epoch": 0.05744520030234316, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01413390040397644, "fcm_dpo/q_t": 0.49964767694473267, "grad_norm": 29.705196380615234, "learning_rate": 2.761194029850746e-07, "logits/chosen": 0.06554099917411804, "logits/rejected": 0.05162451043725014, "logps/chosen": -65.85787200927734, "logps/ref_chosen": -65.78836059570312, "logps/ref_rejected": -76.1619873046875, "logps/rejected": -76.24563598632812, "loss": 1.3853, "margin_dpo/margin_mean": 0.014134079217910767, "margin_dpo/margin_std": 0.4092469811439514, "step": 38 }, { "KL/chosen_KL_mean": -0.09235954284667969, "KL/mean": -0.12527738511562347, "KL/rejected_KL_mean": -0.15819931030273438, "KL/std": 0.21432055532932281, "epoch": 0.05895691609977324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06583818793296814, "fcm_dpo/q_t": 0.4983542561531067, "grad_norm": 29.24547576904297, "learning_rate": 2.8358208955223876e-07, "logits/chosen": 0.15803761780261993, "logits/rejected": 0.1312660574913025, "logps/chosen": -57.269168853759766, "logps/ref_chosen": -57.17681121826172, "logps/ref_rejected": -79.486328125, "logps/rejected": -79.64453125, "loss": 1.3799, "margin_dpo/margin_mean": 0.0658378005027771, "margin_dpo/margin_std": 0.28493544459342957, "step": 39 }, { "KL/chosen_KL_mean": -0.0874176025390625, "KL/mean": -0.1004403755068779, "KL/rejected_KL_mean": -0.11346435546875, "KL/std": 0.251811683177948, "epoch": 0.06046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02604334056377411, "fcm_dpo/q_t": 0.4993489384651184, "grad_norm": 31.375804901123047, "learning_rate": 2.9104477611940296e-07, "logits/chosen": 0.09894993901252747, "logits/rejected": 0.04999292641878128, "logps/chosen": -61.42158508300781, "logps/ref_chosen": -61.33416748046875, "logps/ref_rejected": -79.10697174072266, "logps/rejected": -79.2204360961914, "loss": 1.384, "margin_dpo/margin_mean": 0.026043608784675598, "margin_dpo/margin_std": 0.3506956100463867, "step": 40 }, { "KL/chosen_KL_mean": -0.10820770263671875, "KL/mean": -0.1342216283082962, "KL/rejected_KL_mean": -0.16022872924804688, "KL/std": 0.24819834530353546, "epoch": 0.06198034769463341, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05201807618141174, "fcm_dpo/q_t": 0.49869972467422485, "grad_norm": 30.542781829833984, "learning_rate": 2.985074626865671e-07, "logits/chosen": 0.05286967754364014, "logits/rejected": 0.03276796638965607, "logps/chosen": -67.65493774414062, "logps/ref_chosen": -67.5467300415039, "logps/ref_rejected": -83.87788391113281, "logps/rejected": -84.03811645507812, "loss": 1.3814, "margin_dpo/margin_mean": 0.05201798677444458, "margin_dpo/margin_std": 0.33188194036483765, "step": 41 }, { "KL/chosen_KL_mean": -0.108795166015625, "KL/mean": -0.12974503636360168, "KL/rejected_KL_mean": -0.15069580078125, "KL/std": 0.30932196974754333, "epoch": 0.06349206349206349, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0419042706489563, "fcm_dpo/q_t": 0.49895286560058594, "grad_norm": 29.216102600097656, "learning_rate": 3.059701492537313e-07, "logits/chosen": 0.03016223944723606, "logits/rejected": 0.00932481698691845, "logps/chosen": -61.373653411865234, "logps/ref_chosen": -61.26485824584961, "logps/ref_rejected": -76.3629150390625, "logps/rejected": -76.51361083984375, "loss": 1.3826, "margin_dpo/margin_mean": 0.04190385341644287, "margin_dpo/margin_std": 0.43035653233528137, "step": 42 }, { "KL/chosen_KL_mean": -0.09076690673828125, "KL/mean": -0.135238379240036, "KL/rejected_KL_mean": -0.17971420288085938, "KL/std": 0.2912582755088806, "epoch": 0.06500377928949358, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08895289897918701, "fcm_dpo/q_t": 0.49777668714523315, "grad_norm": 34.40074157714844, "learning_rate": 3.134328358208955e-07, "logits/chosen": 0.09243658185005188, "logits/rejected": 0.081363745033741, "logps/chosen": -71.89979553222656, "logps/ref_chosen": -71.80902862548828, "logps/ref_rejected": -81.12464141845703, "logps/rejected": -81.30435180664062, "loss": 1.3778, "margin_dpo/margin_mean": 0.08895328640937805, "margin_dpo/margin_std": 0.39062684774398804, "step": 43 }, { "KL/chosen_KL_mean": -0.16825103759765625, "KL/mean": -0.17432071268558502, "KL/rejected_KL_mean": -0.180389404296875, "KL/std": 0.29811811447143555, "epoch": 0.06651549508692366, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012138456106185913, "fcm_dpo/q_t": 0.4996975064277649, "grad_norm": 32.57433319091797, "learning_rate": 3.2089552238805965e-07, "logits/chosen": 0.04532858729362488, "logits/rejected": 0.014902985654771328, "logps/chosen": -66.71868133544922, "logps/ref_chosen": -66.55043029785156, "logps/ref_rejected": -85.06198120117188, "logps/rejected": -85.24237060546875, "loss": 1.3856, "margin_dpo/margin_mean": 0.012138217687606812, "margin_dpo/margin_std": 0.445947527885437, "step": 44 }, { "KL/chosen_KL_mean": -0.12631607055664062, "KL/mean": -0.1697479486465454, "KL/rejected_KL_mean": -0.2131805419921875, "KL/std": 0.2817876935005188, "epoch": 0.06802721088435375, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08686672151088715, "fcm_dpo/q_t": 0.49782922863960266, "grad_norm": 31.86672592163086, "learning_rate": 3.2835820895522385e-07, "logits/chosen": 0.10761643946170807, "logits/rejected": 0.05567549169063568, "logps/chosen": -62.37017059326172, "logps/ref_chosen": -62.24385452270508, "logps/ref_rejected": -92.96665954589844, "logps/rejected": -93.17984008789062, "loss": 1.378, "margin_dpo/margin_mean": 0.08686742186546326, "margin_dpo/margin_std": 0.36542221903800964, "step": 45 }, { "KL/chosen_KL_mean": -0.10385322570800781, "KL/mean": -0.17652657628059387, "KL/rejected_KL_mean": -0.24920272827148438, "KL/std": 0.34992483258247375, "epoch": 0.06953892668178382, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14534282684326172, "fcm_dpo/q_t": 0.4963688254356384, "grad_norm": 31.092803955078125, "learning_rate": 3.3582089552238805e-07, "logits/chosen": 0.10892710834741592, "logits/rejected": 0.06376945227384567, "logps/chosen": -61.602760314941406, "logps/ref_chosen": -61.498905181884766, "logps/ref_rejected": -78.91172790527344, "logps/rejected": -79.16093444824219, "loss": 1.3723, "margin_dpo/margin_mean": 0.1453423798084259, "margin_dpo/margin_std": 0.44881075620651245, "step": 46 }, { "KL/chosen_KL_mean": -0.12169647216796875, "KL/mean": -0.2103060632944107, "KL/rejected_KL_mean": -0.2989158630371094, "KL/std": 0.2522382140159607, "epoch": 0.0710506424792139, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17722073197364807, "fcm_dpo/q_t": 0.49557074904441833, "grad_norm": 28.476625442504883, "learning_rate": 3.432835820895522e-07, "logits/chosen": 0.033945608884096146, "logits/rejected": -0.008820239454507828, "logps/chosen": -51.700042724609375, "logps/ref_chosen": -51.578346252441406, "logps/ref_rejected": -68.2215576171875, "logps/rejected": -68.52047729492188, "loss": 1.3689, "margin_dpo/margin_mean": 0.17722094058990479, "margin_dpo/margin_std": 0.31175607442855835, "step": 47 }, { "KL/chosen_KL_mean": -0.21759986877441406, "KL/mean": -0.234956756234169, "KL/rejected_KL_mean": -0.25231361389160156, "KL/std": 0.3202669024467468, "epoch": 0.07256235827664399, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.034711629152297974, "fcm_dpo/q_t": 0.49913373589515686, "grad_norm": 26.849285125732422, "learning_rate": 3.507462686567164e-07, "logits/chosen": 0.1537940800189972, "logits/rejected": 0.12394692003726959, "logps/chosen": -52.011253356933594, "logps/ref_chosen": -51.79365158081055, "logps/ref_rejected": -64.22503662109375, "logps/rejected": -64.47735595703125, "loss": 1.3833, "margin_dpo/margin_mean": 0.034712135791778564, "margin_dpo/margin_std": 0.4351728558540344, "step": 48 }, { "KL/chosen_KL_mean": -0.19132232666015625, "KL/mean": -0.24339309334754944, "KL/rejected_KL_mean": -0.29546356201171875, "KL/std": 0.35654354095458984, "epoch": 0.07407407407407407, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10414212942123413, "fcm_dpo/q_t": 0.49739664793014526, "grad_norm": 27.24091911315918, "learning_rate": 3.5820895522388055e-07, "logits/chosen": 0.01570693962275982, "logits/rejected": -0.00523674301803112, "logps/chosen": -58.325923919677734, "logps/ref_chosen": -58.13460159301758, "logps/ref_rejected": -64.63206481933594, "logps/rejected": -64.92753601074219, "loss": 1.3766, "margin_dpo/margin_mean": 0.10414272546768188, "margin_dpo/margin_std": 0.5124611854553223, "step": 49 }, { "KL/chosen_KL_mean": -0.2684955596923828, "KL/mean": -0.32318902015686035, "KL/rejected_KL_mean": -0.3778800964355469, "KL/std": 0.35013025999069214, "epoch": 0.07558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10938182473182678, "fcm_dpo/q_t": 0.4972665309906006, "grad_norm": 28.025882720947266, "learning_rate": 3.6567164179104475e-07, "logits/chosen": 0.11795582622289658, "logits/rejected": 0.08789660781621933, "logps/chosen": -53.124935150146484, "logps/ref_chosen": -52.85643768310547, "logps/ref_rejected": -72.17460632324219, "logps/rejected": -72.55248260498047, "loss": 1.3758, "margin_dpo/margin_mean": 0.10938182473182678, "margin_dpo/margin_std": 0.4207468628883362, "step": 50 }, { "KL/chosen_KL_mean": -0.25263214111328125, "KL/mean": -0.3954651355743408, "KL/rejected_KL_mean": -0.5383033752441406, "KL/std": 0.3872417211532593, "epoch": 0.07709750566893424, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2856694459915161, "fcm_dpo/q_t": 0.4928652048110962, "grad_norm": 30.83257293701172, "learning_rate": 3.7313432835820895e-07, "logits/chosen": 0.11152546107769012, "logits/rejected": 0.08300299197435379, "logps/chosen": -63.90907287597656, "logps/ref_chosen": -63.65644073486328, "logps/ref_rejected": -86.13229370117188, "logps/rejected": -86.67060089111328, "loss": 1.3585, "margin_dpo/margin_mean": 0.28566935658454895, "margin_dpo/margin_std": 0.4884032607078552, "step": 51 }, { "KL/chosen_KL_mean": -0.3018016815185547, "KL/mean": -0.4102283716201782, "KL/rejected_KL_mean": -0.5186576843261719, "KL/std": 0.4544343650341034, "epoch": 0.07860922146636433, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21685557067394257, "fcm_dpo/q_t": 0.494590163230896, "grad_norm": 32.574974060058594, "learning_rate": 3.805970149253731e-07, "logits/chosen": 0.07611118257045746, "logits/rejected": 0.027582643553614616, "logps/chosen": -68.14201354980469, "logps/ref_chosen": -67.8402099609375, "logps/ref_rejected": -96.97090911865234, "logps/rejected": -97.48956298828125, "loss": 1.3657, "margin_dpo/margin_mean": 0.21685530245304108, "margin_dpo/margin_std": 0.604822039604187, "step": 52 }, { "KL/chosen_KL_mean": -0.3174476623535156, "KL/mean": -0.4145428538322449, "KL/rejected_KL_mean": -0.5116405487060547, "KL/std": 0.40250444412231445, "epoch": 0.0801209372637944, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19419114291667938, "fcm_dpo/q_t": 0.49514681100845337, "grad_norm": 27.431989669799805, "learning_rate": 3.880597014925373e-07, "logits/chosen": 0.09542200714349747, "logits/rejected": 0.08459050208330154, "logps/chosen": -57.195587158203125, "logps/ref_chosen": -56.87813949584961, "logps/ref_rejected": -60.75569152832031, "logps/rejected": -61.267333984375, "loss": 1.3677, "margin_dpo/margin_mean": 0.1941910982131958, "margin_dpo/margin_std": 0.5309941172599792, "step": 53 }, { "KL/chosen_KL_mean": -0.3586311340332031, "KL/mean": -0.45110976696014404, "KL/rejected_KL_mean": -0.5435848236083984, "KL/std": 0.38474512100219727, "epoch": 0.08163265306122448, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18495365977287292, "fcm_dpo/q_t": 0.4953802227973938, "grad_norm": 26.548255920410156, "learning_rate": 3.9552238805970144e-07, "logits/chosen": 0.06870827078819275, "logits/rejected": 0.05323575437068939, "logps/chosen": -47.62554931640625, "logps/ref_chosen": -47.26692199707031, "logps/ref_rejected": -62.19426727294922, "logps/rejected": -62.737850189208984, "loss": 1.3685, "margin_dpo/margin_mean": 0.18495362997055054, "margin_dpo/margin_std": 0.4835350811481476, "step": 54 }, { "KL/chosen_KL_mean": -0.4072132110595703, "KL/mean": -0.5418218374252319, "KL/rejected_KL_mean": -0.6764335632324219, "KL/std": 0.5083746910095215, "epoch": 0.08314436885865457, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2692183256149292, "fcm_dpo/q_t": 0.49328285455703735, "grad_norm": 31.305131912231445, "learning_rate": 4.0298507462686564e-07, "logits/chosen": 0.05396968498826027, "logits/rejected": -0.02452990412712097, "logps/chosen": -50.73340606689453, "logps/ref_chosen": -50.32619094848633, "logps/ref_rejected": -92.44389343261719, "logps/rejected": -93.12032318115234, "loss": 1.3607, "margin_dpo/margin_mean": 0.26921796798706055, "margin_dpo/margin_std": 0.654507577419281, "step": 55 }, { "KL/chosen_KL_mean": -0.3615074157714844, "KL/mean": -0.4682575464248657, "KL/rejected_KL_mean": -0.5750026702880859, "KL/std": 0.52659672498703, "epoch": 0.08465608465608465, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21349409222602844, "fcm_dpo/q_t": 0.4946634769439697, "grad_norm": 27.466205596923828, "learning_rate": 4.1044776119402984e-07, "logits/chosen": 0.12170642614364624, "logits/rejected": 0.09938912093639374, "logps/chosen": -57.12847900390625, "logps/ref_chosen": -56.766971588134766, "logps/ref_rejected": -66.30504608154297, "logps/rejected": -66.88005065917969, "loss": 1.3663, "margin_dpo/margin_mean": 0.21349415183067322, "margin_dpo/margin_std": 0.6816796064376831, "step": 56 }, { "KL/chosen_KL_mean": -0.5067310333251953, "KL/mean": -0.6605712175369263, "KL/rejected_KL_mean": -0.8144111633300781, "KL/std": 0.5967442393302917, "epoch": 0.08616780045351474, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3076814115047455, "fcm_dpo/q_t": 0.4923190772533417, "grad_norm": 30.64410972595215, "learning_rate": 4.17910447761194e-07, "logits/chosen": 0.10049479454755783, "logits/rejected": 0.03633933141827583, "logps/chosen": -58.27447509765625, "logps/ref_chosen": -57.76774597167969, "logps/ref_rejected": -82.75698852539062, "logps/rejected": -83.57139587402344, "loss": 1.357, "margin_dpo/margin_mean": 0.3076811134815216, "margin_dpo/margin_std": 0.6833426356315613, "step": 57 }, { "KL/chosen_KL_mean": -0.5886154174804688, "KL/mean": -0.6974492073059082, "KL/rejected_KL_mean": -0.8062820434570312, "KL/std": 0.6851860284805298, "epoch": 0.08767951625094482, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21767285466194153, "fcm_dpo/q_t": 0.4946228861808777, "grad_norm": 30.1894588470459, "learning_rate": 4.253731343283582e-07, "logits/chosen": 0.04159823805093765, "logits/rejected": 0.026745518669486046, "logps/chosen": -73.35270690917969, "logps/ref_chosen": -72.76408386230469, "logps/ref_rejected": -84.49275207519531, "logps/rejected": -85.29903411865234, "loss": 1.3674, "margin_dpo/margin_mean": 0.2176724672317505, "margin_dpo/margin_std": 0.980886697769165, "step": 58 }, { "KL/chosen_KL_mean": -0.5488433837890625, "KL/mean": -0.6876722574234009, "KL/rejected_KL_mean": -0.8265056610107422, "KL/std": 0.6126680374145508, "epoch": 0.08919123204837491, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2776602506637573, "fcm_dpo/q_t": 0.4930819869041443, "grad_norm": 26.609149932861328, "learning_rate": 4.3283582089552234e-07, "logits/chosen": 0.13217273354530334, "logits/rejected": 0.0661400556564331, "logps/chosen": -50.36962127685547, "logps/ref_chosen": -49.820777893066406, "logps/ref_rejected": -77.14368438720703, "logps/rejected": -77.9701919555664, "loss": 1.3605, "margin_dpo/margin_mean": 0.2776605784893036, "margin_dpo/margin_std": 0.7779598832130432, "step": 59 }, { "KL/chosen_KL_mean": -0.6309585571289062, "KL/mean": -0.6522994041442871, "KL/rejected_KL_mean": -0.6736431121826172, "KL/std": 0.6409569978713989, "epoch": 0.09070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04268404841423035, "fcm_dpo/q_t": 0.49893319606781006, "grad_norm": 29.606698989868164, "learning_rate": 4.4029850746268654e-07, "logits/chosen": 0.14147618412971497, "logits/rejected": 0.14005886018276215, "logps/chosen": -63.85573196411133, "logps/ref_chosen": -63.22477340698242, "logps/ref_rejected": -61.360477447509766, "logps/rejected": -62.03411865234375, "loss": 1.3837, "margin_dpo/margin_mean": 0.04268389940261841, "margin_dpo/margin_std": 0.8073387145996094, "step": 60 }, { "KL/chosen_KL_mean": -0.7385311126708984, "KL/mean": -0.7797211408615112, "KL/rejected_KL_mean": -0.8209114074707031, "KL/std": 0.724621057510376, "epoch": 0.09221466364323508, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.082376629114151, "fcm_dpo/q_t": 0.49793773889541626, "grad_norm": 28.157575607299805, "learning_rate": 4.4776119402985074e-07, "logits/chosen": 0.1449713408946991, "logits/rejected": 0.1120869517326355, "logps/chosen": -49.75532913208008, "logps/ref_chosen": -49.01679992675781, "logps/ref_rejected": -74.90817260742188, "logps/rejected": -75.72908020019531, "loss": 1.3806, "margin_dpo/margin_mean": 0.08237650990486145, "margin_dpo/margin_std": 0.9990288019180298, "step": 61 }, { "KL/chosen_KL_mean": -0.75982666015625, "KL/mean": -0.855846107006073, "KL/rejected_KL_mean": -0.9518661499023438, "KL/std": 0.6547791361808777, "epoch": 0.09372637944066516, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19204318523406982, "fcm_dpo/q_t": 0.4952077269554138, "grad_norm": 28.638532638549805, "learning_rate": 4.552238805970149e-07, "logits/chosen": 0.0987754687666893, "logits/rejected": 0.06032714992761612, "logps/chosen": -63.511695861816406, "logps/ref_chosen": -62.751869201660156, "logps/ref_rejected": -78.93360900878906, "logps/rejected": -79.8854751586914, "loss": 1.3693, "margin_dpo/margin_mean": 0.19204312562942505, "margin_dpo/margin_std": 0.923231840133667, "step": 62 }, { "KL/chosen_KL_mean": -0.5642757415771484, "KL/mean": -0.8418329954147339, "KL/rejected_KL_mean": -1.1193962097167969, "KL/std": 0.6890658140182495, "epoch": 0.09523809523809523, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5551224946975708, "fcm_dpo/q_t": 0.48615702986717224, "grad_norm": 31.38890838623047, "learning_rate": 4.626865671641791e-07, "logits/chosen": 0.20929303765296936, "logits/rejected": 0.18352961540222168, "logps/chosen": -61.079524993896484, "logps/ref_chosen": -60.51525115966797, "logps/ref_rejected": -85.11021423339844, "logps/rejected": -86.2296142578125, "loss": 1.3334, "margin_dpo/margin_mean": 0.5551224946975708, "margin_dpo/margin_std": 0.8649230003356934, "step": 63 }, { "KL/chosen_KL_mean": -0.8340473175048828, "KL/mean": -0.874204695224762, "KL/rejected_KL_mean": -0.9143638610839844, "KL/std": 0.7256166934967041, "epoch": 0.09674981103552532, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08031821250915527, "fcm_dpo/q_t": 0.4979880452156067, "grad_norm": 26.190099716186523, "learning_rate": 4.701492537313433e-07, "logits/chosen": 0.0871717631816864, "logits/rejected": 0.062399592250585556, "logps/chosen": -52.0408935546875, "logps/ref_chosen": -51.20684814453125, "logps/ref_rejected": -66.93081665039062, "logps/rejected": -67.84518432617188, "loss": 1.3803, "margin_dpo/margin_mean": 0.08031848073005676, "margin_dpo/margin_std": 0.8882924318313599, "step": 64 }, { "KL/chosen_KL_mean": -0.8912067413330078, "KL/mean": -1.1177582740783691, "KL/rejected_KL_mean": -1.3443145751953125, "KL/std": 0.8927726149559021, "epoch": 0.0982615268329554, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.45310866832733154, "fcm_dpo/q_t": 0.4887198805809021, "grad_norm": 30.947498321533203, "learning_rate": 4.776119402985074e-07, "logits/chosen": 0.20429706573486328, "logits/rejected": 0.1743599772453308, "logps/chosen": -68.17990112304688, "logps/ref_chosen": -67.2886962890625, "logps/ref_rejected": -74.44281005859375, "logps/rejected": -75.78712463378906, "loss": 1.3447, "margin_dpo/margin_mean": 0.4531082510948181, "margin_dpo/margin_std": 1.1238809823989868, "step": 65 }, { "KL/chosen_KL_mean": -0.9344940185546875, "KL/mean": -1.0839293003082275, "KL/rejected_KL_mean": -1.2333641052246094, "KL/std": 0.8180510997772217, "epoch": 0.09977324263038549, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.298869788646698, "fcm_dpo/q_t": 0.49254971742630005, "grad_norm": 29.427141189575195, "learning_rate": 4.850746268656717e-07, "logits/chosen": 0.07761037349700928, "logits/rejected": 0.05442436411976814, "logps/chosen": -71.67790222167969, "logps/ref_chosen": -70.743408203125, "logps/ref_rejected": -77.26499938964844, "logps/rejected": -78.49836730957031, "loss": 1.3598, "margin_dpo/margin_mean": 0.29886963963508606, "margin_dpo/margin_std": 1.1081165075302124, "step": 66 }, { "KL/chosen_KL_mean": -0.74542236328125, "KL/mean": -0.9272383451461792, "KL/rejected_KL_mean": -1.1090545654296875, "KL/std": 0.750342845916748, "epoch": 0.10128495842781557, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.36363154649734497, "fcm_dpo/q_t": 0.49093663692474365, "grad_norm": 28.98794937133789, "learning_rate": 4.925373134328357e-07, "logits/chosen": 0.09056040644645691, "logits/rejected": 0.03486517444252968, "logps/chosen": -61.3480224609375, "logps/ref_chosen": -60.60260009765625, "logps/ref_rejected": -75.22235870361328, "logps/rejected": -76.33141326904297, "loss": 1.3523, "margin_dpo/margin_mean": 0.3636314272880554, "margin_dpo/margin_std": 0.9072555303573608, "step": 67 }, { "KL/chosen_KL_mean": -1.2406959533691406, "KL/mean": -1.4078385829925537, "KL/rejected_KL_mean": -1.574981689453125, "KL/std": 0.9226593971252441, "epoch": 0.10279667422524566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.33428969979286194, "fcm_dpo/q_t": 0.4916718304157257, "grad_norm": 31.385425567626953, "learning_rate": 5e-07, "logits/chosen": 0.06587998569011688, "logits/rejected": 0.035971544682979584, "logps/chosen": -78.76905822753906, "logps/ref_chosen": -77.52836608886719, "logps/ref_rejected": -93.17778015136719, "logps/rejected": -94.75276184082031, "loss": 1.3566, "margin_dpo/margin_mean": 0.3342894911766052, "margin_dpo/margin_std": 1.142065167427063, "step": 68 }, { "KL/chosen_KL_mean": -1.1218318939208984, "KL/mean": -1.4224199056625366, "KL/rejected_KL_mean": -1.7230072021484375, "KL/std": 0.9665278196334839, "epoch": 0.10430839002267574, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6011764407157898, "fcm_dpo/q_t": 0.48505643010139465, "grad_norm": 30.984333038330078, "learning_rate": 4.999965034812934e-07, "logits/chosen": 0.10411858558654785, "logits/rejected": 0.06089819222688675, "logps/chosen": -67.06488037109375, "logps/ref_chosen": -65.94305419921875, "logps/ref_rejected": -89.7735595703125, "logps/rejected": -91.49656677246094, "loss": 1.3307, "margin_dpo/margin_mean": 0.6011757850646973, "margin_dpo/margin_std": 1.1689398288726807, "step": 69 }, { "KL/chosen_KL_mean": -1.2116470336914062, "KL/mean": -1.4070619344711304, "KL/rejected_KL_mean": -1.6024742126464844, "KL/std": 0.9462293386459351, "epoch": 0.10582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.39082786440849304, "fcm_dpo/q_t": 0.49027496576309204, "grad_norm": 28.671159744262695, "learning_rate": 4.999860140229787e-07, "logits/chosen": 0.14850440621376038, "logits/rejected": 0.12535987794399261, "logps/chosen": -63.169559478759766, "logps/ref_chosen": -61.95791244506836, "logps/ref_rejected": -75.80945587158203, "logps/rejected": -77.41193389892578, "loss": 1.3516, "margin_dpo/margin_mean": 0.3908279538154602, "margin_dpo/margin_std": 1.268478274345398, "step": 70 }, { "KL/chosen_KL_mean": -1.4452323913574219, "KL/mean": -1.5527398586273193, "KL/rejected_KL_mean": -1.6602516174316406, "KL/std": 1.0212029218673706, "epoch": 0.1073318216175359, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2150188833475113, "fcm_dpo/q_t": 0.4946098327636719, "grad_norm": 28.61311912536621, "learning_rate": 4.999685319184688e-07, "logits/chosen": 0.09887754917144775, "logits/rejected": 0.08335565775632858, "logps/chosen": -64.79280853271484, "logps/ref_chosen": -63.34757995605469, "logps/ref_rejected": -67.49658203125, "logps/rejected": -69.15682983398438, "loss": 1.3706, "margin_dpo/margin_mean": 0.21501855552196503, "margin_dpo/margin_std": 1.4932568073272705, "step": 71 }, { "KL/chosen_KL_mean": -1.1772842407226562, "KL/mean": -1.5442196130752563, "KL/rejected_KL_mean": -1.9111518859863281, "KL/std": 1.0610476732254028, "epoch": 0.10884353741496598, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7338677644729614, "fcm_dpo/q_t": 0.4817340075969696, "grad_norm": 30.302104949951172, "learning_rate": 4.999440576567755e-07, "logits/chosen": 0.10478617250919342, "logits/rejected": 0.04250642657279968, "logps/chosen": -57.036582946777344, "logps/ref_chosen": -55.85929870605469, "logps/ref_rejected": -68.45423889160156, "logps/rejected": -70.36539459228516, "loss": 1.3182, "margin_dpo/margin_mean": 0.7338675260543823, "margin_dpo/margin_std": 1.2317919731140137, "step": 72 }, { "KL/chosen_KL_mean": -1.8173637390136719, "KL/mean": -1.8717069625854492, "KL/rejected_KL_mean": -1.9260501861572266, "KL/std": 1.2456495761871338, "epoch": 0.11035525321239607, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10868409276008606, "fcm_dpo/q_t": 0.4974502623081207, "grad_norm": 32.11731719970703, "learning_rate": 4.999125919224965e-07, "logits/chosen": 0.09151546657085419, "logits/rejected": 0.07785354554653168, "logps/chosen": -70.95616912841797, "logps/ref_chosen": -69.13880920410156, "logps/ref_rejected": -79.04586791992188, "logps/rejected": -80.971923828125, "loss": 1.3824, "margin_dpo/margin_mean": 0.10868486762046814, "margin_dpo/margin_std": 1.5893125534057617, "step": 73 }, { "KL/chosen_KL_mean": -1.2636852264404297, "KL/mean": -1.5985901355743408, "KL/rejected_KL_mean": -1.9334907531738281, "KL/std": 1.0060906410217285, "epoch": 0.11186696900982615, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6698073744773865, "fcm_dpo/q_t": 0.48337024450302124, "grad_norm": 27.95173454284668, "learning_rate": 4.998741355957963e-07, "logits/chosen": 0.14399270713329315, "logits/rejected": 0.0904250368475914, "logps/chosen": -51.18742370605469, "logps/ref_chosen": -49.923736572265625, "logps/ref_rejected": -81.73213958740234, "logps/rejected": -83.66563415527344, "loss": 1.3251, "margin_dpo/margin_mean": 0.6698077321052551, "margin_dpo/margin_std": 1.332854986190796, "step": 74 }, { "KL/chosen_KL_mean": -1.3782119750976562, "KL/mean": -1.7573962211608887, "KL/rejected_KL_mean": -2.1365795135498047, "KL/std": 1.0745444297790527, "epoch": 0.11337868480725624, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7583687901496887, "fcm_dpo/q_t": 0.4811515808105469, "grad_norm": 26.23224449157715, "learning_rate": 4.998286897523808e-07, "logits/chosen": 0.1194392591714859, "logits/rejected": 0.08680184185504913, "logps/chosen": -47.446964263916016, "logps/ref_chosen": -46.06875228881836, "logps/ref_rejected": -66.1181411743164, "logps/rejected": -68.25472259521484, "loss": 1.3163, "margin_dpo/margin_mean": 0.7583684921264648, "margin_dpo/margin_std": 1.3214094638824463, "step": 75 }, { "KL/chosen_KL_mean": -1.6166152954101562, "KL/mean": -1.7738971710205078, "KL/rejected_KL_mean": -1.931182861328125, "KL/std": 1.227593183517456, "epoch": 0.11489040060468632, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3145681321620941, "fcm_dpo/q_t": 0.49217700958251953, "grad_norm": 29.346458435058594, "learning_rate": 4.997762556634679e-07, "logits/chosen": 0.11621889472007751, "logits/rejected": 0.0730433315038681, "logps/chosen": -55.67936706542969, "logps/ref_chosen": -54.06275177001953, "logps/ref_rejected": -74.87464141845703, "logps/rejected": -76.80582427978516, "loss": 1.3614, "margin_dpo/margin_mean": 0.3145679235458374, "margin_dpo/margin_std": 1.596081256866455, "step": 76 }, { "KL/chosen_KL_mean": -1.6418571472167969, "KL/mean": -2.0117712020874023, "KL/rejected_KL_mean": -2.381683349609375, "KL/std": 1.2005259990692139, "epoch": 0.1164021164021164, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7398277521133423, "fcm_dpo/q_t": 0.48161664605140686, "grad_norm": 29.373077392578125, "learning_rate": 4.99716834795752e-07, "logits/chosen": 0.1848347783088684, "logits/rejected": 0.14166870713233948, "logps/chosen": -54.71794891357422, "logps/ref_chosen": -53.07609176635742, "logps/ref_rejected": -74.45601654052734, "logps/rejected": -76.83769989013672, "loss": 1.3187, "margin_dpo/margin_mean": 0.7398276329040527, "margin_dpo/margin_std": 1.3978897333145142, "step": 77 }, { "KL/chosen_KL_mean": -1.8243026733398438, "KL/mean": -2.026995897293091, "KL/rejected_KL_mean": -2.2296905517578125, "KL/std": 1.23481285572052, "epoch": 0.11791383219954649, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.40538251399993896, "fcm_dpo/q_t": 0.4899021089076996, "grad_norm": 29.415245056152344, "learning_rate": 4.996504288113623e-07, "logits/chosen": 0.09819696843624115, "logits/rejected": 0.07809644937515259, "logps/chosen": -69.54972839355469, "logps/ref_chosen": -67.72541809082031, "logps/ref_rejected": -79.03926849365234, "logps/rejected": -81.26895904541016, "loss": 1.3508, "margin_dpo/margin_mean": 0.40538227558135986, "margin_dpo/margin_std": 1.3632533550262451, "step": 78 }, { "KL/chosen_KL_mean": -1.7598819732666016, "KL/mean": -2.3264784812927246, "KL/rejected_KL_mean": -2.893077850341797, "KL/std": 1.5687503814697266, "epoch": 0.11942554799697656, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.133197546005249, "fcm_dpo/q_t": 0.4722020924091339, "grad_norm": 30.811988830566406, "learning_rate": 4.995770395678171e-07, "logits/chosen": 0.16424202919006348, "logits/rejected": 0.10485352575778961, "logps/chosen": -53.92052459716797, "logps/ref_chosen": -52.16064453125, "logps/ref_rejected": -83.31062316894531, "logps/rejected": -86.20369720458984, "loss": 1.2864, "margin_dpo/margin_mean": 1.133197546005249, "margin_dpo/margin_std": 2.019831895828247, "step": 79 }, { "KL/chosen_KL_mean": -2.059539794921875, "KL/mean": -2.3286733627319336, "KL/rejected_KL_mean": -2.5978050231933594, "KL/std": 1.3776694536209106, "epoch": 0.12093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5382720828056335, "fcm_dpo/q_t": 0.48672568798065186, "grad_norm": 28.456384658813477, "learning_rate": 4.994966691179711e-07, "logits/chosen": 0.16742676496505737, "logits/rejected": 0.10579954087734222, "logps/chosen": -63.47010040283203, "logps/ref_chosen": -61.410560607910156, "logps/ref_rejected": -78.66004943847656, "logps/rejected": -81.25785827636719, "loss": 1.3428, "margin_dpo/margin_mean": 0.5382719039916992, "margin_dpo/margin_std": 1.8884304761886597, "step": 80 }, { "KL/chosen_KL_mean": -2.1660118103027344, "KL/mean": -2.568941831588745, "KL/rejected_KL_mean": -2.9718704223632812, "KL/std": 1.4467761516571045, "epoch": 0.12244897959183673, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8058617115020752, "fcm_dpo/q_t": 0.4801437556743622, "grad_norm": 29.072032928466797, "learning_rate": 4.994093197099587e-07, "logits/chosen": 0.13002906739711761, "logits/rejected": 0.09610335528850555, "logps/chosen": -65.97038269042969, "logps/ref_chosen": -63.80437088012695, "logps/ref_rejected": -79.3484115600586, "logps/rejected": -82.32028198242188, "loss": 1.3155, "margin_dpo/margin_mean": 0.8058614730834961, "margin_dpo/margin_std": 1.7406330108642578, "step": 81 }, { "KL/chosen_KL_mean": -1.7479782104492188, "KL/mean": -2.3002214431762695, "KL/rejected_KL_mean": -2.852466583251953, "KL/std": 1.379155158996582, "epoch": 0.12396069538926682, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1044880151748657, "fcm_dpo/q_t": 0.47257328033447266, "grad_norm": 26.456167221069336, "learning_rate": 4.993149937871306e-07, "logits/chosen": 0.11616471409797668, "logits/rejected": 0.05289188772439957, "logps/chosen": -50.56587219238281, "logps/ref_chosen": -48.817893981933594, "logps/ref_rejected": -70.31497955322266, "logps/rejected": -73.16744995117188, "loss": 1.2853, "margin_dpo/margin_mean": 1.1044878959655762, "margin_dpo/margin_std": 1.5824530124664307, "step": 82 }, { "KL/chosen_KL_mean": -2.026599884033203, "KL/mean": -2.6412153244018555, "KL/rejected_KL_mean": -3.2558326721191406, "KL/std": 1.5034358501434326, "epoch": 0.1254724111866969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.229237675666809, "fcm_dpo/q_t": 0.4697047173976898, "grad_norm": 28.895191192626953, "learning_rate": 4.992136939879856e-07, "logits/chosen": 0.20435649156570435, "logits/rejected": 0.1537850946187973, "logps/chosen": -59.17737579345703, "logps/ref_chosen": -57.15077209472656, "logps/ref_rejected": -75.1710205078125, "logps/rejected": -78.4268569946289, "loss": 1.2759, "margin_dpo/margin_mean": 1.2292377948760986, "margin_dpo/margin_std": 1.876763105392456, "step": 83 }, { "KL/chosen_KL_mean": -2.5615406036376953, "KL/mean": -2.99251127243042, "KL/rejected_KL_mean": -3.4234886169433594, "KL/std": 1.8197745084762573, "epoch": 0.12698412698412698, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8619481325149536, "fcm_dpo/q_t": 0.4786767363548279, "grad_norm": 30.56605339050293, "learning_rate": 4.991054231460969e-07, "logits/chosen": 0.1459577977657318, "logits/rejected": 0.10570729523897171, "logps/chosen": -67.3388442993164, "logps/ref_chosen": -64.77729797363281, "logps/ref_rejected": -84.71949768066406, "logps/rejected": -88.14299011230469, "loss": 1.3109, "margin_dpo/margin_mean": 0.8619476556777954, "margin_dpo/margin_std": 1.8914613723754883, "step": 84 }, { "KL/chosen_KL_mean": -2.258808135986328, "KL/mean": -2.887326955795288, "KL/rejected_KL_mean": -3.5158462524414062, "KL/std": 1.792755365371704, "epoch": 0.12849584278155707, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2570416927337646, "fcm_dpo/q_t": 0.46897968649864197, "grad_norm": 26.82134246826172, "learning_rate": 4.989901842900325e-07, "logits/chosen": 0.17407938838005066, "logits/rejected": 0.12974150478839874, "logps/chosen": -52.51050567626953, "logps/ref_chosen": -50.25169372558594, "logps/ref_rejected": -66.55439758300781, "logps/rejected": -70.07023620605469, "loss": 1.2753, "margin_dpo/margin_mean": 1.2570418119430542, "margin_dpo/margin_std": 2.020524501800537, "step": 85 }, { "KL/chosen_KL_mean": -2.8597545623779297, "KL/mean": -3.328057289123535, "KL/rejected_KL_mean": -3.7963600158691406, "KL/std": 1.9231126308441162, "epoch": 0.13000755857898716, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9366029500961304, "fcm_dpo/q_t": 0.4768456518650055, "grad_norm": 27.291244506835938, "learning_rate": 4.988679806432711e-07, "logits/chosen": 0.1812715232372284, "logits/rejected": 0.16291239857673645, "logps/chosen": -63.58893585205078, "logps/ref_chosen": -60.72917938232422, "logps/ref_rejected": -72.30961608886719, "logps/rejected": -76.10597229003906, "loss": 1.3055, "margin_dpo/margin_mean": 0.936603307723999, "margin_dpo/margin_std": 2.06821346282959, "step": 86 }, { "KL/chosen_KL_mean": -3.0067214965820312, "KL/mean": -3.5237975120544434, "KL/rejected_KL_mean": -4.040866851806641, "KL/std": 2.07621431350708, "epoch": 0.13151927437641722, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.034153699874878, "fcm_dpo/q_t": 0.4745134115219116, "grad_norm": 30.608993530273438, "learning_rate": 4.987388156241114e-07, "logits/chosen": 0.16269248723983765, "logits/rejected": 0.10423875600099564, "logps/chosen": -68.76469421386719, "logps/ref_chosen": -65.75796508789062, "logps/ref_rejected": -84.81159973144531, "logps/rejected": -88.85246276855469, "loss": 1.3017, "margin_dpo/margin_mean": 1.0341542959213257, "margin_dpo/margin_std": 2.5299935340881348, "step": 87 }, { "KL/chosen_KL_mean": -3.0388126373291016, "KL/mean": -3.4782047271728516, "KL/rejected_KL_mean": -3.917591094970703, "KL/std": 2.3414268493652344, "epoch": 0.1330309901738473, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8787797689437866, "fcm_dpo/q_t": 0.4786525368690491, "grad_norm": 30.143423080444336, "learning_rate": 4.986026928455767e-07, "logits/chosen": 0.23746399581432343, "logits/rejected": 0.21029482781887054, "logps/chosen": -65.86283874511719, "logps/ref_chosen": -62.82402801513672, "logps/ref_rejected": -74.9607162475586, "logps/rejected": -78.87831115722656, "loss": 1.3231, "margin_dpo/margin_mean": 0.8787797689437866, "margin_dpo/margin_std": 2.9315881729125977, "step": 88 }, { "KL/chosen_KL_mean": -2.5165977478027344, "KL/mean": -3.214366912841797, "KL/rejected_KL_mean": -3.9121322631835938, "KL/std": 2.134425640106201, "epoch": 0.1345427059712774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3955327272415161, "fcm_dpo/q_t": 0.4659721851348877, "grad_norm": 29.33747100830078, "learning_rate": 4.984596161153135e-07, "logits/chosen": 0.23213061690330505, "logits/rejected": 0.15128636360168457, "logps/chosen": -43.708030700683594, "logps/ref_chosen": -41.191436767578125, "logps/ref_rejected": -85.44769287109375, "logps/rejected": -89.35983276367188, "loss": 1.268, "margin_dpo/margin_mean": 1.3955327272415161, "margin_dpo/margin_std": 2.4990651607513428, "step": 89 }, { "KL/chosen_KL_mean": -3.1267757415771484, "KL/mean": -3.7660017013549805, "KL/rejected_KL_mean": -4.405231475830078, "KL/std": 2.458479404449463, "epoch": 0.1360544217687075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.278447151184082, "fcm_dpo/q_t": 0.4686064124107361, "grad_norm": 29.23085594177246, "learning_rate": 4.983095894354857e-07, "logits/chosen": 0.13286244869232178, "logits/rejected": 0.0797010213136673, "logps/chosen": -59.71068572998047, "logps/ref_chosen": -56.58390808105469, "logps/ref_rejected": -86.86978149414062, "logps/rejected": -91.27500915527344, "loss": 1.2785, "margin_dpo/margin_mean": 1.2784464359283447, "margin_dpo/margin_std": 2.485658645629883, "step": 90 }, { "KL/chosen_KL_mean": -3.142923355102539, "KL/mean": -3.6927480697631836, "KL/rejected_KL_mean": -4.242572784423828, "KL/std": 2.266369342803955, "epoch": 0.13756613756613756, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0996489524841309, "fcm_dpo/q_t": 0.4733736217021942, "grad_norm": 25.786420822143555, "learning_rate": 4.98152617002662e-07, "logits/chosen": 0.17510226368904114, "logits/rejected": 0.13035190105438232, "logps/chosen": -55.5252685546875, "logps/ref_chosen": -52.38234329223633, "logps/ref_rejected": -72.17642211914062, "logps/rejected": -76.41899108886719, "loss": 1.2985, "margin_dpo/margin_mean": 1.09964919090271, "margin_dpo/margin_std": 2.7757925987243652, "step": 91 }, { "KL/chosen_KL_mean": -3.198556900024414, "KL/mean": -4.033766746520996, "KL/rejected_KL_mean": -4.868980407714844, "KL/std": 2.7098605632781982, "epoch": 0.13907785336356765, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.670418381690979, "fcm_dpo/q_t": 0.45981651544570923, "grad_norm": 26.77798080444336, "learning_rate": 4.979887032076988e-07, "logits/chosen": 0.21588903665542603, "logits/rejected": 0.17576783895492554, "logps/chosen": -56.20726013183594, "logps/ref_chosen": -53.00870132446289, "logps/ref_rejected": -79.77812957763672, "logps/rejected": -84.64710998535156, "loss": 1.2519, "margin_dpo/margin_mean": 1.6704179048538208, "margin_dpo/margin_std": 3.2294511795043945, "step": 92 }, { "KL/chosen_KL_mean": -3.550262451171875, "KL/mean": -4.06852912902832, "KL/rejected_KL_mean": -4.586795806884766, "KL/std": 2.6479063034057617, "epoch": 0.14058956916099774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0365362167358398, "fcm_dpo/q_t": 0.47524404525756836, "grad_norm": 24.42961311340332, "learning_rate": 4.978178526356172e-07, "logits/chosen": 0.21999691426753998, "logits/rejected": 0.18996566534042358, "logps/chosen": -48.457313537597656, "logps/ref_chosen": -44.90705108642578, "logps/ref_rejected": -58.7879524230957, "logps/rejected": -63.37474822998047, "loss": 1.3107, "margin_dpo/margin_mean": 1.0365365743637085, "margin_dpo/margin_std": 3.226975679397583, "step": 93 }, { "KL/chosen_KL_mean": -3.274158477783203, "KL/mean": -4.244356632232666, "KL/rejected_KL_mean": -5.214557647705078, "KL/std": 3.2750868797302246, "epoch": 0.1421012849584278, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9403965473175049, "fcm_dpo/q_t": 0.45354098081588745, "grad_norm": 27.23829460144043, "learning_rate": 4.976400700654751e-07, "logits/chosen": 0.2586434483528137, "logits/rejected": 0.21814142167568207, "logps/chosen": -63.21192932128906, "logps/ref_chosen": -59.93777084350586, "logps/ref_rejected": -79.3138427734375, "logps/rejected": -84.52839660644531, "loss": 1.2412, "margin_dpo/margin_mean": 1.9403966665267944, "margin_dpo/margin_std": 4.026358604431152, "step": 94 }, { "KL/chosen_KL_mean": -4.264810562133789, "KL/mean": -5.062840461730957, "KL/rejected_KL_mean": -5.860870361328125, "KL/std": 2.7003493309020996, "epoch": 0.1436130007558579, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5960580110549927, "fcm_dpo/q_t": 0.46138566732406616, "grad_norm": 29.303754806518555, "learning_rate": 4.974553604702332e-07, "logits/chosen": 0.16401472687721252, "logits/rejected": 0.09887714684009552, "logps/chosen": -64.43330383300781, "logps/ref_chosen": -60.168487548828125, "logps/ref_rejected": -90.73665618896484, "logps/rejected": -96.59752655029297, "loss": 1.2612, "margin_dpo/margin_mean": 1.5960575342178345, "margin_dpo/margin_std": 3.384384870529175, "step": 95 }, { "KL/chosen_KL_mean": -4.164896011352539, "KL/mean": -5.096510887145996, "KL/rejected_KL_mean": -6.028129577636719, "KL/std": 3.407775402069092, "epoch": 0.14512471655328799, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.863236904144287, "fcm_dpo/q_t": 0.45675230026245117, "grad_norm": 27.36403465270996, "learning_rate": 4.972637290166157e-07, "logits/chosen": 0.2020627111196518, "logits/rejected": 0.158769428730011, "logps/chosen": -64.83367156982422, "logps/ref_chosen": -60.66877746582031, "logps/ref_rejected": -88.30673217773438, "logps/rejected": -94.3348617553711, "loss": 1.2579, "margin_dpo/margin_mean": 1.863236427307129, "margin_dpo/margin_std": 4.4731011390686035, "step": 96 }, { "KL/chosen_KL_mean": -5.011560440063477, "KL/mean": -5.50346565246582, "KL/rejected_KL_mean": -5.995372772216797, "KL/std": 3.6453664302825928, "epoch": 0.14663643235071808, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9838161468505859, "fcm_dpo/q_t": 0.476479172706604, "grad_norm": 36.27960968017578, "learning_rate": 4.970651810649666e-07, "logits/chosen": 0.14650177955627441, "logits/rejected": 0.10253134369850159, "logps/chosen": -70.05567932128906, "logps/ref_chosen": -65.04412078857422, "logps/ref_rejected": -78.42092895507812, "logps/rejected": -84.41630554199219, "loss": 1.3507, "margin_dpo/margin_mean": 0.9838162660598755, "margin_dpo/margin_std": 5.006588935852051, "step": 97 }, { "KL/chosen_KL_mean": -4.654573440551758, "KL/mean": -4.997580528259277, "KL/rejected_KL_mean": -5.3405914306640625, "KL/std": 3.2552270889282227, "epoch": 0.14814814814814814, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6860167980194092, "fcm_dpo/q_t": 0.4829104542732239, "grad_norm": 30.320602416992188, "learning_rate": 4.968597221690985e-07, "logits/chosen": 0.252997487783432, "logits/rejected": 0.2249354124069214, "logps/chosen": -60.157806396484375, "logps/ref_chosen": -55.503231048583984, "logps/ref_rejected": -72.81553649902344, "logps/rejected": -78.1561279296875, "loss": 1.3452, "margin_dpo/margin_mean": 0.6860172748565674, "margin_dpo/margin_std": 3.23012113571167, "step": 98 }, { "KL/chosen_KL_mean": -4.6743316650390625, "KL/mean": -5.189509391784668, "KL/rejected_KL_mean": -5.704692840576172, "KL/std": 3.4976234436035156, "epoch": 0.14965986394557823, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0303622484207153, "fcm_dpo/q_t": 0.47562655806541443, "grad_norm": 32.572608947753906, "learning_rate": 4.966473580761389e-07, "logits/chosen": 0.24607810378074646, "logits/rejected": 0.2065376341342926, "logps/chosen": -63.249969482421875, "logps/ref_chosen": -58.57563781738281, "logps/ref_rejected": -78.693603515625, "logps/rejected": -84.39830017089844, "loss": 1.3361, "margin_dpo/margin_mean": 1.0303621292114258, "margin_dpo/margin_std": 4.529152870178223, "step": 99 }, { "KL/chosen_KL_mean": -5.290992736816406, "KL/mean": -6.019989967346191, "KL/rejected_KL_mean": -6.748981475830078, "KL/std": 4.169232368469238, "epoch": 0.15117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4579877853393555, "fcm_dpo/q_t": 0.46763741970062256, "grad_norm": 33.273681640625, "learning_rate": 4.964280947263676e-07, "logits/chosen": 0.20944450795650482, "logits/rejected": 0.20199376344680786, "logps/chosen": -84.87442779541016, "logps/ref_chosen": -79.58343505859375, "logps/ref_rejected": -92.152587890625, "logps/rejected": -98.90156555175781, "loss": 1.332, "margin_dpo/margin_mean": 1.4579877853393555, "margin_dpo/margin_std": 5.84016752243042, "step": 100 }, { "KL/chosen_KL_mean": -4.271394729614258, "KL/mean": -5.309800148010254, "KL/rejected_KL_mean": -6.34820556640625, "KL/std": 3.68625545501709, "epoch": 0.15268329554043839, "fcm_dpo/beta": 0.1042826771736145, "fcm_dpo/delta": 0.08218219131231308, "fcm_dpo/margin": 2.076812267303467, "fcm_dpo/q_t": 0.4498174786567688, "grad_norm": 27.07867431640625, "learning_rate": 4.96201938253052e-07, "logits/chosen": 0.23947615921497345, "logits/rejected": 0.19994020462036133, "logps/chosen": -56.60417938232422, "logps/ref_chosen": -52.332786560058594, "logps/ref_rejected": -69.55589294433594, "logps/rejected": -75.90409851074219, "loss": 1.2283, "margin_dpo/margin_mean": 2.076812267303467, "margin_dpo/margin_std": 4.143991470336914, "step": 101 }, { "KL/chosen_KL_mean": -5.339498519897461, "KL/mean": -5.742995262145996, "KL/rejected_KL_mean": -6.146495819091797, "KL/std": 3.6950936317443848, "epoch": 0.15419501133786848, "fcm_dpo/beta": 0.10856536030769348, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8069917559623718, "fcm_dpo/q_t": 0.4828066825866699, "grad_norm": 36.31706619262695, "learning_rate": 4.959688949822748e-07, "logits/chosen": 0.1819576621055603, "logits/rejected": 0.1423925906419754, "logps/chosen": -70.08297729492188, "logps/ref_chosen": -64.74348449707031, "logps/ref_rejected": -69.06132507324219, "logps/rejected": -75.20782470703125, "loss": 1.3832, "margin_dpo/margin_mean": 0.8069913387298584, "margin_dpo/margin_std": 5.488136291503906, "step": 102 }, { "KL/chosen_KL_mean": -4.825580596923828, "KL/mean": -5.593038558959961, "KL/rejected_KL_mean": -6.360500335693359, "KL/std": 3.912938117980957, "epoch": 0.15570672713529857, "fcm_dpo/beta": 0.11347013711929321, "fcm_dpo/delta": 0.08650438487529755, "fcm_dpo/margin": 1.5349235534667969, "fcm_dpo/q_t": 0.46038591861724854, "grad_norm": 33.3403434753418, "learning_rate": 4.957289714327572e-07, "logits/chosen": 0.2608705461025238, "logits/rejected": 0.22703281044960022, "logps/chosen": -68.66222381591797, "logps/ref_chosen": -63.83664321899414, "logps/ref_rejected": -79.32362365722656, "logps/rejected": -85.68412780761719, "loss": 1.2615, "margin_dpo/margin_mean": 1.5349233150482178, "margin_dpo/margin_std": 3.726921558380127, "step": 103 }, { "KL/chosen_KL_mean": -4.706798553466797, "KL/mean": -5.820075035095215, "KL/rejected_KL_mean": -6.933357238769531, "KL/std": 4.9595770835876465, "epoch": 0.15721844293272866, "fcm_dpo/beta": 0.13176845014095306, "fcm_dpo/delta": 0.12116898596286774, "fcm_dpo/margin": 2.2265567779541016, "fcm_dpo/q_t": 0.4444318413734436, "grad_norm": 41.81438446044922, "learning_rate": 4.954821743156767e-07, "logits/chosen": 0.24606768786907196, "logits/rejected": 0.15957264602184296, "logps/chosen": -65.70600891113281, "logps/ref_chosen": -60.99920654296875, "logps/ref_rejected": -98.84645080566406, "logps/rejected": -105.77981567382812, "loss": 1.2672, "margin_dpo/margin_mean": 2.2265567779541016, "margin_dpo/margin_std": 6.326627731323242, "step": 104 }, { "KL/chosen_KL_mean": -5.340984344482422, "KL/mean": -6.228916168212891, "KL/rejected_KL_mean": -7.116844177246094, "KL/std": 3.9524288177490234, "epoch": 0.15873015873015872, "fcm_dpo/beta": 0.13953134417533875, "fcm_dpo/delta": 0.0847325474023819, "fcm_dpo/margin": 1.7758561372756958, "fcm_dpo/q_t": 0.44833284616470337, "grad_norm": 43.252464294433594, "learning_rate": 4.952285105344791e-07, "logits/chosen": 0.1813499629497528, "logits/rejected": 0.12760114669799805, "logps/chosen": -76.29125213623047, "logps/ref_chosen": -70.95027160644531, "logps/ref_rejected": -87.88340759277344, "logps/rejected": -95.00025939941406, "loss": 1.259, "margin_dpo/margin_mean": 1.7758560180664062, "margin_dpo/margin_std": 4.853283882141113, "step": 105 }, { "KL/chosen_KL_mean": -4.989650726318359, "KL/mean": -5.752998352050781, "KL/rejected_KL_mean": -6.5163421630859375, "KL/std": 3.8832058906555176, "epoch": 0.1602418745275888, "fcm_dpo/beta": 0.16595645248889923, "fcm_dpo/delta": 0.16586124897003174, "fcm_dpo/margin": 1.5266880989074707, "fcm_dpo/q_t": 0.44716203212738037, "grad_norm": 55.961551666259766, "learning_rate": 4.949679871846857e-07, "logits/chosen": 0.256155788898468, "logits/rejected": 0.24164044857025146, "logps/chosen": -67.44898223876953, "logps/ref_chosen": -62.45933151245117, "logps/ref_rejected": -67.00595092773438, "logps/rejected": -73.52229309082031, "loss": 1.3149, "margin_dpo/margin_mean": 1.5266880989074707, "margin_dpo/margin_std": 5.232348442077637, "step": 106 }, { "KL/chosen_KL_mean": -6.383148193359375, "KL/mean": -6.799151420593262, "KL/rejected_KL_mean": -7.21514892578125, "KL/std": 4.821557998657227, "epoch": 0.1617535903250189, "fcm_dpo/beta": 0.1813592165708542, "fcm_dpo/delta": 0.05486012250185013, "fcm_dpo/margin": 0.832004725933075, "fcm_dpo/q_t": 0.4626944661140442, "grad_norm": 88.15316009521484, "learning_rate": 4.947006115536947e-07, "logits/chosen": 0.14859333634376526, "logits/rejected": 0.1259216070175171, "logps/chosen": -82.22111511230469, "logps/ref_chosen": -75.83796691894531, "logps/ref_rejected": -87.74038696289062, "logps/rejected": -94.95553588867188, "loss": 1.5013, "margin_dpo/margin_mean": 0.8320047855377197, "margin_dpo/margin_std": 6.031615257263184, "step": 107 }, { "KL/chosen_KL_mean": -5.376251220703125, "KL/mean": -6.28802490234375, "KL/rejected_KL_mean": -7.199794769287109, "KL/std": 3.9641804695129395, "epoch": 0.16326530612244897, "fcm_dpo/beta": 0.19046062231063843, "fcm_dpo/delta": 0.0635533481836319, "fcm_dpo/margin": 1.823544979095459, "fcm_dpo/q_t": 0.432595431804657, "grad_norm": 58.52869415283203, "learning_rate": 4.944263911205772e-07, "logits/chosen": 0.20340561866760254, "logits/rejected": 0.17100761830806732, "logps/chosen": -73.76948547363281, "logps/ref_chosen": -68.39323425292969, "logps/ref_rejected": -83.24267578125, "logps/rejected": -90.44246673583984, "loss": 1.2828, "margin_dpo/margin_mean": 1.8235454559326172, "margin_dpo/margin_std": 5.186200141906738, "step": 108 }, { "KL/chosen_KL_mean": -4.721492767333984, "KL/mean": -6.179901123046875, "KL/rejected_KL_mean": -7.638313293457031, "KL/std": 4.794259071350098, "epoch": 0.16477702191987906, "fcm_dpo/beta": 0.17355400323867798, "fcm_dpo/delta": -0.14321784675121307, "fcm_dpo/margin": 2.916821241378784, "fcm_dpo/q_t": 0.3970474600791931, "grad_norm": 45.6633415222168, "learning_rate": 4.941453335558681e-07, "logits/chosen": 0.2231045961380005, "logits/rejected": 0.16527457535266876, "logps/chosen": -60.24897384643555, "logps/ref_chosen": -55.52748107910156, "logps/ref_rejected": -83.55218505859375, "logps/rejected": -91.19050598144531, "loss": 1.0975, "margin_dpo/margin_mean": 2.916820526123047, "margin_dpo/margin_std": 5.183098793029785, "step": 109 }, { "KL/chosen_KL_mean": -5.953372955322266, "KL/mean": -6.1177167892456055, "KL/rejected_KL_mean": -6.282051086425781, "KL/std": 3.8472280502319336, "epoch": 0.16628873771730915, "fcm_dpo/beta": 0.16747182607650757, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3286726772785187, "fcm_dpo/q_t": 0.4821946620941162, "grad_norm": 79.94758605957031, "learning_rate": 4.938574467213517e-07, "logits/chosen": 0.1649562120437622, "logits/rejected": 0.17404744029045105, "logps/chosen": -87.11212158203125, "logps/ref_chosen": -81.15874481201172, "logps/ref_rejected": -72.56021118164062, "logps/rejected": -78.8422622680664, "loss": 1.4935, "margin_dpo/margin_mean": 0.32867202162742615, "margin_dpo/margin_std": 5.137603759765625, "step": 110 }, { "KL/chosen_KL_mean": -5.388044357299805, "KL/mean": -6.336487770080566, "KL/rejected_KL_mean": -7.284934997558594, "KL/std": 3.9769458770751953, "epoch": 0.16780045351473924, "fcm_dpo/beta": 0.17862893640995026, "fcm_dpo/delta": 0.07212649285793304, "fcm_dpo/margin": 1.8968864679336548, "fcm_dpo/q_t": 0.43422025442123413, "grad_norm": 47.36850357055664, "learning_rate": 4.935627386698418e-07, "logits/chosen": 0.2890339493751526, "logits/rejected": 0.25080791115760803, "logps/chosen": -57.74702835083008, "logps/ref_chosen": -52.358985900878906, "logps/ref_rejected": -77.06150817871094, "logps/rejected": -84.346435546875, "loss": 1.2395, "margin_dpo/margin_mean": 1.8968862295150757, "margin_dpo/margin_std": 4.821080207824707, "step": 111 }, { "KL/chosen_KL_mean": -4.940637588500977, "KL/mean": -6.110255241394043, "KL/rejected_KL_mean": -7.279872894287109, "KL/std": 3.8712010383605957, "epoch": 0.1693121693121693, "fcm_dpo/beta": 0.17799095809459686, "fcm_dpo/delta": -0.020651305094361305, "fcm_dpo/margin": 2.3392438888549805, "fcm_dpo/q_t": 0.40773606300354004, "grad_norm": 54.7916259765625, "learning_rate": 4.932612176449559e-07, "logits/chosen": 0.1744939088821411, "logits/rejected": 0.11035922169685364, "logps/chosen": -67.96070098876953, "logps/ref_chosen": -63.02006530761719, "logps/ref_rejected": -111.36941528320312, "logps/rejected": -118.6492919921875, "loss": 1.1484, "margin_dpo/margin_mean": 2.3392434120178223, "margin_dpo/margin_std": 4.352907657623291, "step": 112 }, { "KL/chosen_KL_mean": -4.952281951904297, "KL/mean": -5.632550239562988, "KL/rejected_KL_mean": -6.312816619873047, "KL/std": 3.2599143981933594, "epoch": 0.1708238851095994, "fcm_dpo/beta": 0.1789640188217163, "fcm_dpo/delta": 0.029570240527391434, "fcm_dpo/margin": 1.3605376482009888, "fcm_dpo/q_t": 0.4449806213378906, "grad_norm": 65.98124694824219, "learning_rate": 4.929528920808854e-07, "logits/chosen": 0.2306734025478363, "logits/rejected": 0.18952682614326477, "logps/chosen": -60.759944915771484, "logps/ref_chosen": -55.80766296386719, "logps/ref_rejected": -69.84014129638672, "logps/rejected": -76.1529541015625, "loss": 1.2834, "margin_dpo/margin_mean": 1.3605376482009888, "margin_dpo/margin_std": 4.0923542976379395, "step": 113 }, { "KL/chosen_KL_mean": -4.631011962890625, "KL/mean": -5.725174427032471, "KL/rejected_KL_mean": -6.819339752197266, "KL/std": 3.478522539138794, "epoch": 0.17233560090702948, "fcm_dpo/beta": 0.1806703805923462, "fcm_dpo/delta": 0.005448013544082642, "fcm_dpo/margin": 2.188323497772217, "fcm_dpo/q_t": 0.41690295934677124, "grad_norm": 52.3661994934082, "learning_rate": 4.92637770602159e-07, "logits/chosen": 0.2191499024629593, "logits/rejected": 0.15943503379821777, "logps/chosen": -70.96378326416016, "logps/ref_chosen": -66.33277130126953, "logps/ref_rejected": -71.61489868164062, "logps/rejected": -78.43423461914062, "loss": 1.1574, "margin_dpo/margin_mean": 2.1883232593536377, "margin_dpo/margin_std": 4.21035623550415, "step": 114 }, { "KL/chosen_KL_mean": -4.939592361450195, "KL/mean": -5.821917533874512, "KL/rejected_KL_mean": -6.704244613647461, "KL/std": 3.1544241905212402, "epoch": 0.17384731670445955, "fcm_dpo/beta": 0.19181868433952332, "fcm_dpo/delta": 0.07429927587509155, "fcm_dpo/margin": 1.7646498680114746, "fcm_dpo/q_t": 0.42928892374038696, "grad_norm": 54.519893646240234, "learning_rate": 4.923158620234019e-07, "logits/chosen": 0.2349502593278885, "logits/rejected": 0.1712397038936615, "logps/chosen": -60.688629150390625, "logps/ref_chosen": -55.74903869628906, "logps/ref_rejected": -79.59849548339844, "logps/rejected": -86.30274200439453, "loss": 1.1963, "margin_dpo/margin_mean": 1.7646501064300537, "margin_dpo/margin_std": 3.7572741508483887, "step": 115 }, { "KL/chosen_KL_mean": -4.6262359619140625, "KL/mean": -5.866840362548828, "KL/rejected_KL_mean": -7.107444763183594, "KL/std": 3.217970848083496, "epoch": 0.17535903250188964, "fcm_dpo/beta": 0.18887044489383698, "fcm_dpo/delta": -0.08994907140731812, "fcm_dpo/margin": 2.481212615966797, "fcm_dpo/q_t": 0.3914183974266052, "grad_norm": 42.89326095581055, "learning_rate": 4.91987175349089e-07, "logits/chosen": 0.2203093022108078, "logits/rejected": 0.15200431644916534, "logps/chosen": -53.99140167236328, "logps/ref_chosen": -49.36516571044922, "logps/ref_rejected": -72.84671020507812, "logps/rejected": -79.95414733886719, "loss": 1.0722, "margin_dpo/margin_mean": 2.481212615966797, "margin_dpo/margin_std": 3.4258229732513428, "step": 116 }, { "KL/chosen_KL_mean": -4.495201110839844, "KL/mean": -5.37297248840332, "KL/rejected_KL_mean": -6.250743865966797, "KL/std": 2.998063325881958, "epoch": 0.17687074829931973, "fcm_dpo/beta": 0.16782402992248535, "fcm_dpo/delta": -0.06874273717403412, "fcm_dpo/margin": 1.755537509918213, "fcm_dpo/q_t": 0.4289935231208801, "grad_norm": 46.18778991699219, "learning_rate": 4.916517197732933e-07, "logits/chosen": 0.2181738018989563, "logits/rejected": 0.18157288432121277, "logps/chosen": -62.20610046386719, "logps/ref_chosen": -57.710899353027344, "logps/ref_rejected": -69.77253723144531, "logps/rejected": -76.02328491210938, "loss": 1.2104, "margin_dpo/margin_mean": 1.7555381059646606, "margin_dpo/margin_std": 3.6577625274658203, "step": 117 }, { "KL/chosen_KL_mean": -4.316884994506836, "KL/mean": -5.443312644958496, "KL/rejected_KL_mean": -6.569736480712891, "KL/std": 3.2011799812316895, "epoch": 0.17838246409674982, "fcm_dpo/beta": 0.1683710217475891, "fcm_dpo/delta": 0.02489166334271431, "fcm_dpo/margin": 2.252856731414795, "fcm_dpo/q_t": 0.4135647118091583, "grad_norm": 44.422664642333984, "learning_rate": 4.913095046794281e-07, "logits/chosen": 0.26736220717430115, "logits/rejected": 0.22663536667823792, "logps/chosen": -56.796783447265625, "logps/ref_chosen": -52.479896545410156, "logps/ref_rejected": -81.359130859375, "logps/rejected": -87.92886352539062, "loss": 1.1169, "margin_dpo/margin_mean": 2.252856492996216, "margin_dpo/margin_std": 3.2952795028686523, "step": 118 }, { "KL/chosen_KL_mean": -5.435281753540039, "KL/mean": -6.261282920837402, "KL/rejected_KL_mean": -7.087284088134766, "KL/std": 3.5549156665802, "epoch": 0.17989417989417988, "fcm_dpo/beta": 0.1748862862586975, "fcm_dpo/delta": 0.03239409253001213, "fcm_dpo/margin": 1.6520025730133057, "fcm_dpo/q_t": 0.4375898838043213, "grad_norm": 50.825462341308594, "learning_rate": 4.909605396399855e-07, "logits/chosen": 0.17994274199008942, "logits/rejected": 0.14161953330039978, "logps/chosen": -66.79295349121094, "logps/ref_chosen": -61.35767364501953, "logps/ref_rejected": -75.71510314941406, "logps/rejected": -82.80238342285156, "loss": 1.2475, "margin_dpo/margin_mean": 1.6520024538040161, "margin_dpo/margin_std": 4.303752899169922, "step": 119 }, { "KL/chosen_KL_mean": -4.451520919799805, "KL/mean": -5.66746711730957, "KL/rejected_KL_mean": -6.883419036865234, "KL/std": 3.1244192123413086, "epoch": 0.18140589569160998, "fcm_dpo/beta": 0.17477713525295258, "fcm_dpo/delta": -0.03177126869559288, "fcm_dpo/margin": 2.4318950176239014, "fcm_dpo/q_t": 0.4027881622314453, "grad_norm": 45.37185287475586, "learning_rate": 4.906048344162676e-07, "logits/chosen": 0.18768131732940674, "logits/rejected": 0.12764175236225128, "logps/chosen": -64.35908508300781, "logps/ref_chosen": -59.907569885253906, "logps/ref_rejected": -79.6910629272461, "logps/rejected": -86.57447814941406, "loss": 1.0784, "margin_dpo/margin_mean": 2.4318952560424805, "margin_dpo/margin_std": 3.188009262084961, "step": 120 }, { "KL/chosen_KL_mean": -4.891788482666016, "KL/mean": -6.08499002456665, "KL/rejected_KL_mean": -7.278194427490234, "KL/std": 3.485968589782715, "epoch": 0.18291761148904007, "fcm_dpo/beta": 0.1698891520500183, "fcm_dpo/delta": -0.0071594808250665665, "fcm_dpo/margin": 2.3864026069641113, "fcm_dpo/q_t": 0.4125250279903412, "grad_norm": 41.51447677612305, "learning_rate": 4.902423989581143e-07, "logits/chosen": 0.2931978404521942, "logits/rejected": 0.20720380544662476, "logps/chosen": -60.557830810546875, "logps/ref_chosen": -55.66604232788086, "logps/ref_rejected": -101.56233978271484, "logps/rejected": -108.84053039550781, "loss": 1.1083, "margin_dpo/margin_mean": 2.3864026069641113, "margin_dpo/margin_std": 3.758132219314575, "step": 121 }, { "KL/chosen_KL_mean": -5.182319641113281, "KL/mean": -6.3793487548828125, "KL/rejected_KL_mean": -7.576377868652344, "KL/std": 3.738175868988037, "epoch": 0.18442932728647016, "fcm_dpo/beta": 0.17170721292495728, "fcm_dpo/delta": -0.014732498675584793, "fcm_dpo/margin": 2.394054889678955, "fcm_dpo/q_t": 0.414374977350235, "grad_norm": 51.34184646606445, "learning_rate": 4.898732434036243e-07, "logits/chosen": 0.18007788062095642, "logits/rejected": 0.14575596153736115, "logps/chosen": -68.51669311523438, "logps/ref_chosen": -63.334373474121094, "logps/ref_rejected": -73.67523193359375, "logps/rejected": -81.25161743164062, "loss": 1.14, "margin_dpo/margin_mean": 2.394054412841797, "margin_dpo/margin_std": 4.409090995788574, "step": 122 }, { "KL/chosen_KL_mean": -4.814731597900391, "KL/mean": -5.987967014312744, "KL/rejected_KL_mean": -7.161205291748047, "KL/std": 3.269620180130005, "epoch": 0.18594104308390022, "fcm_dpo/beta": 0.17096179723739624, "fcm_dpo/delta": -0.0026538409292697906, "fcm_dpo/margin": 2.3464741706848145, "fcm_dpo/q_t": 0.4092286229133606, "grad_norm": 42.88996124267578, "learning_rate": 4.894973780788722e-07, "logits/chosen": 0.2510654628276825, "logits/rejected": 0.20839804410934448, "logps/chosen": -61.71347427368164, "logps/ref_chosen": -56.89874267578125, "logps/ref_rejected": -78.97028350830078, "logps/rejected": -86.13148498535156, "loss": 1.1104, "margin_dpo/margin_mean": 2.3464739322662354, "margin_dpo/margin_std": 3.5289008617401123, "step": 123 }, { "KL/chosen_KL_mean": -5.191936492919922, "KL/mean": -6.721776962280273, "KL/rejected_KL_mean": -8.251617431640625, "KL/std": 3.5442142486572266, "epoch": 0.1874527588813303, "fcm_dpo/beta": 0.15416131913661957, "fcm_dpo/delta": -0.09668783843517303, "fcm_dpo/margin": 3.059680938720703, "fcm_dpo/q_t": 0.3888171315193176, "grad_norm": 37.990482330322266, "learning_rate": 4.89114813497619e-07, "logits/chosen": 0.24223476648330688, "logits/rejected": 0.18118014931678772, "logps/chosen": -62.308021545410156, "logps/ref_chosen": -57.116085052490234, "logps/ref_rejected": -87.93074035644531, "logps/rejected": -96.18235778808594, "loss": 1.0411, "margin_dpo/margin_mean": 3.0596816539764404, "margin_dpo/margin_std": 3.643080949783325, "step": 124 }, { "KL/chosen_KL_mean": -5.501195907592773, "KL/mean": -6.80970573425293, "KL/rejected_KL_mean": -8.118209838867188, "KL/std": 3.810020923614502, "epoch": 0.1889644746787604, "fcm_dpo/beta": 0.14969037473201752, "fcm_dpo/delta": 0.007746942341327667, "fcm_dpo/margin": 2.6170156002044678, "fcm_dpo/q_t": 0.4116974472999573, "grad_norm": 41.211639404296875, "learning_rate": 4.887255603610184e-07, "logits/chosen": 0.26171737909317017, "logits/rejected": 0.201465904712677, "logps/chosen": -71.2073745727539, "logps/ref_chosen": -65.7061767578125, "logps/ref_rejected": -91.72711944580078, "logps/rejected": -99.84532928466797, "loss": 1.112, "margin_dpo/margin_mean": 2.6170156002044678, "margin_dpo/margin_std": 3.844024181365967, "step": 125 }, { "KL/chosen_KL_mean": -5.178997039794922, "KL/mean": -5.999068260192871, "KL/rejected_KL_mean": -6.819145202636719, "KL/std": 3.7093849182128906, "epoch": 0.19047619047619047, "fcm_dpo/beta": 0.17089751362800598, "fcm_dpo/delta": 0.1385306864976883, "fcm_dpo/margin": 1.640149474143982, "fcm_dpo/q_t": 0.4417935013771057, "grad_norm": 46.21920394897461, "learning_rate": 4.883296295573176e-07, "logits/chosen": 0.07007776945829391, "logits/rejected": 0.06446836143732071, "logps/chosen": -73.35508728027344, "logps/ref_chosen": -68.17608642578125, "logps/ref_rejected": -65.1175537109375, "logps/rejected": -71.93669891357422, "loss": 1.2297, "margin_dpo/margin_mean": 1.640149712562561, "margin_dpo/margin_std": 3.9861550331115723, "step": 126 }, { "KL/chosen_KL_mean": -5.413948059082031, "KL/mean": -6.823336601257324, "KL/rejected_KL_mean": -8.232730865478516, "KL/std": 3.28928804397583, "epoch": 0.19198790627362056, "fcm_dpo/beta": 0.160674050450325, "fcm_dpo/delta": -0.0780230164527893, "fcm_dpo/margin": 2.818779945373535, "fcm_dpo/q_t": 0.39240193367004395, "grad_norm": 40.320648193359375, "learning_rate": 4.87927032161552e-07, "logits/chosen": 0.18640990555286407, "logits/rejected": 0.15599671006202698, "logps/chosen": -67.29418182373047, "logps/ref_chosen": -61.88023376464844, "logps/ref_rejected": -68.46012878417969, "logps/rejected": -76.69285583496094, "loss": 1.0581, "margin_dpo/margin_mean": 2.818779945373535, "margin_dpo/margin_std": 3.420637607574463, "step": 127 }, { "KL/chosen_KL_mean": -5.826803207397461, "KL/mean": -7.201224327087402, "KL/rejected_KL_mean": -8.575653076171875, "KL/std": 4.162779808044434, "epoch": 0.19349962207105065, "fcm_dpo/beta": 0.15589803457260132, "fcm_dpo/delta": -0.0385875478386879, "fcm_dpo/margin": 2.7488479614257812, "fcm_dpo/q_t": 0.40568995475769043, "grad_norm": 41.49650192260742, "learning_rate": 4.875177794352363e-07, "logits/chosen": 0.2261795848608017, "logits/rejected": 0.16798809170722961, "logps/chosen": -72.53578186035156, "logps/ref_chosen": -66.708984375, "logps/ref_rejected": -94.97969055175781, "logps/rejected": -103.55534362792969, "loss": 1.1343, "margin_dpo/margin_mean": 2.748847484588623, "margin_dpo/margin_std": 4.877597332000732, "step": 128 }, { "KL/chosen_KL_mean": -6.429250717163086, "KL/mean": -7.367158889770508, "KL/rejected_KL_mean": -8.305068969726562, "KL/std": 3.734823226928711, "epoch": 0.19501133786848074, "fcm_dpo/beta": 0.1696051061153412, "fcm_dpo/delta": 0.09658706188201904, "fcm_dpo/margin": 1.8758163452148438, "fcm_dpo/q_t": 0.43266767263412476, "grad_norm": 54.703407287597656, "learning_rate": 4.871018828260491e-07, "logits/chosen": 0.1982039511203766, "logits/rejected": 0.19206386804580688, "logps/chosen": -71.76808166503906, "logps/ref_chosen": -65.33882904052734, "logps/ref_rejected": -68.06109619140625, "logps/rejected": -76.36616516113281, "loss": 1.2189, "margin_dpo/margin_mean": 1.8758164644241333, "margin_dpo/margin_std": 4.385315895080566, "step": 129 }, { "KL/chosen_KL_mean": -6.053129196166992, "KL/mean": -7.374553680419922, "KL/rejected_KL_mean": -8.695980072021484, "KL/std": 3.675654411315918, "epoch": 0.1965230536659108, "fcm_dpo/beta": 0.16109490394592285, "fcm_dpo/delta": -0.04224683344364166, "fcm_dpo/margin": 2.6428451538085938, "fcm_dpo/q_t": 0.4000886082649231, "grad_norm": 46.596336364746094, "learning_rate": 4.866793539675126e-07, "logits/chosen": 0.17449048161506653, "logits/rejected": 0.1231340616941452, "logps/chosen": -64.71387481689453, "logps/ref_chosen": -58.660743713378906, "logps/ref_rejected": -79.24510192871094, "logps/rejected": -87.94108581542969, "loss": 1.0862, "margin_dpo/margin_mean": 2.642845630645752, "margin_dpo/margin_std": 3.5206475257873535, "step": 130 }, { "KL/chosen_KL_mean": -5.886774063110352, "KL/mean": -7.548995018005371, "KL/rejected_KL_mean": -9.21121597290039, "KL/std": 4.397624969482422, "epoch": 0.1980347694633409, "fcm_dpo/beta": 0.1490076184272766, "fcm_dpo/delta": -0.1296783685684204, "fcm_dpo/margin": 3.324448347091675, "fcm_dpo/q_t": 0.3871789574623108, "grad_norm": 37.18637466430664, "learning_rate": 4.86250204678667e-07, "logits/chosen": 0.17878660559654236, "logits/rejected": 0.11429747194051743, "logps/chosen": -58.401309967041016, "logps/ref_chosen": -52.51453399658203, "logps/ref_rejected": -85.18299865722656, "logps/rejected": -94.39421081542969, "loss": 1.0717, "margin_dpo/margin_mean": 3.324448347091675, "margin_dpo/margin_std": 4.856185436248779, "step": 131 }, { "KL/chosen_KL_mean": -6.430377960205078, "KL/mean": -7.8555006980896, "KL/rejected_KL_mean": -9.28062629699707, "KL/std": 4.197918891906738, "epoch": 0.19954648526077098, "fcm_dpo/beta": 0.14397326111793518, "fcm_dpo/delta": -0.013092327862977982, "fcm_dpo/margin": 2.8502440452575684, "fcm_dpo/q_t": 0.40407687425613403, "grad_norm": 38.280391693115234, "learning_rate": 4.858144469637408e-07, "logits/chosen": 0.258601576089859, "logits/rejected": 0.225815549492836, "logps/chosen": -72.11550903320312, "logps/ref_chosen": -65.68513488769531, "logps/ref_rejected": -69.54120635986328, "logps/rejected": -78.82183837890625, "loss": 1.1043, "margin_dpo/margin_mean": 2.8502445220947266, "margin_dpo/margin_std": 4.208562850952148, "step": 132 }, { "KL/chosen_KL_mean": -6.6004486083984375, "KL/mean": -7.834145545959473, "KL/rejected_KL_mean": -9.06783676147461, "KL/std": 4.116161346435547, "epoch": 0.20105820105820105, "fcm_dpo/beta": 0.14853733777999878, "fcm_dpo/delta": 0.040870919823646545, "fcm_dpo/margin": 2.4673891067504883, "fcm_dpo/q_t": 0.4182808995246887, "grad_norm": 43.48795700073242, "learning_rate": 4.853720930118138e-07, "logits/chosen": 0.17309647798538208, "logits/rejected": 0.1633840650320053, "logps/chosen": -70.19856262207031, "logps/ref_chosen": -63.598114013671875, "logps/ref_rejected": -73.72798156738281, "logps/rejected": -82.79582214355469, "loss": 1.1485, "margin_dpo/margin_mean": 2.4673893451690674, "margin_dpo/margin_std": 4.310044288635254, "step": 133 }, { "KL/chosen_KL_mean": -6.207492828369141, "KL/mean": -8.14185905456543, "KL/rejected_KL_mean": -10.076229095458984, "KL/std": 4.746306419372559, "epoch": 0.20256991685563114, "fcm_dpo/beta": 0.13268497586250305, "fcm_dpo/delta": -0.15481051802635193, "fcm_dpo/margin": 3.868732452392578, "fcm_dpo/q_t": 0.37830227613449097, "grad_norm": 32.207279205322266, "learning_rate": 4.849231551964771e-07, "logits/chosen": 0.3129193186759949, "logits/rejected": 0.25384044647216797, "logps/chosen": -60.0020637512207, "logps/ref_chosen": -53.79457092285156, "logps/ref_rejected": -74.16741943359375, "logps/rejected": -84.24365234375, "loss": 1.0239, "margin_dpo/margin_mean": 3.8687334060668945, "margin_dpo/margin_std": 4.69981050491333, "step": 134 }, { "KL/chosen_KL_mean": -6.265291213989258, "KL/mean": -7.531993389129639, "KL/rejected_KL_mean": -8.798698425292969, "KL/std": 4.477133750915527, "epoch": 0.20408163265306123, "fcm_dpo/beta": 0.1358022391796112, "fcm_dpo/delta": 0.06674190610647202, "fcm_dpo/margin": 2.5334067344665527, "fcm_dpo/q_t": 0.4239792227745056, "grad_norm": 32.82770538330078, "learning_rate": 4.844676460754862e-07, "logits/chosen": 0.20795100927352905, "logits/rejected": 0.17573975026607513, "logps/chosen": -55.70636749267578, "logps/ref_chosen": -49.441078186035156, "logps/ref_rejected": -65.96878051757812, "logps/rejected": -74.7674789428711, "loss": 1.1624, "margin_dpo/margin_mean": 2.5334067344665527, "margin_dpo/margin_std": 4.60736083984375, "step": 135 }, { "KL/chosen_KL_mean": -7.658565521240234, "KL/mean": -9.265897750854492, "KL/rejected_KL_mean": -10.87322998046875, "KL/std": 4.929014682769775, "epoch": 0.20559334845049132, "fcm_dpo/beta": 0.131425142288208, "fcm_dpo/delta": -0.031049178913235664, "fcm_dpo/margin": 3.214667797088623, "fcm_dpo/q_t": 0.4090140461921692, "grad_norm": 39.72063064575195, "learning_rate": 4.840055783904106e-07, "logits/chosen": 0.20339098572731018, "logits/rejected": 0.1307443231344223, "logps/chosen": -74.41783142089844, "logps/ref_chosen": -66.75926208496094, "logps/ref_rejected": -94.61787414550781, "logps/rejected": -105.49110412597656, "loss": 1.1609, "margin_dpo/margin_mean": 3.2146685123443604, "margin_dpo/margin_std": 6.277226448059082, "step": 136 }, { "KL/chosen_KL_mean": -7.224740982055664, "KL/mean": -8.883138656616211, "KL/rejected_KL_mean": -10.54153823852539, "KL/std": 4.694730758666992, "epoch": 0.20710506424792138, "fcm_dpo/beta": 0.13092255592346191, "fcm_dpo/delta": -0.04417610913515091, "fcm_dpo/margin": 3.3167953491210938, "fcm_dpo/q_t": 0.40223658084869385, "grad_norm": 32.55488967895508, "learning_rate": 4.835369650662767e-07, "logits/chosen": 0.2396807074546814, "logits/rejected": 0.21028423309326172, "logps/chosen": -64.00853729248047, "logps/ref_chosen": -56.78379821777344, "logps/ref_rejected": -69.89952087402344, "logps/rejected": -80.44105529785156, "loss": 1.1002, "margin_dpo/margin_mean": 3.316795825958252, "margin_dpo/margin_std": 5.056219100952148, "step": 137 }, { "KL/chosen_KL_mean": -8.548992156982422, "KL/mean": -9.971826553344727, "KL/rejected_KL_mean": -11.394657135009766, "KL/std": 4.893838882446289, "epoch": 0.20861678004535147, "fcm_dpo/beta": 0.1305425614118576, "fcm_dpo/delta": 0.035015083849430084, "fcm_dpo/margin": 2.845663547515869, "fcm_dpo/q_t": 0.41715729236602783, "grad_norm": 34.6871223449707, "learning_rate": 4.830618192112065e-07, "logits/chosen": 0.2283031940460205, "logits/rejected": 0.191786989569664, "logps/chosen": -67.31500244140625, "logps/ref_chosen": -58.766014099121094, "logps/ref_rejected": -68.12371826171875, "logps/rejected": -79.51837158203125, "loss": 1.1657, "margin_dpo/margin_mean": 2.845663070678711, "margin_dpo/margin_std": 5.385343074798584, "step": 138 }, { "KL/chosen_KL_mean": -8.900955200195312, "KL/mean": -10.671952247619629, "KL/rejected_KL_mean": -12.442947387695312, "KL/std": 4.920912742614746, "epoch": 0.21012849584278157, "fcm_dpo/beta": 0.12715420126914978, "fcm_dpo/delta": -0.06484213471412659, "fcm_dpo/margin": 3.5419931411743164, "fcm_dpo/q_t": 0.39593103528022766, "grad_norm": 36.7552490234375, "learning_rate": 4.825801541160509e-07, "logits/chosen": 0.19569626450538635, "logits/rejected": 0.16585242748260498, "logps/chosen": -80.12654113769531, "logps/ref_chosen": -71.2255859375, "logps/ref_rejected": -82.1834716796875, "logps/rejected": -94.62641906738281, "loss": 1.0784, "margin_dpo/margin_mean": 3.5419931411743164, "margin_dpo/margin_std": 4.931921005249023, "step": 139 }, { "KL/chosen_KL_mean": -8.247709274291992, "KL/mean": -10.586565017700195, "KL/rejected_KL_mean": -12.925422668457031, "KL/std": 6.458744049072266, "epoch": 0.21164021164021163, "fcm_dpo/beta": 0.10884112864732742, "fcm_dpo/delta": -0.15088868141174316, "fcm_dpo/margin": 4.67771053314209, "fcm_dpo/q_t": 0.3807474970817566, "grad_norm": 32.42084884643555, "learning_rate": 4.820919832540181e-07, "logits/chosen": 0.19952046871185303, "logits/rejected": 0.1522863507270813, "logps/chosen": -71.52537536621094, "logps/ref_chosen": -63.27766418457031, "logps/ref_rejected": -83.30647277832031, "logps/rejected": -96.23189544677734, "loss": 1.0738, "margin_dpo/margin_mean": 4.67771053314209, "margin_dpo/margin_std": 7.042229652404785, "step": 140 }, { "KL/chosen_KL_mean": -9.049182891845703, "KL/mean": -11.24361515045166, "KL/rejected_KL_mean": -13.43804931640625, "KL/std": 5.461617469787598, "epoch": 0.21315192743764172, "fcm_dpo/beta": 0.0988968014717102, "fcm_dpo/delta": -0.05077613145112991, "fcm_dpo/margin": 4.38886022567749, "fcm_dpo/q_t": 0.4017443358898163, "grad_norm": 28.450496673583984, "learning_rate": 4.815973202802966e-07, "logits/chosen": 0.21967005729675293, "logits/rejected": 0.17520186305046082, "logps/chosen": -70.81594848632812, "logps/ref_chosen": -61.76676940917969, "logps/ref_rejected": -88.60601806640625, "logps/rejected": -102.0440673828125, "loss": 1.1074, "margin_dpo/margin_mean": 4.388860702514648, "margin_dpo/margin_std": 6.8387346267700195, "step": 141 }, { "KL/chosen_KL_mean": -9.000873565673828, "KL/mean": -10.799939155578613, "KL/rejected_KL_mean": -12.599006652832031, "KL/std": 5.402791976928711, "epoch": 0.2146636432350718, "fcm_dpo/beta": 0.10335898399353027, "fcm_dpo/delta": 0.03455838933587074, "fcm_dpo/margin": 3.598137855529785, "fcm_dpo/q_t": 0.41603314876556396, "grad_norm": 28.805465698242188, "learning_rate": 4.810961790316729e-07, "logits/chosen": 0.2335534393787384, "logits/rejected": 0.20831874012947083, "logps/chosen": -74.27565002441406, "logps/ref_chosen": -65.2747802734375, "logps/ref_rejected": -81.1378173828125, "logps/rejected": -93.73682403564453, "loss": 1.1255, "margin_dpo/margin_mean": 3.598137378692627, "margin_dpo/margin_std": 5.554536819458008, "step": 142 }, { "KL/chosen_KL_mean": -9.662801742553711, "KL/mean": -11.23399543762207, "KL/rejected_KL_mean": -12.805187225341797, "KL/std": 6.142293930053711, "epoch": 0.2161753590325019, "fcm_dpo/beta": 0.11018843948841095, "fcm_dpo/delta": 0.06442005187273026, "fcm_dpo/margin": 3.1423885822296143, "fcm_dpo/q_t": 0.42225682735443115, "grad_norm": 38.34531784057617, "learning_rate": 4.805885735261454e-07, "logits/chosen": 0.26950305700302124, "logits/rejected": 0.25179946422576904, "logps/chosen": -72.28062438964844, "logps/ref_chosen": -62.617828369140625, "logps/ref_rejected": -70.39239501953125, "logps/rejected": -83.19757843017578, "loss": 1.2132, "margin_dpo/margin_mean": 3.1423885822296143, "margin_dpo/margin_std": 7.202130317687988, "step": 143 }, { "KL/chosen_KL_mean": -10.057125091552734, "KL/mean": -12.049142837524414, "KL/rejected_KL_mean": -14.041160583496094, "KL/std": 6.3006911277771, "epoch": 0.21768707482993196, "fcm_dpo/beta": 0.10638520121574402, "fcm_dpo/delta": -0.033301614224910736, "fcm_dpo/margin": 3.9840335845947266, "fcm_dpo/q_t": 0.40695470571517944, "grad_norm": 31.39975357055664, "learning_rate": 4.800745179625307e-07, "logits/chosen": 0.24360644817352295, "logits/rejected": 0.21164511144161224, "logps/chosen": -70.85981750488281, "logps/ref_chosen": -60.80268859863281, "logps/ref_rejected": -79.07284545898438, "logps/rejected": -93.11399841308594, "loss": 1.1387, "margin_dpo/margin_mean": 3.9840338230133057, "margin_dpo/margin_std": 7.172069549560547, "step": 144 }, { "KL/chosen_KL_mean": -9.3580322265625, "KL/mean": -11.521739959716797, "KL/rejected_KL_mean": -13.685443878173828, "KL/std": 6.398914813995361, "epoch": 0.21919879062736206, "fcm_dpo/beta": 0.1024373397231102, "fcm_dpo/delta": -0.05667082592844963, "fcm_dpo/margin": 4.327418804168701, "fcm_dpo/q_t": 0.39943933486938477, "grad_norm": 32.25492858886719, "learning_rate": 4.795540267200686e-07, "logits/chosen": 0.20689266920089722, "logits/rejected": 0.22335760295391083, "logps/chosen": -83.96949768066406, "logps/ref_chosen": -74.61146545410156, "logps/ref_rejected": -83.24461364746094, "logps/rejected": -96.9300537109375, "loss": 1.1228, "margin_dpo/margin_mean": 4.327418327331543, "margin_dpo/margin_std": 7.483129501342773, "step": 145 }, { "KL/chosen_KL_mean": -9.144096374511719, "KL/mean": -11.36795425415039, "KL/rejected_KL_mean": -13.591815948486328, "KL/std": 6.335058212280273, "epoch": 0.22071050642479215, "fcm_dpo/beta": 0.09927242994308472, "fcm_dpo/delta": -0.05333557724952698, "fcm_dpo/margin": 4.447718143463135, "fcm_dpo/q_t": 0.40064430236816406, "grad_norm": 28.61855697631836, "learning_rate": 4.790271143580173e-07, "logits/chosen": 0.1993100643157959, "logits/rejected": 0.1848219335079193, "logps/chosen": -66.98507690429688, "logps/ref_chosen": -57.84098434448242, "logps/ref_rejected": -67.47422790527344, "logps/rejected": -81.0660400390625, "loss": 1.1045, "margin_dpo/margin_mean": 4.44771671295166, "margin_dpo/margin_std": 7.0095109939575195, "step": 146 }, { "KL/chosen_KL_mean": -10.900079727172852, "KL/mean": -12.673417091369629, "KL/rejected_KL_mean": -14.446762084960938, "KL/std": 6.889484405517578, "epoch": 0.2222222222222222, "fcm_dpo/beta": 0.1005827784538269, "fcm_dpo/delta": 0.05248674005270004, "fcm_dpo/margin": 3.546682834625244, "fcm_dpo/q_t": 0.42228808999061584, "grad_norm": 35.238861083984375, "learning_rate": 4.784937956152489e-07, "logits/chosen": 0.22990134358406067, "logits/rejected": 0.18562506139278412, "logps/chosen": -77.71354675292969, "logps/ref_chosen": -66.81346893310547, "logps/ref_rejected": -81.1796875, "logps/rejected": -95.62644958496094, "loss": 1.1886, "margin_dpo/margin_mean": 3.546682119369507, "margin_dpo/margin_std": 7.444207191467285, "step": 147 }, { "KL/chosen_KL_mean": -10.144840240478516, "KL/mean": -12.388810157775879, "KL/rejected_KL_mean": -14.632774353027344, "KL/std": 6.4042558670043945, "epoch": 0.2237339380196523, "fcm_dpo/beta": 0.09929439425468445, "fcm_dpo/delta": -0.05904500186443329, "fcm_dpo/margin": 4.487936496734619, "fcm_dpo/q_t": 0.39827683568000793, "grad_norm": 24.603193283081055, "learning_rate": 4.779540854098347e-07, "logits/chosen": 0.36354687809944153, "logits/rejected": 0.28833717107772827, "logps/chosen": -58.83259582519531, "logps/ref_chosen": -48.6877555847168, "logps/ref_rejected": -67.50503540039062, "logps/rejected": -82.13780975341797, "loss": 1.1109, "margin_dpo/margin_mean": 4.487936973571777, "margin_dpo/margin_std": 7.146442413330078, "step": 148 }, { "KL/chosen_KL_mean": -12.343606948852539, "KL/mean": -14.90990161895752, "KL/rejected_KL_mean": -17.4761962890625, "KL/std": 6.945236682891846, "epoch": 0.2252456538170824, "fcm_dpo/beta": 0.09050105512142181, "fcm_dpo/delta": -0.08389414846897125, "fcm_dpo/margin": 5.13259744644165, "fcm_dpo/q_t": 0.3906511962413788, "grad_norm": 24.449920654296875, "learning_rate": 4.774079988386296e-07, "logits/chosen": 0.20603233575820923, "logits/rejected": 0.15894901752471924, "logps/chosen": -67.48738098144531, "logps/ref_chosen": -55.143775939941406, "logps/ref_rejected": -64.79888916015625, "logps/rejected": -82.27508544921875, "loss": 1.0558, "margin_dpo/margin_mean": 5.13259744644165, "margin_dpo/margin_std": 6.536444664001465, "step": 149 }, { "KL/chosen_KL_mean": -10.346208572387695, "KL/mean": -13.742513656616211, "KL/rejected_KL_mean": -17.138816833496094, "KL/std": 7.590294361114502, "epoch": 0.22675736961451248, "fcm_dpo/beta": 0.07703472673892975, "fcm_dpo/delta": -0.17041468620300293, "fcm_dpo/margin": 6.792609214782715, "fcm_dpo/q_t": 0.3750220239162445, "grad_norm": 20.74353790283203, "learning_rate": 4.768555511768486e-07, "logits/chosen": 0.220647931098938, "logits/rejected": 0.17675264179706573, "logps/chosen": -77.81695556640625, "logps/ref_chosen": -67.47074890136719, "logps/ref_rejected": -89.21170806884766, "logps/rejected": -106.35052490234375, "loss": 1.0144, "margin_dpo/margin_mean": 6.792610168457031, "margin_dpo/margin_std": 7.658425331115723, "step": 150 }, { "KL/chosen_KL_mean": -9.255523681640625, "KL/mean": -13.078845977783203, "KL/rejected_KL_mean": -16.90216827392578, "KL/std": 8.135717391967773, "epoch": 0.22826908541194255, "fcm_dpo/beta": 0.066739022731781, "fcm_dpo/delta": -0.14863666892051697, "fcm_dpo/margin": 7.646646022796631, "fcm_dpo/q_t": 0.3767693042755127, "grad_norm": 18.29514503479004, "learning_rate": 4.762967578776406e-07, "logits/chosen": 0.23800373077392578, "logits/rejected": 0.18349212408065796, "logps/chosen": -61.715065002441406, "logps/ref_chosen": -52.45954132080078, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -95.96517944335938, "loss": 1.0081, "margin_dpo/margin_mean": 7.646645545959473, "margin_dpo/margin_std": 8.172331809997559, "step": 151 }, { "KL/chosen_KL_mean": -11.936054229736328, "KL/mean": -14.415749549865723, "KL/rejected_KL_mean": -16.89544677734375, "KL/std": 7.793455123901367, "epoch": 0.22978080120937264, "fcm_dpo/beta": 0.06914802640676498, "fcm_dpo/delta": 0.0631265938282013, "fcm_dpo/margin": 4.959397792816162, "fcm_dpo/q_t": 0.4234861433506012, "grad_norm": 20.811508178710938, "learning_rate": 4.757316345716553e-07, "logits/chosen": 0.30365419387817383, "logits/rejected": 0.2504726052284241, "logps/chosen": -68.48988342285156, "logps/ref_chosen": -56.5538330078125, "logps/ref_rejected": -76.55074310302734, "logps/rejected": -93.4461898803711, "loss": 1.1642, "margin_dpo/margin_mean": 4.959397315979004, "margin_dpo/margin_std": 8.868165969848633, "step": 152 }, { "KL/chosen_KL_mean": -11.271255493164062, "KL/mean": -14.22636604309082, "KL/rejected_KL_mean": -17.181472778320312, "KL/std": 8.153881072998047, "epoch": 0.23129251700680273, "fcm_dpo/beta": 0.06809602677822113, "fcm_dpo/delta": -0.0030866768211126328, "fcm_dpo/margin": 5.910217761993408, "fcm_dpo/q_t": 0.4064687490463257, "grad_norm": 19.777393341064453, "learning_rate": 4.751601970666064e-07, "logits/chosen": 0.23187056183815002, "logits/rejected": 0.19487443566322327, "logps/chosen": -79.27815246582031, "logps/ref_chosen": -68.00689697265625, "logps/ref_rejected": -74.83482360839844, "logps/rejected": -92.01629638671875, "loss": 1.0864, "margin_dpo/margin_mean": 5.910217761993408, "margin_dpo/margin_std": 7.565615653991699, "step": 153 }, { "KL/chosen_KL_mean": -12.515201568603516, "KL/mean": -14.71187973022461, "KL/rejected_KL_mean": -16.90856170654297, "KL/std": 8.529696464538574, "epoch": 0.2328042328042328, "fcm_dpo/beta": 0.07292280346155167, "fcm_dpo/delta": 0.0948781967163086, "fcm_dpo/margin": 4.3933634757995605, "fcm_dpo/q_t": 0.4292556047439575, "grad_norm": 21.461366653442383, "learning_rate": 4.745824613468292e-07, "logits/chosen": 0.3159400224685669, "logits/rejected": 0.31303977966308594, "logps/chosen": -71.73773956298828, "logps/ref_chosen": -59.222537994384766, "logps/ref_rejected": -64.19131469726562, "logps/rejected": -81.09988403320312, "loss": 1.1898, "margin_dpo/margin_mean": 4.393362998962402, "margin_dpo/margin_std": 8.842275619506836, "step": 154 }, { "KL/chosen_KL_mean": -13.313087463378906, "KL/mean": -16.432861328125, "KL/rejected_KL_mean": -19.552631378173828, "KL/std": 8.535072326660156, "epoch": 0.23431594860166288, "fcm_dpo/beta": 0.0700855702161789, "fcm_dpo/delta": -0.05278134346008301, "fcm_dpo/margin": 6.239541053771973, "fcm_dpo/q_t": 0.40147268772125244, "grad_norm": 20.99993133544922, "learning_rate": 4.7399844357283393e-07, "logits/chosen": 0.2972029447555542, "logits/rejected": 0.2771180272102356, "logps/chosen": -81.76778411865234, "logps/ref_chosen": -68.45469665527344, "logps/ref_rejected": -77.91763305664062, "logps/rejected": -97.47026824951172, "loss": 1.1297, "margin_dpo/margin_mean": 6.239541053771973, "margin_dpo/margin_std": 10.832403182983398, "step": 155 }, { "KL/chosen_KL_mean": -13.374895095825195, "KL/mean": -17.081748962402344, "KL/rejected_KL_mean": -20.788604736328125, "KL/std": 9.638997077941895, "epoch": 0.23582766439909297, "fcm_dpo/beta": 0.06551361083984375, "fcm_dpo/delta": -0.11318676173686981, "fcm_dpo/margin": 7.4137067794799805, "fcm_dpo/q_t": 0.3857467770576477, "grad_norm": 20.287338256835938, "learning_rate": 4.7340816008085305e-07, "logits/chosen": 0.2682151794433594, "logits/rejected": 0.22211289405822754, "logps/chosen": -80.64449310302734, "logps/ref_chosen": -67.26959991455078, "logps/ref_rejected": -86.95914459228516, "logps/rejected": -107.74774932861328, "loss": 1.0398, "margin_dpo/margin_mean": 7.4137067794799805, "margin_dpo/margin_std": 8.992185592651367, "step": 156 }, { "KL/chosen_KL_mean": -12.044370651245117, "KL/mean": -15.103776931762695, "KL/rejected_KL_mean": -18.16318130493164, "KL/std": 9.422752380371094, "epoch": 0.23733938019652306, "fcm_dpo/beta": 0.0583278089761734, "fcm_dpo/delta": -0.08355621248483658, "fcm_dpo/margin": 6.118807792663574, "fcm_dpo/q_t": 0.41334038972854614, "grad_norm": 17.29412078857422, "learning_rate": 4.728116273823847e-07, "logits/chosen": 0.2556699812412262, "logits/rejected": 0.2370312660932541, "logps/chosen": -66.81724548339844, "logps/ref_chosen": -54.77287292480469, "logps/ref_rejected": -63.87866973876953, "logps/rejected": -82.04185485839844, "loss": 1.1341, "margin_dpo/margin_mean": 6.118807792663574, "margin_dpo/margin_std": 9.063301086425781, "step": 157 }, { "KL/chosen_KL_mean": -13.51652717590332, "KL/mean": -16.806453704833984, "KL/rejected_KL_mean": -20.096378326416016, "KL/std": 9.68644905090332, "epoch": 0.23885109599395313, "fcm_dpo/beta": 0.05906832590699196, "fcm_dpo/delta": 0.014011925086379051, "fcm_dpo/margin": 6.579849720001221, "fcm_dpo/q_t": 0.4122830927371979, "grad_norm": 18.49722671508789, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 0.32583776116371155, "logits/rejected": 0.28690358996391296, "logps/chosen": -78.43924713134766, "logps/ref_chosen": -64.92271423339844, "logps/ref_rejected": -82.23789978027344, "logps/rejected": -102.33427429199219, "loss": 1.1197, "margin_dpo/margin_mean": 6.579849720001221, "margin_dpo/margin_std": 10.205474853515625, "step": 158 }, { "KL/chosen_KL_mean": -13.864181518554688, "KL/mean": -17.635059356689453, "KL/rejected_KL_mean": -21.40593719482422, "KL/std": 10.578052520751953, "epoch": 0.24036281179138322, "fcm_dpo/beta": 0.058339424431324005, "fcm_dpo/delta": -0.05248873680830002, "fcm_dpo/margin": 7.541755199432373, "fcm_dpo/q_t": 0.39800336956977844, "grad_norm": 19.51651954650879, "learning_rate": 4.715998812855304e-07, "logits/chosen": 0.3037192225456238, "logits/rejected": 0.26449885964393616, "logps/chosen": -70.91117858886719, "logps/ref_chosen": -57.046993255615234, "logps/ref_rejected": -73.32441711425781, "logps/rejected": -94.73035430908203, "loss": 1.097, "margin_dpo/margin_mean": 7.541754722595215, "margin_dpo/margin_std": 11.181962966918945, "step": 159 }, { "KL/chosen_KL_mean": -15.68055534362793, "KL/mean": -18.908214569091797, "KL/rejected_KL_mean": -22.135875701904297, "KL/std": 9.930036544799805, "epoch": 0.2418745275888133, "fcm_dpo/beta": 0.05775478482246399, "fcm_dpo/delta": 0.03315534442663193, "fcm_dpo/margin": 6.455317497253418, "fcm_dpo/q_t": 0.4187636971473694, "grad_norm": 17.91891098022461, "learning_rate": 4.7098470178228755e-07, "logits/chosen": 0.20492243766784668, "logits/rejected": 0.1574675440788269, "logps/chosen": -65.48747253417969, "logps/ref_chosen": -49.806915283203125, "logps/ref_rejected": -68.3370132446289, "logps/rejected": -90.47288513183594, "loss": 1.1475, "margin_dpo/margin_mean": 6.455317497253418, "margin_dpo/margin_std": 11.438055038452148, "step": 160 }, { "KL/chosen_KL_mean": -15.670604705810547, "KL/mean": -19.274505615234375, "KL/rejected_KL_mean": -22.878414154052734, "KL/std": 9.855175971984863, "epoch": 0.24338624338624337, "fcm_dpo/beta": 0.05751575529575348, "fcm_dpo/delta": -0.01837945356965065, "fcm_dpo/margin": 7.207810878753662, "fcm_dpo/q_t": 0.40520739555358887, "grad_norm": 17.27115249633789, "learning_rate": 4.703633408618955e-07, "logits/chosen": 0.34232738614082336, "logits/rejected": 0.302024781703949, "logps/chosen": -68.17109680175781, "logps/ref_chosen": -52.50048828125, "logps/ref_rejected": -66.04540252685547, "logps/rejected": -88.92381286621094, "loss": 1.1097, "margin_dpo/margin_mean": 7.20781135559082, "margin_dpo/margin_std": 10.996399879455566, "step": 161 }, { "KL/chosen_KL_mean": -16.62833023071289, "KL/mean": -21.415283203125, "KL/rejected_KL_mean": -26.202239990234375, "KL/std": 11.221863746643066, "epoch": 0.24489795918367346, "fcm_dpo/beta": 0.05156968533992767, "fcm_dpo/delta": -0.1260819137096405, "fcm_dpo/margin": 9.573906898498535, "fcm_dpo/q_t": 0.38108617067337036, "grad_norm": 17.9483585357666, "learning_rate": 4.697358159051549e-07, "logits/chosen": 0.37953609228134155, "logits/rejected": 0.3276635408401489, "logps/chosen": -86.09751892089844, "logps/ref_chosen": -69.46919250488281, "logps/ref_rejected": -92.00952911376953, "logps/rejected": -118.2117691040039, "loss": 1.0347, "margin_dpo/margin_mean": 9.573907852172852, "margin_dpo/margin_std": 11.599828720092773, "step": 162 }, { "KL/chosen_KL_mean": -14.487728118896484, "KL/mean": -19.402671813964844, "KL/rejected_KL_mean": -24.317615509033203, "KL/std": 11.253974914550781, "epoch": 0.24640967498110355, "fcm_dpo/beta": 0.048833370208740234, "fcm_dpo/delta": -0.11060789227485657, "fcm_dpo/margin": 9.829888343811035, "fcm_dpo/q_t": 0.38493412733078003, "grad_norm": 16.407407760620117, "learning_rate": 4.691021444652876e-07, "logits/chosen": 0.29747360944747925, "logits/rejected": 0.24846696853637695, "logps/chosen": -65.1015625, "logps/ref_chosen": -50.613834381103516, "logps/ref_rejected": -74.62033081054688, "logps/rejected": -98.93794250488281, "loss": 1.0578, "margin_dpo/margin_mean": 9.829889297485352, "margin_dpo/margin_std": 11.631856918334961, "step": 163 }, { "KL/chosen_KL_mean": -15.621034622192383, "KL/mean": -20.437870025634766, "KL/rejected_KL_mean": -25.25470733642578, "KL/std": 12.038859367370605, "epoch": 0.24792139077853365, "fcm_dpo/beta": 0.044747721403837204, "fcm_dpo/delta": -0.04132752865552902, "fcm_dpo/margin": 9.633673667907715, "fcm_dpo/q_t": 0.39967209100723267, "grad_norm": 14.316239356994629, "learning_rate": 4.6846234426744624e-07, "logits/chosen": 0.28174513578414917, "logits/rejected": 0.21716471016407013, "logps/chosen": -70.46914672851562, "logps/ref_chosen": -54.848114013671875, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -104.31771850585938, "loss": 1.0909, "margin_dpo/margin_mean": 9.633672714233398, "margin_dpo/margin_std": 13.165080070495605, "step": 164 }, { "KL/chosen_KL_mean": -17.53729248046875, "KL/mean": -22.0023193359375, "KL/rejected_KL_mean": -26.467342376708984, "KL/std": 12.443459510803223, "epoch": 0.2494331065759637, "fcm_dpo/beta": 0.04426497220993042, "fcm_dpo/delta": 0.0033883340656757355, "fcm_dpo/margin": 8.9300537109375, "fcm_dpo/q_t": 0.4085206389427185, "grad_norm": 14.935009002685547, "learning_rate": 4.678164332082175e-07, "logits/chosen": 0.35460126399993896, "logits/rejected": 0.29810017347335815, "logps/chosen": -68.62650299072266, "logps/ref_chosen": -51.089210510253906, "logps/ref_rejected": -71.23370361328125, "logps/rejected": -97.70104217529297, "loss": 1.1201, "margin_dpo/margin_mean": 8.930052757263184, "margin_dpo/margin_std": 13.425127029418945, "step": 165 }, { "KL/chosen_KL_mean": -16.783632278442383, "KL/mean": -20.313011169433594, "KL/rejected_KL_mean": -23.842388153076172, "KL/std": 12.600139617919922, "epoch": 0.2509448223733938, "fcm_dpo/beta": 0.046010538935661316, "fcm_dpo/delta": 0.09013964980840683, "fcm_dpo/margin": 7.058765411376953, "fcm_dpo/q_t": 0.4290624260902405, "grad_norm": 18.119962692260742, "learning_rate": 4.6716442935512214e-07, "logits/chosen": 0.3217995762825012, "logits/rejected": 0.2264447808265686, "logps/chosen": -79.97444152832031, "logps/ref_chosen": -63.19081115722656, "logps/ref_rejected": -93.8402099609375, "logps/rejected": -117.68260192871094, "loss": 1.1621, "margin_dpo/margin_mean": 7.058765411376953, "margin_dpo/margin_std": 12.138998985290527, "step": 166 }, { "KL/chosen_KL_mean": -13.947896957397461, "KL/mean": -19.46517562866211, "KL/rejected_KL_mean": -24.98244857788086, "KL/std": 12.885375022888184, "epoch": 0.25245653817082386, "fcm_dpo/beta": 0.04268675297498703, "fcm_dpo/delta": -0.10074665397405624, "fcm_dpo/margin": 11.034552574157715, "fcm_dpo/q_t": 0.38690847158432007, "grad_norm": 15.102579116821289, "learning_rate": 4.6650635094610966e-07, "logits/chosen": 0.25539228320121765, "logits/rejected": 0.22091025114059448, "logps/chosen": -72.8721694946289, "logps/ref_chosen": -58.92427062988281, "logps/ref_rejected": -72.97377014160156, "logps/rejected": -97.95622253417969, "loss": 1.039, "margin_dpo/margin_mean": 11.034552574157715, "margin_dpo/margin_std": 12.66136360168457, "step": 167 }, { "KL/chosen_KL_mean": -16.616640090942383, "KL/mean": -21.07298469543457, "KL/rejected_KL_mean": -25.529327392578125, "KL/std": 14.074921607971191, "epoch": 0.25396825396825395, "fcm_dpo/beta": 0.04318401217460632, "fcm_dpo/delta": 0.01869083382189274, "fcm_dpo/margin": 8.912687301635742, "fcm_dpo/q_t": 0.41359812021255493, "grad_norm": 16.315370559692383, "learning_rate": 4.6584221638904767e-07, "logits/chosen": 0.28571587800979614, "logits/rejected": 0.24888527393341064, "logps/chosen": -82.26802062988281, "logps/ref_chosen": -65.65138244628906, "logps/ref_rejected": -79.71418762207031, "logps/rejected": -105.24351501464844, "loss": 1.1132, "margin_dpo/margin_mean": 8.912687301635742, "margin_dpo/margin_std": 13.038930892944336, "step": 168 }, { "KL/chosen_KL_mean": -13.18602180480957, "KL/mean": -18.36703109741211, "KL/rejected_KL_mean": -23.54804229736328, "KL/std": 13.908943176269531, "epoch": 0.25547996976568405, "fcm_dpo/beta": 0.04207727313041687, "fcm_dpo/delta": -0.04623054713010788, "fcm_dpo/margin": 10.362018585205078, "fcm_dpo/q_t": 0.4016047418117523, "grad_norm": 16.790719985961914, "learning_rate": 4.651720442612075e-07, "logits/chosen": 0.36561042070388794, "logits/rejected": 0.33055049180984497, "logps/chosen": -74.61189270019531, "logps/ref_chosen": -61.425865173339844, "logps/ref_rejected": -76.09590148925781, "logps/rejected": -99.64393615722656, "loss": 1.1074, "margin_dpo/margin_mean": 10.362018585205078, "margin_dpo/margin_std": 16.55835723876953, "step": 169 }, { "KL/chosen_KL_mean": -12.52975082397461, "KL/mean": -17.670759201049805, "KL/rejected_KL_mean": -22.811765670776367, "KL/std": 13.753107070922852, "epoch": 0.25699168556311414, "fcm_dpo/beta": 0.041177112609148026, "fcm_dpo/delta": -0.02979334443807602, "fcm_dpo/margin": 10.28201675415039, "fcm_dpo/q_t": 0.4035836458206177, "grad_norm": 14.086734771728516, "learning_rate": 4.6449585330874425e-07, "logits/chosen": 0.3422015607357025, "logits/rejected": 0.3399706482887268, "logps/chosen": -69.18293762207031, "logps/ref_chosen": -56.65319061279297, "logps/ref_rejected": -63.45965576171875, "logps/rejected": -86.27142333984375, "loss": 1.1271, "margin_dpo/margin_mean": 10.28201675415039, "margin_dpo/margin_std": 17.14507293701172, "step": 170 }, { "KL/chosen_KL_mean": -14.82802963256836, "KL/mean": -20.42302703857422, "KL/rejected_KL_mean": -26.018028259277344, "KL/std": 15.890604019165039, "epoch": 0.2585034013605442, "fcm_dpo/beta": 0.03773067146539688, "fcm_dpo/delta": -0.037932828068733215, "fcm_dpo/margin": 11.190006256103516, "fcm_dpo/q_t": 0.4020886719226837, "grad_norm": 13.957395553588867, "learning_rate": 4.6381366244617224e-07, "logits/chosen": 0.4011567234992981, "logits/rejected": 0.3479583263397217, "logps/chosen": -78.56278991699219, "logps/ref_chosen": -63.73476028442383, "logps/ref_rejected": -78.50328063964844, "logps/rejected": -104.52130126953125, "loss": 1.1247, "margin_dpo/margin_mean": 11.1900053024292, "margin_dpo/margin_std": 18.119037628173828, "step": 171 }, { "KL/chosen_KL_mean": -16.471878051757812, "KL/mean": -21.892181396484375, "KL/rejected_KL_mean": -27.312484741210938, "KL/std": 16.332107543945312, "epoch": 0.2600151171579743, "fcm_dpo/beta": 0.03937235847115517, "fcm_dpo/delta": -0.03921569883823395, "fcm_dpo/margin": 10.840606689453125, "fcm_dpo/q_t": 0.4032423496246338, "grad_norm": 15.800121307373047, "learning_rate": 4.631254907558365e-07, "logits/chosen": 0.41245076060295105, "logits/rejected": 0.3497459888458252, "logps/chosen": -68.67363739013672, "logps/ref_chosen": -52.201759338378906, "logps/ref_rejected": -82.85285949707031, "logps/rejected": -110.16534423828125, "loss": 1.1128, "margin_dpo/margin_mean": 10.840605735778809, "margin_dpo/margin_std": 16.780426025390625, "step": 172 }, { "KL/chosen_KL_mean": -15.000802993774414, "KL/mean": -20.906818389892578, "KL/rejected_KL_mean": -26.81283187866211, "KL/std": 16.735857009887695, "epoch": 0.2615268329554044, "fcm_dpo/beta": 0.03506167232990265, "fcm_dpo/delta": -0.028556950390338898, "fcm_dpo/margin": 11.812032699584961, "fcm_dpo/q_t": 0.40625429153442383, "grad_norm": 14.270763397216797, "learning_rate": 4.624313574873786e-07, "logits/chosen": 0.392696350812912, "logits/rejected": 0.3005034327507019, "logps/chosen": -70.43553161621094, "logps/ref_chosen": -55.434722900390625, "logps/ref_rejected": -77.81967163085938, "logps/rejected": -104.63250732421875, "loss": 1.1447, "margin_dpo/margin_mean": 11.812033653259277, "margin_dpo/margin_std": 20.58139419555664, "step": 173 }, { "KL/chosen_KL_mean": -17.099306106567383, "KL/mean": -23.63693618774414, "KL/rejected_KL_mean": -30.17456817626953, "KL/std": 17.15781593322754, "epoch": 0.26303854875283444, "fcm_dpo/beta": 0.034231267869472504, "fcm_dpo/delta": -0.0633653849363327, "fcm_dpo/margin": 13.075263023376465, "fcm_dpo/q_t": 0.3955170512199402, "grad_norm": 14.480879783630371, "learning_rate": 4.61731282057198e-07, "logits/chosen": 0.402925968170166, "logits/rejected": 0.33117085695266724, "logps/chosen": -74.27125549316406, "logps/ref_chosen": -57.17195129394531, "logps/ref_rejected": -85.47578430175781, "logps/rejected": -115.65034484863281, "loss": 1.0822, "margin_dpo/margin_mean": 13.075262069702148, "margin_dpo/margin_std": 18.593975067138672, "step": 174 }, { "KL/chosen_KL_mean": -14.67591667175293, "KL/mean": -22.01688003540039, "KL/rejected_KL_mean": -29.35784912109375, "KL/std": 17.625797271728516, "epoch": 0.26455026455026454, "fcm_dpo/beta": 0.032175906002521515, "fcm_dpo/delta": -0.09433892369270325, "fcm_dpo/margin": 14.681928634643555, "fcm_dpo/q_t": 0.39160484075546265, "grad_norm": 13.93406867980957, "learning_rate": 4.6102528404790965e-07, "logits/chosen": 0.38863056898117065, "logits/rejected": 0.3585554361343384, "logps/chosen": -82.34154510498047, "logps/ref_chosen": -67.6656265258789, "logps/ref_rejected": -84.36766815185547, "logps/rejected": -113.72551727294922, "loss": 1.071, "margin_dpo/margin_mean": 14.681928634643555, "margin_dpo/margin_std": 20.618019104003906, "step": 175 }, { "KL/chosen_KL_mean": -19.09964370727539, "KL/mean": -23.8125, "KL/rejected_KL_mean": -28.52536392211914, "KL/std": 18.95433807373047, "epoch": 0.2660619803476946, "fcm_dpo/beta": 0.031624685972929, "fcm_dpo/delta": 0.017538614571094513, "fcm_dpo/margin": 9.425718307495117, "fcm_dpo/q_t": 0.4338553547859192, "grad_norm": 16.63987159729004, "learning_rate": 4.603133832077953e-07, "logits/chosen": 0.34582072496414185, "logits/rejected": 0.318925678730011, "logps/chosen": -96.95840454101562, "logps/ref_chosen": -77.8587646484375, "logps/ref_rejected": -81.08732604980469, "logps/rejected": -109.61268615722656, "loss": 1.2205, "margin_dpo/margin_mean": 9.425718307495117, "margin_dpo/margin_std": 21.521808624267578, "step": 176 }, { "KL/chosen_KL_mean": -14.09356689453125, "KL/mean": -23.430145263671875, "KL/rejected_KL_mean": -32.766727447509766, "KL/std": 19.354829788208008, "epoch": 0.2675736961451247, "fcm_dpo/beta": 0.02757161483168602, "fcm_dpo/delta": -0.16072139143943787, "fcm_dpo/margin": 18.673160552978516, "fcm_dpo/q_t": 0.37375608086586, "grad_norm": 15.990117073059082, "learning_rate": 4.5959559945025183e-07, "logits/chosen": 0.48053431510925293, "logits/rejected": 0.3819437026977539, "logps/chosen": -69.31396484375, "logps/ref_chosen": -55.22039794921875, "logps/ref_rejected": -92.54973602294922, "logps/rejected": -125.31646728515625, "loss": 1.0054, "margin_dpo/margin_mean": 18.673158645629883, "margin_dpo/margin_std": 19.863622665405273, "step": 177 }, { "KL/chosen_KL_mean": -17.601396560668945, "KL/mean": -23.378013610839844, "KL/rejected_KL_mean": -29.15462875366211, "KL/std": 18.475444793701172, "epoch": 0.2690854119425548, "fcm_dpo/beta": 0.02648773603141308, "fcm_dpo/delta": -0.016534805297851562, "fcm_dpo/margin": 11.553230285644531, "fcm_dpo/q_t": 0.42779386043548584, "grad_norm": 15.744258880615234, "learning_rate": 4.588719528532341e-07, "logits/chosen": 0.3623722791671753, "logits/rejected": 0.3107893764972687, "logps/chosen": -78.4118881225586, "logps/ref_chosen": -60.81049346923828, "logps/ref_rejected": -81.12973022460938, "logps/rejected": -110.28435516357422, "loss": 1.1656, "margin_dpo/margin_mean": 11.553229331970215, "margin_dpo/margin_std": 18.718524932861328, "step": 178 }, { "KL/chosen_KL_mean": -18.142658233642578, "KL/mean": -25.477218627929688, "KL/rejected_KL_mean": -32.81178283691406, "KL/std": 19.89893913269043, "epoch": 0.2705971277399849, "fcm_dpo/beta": 0.02664315700531006, "fcm_dpo/delta": 0.011396395973861217, "fcm_dpo/margin": 14.669126510620117, "fcm_dpo/q_t": 0.41117656230926514, "grad_norm": 13.520777702331543, "learning_rate": 4.581424636586928e-07, "logits/chosen": 0.4051831066608429, "logits/rejected": 0.38962996006011963, "logps/chosen": -83.81437683105469, "logps/ref_chosen": -65.67171478271484, "logps/ref_rejected": -75.32586669921875, "logps/rejected": -108.13764953613281, "loss": 1.1254, "margin_dpo/margin_mean": 14.669126510620117, "margin_dpo/margin_std": 23.581867218017578, "step": 179 }, { "KL/chosen_KL_mean": -13.859926223754883, "KL/mean": -19.828092575073242, "KL/rejected_KL_mean": -25.796255111694336, "KL/std": 19.319965362548828, "epoch": 0.272108843537415, "fcm_dpo/beta": 0.02849998138844967, "fcm_dpo/delta": 0.07087843120098114, "fcm_dpo/margin": 11.936328887939453, "fcm_dpo/q_t": 0.42550933361053467, "grad_norm": 12.52182388305664, "learning_rate": 4.5740715227200897e-07, "logits/chosen": 0.27053821086883545, "logits/rejected": 0.2509544789791107, "logps/chosen": -70.54273223876953, "logps/ref_chosen": -56.68280792236328, "logps/ref_rejected": -64.94414520263672, "logps/rejected": -90.74040222167969, "loss": 1.1686, "margin_dpo/margin_mean": 11.936328887939453, "margin_dpo/margin_std": 22.309284210205078, "step": 180 }, { "KL/chosen_KL_mean": -11.871850967407227, "KL/mean": -20.5865478515625, "KL/rejected_KL_mean": -29.301239013671875, "KL/std": 20.67347526550293, "epoch": 0.273620559334845, "fcm_dpo/beta": 0.026625443249940872, "fcm_dpo/delta": -0.08586108684539795, "fcm_dpo/margin": 17.429393768310547, "fcm_dpo/q_t": 0.3906833529472351, "grad_norm": 15.263794898986816, "learning_rate": 4.566660392614228e-07, "logits/chosen": 0.4320031404495239, "logits/rejected": 0.38959449529647827, "logps/chosen": -72.64790344238281, "logps/ref_chosen": -60.77604675292969, "logps/ref_rejected": -83.98361206054688, "logps/rejected": -113.28485107421875, "loss": 1.0434, "margin_dpo/margin_mean": 17.429393768310547, "margin_dpo/margin_std": 20.527790069580078, "step": 181 }, { "KL/chosen_KL_mean": -18.988346099853516, "KL/mean": -27.982421875, "KL/rejected_KL_mean": -36.97650146484375, "KL/std": 22.851451873779297, "epoch": 0.2751322751322751, "fcm_dpo/beta": 0.025137335062026978, "fcm_dpo/delta": -0.06744768470525742, "fcm_dpo/margin": 17.988162994384766, "fcm_dpo/q_t": 0.39542728662490845, "grad_norm": 16.23828125, "learning_rate": 4.5591914535745817e-07, "logits/chosen": 0.39073818922042847, "logits/rejected": 0.3045911192893982, "logps/chosen": -79.24212646484375, "logps/ref_chosen": -60.2537841796875, "logps/ref_rejected": -89.7706298828125, "logps/rejected": -126.74713134765625, "loss": 1.0819, "margin_dpo/margin_mean": 17.988162994384766, "margin_dpo/margin_std": 25.733863830566406, "step": 182 }, { "KL/chosen_KL_mean": -23.74131202697754, "KL/mean": -27.4285888671875, "KL/rejected_KL_mean": -31.115867614746094, "KL/std": 22.018535614013672, "epoch": 0.2766439909297052, "fcm_dpo/beta": 0.026868050917983055, "fcm_dpo/delta": 0.0852898508310318, "fcm_dpo/margin": 7.374558448791504, "fcm_dpo/q_t": 0.45595434308052063, "grad_norm": 14.200604438781738, "learning_rate": 4.551664914523433e-07, "logits/chosen": 0.3912127912044525, "logits/rejected": 0.368286669254303, "logps/chosen": -85.50273132324219, "logps/ref_chosen": -61.76142120361328, "logps/ref_rejected": -72.54627990722656, "logps/rejected": -103.66215515136719, "loss": 1.2869, "margin_dpo/margin_mean": 7.374558925628662, "margin_dpo/margin_std": 22.744346618652344, "step": 183 }, { "KL/chosen_KL_mean": -13.689094543457031, "KL/mean": -20.588485717773438, "KL/rejected_KL_mean": -27.48788070678711, "KL/std": 19.24828338623047, "epoch": 0.2781557067271353, "fcm_dpo/beta": 0.026989229023456573, "fcm_dpo/delta": 0.032808832824230194, "fcm_dpo/margin": 13.798786163330078, "fcm_dpo/q_t": 0.4167749881744385, "grad_norm": 16.775178909301758, "learning_rate": 4.544080985994258e-07, "logits/chosen": 0.5024610757827759, "logits/rejected": 0.43457919359207153, "logps/chosen": -60.529815673828125, "logps/ref_chosen": -46.840721130371094, "logps/ref_rejected": -69.3609390258789, "logps/rejected": -96.84881591796875, "loss": 1.1215, "margin_dpo/margin_mean": 13.798786163330078, "margin_dpo/margin_std": 20.367998123168945, "step": 184 }, { "KL/chosen_KL_mean": -18.885101318359375, "KL/mean": -27.216163635253906, "KL/rejected_KL_mean": -35.547218322753906, "KL/std": 23.988807678222656, "epoch": 0.2796674225245654, "fcm_dpo/beta": 0.025701653212308884, "fcm_dpo/delta": -0.04676172509789467, "fcm_dpo/margin": 16.66212272644043, "fcm_dpo/q_t": 0.4024374485015869, "grad_norm": 14.15645980834961, "learning_rate": 4.5364398801258394e-07, "logits/chosen": 0.4315561056137085, "logits/rejected": 0.38441091775894165, "logps/chosen": -71.20623779296875, "logps/ref_chosen": -52.32114028930664, "logps/ref_rejected": -68.3885726928711, "logps/rejected": -103.935791015625, "loss": 1.1383, "margin_dpo/margin_mean": 16.66212272644043, "margin_dpo/margin_std": 28.837909698486328, "step": 185 }, { "KL/chosen_KL_mean": -16.665929794311523, "KL/mean": -25.744937896728516, "KL/rejected_KL_mean": -34.823944091796875, "KL/std": 23.925195693969727, "epoch": 0.2811791383219955, "fcm_dpo/beta": 0.02497974783182144, "fcm_dpo/delta": -0.07023215293884277, "fcm_dpo/margin": 18.158008575439453, "fcm_dpo/q_t": 0.39706262946128845, "grad_norm": 17.287595748901367, "learning_rate": 4.5287418106563354e-07, "logits/chosen": 0.3560236692428589, "logits/rejected": 0.312433660030365, "logps/chosen": -84.0860595703125, "logps/ref_chosen": -67.42012786865234, "logps/ref_rejected": -82.50968933105469, "logps/rejected": -117.33363342285156, "loss": 1.104, "margin_dpo/margin_mean": 18.158010482788086, "margin_dpo/margin_std": 29.182445526123047, "step": 186 }, { "KL/chosen_KL_mean": -22.537792205810547, "KL/mean": -30.65770721435547, "KL/rejected_KL_mean": -38.77762985229492, "KL/std": 25.62303924560547, "epoch": 0.28269085411942557, "fcm_dpo/beta": 0.02443239837884903, "fcm_dpo/delta": 0.0032668951898813248, "fcm_dpo/margin": 16.239839553833008, "fcm_dpo/q_t": 0.4135947823524475, "grad_norm": 15.863929748535156, "learning_rate": 4.520986992917297e-07, "logits/chosen": 0.41961655020713806, "logits/rejected": 0.363817036151886, "logps/chosen": -98.06328582763672, "logps/ref_chosen": -75.52549743652344, "logps/ref_rejected": -94.76289367675781, "logps/rejected": -133.54052734375, "loss": 1.1545, "margin_dpo/margin_mean": 16.239839553833008, "margin_dpo/margin_std": 30.24182891845703, "step": 187 }, { "KL/chosen_KL_mean": -20.841873168945312, "KL/mean": -30.49065399169922, "KL/rejected_KL_mean": -40.139442443847656, "KL/std": 26.88828468322754, "epoch": 0.2842025699168556, "fcm_dpo/beta": 0.023507175967097282, "fcm_dpo/delta": -0.06960050761699677, "fcm_dpo/margin": 19.297565460205078, "fcm_dpo/q_t": 0.39809972047805786, "grad_norm": 12.509038925170898, "learning_rate": 4.5131756438276466e-07, "logits/chosen": 0.4281846880912781, "logits/rejected": 0.384998619556427, "logps/chosen": -92.36520385742188, "logps/ref_chosen": -71.52333068847656, "logps/ref_rejected": -78.29949951171875, "logps/rejected": -118.4389419555664, "loss": 1.0881, "margin_dpo/margin_mean": 19.29756736755371, "margin_dpo/margin_std": 28.976045608520508, "step": 188 }, { "KL/chosen_KL_mean": -20.803133010864258, "KL/mean": -29.196277618408203, "KL/rejected_KL_mean": -37.58942413330078, "KL/std": 26.68855094909668, "epoch": 0.2857142857142857, "fcm_dpo/beta": 0.02266472578048706, "fcm_dpo/delta": 0.0194806307554245, "fcm_dpo/margin": 16.786296844482422, "fcm_dpo/q_t": 0.4146800935268402, "grad_norm": 14.953192710876465, "learning_rate": 4.5053079818876096e-07, "logits/chosen": 0.4511597156524658, "logits/rejected": 0.46751925349235535, "logps/chosen": -92.97940063476562, "logps/ref_chosen": -72.17626953125, "logps/ref_rejected": -75.26313781738281, "logps/rejected": -112.85255432128906, "loss": 1.1438, "margin_dpo/margin_mean": 16.786296844482422, "margin_dpo/margin_std": 27.98157501220703, "step": 189 }, { "KL/chosen_KL_mean": -19.941322326660156, "KL/mean": -32.42082977294922, "KL/rejected_KL_mean": -44.90033721923828, "KL/std": 28.068832397460938, "epoch": 0.2872260015117158, "fcm_dpo/beta": 0.02070463076233864, "fcm_dpo/delta": -0.16096393764019012, "fcm_dpo/margin": 24.95901870727539, "fcm_dpo/q_t": 0.37751439213752747, "grad_norm": 18.85645866394043, "learning_rate": 4.4973842271726024e-07, "logits/chosen": 0.5245158672332764, "logits/rejected": 0.36903607845306396, "logps/chosen": -74.56559753417969, "logps/ref_chosen": -54.624271392822266, "logps/ref_rejected": -101.47068786621094, "logps/rejected": -146.37103271484375, "loss": 1.0289, "margin_dpo/margin_mean": 24.95901870727539, "margin_dpo/margin_std": 31.038402557373047, "step": 190 }, { "KL/chosen_KL_mean": -29.806278228759766, "KL/mean": -39.38099670410156, "KL/rejected_KL_mean": -48.95571517944336, "KL/std": 29.71709442138672, "epoch": 0.2887377173091459, "fcm_dpo/beta": 0.020269766449928284, "fcm_dpo/delta": 0.014703525230288506, "fcm_dpo/margin": 19.14944076538086, "fcm_dpo/q_t": 0.41183042526245117, "grad_norm": 14.513212203979492, "learning_rate": 4.48940460132708e-07, "logits/chosen": 0.5108237266540527, "logits/rejected": 0.4802900552749634, "logps/chosen": -102.7387924194336, "logps/ref_chosen": -72.93251037597656, "logps/ref_rejected": -89.95103454589844, "logps/rejected": -138.90675354003906, "loss": 1.1321, "margin_dpo/margin_mean": 19.14944076538086, "margin_dpo/margin_std": 31.697952270507812, "step": 191 }, { "KL/chosen_KL_mean": -34.189476013183594, "KL/mean": -40.280208587646484, "KL/rejected_KL_mean": -46.37093734741211, "KL/std": 31.34961700439453, "epoch": 0.29024943310657597, "fcm_dpo/beta": 0.02095700055360794, "fcm_dpo/delta": 0.056858569383621216, "fcm_dpo/margin": 12.181463241577148, "fcm_dpo/q_t": 0.44299930334091187, "grad_norm": 14.316689491271973, "learning_rate": 4.481369327558329e-07, "logits/chosen": 0.5008162260055542, "logits/rejected": 0.47292008996009827, "logps/chosen": -88.19059753417969, "logps/ref_chosen": -54.001121520996094, "logps/ref_rejected": -63.531551361083984, "logps/rejected": -109.9024887084961, "loss": 1.2285, "margin_dpo/margin_mean": 12.181463241577148, "margin_dpo/margin_std": 28.081724166870117, "step": 192 }, { "KL/chosen_KL_mean": -27.229509353637695, "KL/mean": -38.814231872558594, "KL/rejected_KL_mean": -50.398956298828125, "KL/std": 29.14958953857422, "epoch": 0.29176114890400606, "fcm_dpo/beta": 0.01980617642402649, "fcm_dpo/delta": -0.08162431418895721, "fcm_dpo/margin": 23.16944122314453, "fcm_dpo/q_t": 0.39218446612358093, "grad_norm": 12.384896278381348, "learning_rate": 4.47327863063023e-07, "logits/chosen": 0.46182727813720703, "logits/rejected": 0.4414292573928833, "logps/chosen": -83.97879028320312, "logps/ref_chosen": -56.74927520751953, "logps/ref_rejected": -58.80629348754883, "logps/rejected": -109.20524597167969, "loss": 1.0637, "margin_dpo/margin_mean": 23.16944122314453, "margin_dpo/margin_std": 30.30309295654297, "step": 193 }, { "KL/chosen_KL_mean": -30.627708435058594, "KL/mean": -39.823089599609375, "KL/rejected_KL_mean": -49.018470764160156, "KL/std": 30.80691146850586, "epoch": 0.29327286470143615, "fcm_dpo/beta": 0.019777745008468628, "fcm_dpo/delta": 0.04038870707154274, "fcm_dpo/margin": 18.390762329101562, "fcm_dpo/q_t": 0.4192665219306946, "grad_norm": 17.1464786529541, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 0.5140712857246399, "logits/rejected": 0.48340654373168945, "logps/chosen": -87.27714538574219, "logps/ref_chosen": -56.64944076538086, "logps/ref_rejected": -69.98954772949219, "logps/rejected": -119.00801086425781, "loss": 1.181, "margin_dpo/margin_mean": 18.390764236450195, "margin_dpo/margin_std": 36.06867980957031, "step": 194 }, { "KL/chosen_KL_mean": -36.239933013916016, "KL/mean": -48.81350326538086, "KL/rejected_KL_mean": -61.38707733154297, "KL/std": 33.777400970458984, "epoch": 0.2947845804988662, "fcm_dpo/beta": 0.018797680735588074, "fcm_dpo/delta": -0.10017755627632141, "fcm_dpo/margin": 25.147144317626953, "fcm_dpo/q_t": 0.38978683948516846, "grad_norm": 15.896224021911621, "learning_rate": 4.4569318740967043e-07, "logits/chosen": 0.4080420136451721, "logits/rejected": 0.41872015595436096, "logps/chosen": -106.64971160888672, "logps/ref_chosen": -70.40977478027344, "logps/ref_rejected": -74.39448547363281, "logps/rejected": -135.78155517578125, "loss": 1.0756, "margin_dpo/margin_mean": 25.147144317626953, "margin_dpo/margin_std": 36.095008850097656, "step": 195 }, { "KL/chosen_KL_mean": -37.003265380859375, "KL/mean": -46.09539794921875, "KL/rejected_KL_mean": -55.18753433227539, "KL/std": 35.14692687988281, "epoch": 0.2962962962962963, "fcm_dpo/beta": 0.01977820321917534, "fcm_dpo/delta": 0.046010956168174744, "fcm_dpo/margin": 18.18426513671875, "fcm_dpo/q_t": 0.41924434900283813, "grad_norm": 17.794841766357422, "learning_rate": 4.448676271745197e-07, "logits/chosen": 0.5215525031089783, "logits/rejected": 0.4756397604942322, "logps/chosen": -96.23084259033203, "logps/ref_chosen": -59.227577209472656, "logps/ref_rejected": -83.54757690429688, "logps/rejected": -138.735107421875, "loss": 1.1459, "margin_dpo/margin_mean": 18.18426513671875, "margin_dpo/margin_std": 30.57408905029297, "step": 196 }, { "KL/chosen_KL_mean": -32.982200622558594, "KL/mean": -44.936031341552734, "KL/rejected_KL_mean": -56.88986587524414, "KL/std": 33.14744567871094, "epoch": 0.29780801209372637, "fcm_dpo/beta": 0.019095640629529953, "fcm_dpo/delta": -0.07451824843883514, "fcm_dpo/margin": 23.90766143798828, "fcm_dpo/q_t": 0.39633709192276, "grad_norm": 16.9836483001709, "learning_rate": 4.440366160729392e-07, "logits/chosen": 0.6078462600708008, "logits/rejected": 0.5525588989257812, "logps/chosen": -84.5113296508789, "logps/ref_chosen": -51.52912902832031, "logps/ref_rejected": -73.70631408691406, "logps/rejected": -130.59617614746094, "loss": 1.134, "margin_dpo/margin_mean": 23.90765953063965, "margin_dpo/margin_std": 41.63103485107422, "step": 197 }, { "KL/chosen_KL_mean": -35.99111557006836, "KL/mean": -48.80699920654297, "KL/rejected_KL_mean": -61.62289047241211, "KL/std": 33.919517517089844, "epoch": 0.29931972789115646, "fcm_dpo/beta": 0.017618469893932343, "fcm_dpo/delta": -0.06671290844678879, "fcm_dpo/margin": 25.63176727294922, "fcm_dpo/q_t": 0.3941599726676941, "grad_norm": 14.554643630981445, "learning_rate": 4.432001773500957e-07, "logits/chosen": 0.5819200873374939, "logits/rejected": 0.5387852191925049, "logps/chosen": -95.77379608154297, "logps/ref_chosen": -59.78268051147461, "logps/ref_rejected": -72.24533081054688, "logps/rejected": -133.86822509765625, "loss": 1.0748, "margin_dpo/margin_mean": 25.63176727294922, "margin_dpo/margin_std": 34.73223114013672, "step": 198 }, { "KL/chosen_KL_mean": -43.49638366699219, "KL/mean": -53.40158462524414, "KL/rejected_KL_mean": -63.306785583496094, "KL/std": 34.76414108276367, "epoch": 0.30083144368858655, "fcm_dpo/beta": 0.01662040874361992, "fcm_dpo/delta": -0.04701909050345421, "fcm_dpo/margin": 19.810405731201172, "fcm_dpo/q_t": 0.4231904149055481, "grad_norm": 12.835299491882324, "learning_rate": 4.4235833440297856e-07, "logits/chosen": 0.529554009437561, "logits/rejected": 0.437679648399353, "logps/chosen": -99.88314819335938, "logps/ref_chosen": -56.38677215576172, "logps/ref_rejected": -74.56779479980469, "logps/rejected": -137.8745880126953, "loss": 1.1928, "margin_dpo/margin_mean": 19.810405731201172, "margin_dpo/margin_std": 39.37937927246094, "step": 199 }, { "KL/chosen_KL_mean": -35.22080993652344, "KL/mean": -49.63976287841797, "KL/rejected_KL_mean": -64.05872344970703, "KL/std": 36.015777587890625, "epoch": 0.30234315948601664, "fcm_dpo/beta": 0.015300557017326355, "fcm_dpo/delta": -0.05560305714607239, "fcm_dpo/margin": 28.837909698486328, "fcm_dpo/q_t": 0.397291362285614, "grad_norm": 12.404858589172363, "learning_rate": 4.415111107797445e-07, "logits/chosen": 0.5855056047439575, "logits/rejected": 0.5044532418251038, "logps/chosen": -93.04513549804688, "logps/ref_chosen": -57.82432556152344, "logps/ref_rejected": -89.28246307373047, "logps/rejected": -153.3411865234375, "loss": 1.0938, "margin_dpo/margin_mean": 28.837909698486328, "margin_dpo/margin_std": 42.91114044189453, "step": 200 }, { "KL/chosen_KL_mean": -48.78852081298828, "KL/mean": -60.213199615478516, "KL/rejected_KL_mean": -71.63787841796875, "KL/std": 37.877784729003906, "epoch": 0.30385487528344673, "fcm_dpo/beta": 0.015576442703604698, "fcm_dpo/delta": 0.052298612892627716, "fcm_dpo/margin": 22.84935760498047, "fcm_dpo/q_t": 0.42080944776535034, "grad_norm": 14.842419624328613, "learning_rate": 4.4065853017905953e-07, "logits/chosen": 0.6083788871765137, "logits/rejected": 0.5580486059188843, "logps/chosen": -107.78828430175781, "logps/ref_chosen": -58.999759674072266, "logps/ref_rejected": -84.67575073242188, "logps/rejected": -156.31362915039062, "loss": 1.157, "margin_dpo/margin_mean": 22.84935760498047, "margin_dpo/margin_std": 39.762603759765625, "step": 201 }, { "KL/chosen_KL_mean": -40.75391387939453, "KL/mean": -55.758758544921875, "KL/rejected_KL_mean": -70.76358795166016, "KL/std": 36.27582550048828, "epoch": 0.30536659108087677, "fcm_dpo/beta": 0.015416000038385391, "fcm_dpo/delta": -0.08111350238323212, "fcm_dpo/margin": 30.009674072265625, "fcm_dpo/q_t": 0.39005911350250244, "grad_norm": 13.22191047668457, "learning_rate": 4.3980061644943575e-07, "logits/chosen": 0.5110350847244263, "logits/rejected": 0.4323844313621521, "logps/chosen": -88.41456604003906, "logps/ref_chosen": -47.660648345947266, "logps/ref_rejected": -73.63249969482422, "logps/rejected": -144.39608764648438, "loss": 1.0704, "margin_dpo/margin_mean": 30.009674072265625, "margin_dpo/margin_std": 40.26255798339844, "step": 202 }, { "KL/chosen_KL_mean": -51.74449157714844, "KL/mean": -64.84964752197266, "KL/rejected_KL_mean": -77.95479583740234, "KL/std": 37.77312469482422, "epoch": 0.30687830687830686, "fcm_dpo/beta": 0.01509188860654831, "fcm_dpo/delta": 0.005205655470490456, "fcm_dpo/margin": 26.210308074951172, "fcm_dpo/q_t": 0.4106394946575165, "grad_norm": 15.703864097595215, "learning_rate": 4.3893739358856455e-07, "logits/chosen": 0.5600358843803406, "logits/rejected": 0.47498154640197754, "logps/chosen": -114.07002258300781, "logps/ref_chosen": -62.32553482055664, "logps/ref_rejected": -99.37226104736328, "logps/rejected": -177.32705688476562, "loss": 1.1234, "margin_dpo/margin_mean": 26.21030616760254, "margin_dpo/margin_std": 42.12183380126953, "step": 203 }, { "KL/chosen_KL_mean": -46.200294494628906, "KL/mean": -60.380897521972656, "KL/rejected_KL_mean": -74.56149291992188, "KL/std": 38.21998596191406, "epoch": 0.30839002267573695, "fcm_dpo/beta": 0.01425536535680294, "fcm_dpo/delta": -0.013648569583892822, "fcm_dpo/margin": 28.361196517944336, "fcm_dpo/q_t": 0.40619611740112305, "grad_norm": 14.456401824951172, "learning_rate": 4.380688857426449e-07, "logits/chosen": 0.552179217338562, "logits/rejected": 0.4786151647567749, "logps/chosen": -96.8296127319336, "logps/ref_chosen": -50.62931823730469, "logps/ref_rejected": -66.60475158691406, "logps/rejected": -141.16624450683594, "loss": 1.1059, "margin_dpo/margin_mean": 28.36119842529297, "margin_dpo/margin_std": 39.7681770324707, "step": 204 }, { "KL/chosen_KL_mean": -54.09760665893555, "KL/mean": -67.47254943847656, "KL/rejected_KL_mean": -80.84750366210938, "KL/std": 40.442012786865234, "epoch": 0.30990173847316704, "fcm_dpo/beta": 0.01479810755699873, "fcm_dpo/delta": 0.0051469397731125355, "fcm_dpo/margin": 26.74989128112793, "fcm_dpo/q_t": 0.4112912714481354, "grad_norm": 16.738981246948242, "learning_rate": 4.3719511720570814e-07, "logits/chosen": 0.6002988815307617, "logits/rejected": 0.5250147581100464, "logps/chosen": -124.45378112792969, "logps/ref_chosen": -70.3561782836914, "logps/ref_rejected": -93.39848327636719, "logps/rejected": -174.2459716796875, "loss": 1.1334, "margin_dpo/margin_mean": 26.749893188476562, "margin_dpo/margin_std": 45.088653564453125, "step": 205 }, { "KL/chosen_KL_mean": -56.70134735107422, "KL/mean": -65.66264343261719, "KL/rejected_KL_mean": -74.62395477294922, "KL/std": 39.392059326171875, "epoch": 0.31141345427059713, "fcm_dpo/beta": 0.015088488347828388, "fcm_dpo/delta": 0.029385201632976532, "fcm_dpo/margin": 17.922607421875, "fcm_dpo/q_t": 0.4415813684463501, "grad_norm": 14.734772682189941, "learning_rate": 4.363161124189387e-07, "logits/chosen": 0.6411951780319214, "logits/rejected": 0.6234967708587646, "logps/chosen": -124.34683227539062, "logps/ref_chosen": -67.64547729492188, "logps/ref_rejected": -79.89584350585938, "logps/rejected": -154.51980590820312, "loss": 1.2593, "margin_dpo/margin_mean": 17.922609329223633, "margin_dpo/margin_std": 48.28623580932617, "step": 206 }, { "KL/chosen_KL_mean": -65.2525863647461, "KL/mean": -77.85491943359375, "KL/rejected_KL_mean": -90.45724487304688, "KL/std": 40.893856048583984, "epoch": 0.3129251700680272, "fcm_dpo/beta": 0.015518700703978539, "fcm_dpo/delta": 0.010801279917359352, "fcm_dpo/margin": 25.20465087890625, "fcm_dpo/q_t": 0.4146854877471924, "grad_norm": 13.349529266357422, "learning_rate": 4.3543189596998986e-07, "logits/chosen": 0.5421028137207031, "logits/rejected": 0.4715673327445984, "logps/chosen": -132.91677856445312, "logps/ref_chosen": -67.66419219970703, "logps/ref_rejected": -85.10249328613281, "logps/rejected": -175.55972290039062, "loss": 1.1394, "margin_dpo/margin_mean": 25.20465087890625, "margin_dpo/margin_std": 43.88922119140625, "step": 207 }, { "KL/chosen_KL_mean": -55.959251403808594, "KL/mean": -62.835411071777344, "KL/rejected_KL_mean": -69.71156311035156, "KL/std": 39.03756332397461, "epoch": 0.3144368858654573, "fcm_dpo/beta": 0.016686484217643738, "fcm_dpo/delta": 0.07528108358383179, "fcm_dpo/margin": 13.752311706542969, "fcm_dpo/q_t": 0.4518064856529236, "grad_norm": 22.71687126159668, "learning_rate": 4.3454249259229664e-07, "logits/chosen": 0.6001783609390259, "logits/rejected": 0.5711269974708557, "logps/chosen": -113.69096374511719, "logps/ref_chosen": -57.731712341308594, "logps/ref_rejected": -74.19276428222656, "logps/rejected": -143.90432739257812, "loss": 1.3074, "margin_dpo/margin_mean": 13.752311706542969, "margin_dpo/margin_std": 45.60139846801758, "step": 208 }, { "KL/chosen_KL_mean": -55.48735809326172, "KL/mean": -71.57313537597656, "KL/rejected_KL_mean": -87.65892028808594, "KL/std": 43.753570556640625, "epoch": 0.31594860166288735, "fcm_dpo/beta": 0.015102138742804527, "fcm_dpo/delta": -0.11657831072807312, "fcm_dpo/margin": 32.17155838012695, "fcm_dpo/q_t": 0.3876298666000366, "grad_norm": 15.37290096282959, "learning_rate": 4.336479271643833e-07, "logits/chosen": 0.5027754306793213, "logits/rejected": 0.4479963183403015, "logps/chosen": -124.03743743896484, "logps/ref_chosen": -68.55007934570312, "logps/ref_rejected": -87.90541076660156, "logps/rejected": -175.5643310546875, "loss": 1.0976, "margin_dpo/margin_mean": 32.17155838012695, "margin_dpo/margin_std": 51.997520446777344, "step": 209 }, { "KL/chosen_KL_mean": -53.723114013671875, "KL/mean": -70.12528228759766, "KL/rejected_KL_mean": -86.52745056152344, "KL/std": 44.81327819824219, "epoch": 0.31746031746031744, "fcm_dpo/beta": 0.014353602193295956, "fcm_dpo/delta": -0.0943194180727005, "fcm_dpo/margin": 32.804325103759766, "fcm_dpo/q_t": 0.39221110939979553, "grad_norm": 16.092060089111328, "learning_rate": 4.327482247091679e-07, "logits/chosen": 0.6441676616668701, "logits/rejected": 0.5358040928840637, "logps/chosen": -110.99138641357422, "logps/ref_chosen": -57.268272399902344, "logps/ref_rejected": -85.72807312011719, "logps/rejected": -172.25552368164062, "loss": 1.0813, "margin_dpo/margin_mean": 32.80432891845703, "margin_dpo/margin_std": 46.69495391845703, "step": 210 }, { "KL/chosen_KL_mean": -49.60154724121094, "KL/mean": -65.75942993164062, "KL/rejected_KL_mean": -81.91730499267578, "KL/std": 42.09577941894531, "epoch": 0.31897203325774753, "fcm_dpo/beta": 0.013455729931592941, "fcm_dpo/delta": -0.046873997896909714, "fcm_dpo/margin": 32.31575393676758, "fcm_dpo/q_t": 0.40088406205177307, "grad_norm": 17.506433486938477, "learning_rate": 4.3184341039326217e-07, "logits/chosen": 0.6236763000488281, "logits/rejected": 0.517917275428772, "logps/chosen": -103.24225616455078, "logps/ref_chosen": -53.640708923339844, "logps/ref_rejected": -93.0387954711914, "logps/rejected": -174.9561004638672, "loss": 1.0823, "margin_dpo/margin_mean": 32.31575012207031, "margin_dpo/margin_std": 43.71531677246094, "step": 211 }, { "KL/chosen_KL_mean": -55.83790588378906, "KL/mean": -72.02022552490234, "KL/rejected_KL_mean": -88.2025375366211, "KL/std": 44.30055236816406, "epoch": 0.3204837490551776, "fcm_dpo/beta": 0.012746179476380348, "fcm_dpo/delta": -0.015783989802002907, "fcm_dpo/margin": 32.36463165283203, "fcm_dpo/q_t": 0.4043083190917969, "grad_norm": 10.171900749206543, "learning_rate": 4.309335095262675e-07, "logits/chosen": 0.6819473505020142, "logits/rejected": 0.5994547605514526, "logps/chosen": -113.20465087890625, "logps/ref_chosen": -57.36674499511719, "logps/ref_rejected": -79.89643096923828, "logps/rejected": -168.09896850585938, "loss": 1.105, "margin_dpo/margin_mean": 32.36463165283203, "margin_dpo/margin_std": 47.924339294433594, "step": 212 }, { "KL/chosen_KL_mean": -44.525672912597656, "KL/mean": -63.037479400634766, "KL/rejected_KL_mean": -81.54928588867188, "KL/std": 44.70442199707031, "epoch": 0.3219954648526077, "fcm_dpo/beta": 0.011922631412744522, "fcm_dpo/delta": -0.05658069625496864, "fcm_dpo/margin": 37.023616790771484, "fcm_dpo/q_t": 0.396555632352829, "grad_norm": 11.255715370178223, "learning_rate": 4.3001854756006724e-07, "logits/chosen": 0.6020195484161377, "logits/rejected": 0.5774927139282227, "logps/chosen": -109.74678802490234, "logps/ref_chosen": -65.22111511230469, "logps/ref_rejected": -80.1810302734375, "logps/rejected": -161.73031616210938, "loss": 1.0767, "margin_dpo/margin_mean": 37.02362060546875, "margin_dpo/margin_std": 50.05058288574219, "step": 213 }, { "KL/chosen_KL_mean": -52.843170166015625, "KL/mean": -70.69467163085938, "KL/rejected_KL_mean": -88.5461654663086, "KL/std": 45.061424255371094, "epoch": 0.3235071806500378, "fcm_dpo/beta": 0.012001337483525276, "fcm_dpo/delta": -0.0386454313993454, "fcm_dpo/margin": 35.7030029296875, "fcm_dpo/q_t": 0.4016464650630951, "grad_norm": 13.126012802124023, "learning_rate": 4.290985500881143e-07, "logits/chosen": 0.45403045415878296, "logits/rejected": 0.4404328465461731, "logps/chosen": -114.135498046875, "logps/ref_chosen": -61.292327880859375, "logps/ref_rejected": -67.69841003417969, "logps/rejected": -156.24456787109375, "loss": 1.101, "margin_dpo/margin_mean": 35.7030029296875, "margin_dpo/margin_std": 52.66297149658203, "step": 214 }, { "KL/chosen_KL_mean": -58.013763427734375, "KL/mean": -76.46696472167969, "KL/rejected_KL_mean": -94.920166015625, "KL/std": 46.602779388427734, "epoch": 0.3250188964474679, "fcm_dpo/beta": 0.011644062586128712, "fcm_dpo/delta": -0.04298893362283707, "fcm_dpo/margin": 36.906394958496094, "fcm_dpo/q_t": 0.4008602797985077, "grad_norm": 12.266192436218262, "learning_rate": 4.281735428447157e-07, "logits/chosen": 0.5056573748588562, "logits/rejected": 0.38637077808380127, "logps/chosen": -121.88290405273438, "logps/ref_chosen": -63.869136810302734, "logps/ref_rejected": -98.7657241821289, "logps/rejected": -193.68588256835938, "loss": 1.1084, "margin_dpo/margin_mean": 36.90639114379883, "margin_dpo/margin_std": 54.872398376464844, "step": 215 }, { "KL/chosen_KL_mean": -49.014102935791016, "KL/mean": -68.88029479980469, "KL/rejected_KL_mean": -88.74649047851562, "KL/std": 45.22344207763672, "epoch": 0.32653061224489793, "fcm_dpo/beta": 0.010737748816609383, "fcm_dpo/delta": -0.03410545364022255, "fcm_dpo/margin": 39.732391357421875, "fcm_dpo/q_t": 0.3984612226486206, "grad_norm": 21.626602172851562, "learning_rate": 4.2724355170431247e-07, "logits/chosen": 0.6049730181694031, "logits/rejected": 0.5137053728103638, "logps/chosen": -116.83905792236328, "logps/ref_chosen": -67.824951171875, "logps/ref_rejected": -96.40231323242188, "logps/rejected": -185.1488037109375, "loss": 1.0616, "margin_dpo/margin_mean": 39.732391357421875, "margin_dpo/margin_std": 45.669105529785156, "step": 216 }, { "KL/chosen_KL_mean": -53.43660354614258, "KL/mean": -74.12617492675781, "KL/rejected_KL_mean": -94.81575775146484, "KL/std": 46.69729995727539, "epoch": 0.328042328042328, "fcm_dpo/beta": 0.010264039039611816, "fcm_dpo/delta": -0.03311503306031227, "fcm_dpo/margin": 41.379146575927734, "fcm_dpo/q_t": 0.3996722102165222, "grad_norm": 10.103537559509277, "learning_rate": 4.26308602680756e-07, "logits/chosen": 0.5727427005767822, "logits/rejected": 0.4670327305793762, "logps/chosen": -113.94159698486328, "logps/ref_chosen": -60.5049934387207, "logps/ref_rejected": -84.26618194580078, "logps/rejected": -179.08193969726562, "loss": 1.0655, "margin_dpo/margin_mean": 41.379146575927734, "margin_dpo/margin_std": 48.67888641357422, "step": 217 }, { "KL/chosen_KL_mean": -59.745277404785156, "KL/mean": -71.58668518066406, "KL/rejected_KL_mean": -83.42808532714844, "KL/std": 46.134681701660156, "epoch": 0.3295540438397581, "fcm_dpo/beta": 0.010512400418519974, "fcm_dpo/delta": 0.033613234758377075, "fcm_dpo/margin": 23.68280601501465, "fcm_dpo/q_t": 0.44248485565185547, "grad_norm": 11.078608512878418, "learning_rate": 4.253687219265803e-07, "logits/chosen": 0.46339693665504456, "logits/rejected": 0.4667159914970398, "logps/chosen": -130.339599609375, "logps/ref_chosen": -70.59431457519531, "logps/ref_rejected": -73.89038848876953, "logps/rejected": -157.3184814453125, "loss": 1.2384, "margin_dpo/margin_mean": 23.68280601501465, "margin_dpo/margin_std": 55.93402099609375, "step": 218 }, { "KL/chosen_KL_mean": -55.8802490234375, "KL/mean": -68.90615844726562, "KL/rejected_KL_mean": -81.93206787109375, "KL/std": 44.350650787353516, "epoch": 0.3310657596371882, "fcm_dpo/beta": 0.011553551070392132, "fcm_dpo/delta": 0.11722569167613983, "fcm_dpo/margin": 26.05181884765625, "fcm_dpo/q_t": 0.43380120396614075, "grad_norm": 12.027776718139648, "learning_rate": 4.2442393573227043e-07, "logits/chosen": 0.5445624589920044, "logits/rejected": 0.5009948015213013, "logps/chosen": -116.3711929321289, "logps/ref_chosen": -60.490943908691406, "logps/ref_rejected": -75.85001373291016, "logps/rejected": -157.78207397460938, "loss": 1.1777, "margin_dpo/margin_mean": 26.05181884765625, "margin_dpo/margin_std": 46.083717346191406, "step": 219 }, { "KL/chosen_KL_mean": -47.478050231933594, "KL/mean": -63.36997985839844, "KL/rejected_KL_mean": -79.26191711425781, "KL/std": 47.789817810058594, "epoch": 0.3325774754346183, "fcm_dpo/beta": 0.011883954517543316, "fcm_dpo/delta": 0.024295464158058167, "fcm_dpo/margin": 31.783857345581055, "fcm_dpo/q_t": 0.4159383475780487, "grad_norm": 13.699325561523438, "learning_rate": 4.234742705255272e-07, "logits/chosen": 0.6039453148841858, "logits/rejected": 0.5356512069702148, "logps/chosen": -92.491455078125, "logps/ref_chosen": -45.013397216796875, "logps/ref_rejected": -70.49369812011719, "logps/rejected": -149.755615234375, "loss": 1.1543, "margin_dpo/margin_mean": 31.783859252929688, "margin_dpo/margin_std": 55.53855514526367, "step": 220 }, { "KL/chosen_KL_mean": -47.119110107421875, "KL/mean": -65.26559448242188, "KL/rejected_KL_mean": -83.41207885742188, "KL/std": 47.285316467285156, "epoch": 0.3340891912320484, "fcm_dpo/beta": 0.011956527829170227, "fcm_dpo/delta": -0.04328383877873421, "fcm_dpo/margin": 36.29296112060547, "fcm_dpo/q_t": 0.40149760246276855, "grad_norm": 13.390650749206543, "learning_rate": 4.22519752870528e-07, "logits/chosen": 0.5664784908294678, "logits/rejected": 0.4844658374786377, "logps/chosen": -106.21495056152344, "logps/ref_chosen": -59.09584045410156, "logps/ref_rejected": -88.64388275146484, "logps/rejected": -172.05596923828125, "loss": 1.0937, "margin_dpo/margin_mean": 36.29296112060547, "margin_dpo/margin_std": 53.84539031982422, "step": 221 }, { "KL/chosen_KL_mean": -47.29530334472656, "KL/mean": -71.27445983886719, "KL/rejected_KL_mean": -95.25361633300781, "KL/std": 53.57499694824219, "epoch": 0.3356009070294785, "fcm_dpo/beta": 0.010547778569161892, "fcm_dpo/delta": -0.14348104596138, "fcm_dpo/margin": 47.95830535888672, "fcm_dpo/q_t": 0.3792991638183594, "grad_norm": 11.575348854064941, "learning_rate": 4.2156040946718343e-07, "logits/chosen": 0.6499016284942627, "logits/rejected": 0.5500251054763794, "logps/chosen": -103.29299926757812, "logps/ref_chosen": -55.9976921081543, "logps/ref_rejected": -111.94727325439453, "logps/rejected": -207.20089721679688, "loss": 1.0293, "margin_dpo/margin_mean": 47.95830535888672, "margin_dpo/margin_std": 57.93938446044922, "step": 222 }, { "KL/chosen_KL_mean": -56.65371322631836, "KL/mean": -75.22976684570312, "KL/rejected_KL_mean": -93.80583190917969, "KL/std": 51.016075134277344, "epoch": 0.3371126228269085, "fcm_dpo/beta": 0.010310441255569458, "fcm_dpo/delta": 0.02092297561466694, "fcm_dpo/margin": 37.15211486816406, "fcm_dpo/q_t": 0.4130101799964905, "grad_norm": 12.59165096282959, "learning_rate": 4.2059626715039065e-07, "logits/chosen": 0.6236449480056763, "logits/rejected": 0.5647650361061096, "logps/chosen": -116.54513549804688, "logps/ref_chosen": -59.891422271728516, "logps/ref_rejected": -86.28954315185547, "logps/rejected": -180.09536743164062, "loss": 1.1099, "margin_dpo/margin_mean": 37.1521110534668, "margin_dpo/margin_std": 52.98957061767578, "step": 223 }, { "KL/chosen_KL_mean": -65.9832763671875, "KL/mean": -76.13775634765625, "KL/rejected_KL_mean": -86.292236328125, "KL/std": 50.740623474121094, "epoch": 0.3386243386243386, "fcm_dpo/beta": 0.011297405697405338, "fcm_dpo/delta": 0.07967288047075272, "fcm_dpo/margin": 20.30895233154297, "fcm_dpo/q_t": 0.44954442977905273, "grad_norm": 17.632829666137695, "learning_rate": 4.1962735288928304e-07, "logits/chosen": 0.6315662264823914, "logits/rejected": 0.6085413098335266, "logps/chosen": -130.02792358398438, "logps/ref_chosen": -64.04463195800781, "logps/ref_rejected": -75.05450439453125, "logps/rejected": -161.34674072265625, "loss": 1.26, "margin_dpo/margin_mean": 20.3089542388916, "margin_dpo/margin_std": 54.537742614746094, "step": 224 }, { "KL/chosen_KL_mean": -59.97877502441406, "KL/mean": -81.12055206298828, "KL/rejected_KL_mean": -102.2623291015625, "KL/std": 55.81074905395508, "epoch": 0.3401360544217687, "fcm_dpo/beta": 0.010476674884557724, "fcm_dpo/delta": -0.061587173491716385, "fcm_dpo/margin": 42.28356170654297, "fcm_dpo/q_t": 0.3950624465942383, "grad_norm": 12.086466789245605, "learning_rate": 4.186536937864752e-07, "logits/chosen": 0.6357635259628296, "logits/rejected": 0.5179706811904907, "logps/chosen": -126.07460021972656, "logps/ref_chosen": -66.0958251953125, "logps/ref_rejected": -97.68675231933594, "logps/rejected": -199.94908142089844, "loss": 1.0622, "margin_dpo/margin_mean": 42.28356170654297, "margin_dpo/margin_std": 51.27521896362305, "step": 225 }, { "KL/chosen_KL_mean": -58.04826354980469, "KL/mean": -74.63319396972656, "KL/rejected_KL_mean": -91.21812438964844, "KL/std": 50.970638275146484, "epoch": 0.3416477702191988, "fcm_dpo/beta": 0.011192118749022484, "fcm_dpo/delta": 0.030396804213523865, "fcm_dpo/margin": 33.16985321044922, "fcm_dpo/q_t": 0.4161601662635803, "grad_norm": 13.262188911437988, "learning_rate": 4.176753170773052e-07, "logits/chosen": 0.672427773475647, "logits/rejected": 0.6251287460327148, "logps/chosen": -109.46513366699219, "logps/ref_chosen": -51.4168701171875, "logps/ref_rejected": -66.30068969726562, "logps/rejected": -157.518798828125, "loss": 1.1669, "margin_dpo/margin_mean": 33.16985321044922, "margin_dpo/margin_std": 62.296966552734375, "step": 226 }, { "KL/chosen_KL_mean": -62.35593795776367, "KL/mean": -79.04609680175781, "KL/rejected_KL_mean": -95.73623657226562, "KL/std": 55.52360153198242, "epoch": 0.3431594860166289, "fcm_dpo/beta": 0.011107344180345535, "fcm_dpo/delta": 0.03568103536963463, "fcm_dpo/margin": 33.38031005859375, "fcm_dpo/q_t": 0.4179414212703705, "grad_norm": 13.083468437194824, "learning_rate": 4.166922501290729e-07, "logits/chosen": 0.6802507638931274, "logits/rejected": 0.6432693004608154, "logps/chosen": -120.34571838378906, "logps/ref_chosen": -57.989776611328125, "logps/ref_rejected": -75.05464172363281, "logps/rejected": -170.7908935546875, "loss": 1.1644, "margin_dpo/margin_mean": 33.38031005859375, "margin_dpo/margin_std": 62.746315002441406, "step": 227 }, { "KL/chosen_KL_mean": -65.89985656738281, "KL/mean": -85.34629821777344, "KL/rejected_KL_mean": -104.79273223876953, "KL/std": 52.757869720458984, "epoch": 0.34467120181405897, "fcm_dpo/beta": 0.011195195838809013, "fcm_dpo/delta": -0.04603715240955353, "fcm_dpo/margin": 38.89287567138672, "fcm_dpo/q_t": 0.399330198764801, "grad_norm": 14.76215648651123, "learning_rate": 4.1570452044027405e-07, "logits/chosen": 0.6884451508522034, "logits/rejected": 0.6041937470436096, "logps/chosen": -121.45922088623047, "logps/ref_chosen": -55.55936813354492, "logps/ref_rejected": -77.02364349365234, "logps/rejected": -181.81637573242188, "loss": 1.093, "margin_dpo/margin_mean": 38.89287567138672, "margin_dpo/margin_std": 56.33568572998047, "step": 228 }, { "KL/chosen_KL_mean": -56.05231475830078, "KL/mean": -73.21758270263672, "KL/rejected_KL_mean": -90.38285064697266, "KL/std": 51.87107849121094, "epoch": 0.34618291761148906, "fcm_dpo/beta": 0.011033633723855019, "fcm_dpo/delta": 0.02618766576051712, "fcm_dpo/margin": 34.330543518066406, "fcm_dpo/q_t": 0.4163823127746582, "grad_norm": 16.91952133178711, "learning_rate": 4.147121556398312e-07, "logits/chosen": 0.7776898741722107, "logits/rejected": 0.6957346200942993, "logps/chosen": -106.84697723388672, "logps/ref_chosen": -50.79466247558594, "logps/ref_rejected": -78.4474105834961, "logps/rejected": -168.83026123046875, "loss": 1.1559, "margin_dpo/margin_mean": 34.330543518066406, "margin_dpo/margin_std": 63.17416763305664, "step": 229 }, { "KL/chosen_KL_mean": -60.39435577392578, "KL/mean": -79.12918090820312, "KL/rejected_KL_mean": -97.86400604248047, "KL/std": 55.7061653137207, "epoch": 0.3476946334089191, "fcm_dpo/beta": 0.011317807249724865, "fcm_dpo/delta": -0.03716056048870087, "fcm_dpo/margin": 37.46965408325195, "fcm_dpo/q_t": 0.4025961756706238, "grad_norm": 13.502501487731934, "learning_rate": 4.137151834863213e-07, "logits/chosen": 0.6553382873535156, "logits/rejected": 0.658027172088623, "logps/chosen": -117.12358093261719, "logps/ref_chosen": -56.729225158691406, "logps/ref_rejected": -62.99180603027344, "logps/rejected": -160.85580444335938, "loss": 1.1148, "margin_dpo/margin_mean": 37.46965408325195, "margin_dpo/margin_std": 57.32815170288086, "step": 230 }, { "KL/chosen_KL_mean": -68.11759185791016, "KL/mean": -95.83013916015625, "KL/rejected_KL_mean": -123.5427017211914, "KL/std": 54.258819580078125, "epoch": 0.3492063492063492, "fcm_dpo/beta": 0.009298204444348812, "fcm_dpo/delta": -0.16286741197109222, "fcm_dpo/margin": 55.42510986328125, "fcm_dpo/q_t": 0.3714408874511719, "grad_norm": 12.732369422912598, "learning_rate": 4.1271363186719835e-07, "logits/chosen": 0.5794389247894287, "logits/rejected": 0.5769394636154175, "logps/chosen": -140.71469116210938, "logps/ref_chosen": -72.59709930419922, "logps/ref_rejected": -86.2322998046875, "logps/rejected": -209.77499389648438, "loss": 1.0075, "margin_dpo/margin_mean": 55.42510986328125, "margin_dpo/margin_std": 59.240150451660156, "step": 231 }, { "KL/chosen_KL_mean": -68.00968933105469, "KL/mean": -86.72840881347656, "KL/rejected_KL_mean": -105.44712829589844, "KL/std": 59.867862701416016, "epoch": 0.3507180650037793, "fcm_dpo/beta": 0.00953882560133934, "fcm_dpo/delta": 0.051407087594270706, "fcm_dpo/margin": 37.43744659423828, "fcm_dpo/q_t": 0.4222131371498108, "grad_norm": 12.651902198791504, "learning_rate": 4.1170752879801436e-07, "logits/chosen": 0.6084052324295044, "logits/rejected": 0.5753206014633179, "logps/chosen": -136.1282196044922, "logps/ref_chosen": -68.1185302734375, "logps/ref_rejected": -83.79415893554688, "logps/rejected": -189.24130249023438, "loss": 1.1621, "margin_dpo/margin_mean": 37.43744659423828, "margin_dpo/margin_std": 68.7430191040039, "step": 232 }, { "KL/chosen_KL_mean": -77.7924575805664, "KL/mean": -91.54695129394531, "KL/rejected_KL_mean": -105.30143737792969, "KL/std": 54.55568313598633, "epoch": 0.35222978080120937, "fcm_dpo/beta": 0.00945155881345272, "fcm_dpo/delta": -0.014413751661777496, "fcm_dpo/margin": 27.508983612060547, "fcm_dpo/q_t": 0.4402283728122711, "grad_norm": 12.321041107177734, "learning_rate": 4.106969024216348e-07, "logits/chosen": 0.7221860289573669, "logits/rejected": 0.6576840877532959, "logps/chosen": -132.86260986328125, "logps/ref_chosen": -55.070152282714844, "logps/ref_rejected": -66.61845397949219, "logps/rejected": -171.91989135742188, "loss": 1.2193, "margin_dpo/margin_mean": 27.508983612060547, "margin_dpo/margin_std": 57.265926361083984, "step": 233 }, { "KL/chosen_KL_mean": -68.60944366455078, "KL/mean": -81.6121597290039, "KL/rejected_KL_mean": -94.61487579345703, "KL/std": 52.89654541015625, "epoch": 0.35374149659863946, "fcm_dpo/beta": 0.009320363402366638, "fcm_dpo/delta": -0.028154423460364342, "fcm_dpo/margin": 26.00543975830078, "fcm_dpo/q_t": 0.44363802671432495, "grad_norm": 18.08864974975586, "learning_rate": 4.09681781007452e-07, "logits/chosen": 0.606266975402832, "logits/rejected": 0.6039628982543945, "logps/chosen": -124.53533935546875, "logps/ref_chosen": -55.92589569091797, "logps/ref_rejected": -51.11608123779297, "logps/rejected": -145.73095703125, "loss": 1.2432, "margin_dpo/margin_mean": 26.00543975830078, "margin_dpo/margin_std": 58.44346618652344, "step": 234 }, { "KL/chosen_KL_mean": -62.99231719970703, "KL/mean": -86.39044189453125, "KL/rejected_KL_mean": -109.78857421875, "KL/std": 58.375823974609375, "epoch": 0.35525321239606955, "fcm_dpo/beta": 0.009040637873113155, "fcm_dpo/delta": -0.02921903133392334, "fcm_dpo/margin": 46.7962646484375, "fcm_dpo/q_t": 0.4005424380302429, "grad_norm": 13.630228996276855, "learning_rate": 4.08662192950594e-07, "logits/chosen": 0.6960441470146179, "logits/rejected": 0.6833631992340088, "logps/chosen": -127.53204345703125, "logps/ref_chosen": -64.53972625732422, "logps/ref_rejected": -77.69151306152344, "logps/rejected": -187.48007202148438, "loss": 1.0687, "margin_dpo/margin_mean": 46.7962646484375, "margin_dpo/margin_std": 56.326026916503906, "step": 235 }, { "KL/chosen_KL_mean": -89.99750518798828, "KL/mean": -107.64643859863281, "KL/rejected_KL_mean": -125.29537200927734, "KL/std": 56.224552154541016, "epoch": 0.35676492819349964, "fcm_dpo/beta": 0.009423347190022469, "fcm_dpo/delta": 0.08110842108726501, "fcm_dpo/margin": 35.29786682128906, "fcm_dpo/q_t": 0.42564913630485535, "grad_norm": 11.200446128845215, "learning_rate": 4.076381667711306e-07, "logits/chosen": 0.6303587555885315, "logits/rejected": 0.6174304485321045, "logps/chosen": -161.15225219726562, "logps/ref_chosen": -71.15473937988281, "logps/ref_rejected": -84.88541412353516, "logps/rejected": -210.1807861328125, "loss": 1.1709, "margin_dpo/margin_mean": 35.29786682128906, "margin_dpo/margin_std": 64.950927734375, "step": 236 }, { "KL/chosen_KL_mean": -75.23811340332031, "KL/mean": -97.05203247070312, "KL/rejected_KL_mean": -118.8659439086914, "KL/std": 56.13910675048828, "epoch": 0.35827664399092973, "fcm_dpo/beta": 0.009628495201468468, "fcm_dpo/delta": -0.02585686929523945, "fcm_dpo/margin": 43.6278190612793, "fcm_dpo/q_t": 0.4026941657066345, "grad_norm": 14.894103050231934, "learning_rate": 4.066097311132753e-07, "logits/chosen": 0.6865165829658508, "logits/rejected": 0.6780955791473389, "logps/chosen": -151.380126953125, "logps/ref_chosen": -76.14201354980469, "logps/ref_rejected": -80.88479614257812, "logps/rejected": -199.750732421875, "loss": 1.1072, "margin_dpo/margin_mean": 43.62782287597656, "margin_dpo/margin_std": 65.629150390625, "step": 237 }, { "KL/chosen_KL_mean": -66.66596984863281, "KL/mean": -87.54930877685547, "KL/rejected_KL_mean": -108.43265533447266, "KL/std": 57.61941909790039, "epoch": 0.35978835978835977, "fcm_dpo/beta": 0.00932412687689066, "fcm_dpo/delta": 0.011540941894054413, "fcm_dpo/margin": 41.76669692993164, "fcm_dpo/q_t": 0.4105595350265503, "grad_norm": 15.398371696472168, "learning_rate": 4.0557691474458414e-07, "logits/chosen": 0.6271650791168213, "logits/rejected": 0.6165032386779785, "logps/chosen": -135.55081176757812, "logps/ref_chosen": -68.88484954833984, "logps/ref_rejected": -75.8946304321289, "logps/rejected": -184.32728576660156, "loss": 1.1145, "margin_dpo/margin_mean": 41.766693115234375, "margin_dpo/margin_std": 60.80207061767578, "step": 238 }, { "KL/chosen_KL_mean": -80.04875183105469, "KL/mean": -102.63557434082031, "KL/rejected_KL_mean": -125.22238159179688, "KL/std": 63.35125732421875, "epoch": 0.36130007558578986, "fcm_dpo/beta": 0.009503120556473732, "fcm_dpo/delta": -0.0388723723590374, "fcm_dpo/margin": 45.17362976074219, "fcm_dpo/q_t": 0.40100669860839844, "grad_norm": 18.990455627441406, "learning_rate": 4.045397465551513e-07, "logits/chosen": 0.753252387046814, "logits/rejected": 0.6065235733985901, "logps/chosen": -136.82058715820312, "logps/ref_chosen": -56.771827697753906, "logps/ref_rejected": -116.23050689697266, "logps/rejected": -241.452880859375, "loss": 1.106, "margin_dpo/margin_mean": 45.17362976074219, "margin_dpo/margin_std": 67.95465850830078, "step": 239 }, { "KL/chosen_KL_mean": -79.03121948242188, "KL/mean": -106.46686553955078, "KL/rejected_KL_mean": -133.90252685546875, "KL/std": 63.37854766845703, "epoch": 0.36281179138321995, "fcm_dpo/beta": 0.00855420995503664, "fcm_dpo/delta": -0.09123983979225159, "fcm_dpo/margin": 54.871299743652344, "fcm_dpo/q_t": 0.38927310705184937, "grad_norm": 11.540243148803711, "learning_rate": 4.0349825555680045e-07, "logits/chosen": 0.6777857542037964, "logits/rejected": 0.5775108337402344, "logps/chosen": -132.3853302001953, "logps/ref_chosen": -53.35411071777344, "logps/ref_rejected": -80.12019348144531, "logps/rejected": -214.02272033691406, "loss": 1.0507, "margin_dpo/margin_mean": 54.871299743652344, "margin_dpo/margin_std": 68.62873840332031, "step": 240 }, { "KL/chosen_KL_mean": -74.54257202148438, "KL/mean": -92.24656677246094, "KL/rejected_KL_mean": -109.95057678222656, "KL/std": 60.10264587402344, "epoch": 0.36432350718065004, "fcm_dpo/beta": 0.009079969488084316, "fcm_dpo/delta": 0.0913933590054512, "fcm_dpo/margin": 35.40801239013672, "fcm_dpo/q_t": 0.42948025465011597, "grad_norm": 18.661176681518555, "learning_rate": 4.0245247088227377e-07, "logits/chosen": 0.6268007159233093, "logits/rejected": 0.5906528234481812, "logps/chosen": -146.43798828125, "logps/ref_chosen": -71.89541625976562, "logps/ref_rejected": -83.03492736816406, "logps/rejected": -192.98550415039062, "loss": 1.1824, "margin_dpo/margin_mean": 35.40801239013672, "margin_dpo/margin_std": 68.74810791015625, "step": 241 }, { "KL/chosen_KL_mean": -76.36582946777344, "KL/mean": -98.97297668457031, "KL/rejected_KL_mean": -121.58012390136719, "KL/std": 61.90785598754883, "epoch": 0.36583522297808013, "fcm_dpo/beta": 0.00890383217483759, "fcm_dpo/delta": -0.006449941545724869, "fcm_dpo/margin": 45.21430206298828, "fcm_dpo/q_t": 0.40826159715652466, "grad_norm": 11.749044418334961, "learning_rate": 4.0140242178441665e-07, "logits/chosen": 0.6703622341156006, "logits/rejected": 0.6499079465866089, "logps/chosen": -134.2932586669922, "logps/ref_chosen": -57.927433013916016, "logps/ref_rejected": -67.838623046875, "logps/rejected": -189.4187469482422, "loss": 1.1081, "margin_dpo/margin_mean": 45.21430206298828, "margin_dpo/margin_std": 66.5533218383789, "step": 242 }, { "KL/chosen_KL_mean": -75.65873718261719, "KL/mean": -95.70713806152344, "KL/rejected_KL_mean": -115.75553894042969, "KL/std": 61.08662414550781, "epoch": 0.3673469387755102, "fcm_dpo/beta": 0.009353440254926682, "fcm_dpo/delta": 0.03057572804391384, "fcm_dpo/margin": 40.096805572509766, "fcm_dpo/q_t": 0.41490113735198975, "grad_norm": 16.917295455932617, "learning_rate": 4.003481376353596e-07, "logits/chosen": 0.6367839574813843, "logits/rejected": 0.6507890820503235, "logps/chosen": -149.93540954589844, "logps/ref_chosen": -74.27667236328125, "logps/ref_rejected": -73.24340057373047, "logps/rejected": -188.99893188476562, "loss": 1.1307, "margin_dpo/margin_mean": 40.0968017578125, "margin_dpo/margin_std": 63.99544906616211, "step": 243 }, { "KL/chosen_KL_mean": -68.39207458496094, "KL/mean": -95.18928527832031, "KL/rejected_KL_mean": -121.98651123046875, "KL/std": 61.04261779785156, "epoch": 0.3688586545729403, "fcm_dpo/beta": 0.008910516276955605, "fcm_dpo/delta": -0.10152090340852737, "fcm_dpo/margin": 53.59442138671875, "fcm_dpo/q_t": 0.38667529821395874, "grad_norm": 11.596921920776367, "learning_rate": 3.9928964792569654e-07, "logits/chosen": 0.66209876537323, "logits/rejected": 0.5738247632980347, "logps/chosen": -121.7559814453125, "logps/ref_chosen": -53.36390686035156, "logps/ref_rejected": -71.10276794433594, "logps/rejected": -193.0892791748047, "loss": 1.0465, "margin_dpo/margin_mean": 53.59442138671875, "margin_dpo/margin_std": 64.69428253173828, "step": 244 }, { "KL/chosen_KL_mean": -77.90817260742188, "KL/mean": -108.65547943115234, "KL/rejected_KL_mean": -139.40280151367188, "KL/std": 63.35409164428711, "epoch": 0.37037037037037035, "fcm_dpo/beta": 0.007787792012095451, "fcm_dpo/delta": -0.10556904971599579, "fcm_dpo/margin": 61.49463653564453, "fcm_dpo/q_t": 0.3828907907009125, "grad_norm": 12.549822807312012, "learning_rate": 3.982269822636601e-07, "logits/chosen": 0.6861937046051025, "logits/rejected": 0.660259485244751, "logps/chosen": -149.103271484375, "logps/ref_chosen": -71.19510650634766, "logps/ref_rejected": -80.76235961914062, "logps/rejected": -220.1651611328125, "loss": 1.0244, "margin_dpo/margin_mean": 61.49463653564453, "margin_dpo/margin_std": 65.15496063232422, "step": 245 }, { "KL/chosen_KL_mean": -92.9063720703125, "KL/mean": -114.83541870117188, "KL/rejected_KL_mean": -136.76446533203125, "KL/std": 63.131744384765625, "epoch": 0.37188208616780044, "fcm_dpo/beta": 0.0081062326207757, "fcm_dpo/delta": 0.050879672169685364, "fcm_dpo/margin": 43.858089447021484, "fcm_dpo/q_t": 0.42029231786727905, "grad_norm": 11.98619556427002, "learning_rate": 3.971601703742932e-07, "logits/chosen": 0.7157027721405029, "logits/rejected": 0.6501913070678711, "logps/chosen": -164.5274200439453, "logps/ref_chosen": -71.62104797363281, "logps/ref_rejected": -94.03392028808594, "logps/rejected": -230.7983856201172, "loss": 1.1625, "margin_dpo/margin_mean": 43.858089447021484, "margin_dpo/margin_std": 80.09846496582031, "step": 246 }, { "KL/chosen_KL_mean": -96.20511627197266, "KL/mean": -111.12162780761719, "KL/rejected_KL_mean": -126.03813171386719, "KL/std": 64.35607147216797, "epoch": 0.37339380196523053, "fcm_dpo/beta": 0.008263746276497841, "fcm_dpo/delta": 0.05345098674297333, "fcm_dpo/margin": 29.833005905151367, "fcm_dpo/q_t": 0.4449678957462311, "grad_norm": 15.636235237121582, "learning_rate": 3.960892420986177e-07, "logits/chosen": 0.7111119031906128, "logits/rejected": 0.7009575366973877, "logps/chosen": -176.2276611328125, "logps/ref_chosen": -80.02254486083984, "logps/ref_rejected": -89.22705841064453, "logps/rejected": -215.26519775390625, "loss": 1.2462, "margin_dpo/margin_mean": 29.8330078125, "margin_dpo/margin_std": 75.14302825927734, "step": 247 }, { "KL/chosen_KL_mean": -81.6597900390625, "KL/mean": -106.406494140625, "KL/rejected_KL_mean": -131.1531982421875, "KL/std": 66.31968688964844, "epoch": 0.3749055177626606, "fcm_dpo/beta": 0.008432338945567608, "fcm_dpo/delta": -0.02212671749293804, "fcm_dpo/margin": 49.493408203125, "fcm_dpo/q_t": 0.4049510359764099, "grad_norm": 17.001266479492188, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 0.6953562498092651, "logits/rejected": 0.7540360689163208, "logps/chosen": -147.03775024414062, "logps/ref_chosen": -65.37796020507812, "logps/ref_rejected": -61.365787506103516, "logps/rejected": -192.51898193359375, "loss": 1.1098, "margin_dpo/margin_mean": 49.493408203125, "margin_dpo/margin_std": 76.3125, "step": 248 }, { "KL/chosen_KL_mean": -103.32197570800781, "KL/mean": -111.87638854980469, "KL/rejected_KL_mean": -120.43080139160156, "KL/std": 62.08591079711914, "epoch": 0.3764172335600907, "fcm_dpo/beta": 0.008548267185688019, "fcm_dpo/delta": 0.058366335928440094, "fcm_dpo/margin": 17.108829498291016, "fcm_dpo/q_t": 0.46602004766464233, "grad_norm": 15.90197467803955, "learning_rate": 3.9393515632731094e-07, "logits/chosen": 0.6849374771118164, "logits/rejected": 0.7285729646682739, "logps/chosen": -177.92343139648438, "logps/ref_chosen": -74.60145568847656, "logps/ref_rejected": -63.79338455200195, "logps/rejected": -184.22418212890625, "loss": 1.3511, "margin_dpo/margin_mean": 17.108827590942383, "margin_dpo/margin_std": 76.58978271484375, "step": 249 }, { "KL/chosen_KL_mean": -90.82321166992188, "KL/mean": -116.01731872558594, "KL/rejected_KL_mean": -141.21141052246094, "KL/std": 69.73461151123047, "epoch": 0.3779289493575208, "fcm_dpo/beta": 0.008761906996369362, "fcm_dpo/delta": -0.056977760046720505, "fcm_dpo/margin": 50.38819122314453, "fcm_dpo/q_t": 0.3973717987537384, "grad_norm": 15.959137916564941, "learning_rate": 3.9285205908608934e-07, "logits/chosen": 0.7684859037399292, "logits/rejected": 0.7283642888069153, "logps/chosen": -152.7614288330078, "logps/ref_chosen": -61.938209533691406, "logps/ref_rejected": -72.21602630615234, "logps/rejected": -213.42742919921875, "loss": 1.0926, "margin_dpo/margin_mean": 50.38819122314453, "margin_dpo/margin_std": 71.78437805175781, "step": 250 }, { "KL/chosen_KL_mean": -98.59217834472656, "KL/mean": -115.84170532226562, "KL/rejected_KL_mean": -133.0912322998047, "KL/std": 61.36365509033203, "epoch": 0.3794406651549509, "fcm_dpo/beta": 0.008924310095608234, "fcm_dpo/delta": 0.10951556265354156, "fcm_dpo/margin": 34.499053955078125, "fcm_dpo/q_t": 0.4327406883239746, "grad_norm": 23.957935333251953, "learning_rate": 3.9176496596569265e-07, "logits/chosen": 0.7150743007659912, "logits/rejected": 0.6731328964233398, "logps/chosen": -165.44912719726562, "logps/ref_chosen": -66.85694885253906, "logps/ref_rejected": -84.83396911621094, "logps/rejected": -217.92520141601562, "loss": 1.1986, "margin_dpo/margin_mean": 34.499053955078125, "margin_dpo/margin_std": 70.83842468261719, "step": 251 }, { "KL/chosen_KL_mean": -93.67896270751953, "KL/mean": -106.87741088867188, "KL/rejected_KL_mean": -120.07587432861328, "KL/std": 63.966712951660156, "epoch": 0.38095238095238093, "fcm_dpo/beta": 0.009195334278047085, "fcm_dpo/delta": -0.016725320369005203, "fcm_dpo/margin": 26.396902084350586, "fcm_dpo/q_t": 0.4438965320587158, "grad_norm": 21.82607078552246, "learning_rate": 3.9067390737445254e-07, "logits/chosen": 0.6887344121932983, "logits/rejected": 0.6316691040992737, "logps/chosen": -149.90289306640625, "logps/ref_chosen": -56.22393035888672, "logps/ref_rejected": -77.1136245727539, "logps/rejected": -197.1894989013672, "loss": 1.2812, "margin_dpo/margin_mean": 26.396900177001953, "margin_dpo/margin_std": 75.36380767822266, "step": 252 }, { "KL/chosen_KL_mean": -93.51123046875, "KL/mean": -111.86512756347656, "KL/rejected_KL_mean": -130.21900939941406, "KL/std": 61.31932067871094, "epoch": 0.382464096749811, "fcm_dpo/beta": 0.008835416287183762, "fcm_dpo/delta": -0.03153040632605553, "fcm_dpo/margin": 36.70779800415039, "fcm_dpo/q_t": 0.4239751100540161, "grad_norm": 16.764225006103516, "learning_rate": 3.8957891383162304e-07, "logits/chosen": 0.7815088033676147, "logits/rejected": 0.7435786724090576, "logps/chosen": -145.7212371826172, "logps/ref_chosen": -52.21001434326172, "logps/ref_rejected": -58.75764846801758, "logps/rejected": -188.97665405273438, "loss": 1.1675, "margin_dpo/margin_mean": 36.70779800415039, "margin_dpo/margin_std": 63.84772491455078, "step": 253 }, { "KL/chosen_KL_mean": -101.70146179199219, "KL/mean": -121.41376495361328, "KL/rejected_KL_mean": -141.1260528564453, "KL/std": 67.48922729492188, "epoch": 0.3839758125472411, "fcm_dpo/beta": 0.009037662297487259, "fcm_dpo/delta": 0.052855074405670166, "fcm_dpo/margin": 39.424591064453125, "fcm_dpo/q_t": 0.42191147804260254, "grad_norm": 13.55975341796875, "learning_rate": 3.884800159665276e-07, "logits/chosen": 0.6931102275848389, "logits/rejected": 0.6379691958427429, "logps/chosen": -167.33779907226562, "logps/ref_chosen": -65.63632202148438, "logps/ref_rejected": -82.34425354003906, "logps/rejected": -223.47030639648438, "loss": 1.1501, "margin_dpo/margin_mean": 39.424591064453125, "margin_dpo/margin_std": 67.18295288085938, "step": 254 }, { "KL/chosen_KL_mean": -92.79487609863281, "KL/mean": -115.30006408691406, "KL/rejected_KL_mean": -137.80523681640625, "KL/std": 64.47406005859375, "epoch": 0.3854875283446712, "fcm_dpo/beta": 0.009151535108685493, "fcm_dpo/delta": -0.015306653454899788, "fcm_dpo/margin": 45.01036834716797, "fcm_dpo/q_t": 0.4061132073402405, "grad_norm": 19.183948516845703, "learning_rate": 3.873772445177015e-07, "logits/chosen": 0.6413042545318604, "logits/rejected": 0.6087009906768799, "logps/chosen": -160.70596313476562, "logps/ref_chosen": -67.91108703613281, "logps/ref_rejected": -83.89114379882812, "logps/rejected": -221.69639587402344, "loss": 1.1178, "margin_dpo/margin_mean": 45.01036834716797, "margin_dpo/margin_std": 72.30084228515625, "step": 255 }, { "KL/chosen_KL_mean": -110.49552917480469, "KL/mean": -132.25213623046875, "KL/rejected_KL_mean": -154.00872802734375, "KL/std": 61.51966094970703, "epoch": 0.3869992441421013, "fcm_dpo/beta": 0.009058517403900623, "fcm_dpo/delta": 0.006048116832971573, "fcm_dpo/margin": 43.51321029663086, "fcm_dpo/q_t": 0.4129956364631653, "grad_norm": 16.235132217407227, "learning_rate": 3.862706303320329e-07, "logits/chosen": 0.7141859531402588, "logits/rejected": 0.6413957476615906, "logps/chosen": -173.99551391601562, "logps/ref_chosen": -63.49998474121094, "logps/ref_rejected": -90.77104187011719, "logps/rejected": -244.77978515625, "loss": 1.1552, "margin_dpo/margin_mean": 43.513206481933594, "margin_dpo/margin_std": 81.13447570800781, "step": 256 }, { "KL/chosen_KL_mean": -102.68710327148438, "KL/mean": -131.61746215820312, "KL/rejected_KL_mean": -160.54782104492188, "KL/std": 69.95053100585938, "epoch": 0.3885109599395314, "fcm_dpo/beta": 0.008318210951983929, "fcm_dpo/delta": -0.11226293444633484, "fcm_dpo/margin": 57.86071014404297, "fcm_dpo/q_t": 0.3880605101585388, "grad_norm": 11.517922401428223, "learning_rate": 3.851602043638994e-07, "logits/chosen": 0.6858439445495605, "logits/rejected": 0.6109893321990967, "logps/chosen": -173.28775024414062, "logps/ref_chosen": -70.60064697265625, "logps/ref_rejected": -108.58313751220703, "logps/rejected": -269.1309509277344, "loss": 1.062, "margin_dpo/margin_mean": 57.86071014404297, "margin_dpo/margin_std": 80.06947326660156, "step": 257 }, { "KL/chosen_KL_mean": -106.1284408569336, "KL/mean": -127.41146087646484, "KL/rejected_KL_mean": -148.6944580078125, "KL/std": 61.25123596191406, "epoch": 0.3900226757369615, "fcm_dpo/beta": 0.008503508754074574, "fcm_dpo/delta": 0.046553075313568115, "fcm_dpo/margin": 42.5660285949707, "fcm_dpo/q_t": 0.4167933762073517, "grad_norm": 14.567258834838867, "learning_rate": 3.840459976743023e-07, "logits/chosen": 0.7221835255622864, "logits/rejected": 0.6666927337646484, "logps/chosen": -165.38259887695312, "logps/ref_chosen": -59.25416564941406, "logps/ref_rejected": -85.58709716796875, "logps/rejected": -234.28155517578125, "loss": 1.1265, "margin_dpo/margin_mean": 42.56603240966797, "margin_dpo/margin_std": 63.77388000488281, "step": 258 }, { "KL/chosen_KL_mean": -93.2396240234375, "KL/mean": -124.42048645019531, "KL/rejected_KL_mean": -155.60134887695312, "KL/std": 67.78335571289062, "epoch": 0.3915343915343915, "fcm_dpo/beta": 0.007766705006361008, "fcm_dpo/delta": -0.11655537784099579, "fcm_dpo/margin": 62.36172103881836, "fcm_dpo/q_t": 0.3838648796081543, "grad_norm": 12.29951286315918, "learning_rate": 3.8292804142999796e-07, "logits/chosen": 0.6192357540130615, "logits/rejected": 0.5121684074401855, "logps/chosen": -158.67449951171875, "logps/ref_chosen": -65.43487548828125, "logps/ref_rejected": -95.41731262207031, "logps/rejected": -251.01866149902344, "loss": 1.0588, "margin_dpo/margin_mean": 62.36172103881836, "margin_dpo/margin_std": 83.14129638671875, "step": 259 }, { "KL/chosen_KL_mean": -93.89341735839844, "KL/mean": -117.76322937011719, "KL/rejected_KL_mean": -141.633056640625, "KL/std": 62.05198669433594, "epoch": 0.3930461073318216, "fcm_dpo/beta": 0.00776186678558588, "fcm_dpo/delta": 0.03546956554055214, "fcm_dpo/margin": 47.73963928222656, "fcm_dpo/q_t": 0.4163569509983063, "grad_norm": 17.107166290283203, "learning_rate": 3.818063669026256e-07, "logits/chosen": 0.7102089524269104, "logits/rejected": 0.6064608097076416, "logps/chosen": -142.98300170898438, "logps/ref_chosen": -49.08958435058594, "logps/ref_rejected": -79.01708221435547, "logps/rejected": -220.65013122558594, "loss": 1.1402, "margin_dpo/margin_mean": 47.73963928222656, "margin_dpo/margin_std": 78.65926361083984, "step": 260 }, { "KL/chosen_KL_mean": -93.87600708007812, "KL/mean": -114.30181884765625, "KL/rejected_KL_mean": -134.7276611328125, "KL/std": 60.913230895996094, "epoch": 0.3945578231292517, "fcm_dpo/beta": 0.008519364520907402, "fcm_dpo/delta": 0.058973412960767746, "fcm_dpo/margin": 40.85166931152344, "fcm_dpo/q_t": 0.422269344329834, "grad_norm": 16.87601661682129, "learning_rate": 3.806810054678331e-07, "logits/chosen": 0.5581406354904175, "logits/rejected": 0.6009823679924011, "logps/chosen": -164.74839782714844, "logps/ref_chosen": -70.87239074707031, "logps/ref_rejected": -65.01522064208984, "logps/rejected": -199.74288940429688, "loss": 1.1551, "margin_dpo/margin_mean": 40.85166931152344, "margin_dpo/margin_std": 70.33143615722656, "step": 261 }, { "KL/chosen_KL_mean": -101.00912475585938, "KL/mean": -121.16682434082031, "KL/rejected_KL_mean": -141.32452392578125, "KL/std": 61.25640869140625, "epoch": 0.3960695389266818, "fcm_dpo/beta": 0.008892569690942764, "fcm_dpo/delta": 0.048768118023872375, "fcm_dpo/margin": 40.31542205810547, "fcm_dpo/q_t": 0.41975611448287964, "grad_norm": 19.799497604370117, "learning_rate": 3.7955198860439887e-07, "logits/chosen": 0.7563103437423706, "logits/rejected": 0.684789776802063, "logps/chosen": -168.87974548339844, "logps/ref_chosen": -67.8706283569336, "logps/ref_rejected": -88.7205810546875, "logps/rejected": -230.04510498046875, "loss": 1.148, "margin_dpo/margin_mean": 40.31542205810547, "margin_dpo/margin_std": 68.66963195800781, "step": 262 }, { "KL/chosen_KL_mean": -95.22956085205078, "KL/mean": -116.42835998535156, "KL/rejected_KL_mean": -137.62718200683594, "KL/std": 62.601688385009766, "epoch": 0.3975812547241119, "fcm_dpo/beta": 0.00906696729362011, "fcm_dpo/delta": 0.01888187974691391, "fcm_dpo/margin": 42.397621154785156, "fcm_dpo/q_t": 0.41246914863586426, "grad_norm": 16.852615356445312, "learning_rate": 3.784193478933516e-07, "logits/chosen": 0.6627376675605774, "logits/rejected": 0.5485380291938782, "logps/chosen": -150.42413330078125, "logps/ref_chosen": -55.194583892822266, "logps/ref_rejected": -80.54048156738281, "logps/rejected": -218.16766357421875, "loss": 1.1307, "margin_dpo/margin_mean": 42.397621154785156, "margin_dpo/margin_std": 68.78619384765625, "step": 263 }, { "KL/chosen_KL_mean": -97.0634765625, "KL/mean": -120.03509521484375, "KL/rejected_KL_mean": -143.00674438476562, "KL/std": 69.64917755126953, "epoch": 0.39909297052154197, "fcm_dpo/beta": 0.008921649307012558, "fcm_dpo/delta": -0.012593775056302547, "fcm_dpo/margin": 45.94325637817383, "fcm_dpo/q_t": 0.40747833251953125, "grad_norm": 15.435047149658203, "learning_rate": 3.7728311501708674e-07, "logits/chosen": 0.5868898630142212, "logits/rejected": 0.5499594211578369, "logps/chosen": -180.23416137695312, "logps/ref_chosen": -83.17068481445312, "logps/ref_rejected": -88.33625793457031, "logps/rejected": -231.34298706054688, "loss": 1.1283, "margin_dpo/margin_mean": 45.94325256347656, "margin_dpo/margin_std": 76.74105072021484, "step": 264 }, { "KL/chosen_KL_mean": -92.12577819824219, "KL/mean": -115.92205810546875, "KL/rejected_KL_mean": -139.7183380126953, "KL/std": 64.89045715332031, "epoch": 0.40060468631897206, "fcm_dpo/beta": 0.008840169757604599, "fcm_dpo/delta": -0.02635415829718113, "fcm_dpo/margin": 47.59254837036133, "fcm_dpo/q_t": 0.404721736907959, "grad_norm": 12.38251781463623, "learning_rate": 3.7614332175848027e-07, "logits/chosen": 0.7509992122650146, "logits/rejected": 0.6839336156845093, "logps/chosen": -143.78863525390625, "logps/ref_chosen": -51.66284942626953, "logps/ref_rejected": -67.1720962524414, "logps/rejected": -206.89044189453125, "loss": 1.1367, "margin_dpo/margin_mean": 47.59254455566406, "margin_dpo/margin_std": 83.42626953125, "step": 265 }, { "KL/chosen_KL_mean": -87.49989318847656, "KL/mean": -111.67654418945312, "KL/rejected_KL_mean": -135.8531951904297, "KL/std": 65.83747863769531, "epoch": 0.4021164021164021, "fcm_dpo/beta": 0.008526175282895565, "fcm_dpo/delta": -0.015927668660879135, "fcm_dpo/margin": 48.353309631347656, "fcm_dpo/q_t": 0.40604427456855774, "grad_norm": 22.090471267700195, "learning_rate": 3.75e-07, "logits/chosen": 0.6847240924835205, "logits/rejected": 0.6065027713775635, "logps/chosen": -144.9503936767578, "logps/ref_chosen": -57.45049285888672, "logps/ref_rejected": -77.60826110839844, "logps/rejected": -213.46145629882812, "loss": 1.1181, "margin_dpo/margin_mean": 48.353309631347656, "margin_dpo/margin_std": 77.61994934082031, "step": 266 }, { "KL/chosen_KL_mean": -80.65776824951172, "KL/mean": -99.42727661132812, "KL/rejected_KL_mean": -118.19677734375, "KL/std": 63.5875358581543, "epoch": 0.4036281179138322, "fcm_dpo/beta": 0.008297070860862732, "fcm_dpo/delta": -0.06384800374507904, "fcm_dpo/margin": 37.53900909423828, "fcm_dpo/q_t": 0.4270240068435669, "grad_norm": 15.486305236816406, "learning_rate": 3.738531817228131e-07, "logits/chosen": 0.7166332006454468, "logits/rejected": 0.697598934173584, "logps/chosen": -135.693115234375, "logps/ref_chosen": -55.03535079956055, "logps/ref_rejected": -66.0953369140625, "logps/rejected": -184.2921142578125, "loss": 1.2079, "margin_dpo/margin_mean": 37.539005279541016, "margin_dpo/margin_std": 74.69694519042969, "step": 267 }, { "KL/chosen_KL_mean": -77.2738037109375, "KL/mean": -101.48638153076172, "KL/rejected_KL_mean": -125.69896697998047, "KL/std": 66.79031372070312, "epoch": 0.4051398337112623, "fcm_dpo/beta": 0.008188547566533089, "fcm_dpo/delta": 0.0029786527156829834, "fcm_dpo/margin": 48.42516326904297, "fcm_dpo/q_t": 0.4089292287826538, "grad_norm": 14.64191722869873, "learning_rate": 3.7270289900589204e-07, "logits/chosen": 0.5708452463150024, "logits/rejected": 0.5601274371147156, "logps/chosen": -142.34555053710938, "logps/ref_chosen": -65.07174682617188, "logps/ref_rejected": -71.42485809326172, "logps/rejected": -197.1238250732422, "loss": 1.0983, "margin_dpo/margin_mean": 48.42516326904297, "margin_dpo/margin_std": 65.15290832519531, "step": 268 }, { "KL/chosen_KL_mean": -91.49454498291016, "KL/mean": -116.61368560791016, "KL/rejected_KL_mean": -141.73281860351562, "KL/std": 70.12186431884766, "epoch": 0.40665154950869237, "fcm_dpo/beta": 0.008165009319782257, "fcm_dpo/delta": -0.015037402510643005, "fcm_dpo/margin": 50.23826599121094, "fcm_dpo/q_t": 0.40580272674560547, "grad_norm": 14.160943031311035, "learning_rate": 3.7154918402511714e-07, "logits/chosen": 0.7557035088539124, "logits/rejected": 0.7087898254394531, "logps/chosen": -158.63075256347656, "logps/ref_chosen": -67.1362075805664, "logps/ref_rejected": -82.55778503417969, "logps/rejected": -224.2906036376953, "loss": 1.0967, "margin_dpo/margin_mean": 50.2382698059082, "margin_dpo/margin_std": 69.37471771240234, "step": 269 }, { "KL/chosen_KL_mean": -90.2132568359375, "KL/mean": -112.82662200927734, "KL/rejected_KL_mean": -135.4399871826172, "KL/std": 68.86807250976562, "epoch": 0.40816326530612246, "fcm_dpo/beta": 0.00815459992736578, "fcm_dpo/delta": 0.0382687933743, "fcm_dpo/margin": 45.226722717285156, "fcm_dpo/q_t": 0.41671714186668396, "grad_norm": 15.69405460357666, "learning_rate": 3.7039206905237656e-07, "logits/chosen": 0.7264769673347473, "logits/rejected": 0.6401715278625488, "logps/chosen": -156.90194702148438, "logps/ref_chosen": -66.6886978149414, "logps/ref_rejected": -85.16129302978516, "logps/rejected": -220.60128784179688, "loss": 1.1389, "margin_dpo/margin_mean": 45.22671890258789, "margin_dpo/margin_std": 75.06895446777344, "step": 270 }, { "KL/chosen_KL_mean": -92.40959167480469, "KL/mean": -108.70755004882812, "KL/rejected_KL_mean": -125.00550842285156, "KL/std": 74.48558044433594, "epoch": 0.40967498110355255, "fcm_dpo/beta": 0.008555498905479908, "fcm_dpo/delta": 0.03689521178603172, "fcm_dpo/margin": 32.59592056274414, "fcm_dpo/q_t": 0.44212037324905396, "grad_norm": 18.4699649810791, "learning_rate": 3.692315864546635e-07, "logits/chosen": 0.7211564779281616, "logits/rejected": 0.6540436744689941, "logps/chosen": -164.817138671875, "logps/ref_chosen": -72.40754699707031, "logps/ref_rejected": -92.06311798095703, "logps/rejected": -217.06863403320312, "loss": 1.2611, "margin_dpo/margin_mean": 32.595916748046875, "margin_dpo/margin_std": 88.71417236328125, "step": 271 }, { "KL/chosen_KL_mean": -76.29147338867188, "KL/mean": -107.88369750976562, "KL/rejected_KL_mean": -139.47592163085938, "KL/std": 63.384490966796875, "epoch": 0.41118669690098264, "fcm_dpo/beta": 0.007848805747926235, "fcm_dpo/delta": -0.127114936709404, "fcm_dpo/margin": 63.18445587158203, "fcm_dpo/q_t": 0.3781195282936096, "grad_norm": 15.043636322021484, "learning_rate": 3.6806776869317067e-07, "logits/chosen": 0.6831435561180115, "logits/rejected": 0.7051393985748291, "logps/chosen": -142.8928680419922, "logps/ref_chosen": -66.60140228271484, "logps/ref_rejected": -67.74340057373047, "logps/rejected": -207.2193145751953, "loss": 1.0052, "margin_dpo/margin_mean": 63.184452056884766, "margin_dpo/margin_std": 60.51321029663086, "step": 272 }, { "KL/chosen_KL_mean": -97.41845703125, "KL/mean": -120.54800415039062, "KL/rejected_KL_mean": -143.67755126953125, "KL/std": 70.76217651367188, "epoch": 0.4126984126984127, "fcm_dpo/beta": 0.007727320306003094, "fcm_dpo/delta": 0.05174464359879494, "fcm_dpo/margin": 46.25908279418945, "fcm_dpo/q_t": 0.42130160331726074, "grad_norm": 19.00057601928711, "learning_rate": 3.669006483223828e-07, "logits/chosen": 0.6799752712249756, "logits/rejected": 0.6046786904335022, "logps/chosen": -154.77334594726562, "logps/ref_chosen": -57.35487747192383, "logps/ref_rejected": -84.17168426513672, "logps/rejected": -227.84922790527344, "loss": 1.1757, "margin_dpo/margin_mean": 46.25908279418945, "margin_dpo/margin_std": 89.17124938964844, "step": 273 }, { "KL/chosen_KL_mean": -96.37403106689453, "KL/mean": -121.2509765625, "KL/rejected_KL_mean": -146.1279296875, "KL/std": 67.85552978515625, "epoch": 0.41421012849584277, "fcm_dpo/beta": 0.008015267550945282, "fcm_dpo/delta": 0.001110810786485672, "fcm_dpo/margin": 49.753902435302734, "fcm_dpo/q_t": 0.40851789712905884, "grad_norm": 13.290173530578613, "learning_rate": 3.657302579891656e-07, "logits/chosen": 0.5827606320381165, "logits/rejected": 0.5658946633338928, "logps/chosen": -156.01553344726562, "logps/ref_chosen": -59.64149475097656, "logps/ref_rejected": -68.29348754882812, "logps/rejected": -214.42141723632812, "loss": 1.1243, "margin_dpo/margin_mean": 49.75389862060547, "margin_dpo/margin_std": 80.14501190185547, "step": 274 }, { "KL/chosen_KL_mean": -93.57954406738281, "KL/mean": -118.13485717773438, "KL/rejected_KL_mean": -142.69017028808594, "KL/std": 64.97964477539062, "epoch": 0.41572184429327286, "fcm_dpo/beta": 0.008048377931118011, "fcm_dpo/delta": 0.005478540435433388, "fcm_dpo/margin": 49.110626220703125, "fcm_dpo/q_t": 0.4092429578304291, "grad_norm": 16.686729431152344, "learning_rate": 3.645566304318526e-07, "logits/chosen": 0.666012167930603, "logits/rejected": 0.5821614265441895, "logps/chosen": -146.84619140625, "logps/ref_chosen": -53.26664352416992, "logps/ref_rejected": -73.84062194824219, "logps/rejected": -216.53079223632812, "loss": 1.0982, "margin_dpo/margin_mean": 49.110626220703125, "margin_dpo/margin_std": 66.99764251708984, "step": 275 }, { "KL/chosen_KL_mean": -87.14787292480469, "KL/mean": -111.3336181640625, "KL/rejected_KL_mean": -135.51934814453125, "KL/std": 59.877315521240234, "epoch": 0.41723356009070295, "fcm_dpo/beta": 0.008093775250017643, "fcm_dpo/delta": 0.010346898809075356, "fcm_dpo/margin": 48.37148666381836, "fcm_dpo/q_t": 0.40958449244499207, "grad_norm": 18.336421966552734, "learning_rate": 3.633797984793294e-07, "logits/chosen": 0.6678320169448853, "logits/rejected": 0.6339179277420044, "logps/chosen": -140.16867065429688, "logps/ref_chosen": -53.02079772949219, "logps/ref_rejected": -61.56678771972656, "logps/rejected": -197.08615112304688, "loss": 1.1051, "margin_dpo/margin_mean": 48.37148666381836, "margin_dpo/margin_std": 68.28596496582031, "step": 276 }, { "KL/chosen_KL_mean": -101.01702880859375, "KL/mean": -112.96041870117188, "KL/rejected_KL_mean": -124.90379333496094, "KL/std": 69.72208404541016, "epoch": 0.41874527588813304, "fcm_dpo/beta": 0.008755484595894814, "fcm_dpo/delta": 0.08153992146253586, "fcm_dpo/margin": 23.88676643371582, "fcm_dpo/q_t": 0.45458197593688965, "grad_norm": 21.524913787841797, "learning_rate": 3.6219979505011555e-07, "logits/chosen": 0.7365692853927612, "logits/rejected": 0.7629575133323669, "logps/chosen": -172.45001220703125, "logps/ref_chosen": -71.43299102783203, "logps/ref_rejected": -67.65852355957031, "logps/rejected": -192.56231689453125, "loss": 1.2996, "margin_dpo/margin_mean": 23.886764526367188, "margin_dpo/margin_std": 77.9295654296875, "step": 277 }, { "KL/chosen_KL_mean": -107.86801147460938, "KL/mean": -131.03553771972656, "KL/rejected_KL_mean": -154.20306396484375, "KL/std": 71.07804870605469, "epoch": 0.42025699168556313, "fcm_dpo/beta": 0.008978527039289474, "fcm_dpo/delta": -0.027673780918121338, "fcm_dpo/margin": 46.335052490234375, "fcm_dpo/q_t": 0.4027869999408722, "grad_norm": 23.630117416381836, "learning_rate": 3.6101665315144353e-07, "logits/chosen": 0.651962399482727, "logits/rejected": 0.5864707827568054, "logps/chosen": -174.97877502441406, "logps/ref_chosen": -67.11076354980469, "logps/ref_rejected": -88.74851989746094, "logps/rejected": -242.9515838623047, "loss": 1.1303, "margin_dpo/margin_mean": 46.33504867553711, "margin_dpo/margin_std": 74.0306396484375, "step": 278 }, { "KL/chosen_KL_mean": -84.87873840332031, "KL/mean": -117.05733489990234, "KL/rejected_KL_mean": -149.23593139648438, "KL/std": 68.21269226074219, "epoch": 0.4217687074829932, "fcm_dpo/beta": 0.007804807275533676, "fcm_dpo/delta": -0.13575728237628937, "fcm_dpo/margin": 64.35718536376953, "fcm_dpo/q_t": 0.3770613670349121, "grad_norm": 15.381987571716309, "learning_rate": 3.5983040587833563e-07, "logits/chosen": 0.6404179334640503, "logits/rejected": 0.6012611389160156, "logps/chosen": -139.376220703125, "logps/ref_chosen": -54.49748611450195, "logps/ref_rejected": -70.42373657226562, "logps/rejected": -219.65966796875, "loss": 1.0157, "margin_dpo/margin_mean": 64.35718536376953, "margin_dpo/margin_std": 67.33639526367188, "step": 279 }, { "KL/chosen_KL_mean": -91.69462585449219, "KL/mean": -125.66950225830078, "KL/rejected_KL_mean": -159.64437866210938, "KL/std": 71.94490051269531, "epoch": 0.42328042328042326, "fcm_dpo/beta": 0.006942619103938341, "fcm_dpo/delta": -0.09404292702674866, "fcm_dpo/margin": 67.94975280761719, "fcm_dpo/q_t": 0.385819673538208, "grad_norm": 10.739294052124023, "learning_rate": 3.586410864126781e-07, "logits/chosen": 0.7250418663024902, "logits/rejected": 0.6764532327651978, "logps/chosen": -152.12744140625, "logps/ref_chosen": -60.43281173706055, "logps/ref_rejected": -78.39051818847656, "logps/rejected": -238.034912109375, "loss": 1.0215, "margin_dpo/margin_mean": 67.94975280761719, "margin_dpo/margin_std": 68.12228393554688, "step": 280 }, { "KL/chosen_KL_mean": -94.15901184082031, "KL/mean": -121.11312866210938, "KL/rejected_KL_mean": -148.06723022460938, "KL/std": 70.25721740722656, "epoch": 0.42479213907785335, "fcm_dpo/beta": 0.007024562917649746, "fcm_dpo/delta": 0.02409074455499649, "fcm_dpo/margin": 53.908233642578125, "fcm_dpo/q_t": 0.4124564230442047, "grad_norm": 12.674901008605957, "learning_rate": 3.574487280222929e-07, "logits/chosen": 0.6601166725158691, "logits/rejected": 0.6880499124526978, "logps/chosen": -154.44110107421875, "logps/ref_chosen": -60.2820930480957, "logps/ref_rejected": -62.04009246826172, "logps/rejected": -210.10733032226562, "loss": 1.1157, "margin_dpo/margin_mean": 53.90823745727539, "margin_dpo/margin_std": 77.37689208984375, "step": 281 }, { "KL/chosen_KL_mean": -102.38758087158203, "KL/mean": -126.72209167480469, "KL/rejected_KL_mean": -151.05661010742188, "KL/std": 70.62893676757812, "epoch": 0.42630385487528344, "fcm_dpo/beta": 0.006782787851989269, "fcm_dpo/delta": -0.04430830478668213, "fcm_dpo/margin": 48.669010162353516, "fcm_dpo/q_t": 0.4228168725967407, "grad_norm": 15.216257095336914, "learning_rate": 3.562533640600075e-07, "logits/chosen": 0.6253547668457031, "logits/rejected": 0.5807002782821655, "logps/chosen": -163.01150512695312, "logps/ref_chosen": -60.623924255371094, "logps/ref_rejected": -68.67400360107422, "logps/rejected": -219.73060607910156, "loss": 1.1665, "margin_dpo/margin_mean": 48.66900634765625, "margin_dpo/margin_std": 82.96221923828125, "step": 282 }, { "KL/chosen_KL_mean": -99.55526733398438, "KL/mean": -125.99640655517578, "KL/rejected_KL_mean": -152.4375457763672, "KL/std": 74.96566772460938, "epoch": 0.42781557067271353, "fcm_dpo/beta": 0.006863541901111603, "fcm_dpo/delta": 0.04517936706542969, "fcm_dpo/margin": 52.88227844238281, "fcm_dpo/q_t": 0.41826343536376953, "grad_norm": 13.062129974365234, "learning_rate": 3.550550279627215e-07, "logits/chosen": 0.6494210362434387, "logits/rejected": 0.5451687574386597, "logps/chosen": -167.20301818847656, "logps/ref_chosen": -67.64775085449219, "logps/ref_rejected": -99.96835327148438, "logps/rejected": -252.40589904785156, "loss": 1.1343, "margin_dpo/margin_mean": 52.88227081298828, "margin_dpo/margin_std": 83.99928283691406, "step": 283 }, { "KL/chosen_KL_mean": -95.5714111328125, "KL/mean": -120.84217071533203, "KL/rejected_KL_mean": -146.11294555664062, "KL/std": 73.72892761230469, "epoch": 0.4293272864701436, "fcm_dpo/beta": 0.007086652331054211, "fcm_dpo/delta": 0.050613950937986374, "fcm_dpo/margin": 50.541542053222656, "fcm_dpo/q_t": 0.41856300830841064, "grad_norm": 12.229044914245605, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 0.7038174867630005, "logits/rejected": 0.6362642049789429, "logps/chosen": -152.53883361816406, "logps/ref_chosen": -56.96742630004883, "logps/ref_rejected": -86.36236572265625, "logps/rejected": -232.47531127929688, "loss": 1.124, "margin_dpo/margin_mean": 50.541542053222656, "margin_dpo/margin_std": 71.2654037475586, "step": 284 }, { "KL/chosen_KL_mean": -109.09988403320312, "KL/mean": -130.37918090820312, "KL/rejected_KL_mean": -151.65847778320312, "KL/std": 67.08362579345703, "epoch": 0.4308390022675737, "fcm_dpo/beta": 0.007905875332653522, "fcm_dpo/delta": 0.07085709273815155, "fcm_dpo/margin": 42.5586051940918, "fcm_dpo/q_t": 0.4255504310131073, "grad_norm": 19.009510040283203, "learning_rate": 3.5264957352549375e-07, "logits/chosen": 0.6826480031013489, "logits/rejected": 0.6553780436515808, "logps/chosen": -180.7559814453125, "logps/ref_chosen": -71.65611267089844, "logps/ref_rejected": -81.63829803466797, "logps/rejected": -233.29678344726562, "loss": 1.1658, "margin_dpo/margin_mean": 42.55860900878906, "margin_dpo/margin_std": 75.93214416503906, "step": 285 }, { "KL/chosen_KL_mean": -104.54328918457031, "KL/mean": -139.43801879882812, "KL/rejected_KL_mean": -174.332763671875, "KL/std": 73.45622253417969, "epoch": 0.4323507180650038, "fcm_dpo/beta": 0.007104329764842987, "fcm_dpo/delta": -0.12842434644699097, "fcm_dpo/margin": 69.78947448730469, "fcm_dpo/q_t": 0.380862295627594, "grad_norm": 11.564325332641602, "learning_rate": 3.514425224712835e-07, "logits/chosen": 0.6346836090087891, "logits/rejected": 0.5405333042144775, "logps/chosen": -165.622802734375, "logps/ref_chosen": -61.07952117919922, "logps/ref_rejected": -91.28128051757812, "logps/rejected": -265.6140441894531, "loss": 1.0207, "margin_dpo/margin_mean": 69.78947448730469, "margin_dpo/margin_std": 78.1060791015625, "step": 286 }, { "KL/chosen_KL_mean": -86.993896484375, "KL/mean": -119.77565002441406, "KL/rejected_KL_mean": -152.55740356445312, "KL/std": 72.32278442382812, "epoch": 0.43386243386243384, "fcm_dpo/beta": 0.00666564516723156, "fcm_dpo/delta": -0.04734319448471069, "fcm_dpo/margin": 65.56352233886719, "fcm_dpo/q_t": 0.3966023027896881, "grad_norm": 10.716714859008789, "learning_rate": 3.502326338516534e-07, "logits/chosen": 0.7223142385482788, "logits/rejected": 0.6803905963897705, "logps/chosen": -133.02969360351562, "logps/ref_chosen": -46.035789489746094, "logps/ref_rejected": -59.95293426513672, "logps/rejected": -212.51034545898438, "loss": 1.0565, "margin_dpo/margin_mean": 65.56352233886719, "margin_dpo/margin_std": 75.86518859863281, "step": 287 }, { "KL/chosen_KL_mean": -109.24652099609375, "KL/mean": -134.2801055908203, "KL/rejected_KL_mean": -159.31369018554688, "KL/std": 70.14065551757812, "epoch": 0.43537414965986393, "fcm_dpo/beta": 0.006941050291061401, "fcm_dpo/delta": 0.06258052587509155, "fcm_dpo/margin": 50.067169189453125, "fcm_dpo/q_t": 0.42138350009918213, "grad_norm": 13.64344310760498, "learning_rate": 3.490199415097892e-07, "logits/chosen": 0.544763445854187, "logits/rejected": 0.48616740107536316, "logps/chosen": -174.63735961914062, "logps/ref_chosen": -65.3908462524414, "logps/ref_rejected": -88.53607940673828, "logps/rejected": -247.84976196289062, "loss": 1.135, "margin_dpo/margin_mean": 50.067169189453125, "margin_dpo/margin_std": 76.1004867553711, "step": 288 }, { "KL/chosen_KL_mean": -110.31666564941406, "KL/mean": -134.42742919921875, "KL/rejected_KL_mean": -158.5382080078125, "KL/std": 71.93513488769531, "epoch": 0.436885865457294, "fcm_dpo/beta": 0.007375278510153294, "fcm_dpo/delta": 0.05219840630888939, "fcm_dpo/margin": 48.22156524658203, "fcm_dpo/q_t": 0.42122745513916016, "grad_norm": 12.05275821685791, "learning_rate": 3.4780447936730247e-07, "logits/chosen": 0.7490577101707458, "logits/rejected": 0.7129071950912476, "logps/chosen": -164.9102783203125, "logps/ref_chosen": -54.5936279296875, "logps/ref_rejected": -67.20855712890625, "logps/rejected": -225.74676513671875, "loss": 1.1474, "margin_dpo/margin_mean": 48.22156524658203, "margin_dpo/margin_std": 80.87078857421875, "step": 289 }, { "KL/chosen_KL_mean": -123.2154541015625, "KL/mean": -150.1029815673828, "KL/rejected_KL_mean": -176.99050903320312, "KL/std": 71.00227355957031, "epoch": 0.4383975812547241, "fcm_dpo/beta": 0.007250770926475525, "fcm_dpo/delta": 0.010142248123884201, "fcm_dpo/margin": 53.775054931640625, "fcm_dpo/q_t": 0.41072556376457214, "grad_norm": 14.948026657104492, "learning_rate": 3.465862814232821e-07, "logits/chosen": 0.7731099128723145, "logits/rejected": 0.6953067779541016, "logps/chosen": -184.60003662109375, "logps/ref_chosen": -61.38457489013672, "logps/ref_rejected": -91.92778015136719, "logps/rejected": -268.91827392578125, "loss": 1.1165, "margin_dpo/margin_mean": 53.775054931640625, "margin_dpo/margin_std": 80.15385437011719, "step": 290 }, { "KL/chosen_KL_mean": -111.75973510742188, "KL/mean": -145.4049530029297, "KL/rejected_KL_mean": -179.0501708984375, "KL/std": 73.63685607910156, "epoch": 0.4399092970521542, "fcm_dpo/beta": 0.0071503822691738605, "fcm_dpo/delta": -0.10867234319448471, "fcm_dpo/margin": 67.2904281616211, "fcm_dpo/q_t": 0.38891780376434326, "grad_norm": 14.165699005126953, "learning_rate": 3.4536538175334343e-07, "logits/chosen": 0.8502572774887085, "logits/rejected": 0.7742403745651245, "logps/chosen": -162.62277221679688, "logps/ref_chosen": -50.863037109375, "logps/ref_rejected": -82.20868682861328, "logps/rejected": -261.25885009765625, "loss": 1.059, "margin_dpo/margin_mean": 67.2904281616211, "margin_dpo/margin_std": 87.29713439941406, "step": 291 }, { "KL/chosen_KL_mean": -114.97377014160156, "KL/mean": -137.51162719726562, "KL/rejected_KL_mean": -160.04949951171875, "KL/std": 76.66767883300781, "epoch": 0.4414210128495843, "fcm_dpo/beta": 0.007128318306058645, "fcm_dpo/delta": 0.0940467119216919, "fcm_dpo/margin": 45.07574462890625, "fcm_dpo/q_t": 0.43068501353263855, "grad_norm": 15.699105262756348, "learning_rate": 3.4414181450867465e-07, "logits/chosen": 0.6994675397872925, "logits/rejected": 0.6498559713363647, "logps/chosen": -179.32266235351562, "logps/ref_chosen": -64.34888458251953, "logps/ref_rejected": -72.86434173583984, "logps/rejected": -232.91384887695312, "loss": 1.1944, "margin_dpo/margin_mean": 45.075748443603516, "margin_dpo/margin_std": 93.48008728027344, "step": 292 }, { "KL/chosen_KL_mean": -109.03053283691406, "KL/mean": -144.01535034179688, "KL/rejected_KL_mean": -179.00018310546875, "KL/std": 77.45966339111328, "epoch": 0.4429327286470144, "fcm_dpo/beta": 0.006905644200742245, "fcm_dpo/delta": -0.10910916328430176, "fcm_dpo/margin": 69.96965789794922, "fcm_dpo/q_t": 0.3872179388999939, "grad_norm": 10.040387153625488, "learning_rate": 3.4291561391508185e-07, "logits/chosen": 0.7848567962646484, "logits/rejected": 0.6968656778335571, "logps/chosen": -163.89999389648438, "logps/ref_chosen": -54.869468688964844, "logps/ref_rejected": -81.858642578125, "logps/rejected": -260.85882568359375, "loss": 1.0643, "margin_dpo/margin_mean": 69.96965789794922, "margin_dpo/margin_std": 94.42764282226562, "step": 293 }, { "KL/chosen_KL_mean": -117.87823486328125, "KL/mean": -141.62295532226562, "KL/rejected_KL_mean": -165.36767578125, "KL/std": 79.39311981201172, "epoch": 0.4444444444444444, "fcm_dpo/beta": 0.006892555393278599, "fcm_dpo/delta": 0.08704549074172974, "fcm_dpo/margin": 47.48944091796875, "fcm_dpo/q_t": 0.4273802638053894, "grad_norm": 11.643595695495605, "learning_rate": 3.4168681427203153e-07, "logits/chosen": 0.7372294068336487, "logits/rejected": 0.6890612840652466, "logps/chosen": -174.54913330078125, "logps/ref_chosen": -56.670902252197266, "logps/ref_rejected": -70.32819366455078, "logps/rejected": -235.69586181640625, "loss": 1.1445, "margin_dpo/margin_mean": 47.48944091796875, "margin_dpo/margin_std": 70.49929809570312, "step": 294 }, { "KL/chosen_KL_mean": -120.67965698242188, "KL/mean": -142.65975952148438, "KL/rejected_KL_mean": -164.63983154296875, "KL/std": 73.50402069091797, "epoch": 0.4459561602418745, "fcm_dpo/beta": 0.007594604976475239, "fcm_dpo/delta": 0.07951751351356506, "fcm_dpo/margin": 43.960166931152344, "fcm_dpo/q_t": 0.4269672632217407, "grad_norm": 17.615432739257812, "learning_rate": 3.4045544995169125e-07, "logits/chosen": 0.7029507160186768, "logits/rejected": 0.5973961353302002, "logps/chosen": -171.08055114746094, "logps/ref_chosen": -50.40088653564453, "logps/ref_rejected": -83.43521881103516, "logps/rejected": -248.07505798339844, "loss": 1.1654, "margin_dpo/margin_mean": 43.960166931152344, "margin_dpo/margin_std": 79.29472351074219, "step": 295 }, { "KL/chosen_KL_mean": -125.65110778808594, "KL/mean": -154.50802612304688, "KL/rejected_KL_mean": -183.36492919921875, "KL/std": 81.21266174316406, "epoch": 0.4474678760393046, "fcm_dpo/beta": 0.007408149540424347, "fcm_dpo/delta": -0.03780694305896759, "fcm_dpo/margin": 57.71380615234375, "fcm_dpo/q_t": 0.40250879526138306, "grad_norm": 12.49802017211914, "learning_rate": 3.392215553979679e-07, "logits/chosen": 0.6703150868415833, "logits/rejected": 0.6198123693466187, "logps/chosen": -194.80145263671875, "logps/ref_chosen": -69.15034484863281, "logps/ref_rejected": -89.60166931152344, "logps/rejected": -272.96661376953125, "loss": 1.1177, "margin_dpo/margin_mean": 57.71380615234375, "margin_dpo/margin_std": 94.09014892578125, "step": 296 }, { "KL/chosen_KL_mean": -128.17721557617188, "KL/mean": -156.7912139892578, "KL/rejected_KL_mean": -185.40521240234375, "KL/std": 70.26692962646484, "epoch": 0.4489795918367347, "fcm_dpo/beta": 0.0074169086292386055, "fcm_dpo/delta": -0.031429722905159, "fcm_dpo/margin": 57.227989196777344, "fcm_dpo/q_t": 0.40057173371315, "grad_norm": 13.817586898803711, "learning_rate": 3.3798516512554485e-07, "logits/chosen": 0.7207061052322388, "logits/rejected": 0.6670393943786621, "logps/chosen": -186.1935272216797, "logps/ref_chosen": -58.01630401611328, "logps/ref_rejected": -69.95780944824219, "logps/rejected": -255.36300659179688, "loss": 1.0764, "margin_dpo/margin_mean": 57.22799301147461, "margin_dpo/margin_std": 72.8988037109375, "step": 297 }, { "KL/chosen_KL_mean": -125.60884094238281, "KL/mean": -146.45750427246094, "KL/rejected_KL_mean": -167.30616760253906, "KL/std": 74.95790100097656, "epoch": 0.4504913076341648, "fcm_dpo/beta": 0.007724473252892494, "fcm_dpo/delta": 0.0932346060872078, "fcm_dpo/margin": 41.697330474853516, "fcm_dpo/q_t": 0.4304448068141937, "grad_norm": 13.414716720581055, "learning_rate": 3.367463137189156e-07, "logits/chosen": 0.8223170042037964, "logits/rejected": 0.765424370765686, "logps/chosen": -181.7781524658203, "logps/ref_chosen": -56.1693115234375, "logps/ref_rejected": -68.55052185058594, "logps/rejected": -235.856689453125, "loss": 1.1899, "margin_dpo/margin_mean": 41.697330474853516, "margin_dpo/margin_std": 83.2937240600586, "step": 298 }, { "KL/chosen_KL_mean": -129.3072509765625, "KL/mean": -147.94827270507812, "KL/rejected_KL_mean": -166.58929443359375, "KL/std": 71.92723846435547, "epoch": 0.4520030234315949, "fcm_dpo/beta": 0.008116363547742367, "fcm_dpo/delta": 0.019263437017798424, "fcm_dpo/margin": 37.28206253051758, "fcm_dpo/q_t": 0.43438130617141724, "grad_norm": 21.00090217590332, "learning_rate": 3.355050358314172e-07, "logits/chosen": 0.5886402726173401, "logits/rejected": 0.5575330257415771, "logps/chosen": -191.6250457763672, "logps/ref_chosen": -62.31780242919922, "logps/ref_rejected": -72.60028839111328, "logps/rejected": -239.18959045410156, "loss": 1.2391, "margin_dpo/margin_mean": 37.282066345214844, "margin_dpo/margin_std": 91.93014526367188, "step": 299 }, { "KL/chosen_KL_mean": -128.71337890625, "KL/mean": -152.656982421875, "KL/rejected_KL_mean": -176.60055541992188, "KL/std": 75.02449035644531, "epoch": 0.45351473922902497, "fcm_dpo/beta": 0.008240276016294956, "fcm_dpo/delta": 0.006145725026726723, "fcm_dpo/margin": 47.887184143066406, "fcm_dpo/q_t": 0.4129873514175415, "grad_norm": 16.049564361572266, "learning_rate": 3.3426136618426043e-07, "logits/chosen": 0.7201621532440186, "logits/rejected": 0.6523061990737915, "logps/chosen": -189.09495544433594, "logps/ref_chosen": -60.38157653808594, "logps/ref_rejected": -75.45442199707031, "logps/rejected": -252.0549774169922, "loss": 1.147, "margin_dpo/margin_mean": 47.887184143066406, "margin_dpo/margin_std": 85.44697570800781, "step": 300 }, { "KL/chosen_KL_mean": -128.576171875, "KL/mean": -149.49501037597656, "KL/rejected_KL_mean": -170.41384887695312, "KL/std": 72.66934204101562, "epoch": 0.455026455026455, "fcm_dpo/beta": 0.008374731056392193, "fcm_dpo/delta": 0.05971474200487137, "fcm_dpo/margin": 41.83769989013672, "fcm_dpo/q_t": 0.424157977104187, "grad_norm": 14.166158676147461, "learning_rate": 3.3301533956555885e-07, "logits/chosen": 0.7449129819869995, "logits/rejected": 0.7153865694999695, "logps/chosen": -181.42706298828125, "logps/ref_chosen": -52.85089111328125, "logps/ref_rejected": -69.97584533691406, "logps/rejected": -240.38970947265625, "loss": 1.1846, "margin_dpo/margin_mean": 41.83769989013672, "margin_dpo/margin_std": 84.38682556152344, "step": 301 }, { "KL/chosen_KL_mean": -135.41697692871094, "KL/mean": -151.56887817382812, "KL/rejected_KL_mean": -167.72076416015625, "KL/std": 76.77266693115234, "epoch": 0.4565381708238851, "fcm_dpo/beta": 0.009450605139136314, "fcm_dpo/delta": 0.11110319197177887, "fcm_dpo/margin": 32.30377960205078, "fcm_dpo/q_t": 0.4369601309299469, "grad_norm": 19.52178382873535, "learning_rate": 3.317669908293554e-07, "logits/chosen": 0.5722560882568359, "logits/rejected": 0.5158857107162476, "logps/chosen": -202.38348388671875, "logps/ref_chosen": -66.96650695800781, "logps/ref_rejected": -88.09510803222656, "logps/rejected": -255.81588745117188, "loss": 1.2531, "margin_dpo/margin_mean": 32.30377960205078, "margin_dpo/margin_std": 85.83658599853516, "step": 302 }, { "KL/chosen_KL_mean": -117.91580200195312, "KL/mean": -149.74221801757812, "KL/rejected_KL_mean": -181.56866455078125, "KL/std": 78.72908020019531, "epoch": 0.4580498866213152, "fcm_dpo/beta": 0.008384518325328827, "fcm_dpo/delta": -0.18558809161186218, "fcm_dpo/margin": 63.652870178222656, "fcm_dpo/q_t": 0.3783756494522095, "grad_norm": 12.682239532470703, "learning_rate": 3.3051635489464793e-07, "logits/chosen": 0.6628040671348572, "logits/rejected": 0.5899385213851929, "logps/chosen": -180.03732299804688, "logps/ref_chosen": -62.12152862548828, "logps/ref_rejected": -90.31204223632812, "logps/rejected": -271.8807067871094, "loss": 1.0824, "margin_dpo/margin_mean": 63.652870178222656, "margin_dpo/margin_std": 99.21585083007812, "step": 303 }, { "KL/chosen_KL_mean": -110.49132537841797, "KL/mean": -143.3325653076172, "KL/rejected_KL_mean": -176.17379760742188, "KL/std": 81.67277526855469, "epoch": 0.4595616024187453, "fcm_dpo/beta": 0.007327304221689701, "fcm_dpo/delta": -0.10975846648216248, "fcm_dpo/margin": 65.6824722290039, "fcm_dpo/q_t": 0.3843334913253784, "grad_norm": 11.644968032836914, "learning_rate": 3.292634667444117e-07, "logits/chosen": 0.6995347738265991, "logits/rejected": 0.6476801633834839, "logps/chosen": -171.18641662597656, "logps/ref_chosen": -60.695091247558594, "logps/ref_rejected": -78.2525405883789, "logps/rejected": -254.4263458251953, "loss": 1.0324, "margin_dpo/margin_mean": 65.68247985839844, "margin_dpo/margin_std": 74.8603515625, "step": 304 }, { "KL/chosen_KL_mean": -126.179931640625, "KL/mean": -145.57142639160156, "KL/rejected_KL_mean": -164.96292114257812, "KL/std": 73.00833892822266, "epoch": 0.46107331821617537, "fcm_dpo/beta": 0.007321414537727833, "fcm_dpo/delta": 0.015304024331271648, "fcm_dpo/margin": 38.782997131347656, "fcm_dpo/q_t": 0.43489426374435425, "grad_norm": 16.657503128051758, "learning_rate": 3.280083614246217e-07, "logits/chosen": 0.6606528759002686, "logits/rejected": 0.696006178855896, "logps/chosen": -198.87905883789062, "logps/ref_chosen": -72.69914245605469, "logps/ref_rejected": -65.65670776367188, "logps/rejected": -230.61962890625, "loss": 1.2284, "margin_dpo/margin_mean": 38.78300094604492, "margin_dpo/margin_std": 89.8697280883789, "step": 305 }, { "KL/chosen_KL_mean": -115.84672546386719, "KL/mean": -140.39031982421875, "KL/rejected_KL_mean": -164.9339141845703, "KL/std": 70.68470001220703, "epoch": 0.46258503401360546, "fcm_dpo/beta": 0.007341842632740736, "fcm_dpo/delta": 0.045422591269016266, "fcm_dpo/margin": 49.087181091308594, "fcm_dpo/q_t": 0.41786324977874756, "grad_norm": 11.100536346435547, "learning_rate": 3.267510740432719e-07, "logits/chosen": 0.7595999836921692, "logits/rejected": 0.6553710699081421, "logps/chosen": -169.8172607421875, "logps/ref_chosen": -53.97052764892578, "logps/ref_rejected": -71.02423095703125, "logps/rejected": -235.95814514160156, "loss": 1.1295, "margin_dpo/margin_mean": 49.087181091308594, "margin_dpo/margin_std": 70.42362976074219, "step": 306 }, { "KL/chosen_KL_mean": -111.7839126586914, "KL/mean": -123.70463562011719, "KL/rejected_KL_mean": -135.62535095214844, "KL/std": 71.19670867919922, "epoch": 0.46409674981103555, "fcm_dpo/beta": 0.007661640644073486, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 23.841440200805664, "fcm_dpo/q_t": 0.45939934253692627, "grad_norm": 18.099401473999023, "learning_rate": 3.2549163976939285e-07, "logits/chosen": 0.7359182834625244, "logits/rejected": 0.6884441375732422, "logps/chosen": -169.197021484375, "logps/ref_chosen": -57.413108825683594, "logps/ref_rejected": -68.68010711669922, "logps/rejected": -204.30545043945312, "loss": 1.3311, "margin_dpo/margin_mean": 23.84143829345703, "margin_dpo/margin_std": 92.32785034179688, "step": 307 }, { "KL/chosen_KL_mean": -112.34965515136719, "KL/mean": -136.45751953125, "KL/rejected_KL_mean": -160.56536865234375, "KL/std": 73.75102233886719, "epoch": 0.4656084656084656, "fcm_dpo/beta": 0.007660580798983574, "fcm_dpo/delta": 0.03539668023586273, "fcm_dpo/margin": 48.2157096862793, "fcm_dpo/q_t": 0.41769713163375854, "grad_norm": 11.193486213684082, "learning_rate": 3.2423009383206874e-07, "logits/chosen": 0.7087165117263794, "logits/rejected": 0.694772481918335, "logps/chosen": -178.94845581054688, "logps/ref_chosen": -66.59879302978516, "logps/ref_rejected": -74.337158203125, "logps/rejected": -234.90252685546875, "loss": 1.1604, "margin_dpo/margin_mean": 48.21570587158203, "margin_dpo/margin_std": 87.3203125, "step": 308 }, { "KL/chosen_KL_mean": -126.59266662597656, "KL/mean": -151.807861328125, "KL/rejected_KL_mean": -177.0230712890625, "KL/std": 72.11328125, "epoch": 0.4671201814058957, "fcm_dpo/beta": 0.007747044786810875, "fcm_dpo/delta": 0.007666148245334625, "fcm_dpo/margin": 50.430397033691406, "fcm_dpo/q_t": 0.4117814600467682, "grad_norm": 13.232572555541992, "learning_rate": 3.229664715194511e-07, "logits/chosen": 0.7303632497787476, "logits/rejected": 0.6770374774932861, "logps/chosen": -191.98741149902344, "logps/ref_chosen": -65.39474487304688, "logps/ref_rejected": -75.70930480957031, "logps/rejected": -252.7323760986328, "loss": 1.142, "margin_dpo/margin_mean": 50.43040466308594, "margin_dpo/margin_std": 84.50872039794922, "step": 309 }, { "KL/chosen_KL_mean": -134.52862548828125, "KL/mean": -148.09298706054688, "KL/rejected_KL_mean": -161.65733337402344, "KL/std": 66.22860717773438, "epoch": 0.46863189720332576, "fcm_dpo/beta": 0.00854377169162035, "fcm_dpo/delta": 0.06591358780860901, "fcm_dpo/margin": 27.128713607788086, "fcm_dpo/q_t": 0.4517921209335327, "grad_norm": 15.922846794128418, "learning_rate": 3.2170080817777257e-07, "logits/chosen": 0.7066018581390381, "logits/rejected": 0.6913357377052307, "logps/chosen": -209.1968994140625, "logps/ref_chosen": -74.66827392578125, "logps/ref_rejected": -80.5689697265625, "logps/rejected": -242.22630310058594, "loss": 1.2784, "margin_dpo/margin_mean": 27.128713607788086, "margin_dpo/margin_std": 80.27430725097656, "step": 310 }, { "KL/chosen_KL_mean": -105.25132751464844, "KL/mean": -133.28875732421875, "KL/rejected_KL_mean": -161.326171875, "KL/std": 85.32421875, "epoch": 0.47014361300075586, "fcm_dpo/beta": 0.007903890684247017, "fcm_dpo/delta": -0.0625324696302414, "fcm_dpo/margin": 56.074851989746094, "fcm_dpo/q_t": 0.4021851122379303, "grad_norm": 14.648249626159668, "learning_rate": 3.204331392103574e-07, "logits/chosen": 0.6372594237327576, "logits/rejected": 0.492361843585968, "logps/chosen": -164.98934936523438, "logps/ref_chosen": -59.738033294677734, "logps/ref_rejected": -93.60757446289062, "logps/rejected": -254.93374633789062, "loss": 1.1085, "margin_dpo/margin_mean": 56.074851989746094, "margin_dpo/margin_std": 91.86743927001953, "step": 311 }, { "KL/chosen_KL_mean": -108.83519744873047, "KL/mean": -138.187255859375, "KL/rejected_KL_mean": -167.539306640625, "KL/std": 72.46369934082031, "epoch": 0.47165532879818595, "fcm_dpo/beta": 0.007707584649324417, "fcm_dpo/delta": -0.06755498051643372, "fcm_dpo/margin": 58.70410919189453, "fcm_dpo/q_t": 0.3933003842830658, "grad_norm": 11.331823348999023, "learning_rate": 3.1916350007663176e-07, "logits/chosen": 0.7237418293952942, "logits/rejected": 0.6343299150466919, "logps/chosen": -162.65164184570312, "logps/ref_chosen": -53.816436767578125, "logps/ref_rejected": -68.6575698852539, "logps/rejected": -236.19688415527344, "loss": 1.0551, "margin_dpo/margin_mean": 58.70411682128906, "margin_dpo/margin_std": 71.16246032714844, "step": 312 }, { "KL/chosen_KL_mean": -114.29727935791016, "KL/mean": -130.198974609375, "KL/rejected_KL_mean": -146.10064697265625, "KL/std": 70.88896942138672, "epoch": 0.47316704459561604, "fcm_dpo/beta": 0.0077678910456597805, "fcm_dpo/delta": 0.06859148293733597, "fcm_dpo/margin": 31.803382873535156, "fcm_dpo/q_t": 0.4456871747970581, "grad_norm": 13.516242980957031, "learning_rate": 3.178919262911314e-07, "logits/chosen": 0.7776950597763062, "logits/rejected": 0.75665283203125, "logps/chosen": -174.254638671875, "logps/ref_chosen": -59.957359313964844, "logps/ref_rejected": -69.31729888916016, "logps/rejected": -215.41795349121094, "loss": 1.2415, "margin_dpo/margin_mean": 31.803382873535156, "margin_dpo/margin_std": 78.07221984863281, "step": 313 }, { "KL/chosen_KL_mean": -107.2558822631836, "KL/mean": -136.37551879882812, "KL/rejected_KL_mean": -165.4951171875, "KL/std": 67.8990478515625, "epoch": 0.47467876039304613, "fcm_dpo/beta": 0.007589129731059074, "fcm_dpo/delta": -0.05609302595257759, "fcm_dpo/margin": 58.239227294921875, "fcm_dpo/q_t": 0.397655725479126, "grad_norm": 12.090660095214844, "learning_rate": 3.166184534225087e-07, "logits/chosen": 0.6843063831329346, "logits/rejected": 0.7137875556945801, "logps/chosen": -177.5240478515625, "logps/ref_chosen": -70.26815795898438, "logps/ref_rejected": -69.23971557617188, "logps/rejected": -234.73483276367188, "loss": 1.072, "margin_dpo/margin_mean": 58.239227294921875, "margin_dpo/margin_std": 76.19454956054688, "step": 314 }, { "KL/chosen_KL_mean": -112.75736236572266, "KL/mean": -135.3028564453125, "KL/rejected_KL_mean": -157.84835815429688, "KL/std": 70.18635559082031, "epoch": 0.47619047619047616, "fcm_dpo/beta": 0.008021621033549309, "fcm_dpo/delta": 0.043402545154094696, "fcm_dpo/margin": 45.090999603271484, "fcm_dpo/q_t": 0.4180910587310791, "grad_norm": 12.942291259765625, "learning_rate": 3.1534311709253723e-07, "logits/chosen": 0.622521162033081, "logits/rejected": 0.5911740064620972, "logps/chosen": -180.55206298828125, "logps/ref_chosen": -67.79469299316406, "logps/ref_rejected": -74.55148315429688, "logps/rejected": -232.39984130859375, "loss": 1.1453, "margin_dpo/margin_mean": 45.09100341796875, "margin_dpo/margin_std": 75.40611267089844, "step": 315 }, { "KL/chosen_KL_mean": -104.73755645751953, "KL/mean": -137.7486572265625, "KL/rejected_KL_mean": -170.75973510742188, "KL/std": 74.51022338867188, "epoch": 0.47770219198790626, "fcm_dpo/beta": 0.0075963689014315605, "fcm_dpo/delta": -0.1409759819507599, "fcm_dpo/margin": 66.0221939086914, "fcm_dpo/q_t": 0.3811737895011902, "grad_norm": 13.7676362991333, "learning_rate": 3.1406595297511564e-07, "logits/chosen": 0.5896261930465698, "logits/rejected": 0.4515262842178345, "logps/chosen": -160.02603149414062, "logps/ref_chosen": -55.288482666015625, "logps/ref_rejected": -96.15723419189453, "logps/rejected": -266.9169921875, "loss": 1.0447, "margin_dpo/margin_mean": 66.0221939086914, "margin_dpo/margin_std": 76.85704040527344, "step": 316 }, { "KL/chosen_KL_mean": -96.02757263183594, "KL/mean": -126.60447692871094, "KL/rejected_KL_mean": -157.18138122558594, "KL/std": 66.47987365722656, "epoch": 0.47921390778533635, "fcm_dpo/beta": 0.006697298958897591, "fcm_dpo/delta": -0.01382247731089592, "fcm_dpo/margin": 61.15378189086914, "fcm_dpo/q_t": 0.4028528034687042, "grad_norm": 13.007543563842773, "learning_rate": 3.1278699679526975e-07, "logits/chosen": 0.7125881910324097, "logits/rejected": 0.6669232845306396, "logps/chosen": -150.60894775390625, "logps/ref_chosen": -54.58137512207031, "logps/ref_rejected": -72.77232360839844, "logps/rejected": -229.95370483398438, "loss": 1.0782, "margin_dpo/margin_mean": 61.153785705566406, "margin_dpo/margin_std": 74.21299743652344, "step": 317 }, { "KL/chosen_KL_mean": -105.79743957519531, "KL/mean": -128.85116577148438, "KL/rejected_KL_mean": -151.90487670898438, "KL/std": 72.4944076538086, "epoch": 0.48072562358276644, "fcm_dpo/beta": 0.007166634313762188, "fcm_dpo/delta": 0.08371928334236145, "fcm_dpo/margin": 46.10744857788086, "fcm_dpo/q_t": 0.43039560317993164, "grad_norm": 13.263943672180176, "learning_rate": 3.1150628432815336e-07, "logits/chosen": 0.7046504020690918, "logits/rejected": 0.635469913482666, "logps/chosen": -158.6856689453125, "logps/ref_chosen": -52.88822937011719, "logps/ref_rejected": -80.63988494873047, "logps/rejected": -232.54476928710938, "loss": 1.2019, "margin_dpo/margin_mean": 46.107452392578125, "margin_dpo/margin_std": 100.03981018066406, "step": 318 }, { "KL/chosen_KL_mean": -104.99301147460938, "KL/mean": -137.19601440429688, "KL/rejected_KL_mean": -169.3990478515625, "KL/std": 77.18682861328125, "epoch": 0.48223733938019653, "fcm_dpo/beta": 0.006997345481067896, "fcm_dpo/delta": -0.0660954937338829, "fcm_dpo/margin": 64.40603637695312, "fcm_dpo/q_t": 0.39640793204307556, "grad_norm": 10.057544708251953, "learning_rate": 3.1022385139804707e-07, "logits/chosen": 0.6656143665313721, "logits/rejected": 0.6494824290275574, "logps/chosen": -169.35633850097656, "logps/ref_chosen": -64.36333465576172, "logps/ref_rejected": -79.47296142578125, "logps/rejected": -248.8719940185547, "loss": 1.0744, "margin_dpo/margin_mean": 64.40603637695312, "margin_dpo/margin_std": 89.59465026855469, "step": 319 }, { "KL/chosen_KL_mean": -95.88545227050781, "KL/mean": -121.54087829589844, "KL/rejected_KL_mean": -147.19630432128906, "KL/std": 78.3597412109375, "epoch": 0.4837490551776266, "fcm_dpo/beta": 0.00663637975230813, "fcm_dpo/delta": -0.08432682603597641, "fcm_dpo/margin": 51.31085968017578, "fcm_dpo/q_t": 0.421201229095459, "grad_norm": 11.44666576385498, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 0.5955374240875244, "logits/rejected": 0.556653618812561, "logps/chosen": -145.44419860839844, "logps/ref_chosen": -49.558746337890625, "logps/ref_rejected": -71.23444366455078, "logps/rejected": -218.43075561523438, "loss": 1.1595, "margin_dpo/margin_mean": 51.310855865478516, "margin_dpo/margin_std": 83.80187225341797, "step": 320 }, { "KL/chosen_KL_mean": -100.45033264160156, "KL/mean": -128.28823852539062, "KL/rejected_KL_mean": -156.1261444091797, "KL/std": 66.45053100585938, "epoch": 0.4852607709750567, "fcm_dpo/beta": 0.006418926641345024, "fcm_dpo/delta": 0.049936145544052124, "fcm_dpo/margin": 55.67579650878906, "fcm_dpo/q_t": 0.4175720810890198, "grad_norm": 14.7537202835083, "learning_rate": 3.0765396768561004e-07, "logits/chosen": 0.682221531867981, "logits/rejected": 0.6629788875579834, "logps/chosen": -152.53561401367188, "logps/ref_chosen": -52.08526611328125, "logps/ref_rejected": -55.58674621582031, "logps/rejected": -211.712890625, "loss": 1.1299, "margin_dpo/margin_mean": 55.67579650878906, "margin_dpo/margin_std": 79.13897705078125, "step": 321 }, { "KL/chosen_KL_mean": -112.3656234741211, "KL/mean": -147.36282348632812, "KL/rejected_KL_mean": -182.36004638671875, "KL/std": 75.82512664794922, "epoch": 0.48677248677248675, "fcm_dpo/beta": 0.00645124027505517, "fcm_dpo/delta": -0.0664437934756279, "fcm_dpo/margin": 69.99443054199219, "fcm_dpo/q_t": 0.3910978436470032, "grad_norm": 10.60876178741455, "learning_rate": 3.063665887884511e-07, "logits/chosen": 0.7542526721954346, "logits/rejected": 0.6689043045043945, "logps/chosen": -159.7697296142578, "logps/ref_chosen": -47.404109954833984, "logps/ref_rejected": -73.4260025024414, "logps/rejected": -255.7860565185547, "loss": 1.0393, "margin_dpo/margin_mean": 69.99443054199219, "margin_dpo/margin_std": 72.6507797241211, "step": 322 }, { "KL/chosen_KL_mean": -116.67129516601562, "KL/mean": -138.6676025390625, "KL/rejected_KL_mean": -160.66392517089844, "KL/std": 72.51184844970703, "epoch": 0.48828420256991684, "fcm_dpo/beta": 0.006838475354015827, "fcm_dpo/delta": 0.11610043048858643, "fcm_dpo/margin": 43.99262619018555, "fcm_dpo/q_t": 0.43621429800987244, "grad_norm": 12.70161247253418, "learning_rate": 3.0507763319663517e-07, "logits/chosen": 0.5943468809127808, "logits/rejected": 0.5237321853637695, "logps/chosen": -186.67758178710938, "logps/ref_chosen": -70.00630187988281, "logps/ref_rejected": -86.96690368652344, "logps/rejected": -247.63082885742188, "loss": 1.2035, "margin_dpo/margin_mean": 43.99262619018555, "margin_dpo/margin_std": 93.22569274902344, "step": 323 }, { "KL/chosen_KL_mean": -103.11143493652344, "KL/mean": -133.23056030273438, "KL/rejected_KL_mean": -163.34967041015625, "KL/std": 84.75403594970703, "epoch": 0.4897959183673469, "fcm_dpo/beta": 0.006788309663534164, "fcm_dpo/delta": -0.014123659580945969, "fcm_dpo/margin": 60.23822021484375, "fcm_dpo/q_t": 0.404258668422699, "grad_norm": 13.756731033325195, "learning_rate": 3.0378713696502097e-07, "logits/chosen": 0.7186671495437622, "logits/rejected": 0.6600125432014465, "logps/chosen": -159.00025939941406, "logps/ref_chosen": -55.88882064819336, "logps/ref_rejected": -75.23088073730469, "logps/rejected": -238.58055114746094, "loss": 1.0856, "margin_dpo/margin_mean": 60.23822021484375, "margin_dpo/margin_std": 76.23624420166016, "step": 324 }, { "KL/chosen_KL_mean": -124.19041442871094, "KL/mean": -154.54751586914062, "KL/rejected_KL_mean": -184.90463256835938, "KL/std": 82.73353576660156, "epoch": 0.491307634164777, "fcm_dpo/beta": 0.006723565980792046, "fcm_dpo/delta": -0.012475315481424332, "fcm_dpo/margin": 60.7142219543457, "fcm_dpo/q_t": 0.4078647792339325, "grad_norm": 11.16752815246582, "learning_rate": 3.0249513619156206e-07, "logits/chosen": 0.6296889781951904, "logits/rejected": 0.5697454214096069, "logps/chosen": -188.33743286132812, "logps/ref_chosen": -64.14701843261719, "logps/ref_rejected": -79.91143798828125, "logps/rejected": -264.8160705566406, "loss": 1.1234, "margin_dpo/margin_mean": 60.71421813964844, "margin_dpo/margin_std": 101.00370788574219, "step": 325 }, { "KL/chosen_KL_mean": -145.95529174804688, "KL/mean": -157.24642944335938, "KL/rejected_KL_mean": -168.53758239746094, "KL/std": 74.45347595214844, "epoch": 0.4928193499622071, "fcm_dpo/beta": 0.006838089786469936, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 22.582300186157227, "fcm_dpo/q_t": 0.464555561542511, "grad_norm": 12.780586242675781, "learning_rate": 3.012016670162977e-07, "logits/chosen": 0.5949761271476746, "logits/rejected": 0.6042267084121704, "logps/chosen": -221.48660278320312, "logps/ref_chosen": -75.53131103515625, "logps/ref_rejected": -76.5898666381836, "logps/rejected": -245.12744140625, "loss": 1.3171, "margin_dpo/margin_mean": 22.582300186157227, "margin_dpo/margin_std": 83.8711166381836, "step": 326 }, { "KL/chosen_KL_mean": -128.95672607421875, "KL/mean": -152.0322265625, "KL/rejected_KL_mean": -175.10769653320312, "KL/std": 79.25118255615234, "epoch": 0.4943310657596372, "fcm_dpo/beta": 0.007321531884372234, "fcm_dpo/delta": 0.07252618670463562, "fcm_dpo/margin": 46.15097427368164, "fcm_dpo/q_t": 0.425706684589386, "grad_norm": 13.338642120361328, "learning_rate": 2.99906765620341e-07, "logits/chosen": 0.5909023880958557, "logits/rejected": 0.5673669576644897, "logps/chosen": -198.29388427734375, "logps/ref_chosen": -69.33717346191406, "logps/ref_rejected": -73.37751770019531, "logps/rejected": -248.48521423339844, "loss": 1.1822, "margin_dpo/margin_mean": 46.150970458984375, "margin_dpo/margin_std": 91.36515045166016, "step": 327 }, { "KL/chosen_KL_mean": -113.59530639648438, "KL/mean": -142.98179626464844, "KL/rejected_KL_mean": -172.3682861328125, "KL/std": 80.62422943115234, "epoch": 0.4958427815570673, "fcm_dpo/beta": 0.007260303478688002, "fcm_dpo/delta": -0.034179363399744034, "fcm_dpo/margin": 58.77297592163086, "fcm_dpo/q_t": 0.4015209674835205, "grad_norm": 13.58584213256836, "learning_rate": 2.9861046822486766e-07, "logits/chosen": 0.5832690000534058, "logits/rejected": 0.5415433049201965, "logps/chosen": -175.30154418945312, "logps/ref_chosen": -61.70623016357422, "logps/ref_rejected": -83.73808288574219, "logps/rejected": -256.1063537597656, "loss": 1.0779, "margin_dpo/margin_mean": 58.77297592163086, "margin_dpo/margin_std": 76.87384033203125, "step": 328 }, { "KL/chosen_KL_mean": -127.56649780273438, "KL/mean": -156.63983154296875, "KL/rejected_KL_mean": -185.71316528320312, "KL/std": 86.29991912841797, "epoch": 0.4973544973544973, "fcm_dpo/beta": 0.0070040179416537285, "fcm_dpo/delta": -0.00944619532674551, "fcm_dpo/margin": 58.14664840698242, "fcm_dpo/q_t": 0.4094126522541046, "grad_norm": 14.41002082824707, "learning_rate": 2.9731281109010253e-07, "logits/chosen": 0.7076858282089233, "logits/rejected": 0.6475342512130737, "logps/chosen": -192.06491088867188, "logps/ref_chosen": -64.4984130859375, "logps/ref_rejected": -83.6591796875, "logps/rejected": -269.3723449707031, "loss": 1.1137, "margin_dpo/margin_mean": 58.14665222167969, "margin_dpo/margin_std": 91.29844665527344, "step": 329 }, { "KL/chosen_KL_mean": -111.404541015625, "KL/mean": -139.49217224121094, "KL/rejected_KL_mean": -167.57980346679688, "KL/std": 77.82200622558594, "epoch": 0.4988662131519274, "fcm_dpo/beta": 0.007140832487493753, "fcm_dpo/delta": -0.002552524209022522, "fcm_dpo/margin": 56.17527770996094, "fcm_dpo/q_t": 0.40969616174697876, "grad_norm": 12.402542114257812, "learning_rate": 2.9601383051430505e-07, "logits/chosen": 0.672360360622406, "logits/rejected": 0.6062643527984619, "logps/chosen": -166.2091827392578, "logps/ref_chosen": -54.80464172363281, "logps/ref_rejected": -75.3194351196289, "logps/rejected": -242.89923095703125, "loss": 1.1446, "margin_dpo/margin_mean": 56.17527389526367, "margin_dpo/margin_std": 99.22282409667969, "step": 330 }, { "KL/chosen_KL_mean": -119.64215087890625, "KL/mean": -156.07215881347656, "KL/rejected_KL_mean": -192.50216674804688, "KL/std": 81.64392852783203, "epoch": 0.5003779289493575, "fcm_dpo/beta": 0.0065118009224534035, "fcm_dpo/delta": -0.09822378307580948, "fcm_dpo/margin": 72.86000061035156, "fcm_dpo/q_t": 0.3864233195781708, "grad_norm": 11.534334182739258, "learning_rate": 2.947135628327544e-07, "logits/chosen": 0.76860511302948, "logits/rejected": 0.7442618608474731, "logps/chosen": -178.88473510742188, "logps/ref_chosen": -59.242584228515625, "logps/ref_rejected": -69.87483215332031, "logps/rejected": -262.37701416015625, "loss": 1.0453, "margin_dpo/margin_mean": 72.86000061035156, "margin_dpo/margin_std": 89.18681335449219, "step": 331 }, { "KL/chosen_KL_mean": -124.42134857177734, "KL/mean": -154.72769165039062, "KL/rejected_KL_mean": -185.03404235839844, "KL/std": 78.88076782226562, "epoch": 0.5018896447467877, "fcm_dpo/beta": 0.00656371284276247, "fcm_dpo/delta": -0.0017648041248321533, "fcm_dpo/margin": 60.61271286010742, "fcm_dpo/q_t": 0.40883493423461914, "grad_norm": 11.724095344543457, "learning_rate": 2.934120444167326e-07, "logits/chosen": 0.6075745820999146, "logits/rejected": 0.566921591758728, "logps/chosen": -191.53109741210938, "logps/ref_chosen": -67.10975646972656, "logps/ref_rejected": -77.11839294433594, "logps/rejected": -262.1524353027344, "loss": 1.1018, "margin_dpo/margin_mean": 60.61271286010742, "margin_dpo/margin_std": 83.28633117675781, "step": 332 }, { "KL/chosen_KL_mean": -130.77462768554688, "KL/mean": -165.58177185058594, "KL/rejected_KL_mean": -200.388916015625, "KL/std": 81.94561767578125, "epoch": 0.5034013605442177, "fcm_dpo/beta": 0.006204267032444477, "fcm_dpo/delta": -0.04060637205839157, "fcm_dpo/margin": 69.61427307128906, "fcm_dpo/q_t": 0.39935895800590515, "grad_norm": 10.982404708862305, "learning_rate": 2.921093116725076e-07, "logits/chosen": 0.6704716682434082, "logits/rejected": 0.5928934812545776, "logps/chosen": -189.15576171875, "logps/ref_chosen": -58.381134033203125, "logps/ref_rejected": -85.02839660644531, "logps/rejected": -285.41729736328125, "loss": 1.0682, "margin_dpo/margin_mean": 69.61427307128906, "margin_dpo/margin_std": 87.08734893798828, "step": 333 }, { "KL/chosen_KL_mean": -126.27867126464844, "KL/mean": -151.51400756835938, "KL/rejected_KL_mean": -176.7493438720703, "KL/std": 80.90272521972656, "epoch": 0.5049130763416477, "fcm_dpo/beta": 0.0065368469804525375, "fcm_dpo/delta": 0.08284910768270493, "fcm_dpo/margin": 50.47067642211914, "fcm_dpo/q_t": 0.4275355339050293, "grad_norm": 12.728032112121582, "learning_rate": 2.9080540104031484e-07, "logits/chosen": 0.683144211769104, "logits/rejected": 0.63508141040802, "logps/chosen": -193.17068481445312, "logps/ref_chosen": -66.89199829101562, "logps/ref_rejected": -91.83695220947266, "logps/rejected": -268.5863037109375, "loss": 1.1678, "margin_dpo/margin_mean": 50.470672607421875, "margin_dpo/margin_std": 91.65339660644531, "step": 334 }, { "KL/chosen_KL_mean": -127.95242309570312, "KL/mean": -154.70330810546875, "KL/rejected_KL_mean": -181.45420837402344, "KL/std": 80.90696716308594, "epoch": 0.5064247921390779, "fcm_dpo/beta": 0.0069529893808066845, "fcm_dpo/delta": 0.030666418373584747, "fcm_dpo/margin": 53.501800537109375, "fcm_dpo/q_t": 0.41777414083480835, "grad_norm": 14.211898803710938, "learning_rate": 2.895003489933375e-07, "logits/chosen": 0.6847304105758667, "logits/rejected": 0.6470900774002075, "logps/chosen": -189.4668731689453, "logps/ref_chosen": -61.51445770263672, "logps/ref_rejected": -75.68916320800781, "logps/rejected": -257.14337158203125, "loss": 1.1548, "margin_dpo/margin_mean": 53.501800537109375, "margin_dpo/margin_std": 95.9657974243164, "step": 335 }, { "KL/chosen_KL_mean": -141.7051239013672, "KL/mean": -170.696044921875, "KL/rejected_KL_mean": -199.68698120117188, "KL/std": 86.33305358886719, "epoch": 0.5079365079365079, "fcm_dpo/beta": 0.006664481945335865, "fcm_dpo/delta": 0.011959366500377655, "fcm_dpo/margin": 57.98182678222656, "fcm_dpo/q_t": 0.4162399470806122, "grad_norm": 13.38348388671875, "learning_rate": 2.8819419203668675e-07, "logits/chosen": 0.624375581741333, "logits/rejected": 0.5998581051826477, "logps/chosen": -210.55519104003906, "logps/ref_chosen": -68.85006713867188, "logps/ref_rejected": -92.99603271484375, "logps/rejected": -292.6829833984375, "loss": 1.1397, "margin_dpo/margin_mean": 57.98182678222656, "margin_dpo/margin_std": 98.35008239746094, "step": 336 }, { "KL/chosen_KL_mean": -149.2178955078125, "KL/mean": -172.33078002929688, "KL/rejected_KL_mean": -195.44371032714844, "KL/std": 86.13853454589844, "epoch": 0.509448223733938, "fcm_dpo/beta": 0.0073309652507305145, "fcm_dpo/delta": 0.07309623807668686, "fcm_dpo/margin": 46.225830078125, "fcm_dpo/q_t": 0.4263220429420471, "grad_norm": 12.12402629852295, "learning_rate": 2.8688696670638053e-07, "logits/chosen": 0.5780133605003357, "logits/rejected": 0.5493387579917908, "logps/chosen": -222.40573120117188, "logps/ref_chosen": -73.18783569335938, "logps/ref_rejected": -86.89118957519531, "logps/rejected": -282.33489990234375, "loss": 1.1838, "margin_dpo/margin_mean": 46.225830078125, "margin_dpo/margin_std": 92.9611587524414, "step": 337 }, { "KL/chosen_KL_mean": -146.11094665527344, "KL/mean": -169.81564331054688, "KL/rejected_KL_mean": -193.5203399658203, "KL/std": 78.87342071533203, "epoch": 0.5109599395313681, "fcm_dpo/beta": 0.00764493178576231, "fcm_dpo/delta": 0.045983582735061646, "fcm_dpo/margin": 47.40937805175781, "fcm_dpo/q_t": 0.41997748613357544, "grad_norm": 12.370586395263672, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 0.6543248891830444, "logits/rejected": 0.6287999153137207, "logps/chosen": -210.05056762695312, "logps/ref_chosen": -63.939613342285156, "logps/ref_rejected": -75.34243774414062, "logps/rejected": -268.86279296875, "loss": 1.1633, "margin_dpo/margin_mean": 47.40937805175781, "margin_dpo/margin_std": 88.205810546875, "step": 338 }, { "KL/chosen_KL_mean": -118.83457946777344, "KL/mean": -148.2288818359375, "KL/rejected_KL_mean": -177.6231689453125, "KL/std": 79.79850006103516, "epoch": 0.5124716553287982, "fcm_dpo/beta": 0.007407545577734709, "fcm_dpo/delta": -0.04732739180326462, "fcm_dpo/margin": 58.788604736328125, "fcm_dpo/q_t": 0.3996833562850952, "grad_norm": 15.090683937072754, "learning_rate": 2.842694572172736e-07, "logits/chosen": 0.8103833198547363, "logits/rejected": 0.7248860001564026, "logps/chosen": -164.38369750976562, "logps/ref_chosen": -45.54913330078125, "logps/ref_rejected": -67.0482177734375, "logps/rejected": -244.67138671875, "loss": 1.0939, "margin_dpo/margin_mean": 58.78860092163086, "margin_dpo/margin_std": 85.93904113769531, "step": 339 }, { "KL/chosen_KL_mean": -133.77890014648438, "KL/mean": -161.17971801757812, "KL/rejected_KL_mean": -188.5805206298828, "KL/std": 85.82127380371094, "epoch": 0.5139833711262283, "fcm_dpo/beta": 0.0074435994029045105, "fcm_dpo/delta": -0.010237021371722221, "fcm_dpo/margin": 54.8016242980957, "fcm_dpo/q_t": 0.41037893295288086, "grad_norm": 12.742267608642578, "learning_rate": 2.8295924627584004e-07, "logits/chosen": 0.652264416217804, "logits/rejected": 0.6331235766410828, "logps/chosen": -187.7845458984375, "logps/ref_chosen": -54.00564956665039, "logps/ref_rejected": -61.314430236816406, "logps/rejected": -249.89495849609375, "loss": 1.1512, "margin_dpo/margin_mean": 54.80162811279297, "margin_dpo/margin_std": 101.09391021728516, "step": 340 }, { "KL/chosen_KL_mean": -136.74429321289062, "KL/mean": -168.65585327148438, "KL/rejected_KL_mean": -200.56744384765625, "KL/std": 86.67827606201172, "epoch": 0.5154950869236583, "fcm_dpo/beta": 0.006320003420114517, "fcm_dpo/delta": -0.15153226256370544, "fcm_dpo/margin": 63.823143005371094, "fcm_dpo/q_t": 0.3994399309158325, "grad_norm": 13.01077651977539, "learning_rate": 2.816481133934373e-07, "logits/chosen": 0.6961194276809692, "logits/rejected": 0.6493462324142456, "logps/chosen": -200.13938903808594, "logps/ref_chosen": -63.39509582519531, "logps/ref_rejected": -76.20973205566406, "logps/rejected": -276.77716064453125, "loss": 1.1119, "margin_dpo/margin_mean": 63.823143005371094, "margin_dpo/margin_std": 92.03482055664062, "step": 341 }, { "KL/chosen_KL_mean": -134.47647094726562, "KL/mean": -165.2379150390625, "KL/rejected_KL_mean": -195.99932861328125, "KL/std": 91.90939331054688, "epoch": 0.5170068027210885, "fcm_dpo/beta": 0.006310966797173023, "fcm_dpo/delta": 0.0141034796833992, "fcm_dpo/margin": 61.52287292480469, "fcm_dpo/q_t": 0.4127877652645111, "grad_norm": 10.182599067687988, "learning_rate": 2.8033609524527046e-07, "logits/chosen": 0.7835279107093811, "logits/rejected": 0.7403960227966309, "logps/chosen": -187.5242919921875, "logps/ref_chosen": -53.047813415527344, "logps/ref_rejected": -68.2854232788086, "logps/rejected": -264.2847595214844, "loss": 1.1187, "margin_dpo/margin_mean": 61.52287292480469, "margin_dpo/margin_std": 94.30167388916016, "step": 342 }, { "KL/chosen_KL_mean": -130.44664001464844, "KL/mean": -152.9931640625, "KL/rejected_KL_mean": -175.53968811035156, "KL/std": 85.21757507324219, "epoch": 0.5185185185185185, "fcm_dpo/beta": 0.006350641138851643, "fcm_dpo/delta": -0.009267449378967285, "fcm_dpo/margin": 45.093055725097656, "fcm_dpo/q_t": 0.43408840894699097, "grad_norm": 10.364089965820312, "learning_rate": 2.7902322853130753e-07, "logits/chosen": 0.5760016441345215, "logits/rejected": 0.5660780668258667, "logps/chosen": -201.02517700195312, "logps/ref_chosen": -70.57852935791016, "logps/ref_rejected": -84.73873901367188, "logps/rejected": -260.2784423828125, "loss": 1.1991, "margin_dpo/margin_mean": 45.09305191040039, "margin_dpo/margin_std": 89.2558364868164, "step": 343 }, { "KL/chosen_KL_mean": -141.1691436767578, "KL/mean": -174.16143798828125, "KL/rejected_KL_mean": -207.15371704101562, "KL/std": 86.44344329833984, "epoch": 0.5200302343159486, "fcm_dpo/beta": 0.0062735131941735744, "fcm_dpo/delta": -0.017590418457984924, "fcm_dpo/margin": 65.98458862304688, "fcm_dpo/q_t": 0.40609464049339294, "grad_norm": 11.856791496276855, "learning_rate": 2.7770954997525274e-07, "logits/chosen": 0.7121565937995911, "logits/rejected": 0.6411057710647583, "logps/chosen": -196.9801483154297, "logps/ref_chosen": -55.811004638671875, "logps/ref_rejected": -84.77637481689453, "logps/rejected": -291.93011474609375, "loss": 1.1069, "margin_dpo/margin_mean": 65.98458862304688, "margin_dpo/margin_std": 100.44625854492188, "step": 344 }, { "KL/chosen_KL_mean": -116.9457778930664, "KL/mean": -146.05355834960938, "KL/rejected_KL_mean": -175.1613311767578, "KL/std": 79.33267211914062, "epoch": 0.5215419501133787, "fcm_dpo/beta": 0.0064442697912454605, "fcm_dpo/delta": 0.029752016067504883, "fcm_dpo/margin": 58.21555709838867, "fcm_dpo/q_t": 0.41542765498161316, "grad_norm": 11.773383140563965, "learning_rate": 2.7639509632351927e-07, "logits/chosen": 0.7386205792427063, "logits/rejected": 0.6958231329917908, "logps/chosen": -174.73187255859375, "logps/ref_chosen": -57.78609848022461, "logps/ref_rejected": -78.91847229003906, "logps/rejected": -254.07980346679688, "loss": 1.1319, "margin_dpo/margin_mean": 58.21556091308594, "margin_dpo/margin_std": 93.69883728027344, "step": 345 }, { "KL/chosen_KL_mean": -122.50251007080078, "KL/mean": -158.81753540039062, "KL/rejected_KL_mean": -195.132568359375, "KL/std": 94.08961486816406, "epoch": 0.5230536659108088, "fcm_dpo/beta": 0.0062287356704473495, "fcm_dpo/delta": -0.06783194094896317, "fcm_dpo/margin": 72.63008117675781, "fcm_dpo/q_t": 0.39354556798934937, "grad_norm": 13.235642433166504, "learning_rate": 2.7507990434420123e-07, "logits/chosen": 0.7180252075195312, "logits/rejected": 0.6350604295730591, "logps/chosen": -178.78762817382812, "logps/ref_chosen": -56.285125732421875, "logps/ref_rejected": -91.15303039550781, "logps/rejected": -286.2856140136719, "loss": 1.074, "margin_dpo/margin_mean": 72.63007354736328, "margin_dpo/margin_std": 97.58467102050781, "step": 346 }, { "KL/chosen_KL_mean": -135.82313537597656, "KL/mean": -162.78857421875, "KL/rejected_KL_mean": -189.75399780273438, "KL/std": 86.55868530273438, "epoch": 0.5245653817082389, "fcm_dpo/beta": 0.0063002025708556175, "fcm_dpo/delta": 0.07281588017940521, "fcm_dpo/margin": 53.93086242675781, "fcm_dpo/q_t": 0.4243871867656708, "grad_norm": 12.957653999328613, "learning_rate": 2.737640108260456e-07, "logits/chosen": 0.8214094638824463, "logits/rejected": 0.7735979557037354, "logps/chosen": -189.3226776123047, "logps/ref_chosen": -53.499542236328125, "logps/ref_rejected": -72.52565002441406, "logps/rejected": -262.2796630859375, "loss": 1.1479, "margin_dpo/margin_mean": 53.93086242675781, "margin_dpo/margin_std": 87.56758117675781, "step": 347 }, { "KL/chosen_KL_mean": -122.32320404052734, "KL/mean": -153.75466918945312, "KL/rejected_KL_mean": -185.1861572265625, "KL/std": 81.93644714355469, "epoch": 0.5260770975056689, "fcm_dpo/beta": 0.006337115541100502, "fcm_dpo/delta": 0.0002983175218105316, "fcm_dpo/margin": 62.86295700073242, "fcm_dpo/q_t": 0.4105902910232544, "grad_norm": 12.72779655456543, "learning_rate": 2.724474525774229e-07, "logits/chosen": 0.8158016204833984, "logits/rejected": 0.7839329242706299, "logps/chosen": -173.11004638671875, "logps/ref_chosen": -50.78684997558594, "logps/ref_rejected": -68.63732147216797, "logps/rejected": -253.823486328125, "loss": 1.1154, "margin_dpo/margin_mean": 62.862953186035156, "margin_dpo/margin_std": 96.59371948242188, "step": 348 }, { "KL/chosen_KL_mean": -123.90093994140625, "KL/mean": -156.0067138671875, "KL/rejected_KL_mean": -188.11248779296875, "KL/std": 91.27865600585938, "epoch": 0.527588813303099, "fcm_dpo/beta": 0.0065030986443161964, "fcm_dpo/delta": -0.024084649980068207, "fcm_dpo/margin": 64.21153259277344, "fcm_dpo/q_t": 0.40591293573379517, "grad_norm": 12.217432022094727, "learning_rate": 2.711302664252973e-07, "logits/chosen": 0.7418991923332214, "logits/rejected": 0.646633505821228, "logps/chosen": -177.2259521484375, "logps/ref_chosen": -53.325008392333984, "logps/ref_rejected": -83.21236419677734, "logps/rejected": -271.3248291015625, "loss": 1.1069, "margin_dpo/margin_mean": 64.21154022216797, "margin_dpo/margin_std": 96.88829040527344, "step": 349 }, { "KL/chosen_KL_mean": -126.6453857421875, "KL/mean": -169.14663696289062, "KL/rejected_KL_mean": -211.64791870117188, "KL/std": 97.974609375, "epoch": 0.5291005291005291, "fcm_dpo/beta": 0.005718736909329891, "fcm_dpo/delta": -0.1165238469839096, "fcm_dpo/margin": 85.00253295898438, "fcm_dpo/q_t": 0.3823647201061249, "grad_norm": 12.687152862548828, "learning_rate": 2.698124892141971e-07, "logits/chosen": 0.694975733757019, "logits/rejected": 0.6256704330444336, "logps/chosen": -188.27114868164062, "logps/ref_chosen": -61.625770568847656, "logps/ref_rejected": -87.63627624511719, "logps/rejected": -299.2841796875, "loss": 1.0231, "margin_dpo/margin_mean": 85.00252532958984, "margin_dpo/margin_std": 92.37911224365234, "step": 350 }, { "KL/chosen_KL_mean": -123.06855773925781, "KL/mean": -152.65802001953125, "KL/rejected_KL_mean": -182.24749755859375, "KL/std": 81.3293228149414, "epoch": 0.5306122448979592, "fcm_dpo/beta": 0.005834533832967281, "fcm_dpo/delta": 0.06635048985481262, "fcm_dpo/margin": 59.17893600463867, "fcm_dpo/q_t": 0.42190346121788025, "grad_norm": 10.825620651245117, "learning_rate": 2.6849415780518357e-07, "logits/chosen": 0.6644502878189087, "logits/rejected": 0.5926836133003235, "logps/chosen": -179.32489013671875, "logps/ref_chosen": -56.2563362121582, "logps/ref_rejected": -79.11589813232422, "logps/rejected": -261.3634033203125, "loss": 1.1597, "margin_dpo/margin_mean": 59.17893981933594, "margin_dpo/margin_std": 103.34938049316406, "step": 351 }, { "KL/chosen_KL_mean": -125.21533203125, "KL/mean": -159.1262969970703, "KL/rejected_KL_mean": -193.03726196289062, "KL/std": 92.70458984375, "epoch": 0.5321239606953893, "fcm_dpo/beta": 0.005886302795261145, "fcm_dpo/delta": -0.00063290074467659, "fcm_dpo/margin": 67.82192993164062, "fcm_dpo/q_t": 0.40983670949935913, "grad_norm": 11.221183776855469, "learning_rate": 2.6717530907482024e-07, "logits/chosen": 0.7265364527702332, "logits/rejected": 0.6756460666656494, "logps/chosen": -188.26727294921875, "logps/ref_chosen": -63.05195236206055, "logps/ref_rejected": -85.52035522460938, "logps/rejected": -278.5576171875, "loss": 1.1079, "margin_dpo/margin_mean": 67.82193756103516, "margin_dpo/margin_std": 100.30946350097656, "step": 352 }, { "KL/chosen_KL_mean": -122.37887573242188, "KL/mean": -153.0131072998047, "KL/rejected_KL_mean": -183.64732360839844, "KL/std": 84.29683685302734, "epoch": 0.5336356764928194, "fcm_dpo/beta": 0.006157569587230682, "fcm_dpo/delta": 0.027647539973258972, "fcm_dpo/margin": 61.26842498779297, "fcm_dpo/q_t": 0.4137980043888092, "grad_norm": 10.43450927734375, "learning_rate": 2.658559799141411e-07, "logits/chosen": 0.7220193147659302, "logits/rejected": 0.7308777570724487, "logps/chosen": -191.3880615234375, "logps/ref_chosen": -69.00918579101562, "logps/ref_rejected": -72.65840148925781, "logps/rejected": -256.30572509765625, "loss": 1.114, "margin_dpo/margin_mean": 61.2684326171875, "margin_dpo/margin_std": 87.84406280517578, "step": 353 }, { "KL/chosen_KL_mean": -120.68923950195312, "KL/mean": -157.22747802734375, "KL/rejected_KL_mean": -193.76571655273438, "KL/std": 78.87259674072266, "epoch": 0.5351473922902494, "fcm_dpo/beta": 0.0058960379101336, "fcm_dpo/delta": -0.0409696064889431, "fcm_dpo/margin": 73.07649230957031, "fcm_dpo/q_t": 0.3988759219646454, "grad_norm": 12.436847686767578, "learning_rate": 2.6453620722761895e-07, "logits/chosen": 0.8180617094039917, "logits/rejected": 0.6904994249343872, "logps/chosen": -160.47756958007812, "logps/ref_chosen": -39.78833770751953, "logps/ref_rejected": -69.56885528564453, "logps/rejected": -263.3345947265625, "loss": 1.0792, "margin_dpo/margin_mean": 73.07649230957031, "margin_dpo/margin_std": 97.00477600097656, "step": 354 }, { "KL/chosen_KL_mean": -125.70584106445312, "KL/mean": -166.8114013671875, "KL/rejected_KL_mean": -207.91696166992188, "KL/std": 94.3542251586914, "epoch": 0.5366591080876795, "fcm_dpo/beta": 0.005632805172353983, "fcm_dpo/delta": -0.0817156583070755, "fcm_dpo/margin": 82.21111297607422, "fcm_dpo/q_t": 0.3910810351371765, "grad_norm": 13.240791320800781, "learning_rate": 2.632160279321328e-07, "logits/chosen": 0.7661707401275635, "logits/rejected": 0.6408806443214417, "logps/chosen": -171.9612274169922, "logps/ref_chosen": -46.25537872314453, "logps/ref_rejected": -78.20236206054688, "logps/rejected": -286.11932373046875, "loss": 1.061, "margin_dpo/margin_mean": 82.21110534667969, "margin_dpo/margin_std": 104.74549102783203, "step": 355 }, { "KL/chosen_KL_mean": -120.32354736328125, "KL/mean": -152.593505859375, "KL/rejected_KL_mean": -184.8634796142578, "KL/std": 87.85360717773438, "epoch": 0.5381708238851096, "fcm_dpo/beta": 0.00554704200476408, "fcm_dpo/delta": 0.050387270748615265, "fcm_dpo/margin": 64.53993225097656, "fcm_dpo/q_t": 0.4200926423072815, "grad_norm": 10.86155891418457, "learning_rate": 2.618954789559356e-07, "logits/chosen": 0.7344239950180054, "logits/rejected": 0.6559726595878601, "logps/chosen": -168.22970581054688, "logps/ref_chosen": -47.906158447265625, "logps/ref_rejected": -74.29397583007812, "logps/rejected": -259.1574401855469, "loss": 1.1515, "margin_dpo/margin_mean": 64.53993225097656, "margin_dpo/margin_std": 110.30122375488281, "step": 356 }, { "KL/chosen_KL_mean": -143.30943298339844, "KL/mean": -168.89112854003906, "KL/rejected_KL_mean": -194.47283935546875, "KL/std": 87.86856079101562, "epoch": 0.5396825396825397, "fcm_dpo/beta": 0.005487706046551466, "fcm_dpo/delta": -0.04394787177443504, "fcm_dpo/margin": 51.163387298583984, "fcm_dpo/q_t": 0.4319250285625458, "grad_norm": 11.494694709777832, "learning_rate": 2.6057459723762076e-07, "logits/chosen": 0.7188966274261475, "logits/rejected": 0.7018017768859863, "logps/chosen": -205.94442749023438, "logps/ref_chosen": -62.63500213623047, "logps/ref_rejected": -65.11399841308594, "logps/rejected": -259.5868225097656, "loss": 1.2007, "margin_dpo/margin_mean": 51.16339111328125, "margin_dpo/margin_std": 96.90858459472656, "step": 357 }, { "KL/chosen_KL_mean": -141.5384063720703, "KL/mean": -176.0528564453125, "KL/rejected_KL_mean": -210.56729125976562, "KL/std": 94.40840148925781, "epoch": 0.5411942554799698, "fcm_dpo/beta": 0.00527994055300951, "fcm_dpo/delta": -0.07874049991369247, "fcm_dpo/margin": 69.02886962890625, "fcm_dpo/q_t": 0.41425493359565735, "grad_norm": 14.46906852722168, "learning_rate": 2.5925341972508954e-07, "logits/chosen": 0.7024484872817993, "logits/rejected": 0.7164376974105835, "logps/chosen": -208.74801635742188, "logps/ref_chosen": -67.20960998535156, "logps/ref_rejected": -69.34715270996094, "logps/rejected": -279.9144287109375, "loss": 1.1371, "margin_dpo/margin_mean": 69.02886962890625, "margin_dpo/margin_std": 104.65023040771484, "step": 358 }, { "KL/chosen_KL_mean": -158.43338012695312, "KL/mean": -176.73294067382812, "KL/rejected_KL_mean": -195.03248596191406, "KL/std": 92.38772583007812, "epoch": 0.5427059712773998, "fcm_dpo/beta": 0.0052369264885783195, "fcm_dpo/delta": 0.06293975561857224, "fcm_dpo/margin": 36.59911346435547, "fcm_dpo/q_t": 0.4568448066711426, "grad_norm": 13.132709503173828, "learning_rate": 2.579319833745169e-07, "logits/chosen": 0.703708291053772, "logits/rejected": 0.6700050830841064, "logps/chosen": -220.95916748046875, "logps/ref_chosen": -62.52578353881836, "logps/ref_rejected": -76.63114929199219, "logps/rejected": -271.66363525390625, "loss": 1.2728, "margin_dpo/margin_mean": 36.59911346435547, "margin_dpo/margin_std": 100.44338989257812, "step": 359 }, { "KL/chosen_KL_mean": -151.61996459960938, "KL/mean": -185.01283264160156, "KL/rejected_KL_mean": -218.40570068359375, "KL/std": 98.46240234375, "epoch": 0.54421768707483, "fcm_dpo/beta": 0.005525579676032066, "fcm_dpo/delta": 0.03805600479245186, "fcm_dpo/margin": 66.78570556640625, "fcm_dpo/q_t": 0.41693851351737976, "grad_norm": 11.473810195922852, "learning_rate": 2.5661032514931834e-07, "logits/chosen": 0.6791380643844604, "logits/rejected": 0.5924465656280518, "logps/chosen": -215.10769653320312, "logps/ref_chosen": -63.48772048950195, "logps/ref_rejected": -90.6891098022461, "logps/rejected": -309.09478759765625, "loss": 1.1193, "margin_dpo/margin_mean": 66.78571319580078, "margin_dpo/margin_std": 97.07440185546875, "step": 360 }, { "KL/chosen_KL_mean": -142.7432861328125, "KL/mean": -179.56118774414062, "KL/rejected_KL_mean": -216.37908935546875, "KL/std": 92.99285888671875, "epoch": 0.54572940287226, "fcm_dpo/beta": 0.005594981834292412, "fcm_dpo/delta": -0.015271172858774662, "fcm_dpo/margin": 73.63580322265625, "fcm_dpo/q_t": 0.4040573239326477, "grad_norm": 10.441182136535645, "learning_rate": 2.552884820191154e-07, "logits/chosen": 0.7554844617843628, "logits/rejected": 0.7160251140594482, "logps/chosen": -200.66043090820312, "logps/ref_chosen": -57.917144775390625, "logps/ref_rejected": -72.39089965820312, "logps/rejected": -288.7699890136719, "loss": 1.087, "margin_dpo/margin_mean": 73.63580322265625, "margin_dpo/margin_std": 97.2933349609375, "step": 361 }, { "KL/chosen_KL_mean": -144.31117248535156, "KL/mean": -188.12466430664062, "KL/rejected_KL_mean": -231.93817138671875, "KL/std": 106.7471923828125, "epoch": 0.54724111866969, "fcm_dpo/beta": 0.005310633219778538, "fcm_dpo/delta": -0.08533253520727158, "fcm_dpo/margin": 87.62699890136719, "fcm_dpo/q_t": 0.3944483995437622, "grad_norm": 11.381577491760254, "learning_rate": 2.53966490958702e-07, "logits/chosen": 0.7894976139068604, "logits/rejected": 0.692185640335083, "logps/chosen": -207.754638671875, "logps/ref_chosen": -63.4434700012207, "logps/ref_rejected": -103.45516967773438, "logps/rejected": -335.3933410644531, "loss": 1.0723, "margin_dpo/margin_mean": 87.62699890136719, "margin_dpo/margin_std": 124.65602111816406, "step": 362 }, { "KL/chosen_KL_mean": -148.01797485351562, "KL/mean": -187.11468505859375, "KL/rejected_KL_mean": -226.21136474609375, "KL/std": 92.74317932128906, "epoch": 0.5487528344671202, "fcm_dpo/beta": 0.0050812652334570885, "fcm_dpo/delta": 0.0033413858618587255, "fcm_dpo/margin": 78.19340515136719, "fcm_dpo/q_t": 0.4094967842102051, "grad_norm": 13.362162590026855, "learning_rate": 2.526443889470099e-07, "logits/chosen": 0.793647289276123, "logits/rejected": 0.6671258211135864, "logps/chosen": -196.6697998046875, "logps/ref_chosen": -48.65182876586914, "logps/ref_rejected": -88.65904235839844, "logps/rejected": -314.87042236328125, "loss": 1.0991, "margin_dpo/margin_mean": 78.19339752197266, "margin_dpo/margin_std": 109.21980285644531, "step": 363 }, { "KL/chosen_KL_mean": -131.89614868164062, "KL/mean": -175.02871704101562, "KL/rejected_KL_mean": -218.16127014160156, "KL/std": 102.58001708984375, "epoch": 0.5502645502645502, "fcm_dpo/beta": 0.004936812911182642, "fcm_dpo/delta": -0.033310696482658386, "fcm_dpo/margin": 86.26512908935547, "fcm_dpo/q_t": 0.40305817127227783, "grad_norm": 9.358033180236816, "learning_rate": 2.513222129660744e-07, "logits/chosen": 0.698688268661499, "logits/rejected": 0.6179659366607666, "logps/chosen": -189.76722717285156, "logps/ref_chosen": -57.87107467651367, "logps/ref_rejected": -80.95503234863281, "logps/rejected": -299.1163024902344, "loss": 1.0997, "margin_dpo/margin_mean": 86.26512145996094, "margin_dpo/margin_std": 129.88619995117188, "step": 364 }, { "KL/chosen_KL_mean": -125.53724670410156, "KL/mean": -165.1311798095703, "KL/rejected_KL_mean": -204.72509765625, "KL/std": 109.31954956054688, "epoch": 0.5517762660619804, "fcm_dpo/beta": 0.004892812110483646, "fcm_dpo/delta": 0.014496054500341415, "fcm_dpo/margin": 79.18785095214844, "fcm_dpo/q_t": 0.41075021028518677, "grad_norm": 10.14580249786377, "learning_rate": 2.5e-07, "logits/chosen": 0.745829701423645, "logits/rejected": 0.7439219951629639, "logps/chosen": -190.47943115234375, "logps/ref_chosen": -64.94217681884766, "logps/ref_rejected": -74.8599853515625, "logps/rejected": -279.5850830078125, "loss": 1.0869, "margin_dpo/margin_mean": 79.18785095214844, "margin_dpo/margin_std": 95.73055267333984, "step": 365 }, { "KL/chosen_KL_mean": -124.95919799804688, "KL/mean": -157.53627014160156, "KL/rejected_KL_mean": -190.11334228515625, "KL/std": 95.9143295288086, "epoch": 0.5532879818594104, "fcm_dpo/beta": 0.005323389545083046, "fcm_dpo/delta": 0.06161736696958542, "fcm_dpo/margin": 65.1541519165039, "fcm_dpo/q_t": 0.4233441948890686, "grad_norm": 13.165140151977539, "learning_rate": 2.486777870339255e-07, "logits/chosen": 0.6843109130859375, "logits/rejected": 0.6801989078521729, "logps/chosen": -180.12518310546875, "logps/ref_chosen": -55.16598129272461, "logps/ref_rejected": -65.26121520996094, "logps/rejected": -255.3745574951172, "loss": 1.1685, "margin_dpo/margin_mean": 65.1541519165039, "margin_dpo/margin_std": 121.73723602294922, "step": 366 }, { "KL/chosen_KL_mean": -143.6685791015625, "KL/mean": -176.48419189453125, "KL/rejected_KL_mean": -209.2998046875, "KL/std": 95.0160903930664, "epoch": 0.5547996976568406, "fcm_dpo/beta": 0.005490441806614399, "fcm_dpo/delta": 0.048465244472026825, "fcm_dpo/margin": 65.6312255859375, "fcm_dpo/q_t": 0.4174492359161377, "grad_norm": 12.372570991516113, "learning_rate": 2.4735561105299014e-07, "logits/chosen": 0.7515787482261658, "logits/rejected": 0.6537929773330688, "logps/chosen": -199.67904663085938, "logps/ref_chosen": -56.01046371459961, "logps/ref_rejected": -77.31010437011719, "logps/rejected": -286.60992431640625, "loss": 1.1306, "margin_dpo/margin_mean": 65.6312255859375, "margin_dpo/margin_std": 100.20790100097656, "step": 367 }, { "KL/chosen_KL_mean": -156.02073669433594, "KL/mean": -185.53570556640625, "KL/rejected_KL_mean": -215.05062866210938, "KL/std": 93.59857177734375, "epoch": 0.5563114134542706, "fcm_dpo/beta": 0.005780104547739029, "fcm_dpo/delta": 0.07085883617401123, "fcm_dpo/margin": 59.02989959716797, "fcm_dpo/q_t": 0.4238590598106384, "grad_norm": 13.36587142944336, "learning_rate": 2.46033509041298e-07, "logits/chosen": 0.5935603380203247, "logits/rejected": 0.5959989428520203, "logps/chosen": -230.8500213623047, "logps/ref_chosen": -74.82927703857422, "logps/ref_rejected": -76.11680603027344, "logps/rejected": -291.16741943359375, "loss": 1.1574, "margin_dpo/margin_mean": 59.02989959716797, "margin_dpo/margin_std": 100.53662109375, "step": 368 }, { "KL/chosen_KL_mean": -142.60076904296875, "KL/mean": -169.75457763671875, "KL/rejected_KL_mean": -196.90835571289062, "KL/std": 94.11172485351562, "epoch": 0.5578231292517006, "fcm_dpo/beta": 0.006405709311366081, "fcm_dpo/delta": 0.058929868042469025, "fcm_dpo/margin": 54.307586669921875, "fcm_dpo/q_t": 0.42363277077674866, "grad_norm": 13.711212158203125, "learning_rate": 2.447115179808846e-07, "logits/chosen": 0.7638058662414551, "logits/rejected": 0.7166075110435486, "logps/chosen": -200.927001953125, "logps/ref_chosen": -58.32621765136719, "logps/ref_rejected": -80.92183685302734, "logps/rejected": -277.8302001953125, "loss": 1.1666, "margin_dpo/margin_mean": 54.307586669921875, "margin_dpo/margin_std": 100.56336975097656, "step": 369 }, { "KL/chosen_KL_mean": -135.7542724609375, "KL/mean": -175.9358673095703, "KL/rejected_KL_mean": -216.11746215820312, "KL/std": 98.13227844238281, "epoch": 0.5593348450491308, "fcm_dpo/beta": 0.005805108696222305, "fcm_dpo/delta": -0.09162604808807373, "fcm_dpo/margin": 80.36318969726562, "fcm_dpo/q_t": 0.38908275961875916, "grad_norm": 12.54875659942627, "learning_rate": 2.4338967485068164e-07, "logits/chosen": 0.854151725769043, "logits/rejected": 0.7885351777076721, "logps/chosen": -188.6379852294922, "logps/ref_chosen": -52.88372039794922, "logps/ref_rejected": -79.43692016601562, "logps/rejected": -295.55438232421875, "loss": 1.076, "margin_dpo/margin_mean": 80.36318969726562, "margin_dpo/margin_std": 114.33674621582031, "step": 370 }, { "KL/chosen_KL_mean": -134.52255249023438, "KL/mean": -168.52369689941406, "KL/rejected_KL_mean": -202.5248565673828, "KL/std": 92.02737426757812, "epoch": 0.5608465608465608, "fcm_dpo/beta": 0.00593863520771265, "fcm_dpo/delta": -0.008677050471305847, "fcm_dpo/margin": 68.00228881835938, "fcm_dpo/q_t": 0.4075172543525696, "grad_norm": 16.5703182220459, "learning_rate": 2.420680166254831e-07, "logits/chosen": 0.8438094258308411, "logits/rejected": 0.8158732652664185, "logps/chosen": -183.74676513671875, "logps/ref_chosen": -49.224212646484375, "logps/ref_rejected": -63.348472595214844, "logps/rejected": -265.8733215332031, "loss": 1.1141, "margin_dpo/margin_mean": 68.00228118896484, "margin_dpo/margin_std": 102.23562622070312, "step": 371 }, { "KL/chosen_KL_mean": -142.50531005859375, "KL/mean": -162.75030517578125, "KL/rejected_KL_mean": -182.99530029296875, "KL/std": 92.52728271484375, "epoch": 0.562358276643991, "fcm_dpo/beta": 0.005816627759486437, "fcm_dpo/delta": 0.02477392926812172, "fcm_dpo/margin": 40.48997497558594, "fcm_dpo/q_t": 0.44827184081077576, "grad_norm": 14.055017471313477, "learning_rate": 2.4074658027491044e-07, "logits/chosen": 0.7998120188713074, "logits/rejected": 0.7064263820648193, "logps/chosen": -194.77487182617188, "logps/ref_chosen": -52.269554138183594, "logps/ref_rejected": -72.99522399902344, "logps/rejected": -255.99050903320312, "loss": 1.2831, "margin_dpo/margin_mean": 40.48997497558594, "margin_dpo/margin_std": 119.91127014160156, "step": 372 }, { "KL/chosen_KL_mean": -160.78884887695312, "KL/mean": -188.75387573242188, "KL/rejected_KL_mean": -216.71890258789062, "KL/std": 100.0822982788086, "epoch": 0.563869992441421, "fcm_dpo/beta": 0.006202812306582928, "fcm_dpo/delta": 0.06371521204710007, "fcm_dpo/margin": 55.93007278442383, "fcm_dpo/q_t": 0.4251997470855713, "grad_norm": 13.777511596679688, "learning_rate": 2.394254027623792e-07, "logits/chosen": 0.7657662034034729, "logits/rejected": 0.7004691362380981, "logps/chosen": -221.90184020996094, "logps/ref_chosen": -61.112998962402344, "logps/ref_rejected": -76.24851989746094, "logps/rejected": -292.9674072265625, "loss": 1.2048, "margin_dpo/margin_mean": 55.93006896972656, "margin_dpo/margin_std": 123.24330139160156, "step": 373 }, { "KL/chosen_KL_mean": -137.24179077148438, "KL/mean": -181.21661376953125, "KL/rejected_KL_mean": -225.19142150878906, "KL/std": 102.51628112792969, "epoch": 0.5653817082388511, "fcm_dpo/beta": 0.005645962432026863, "fcm_dpo/delta": -0.13082769513130188, "fcm_dpo/margin": 87.94963073730469, "fcm_dpo/q_t": 0.3829165995121002, "grad_norm": 13.66327953338623, "learning_rate": 2.381045210440644e-07, "logits/chosen": 0.6615780591964722, "logits/rejected": 0.6669573783874512, "logps/chosen": -209.91099548339844, "logps/ref_chosen": -72.66920471191406, "logps/ref_rejected": -76.83158874511719, "logps/rejected": -302.02301025390625, "loss": 1.0671, "margin_dpo/margin_mean": 87.94963073730469, "margin_dpo/margin_std": 126.89126586914062, "step": 374 }, { "KL/chosen_KL_mean": -135.42027282714844, "KL/mean": -167.710693359375, "KL/rejected_KL_mean": -200.0010986328125, "KL/std": 111.11065673828125, "epoch": 0.5668934240362812, "fcm_dpo/beta": 0.005644750315696001, "fcm_dpo/delta": 0.043448496609926224, "fcm_dpo/margin": 64.58082580566406, "fcm_dpo/q_t": 0.42045366764068604, "grad_norm": 15.049660682678223, "learning_rate": 2.3678397206786715e-07, "logits/chosen": 0.76460200548172, "logits/rejected": 0.7118662595748901, "logps/chosen": -193.10357666015625, "logps/ref_chosen": -57.68330383300781, "logps/ref_rejected": -79.34097290039062, "logps/rejected": -279.3420715332031, "loss": 1.1656, "margin_dpo/margin_mean": 64.58082580566406, "margin_dpo/margin_std": 122.09146118164062, "step": 375 }, { "KL/chosen_KL_mean": -143.47830200195312, "KL/mean": -183.77133178710938, "KL/rejected_KL_mean": -224.06434631347656, "KL/std": 112.19872283935547, "epoch": 0.5684051398337112, "fcm_dpo/beta": 0.0054377606138587, "fcm_dpo/delta": -0.0515349879860878, "fcm_dpo/margin": 80.58604431152344, "fcm_dpo/q_t": 0.4034051299095154, "grad_norm": 12.587312698364258, "learning_rate": 2.3546379277238103e-07, "logits/chosen": 0.8178266286849976, "logits/rejected": 0.754265308380127, "logps/chosen": -195.15237426757812, "logps/ref_chosen": -51.674072265625, "logps/ref_rejected": -75.69713592529297, "logps/rejected": -299.761474609375, "loss": 1.1133, "margin_dpo/margin_mean": 80.58604431152344, "margin_dpo/margin_std": 133.2933349609375, "step": 376 }, { "KL/chosen_KL_mean": -144.62603759765625, "KL/mean": -172.3679962158203, "KL/rejected_KL_mean": -200.10995483398438, "KL/std": 88.5212631225586, "epoch": 0.5699168556311414, "fcm_dpo/beta": 0.005908847786486149, "fcm_dpo/delta": 0.08338885009288788, "fcm_dpo/margin": 55.483924865722656, "fcm_dpo/q_t": 0.4264184236526489, "grad_norm": 13.982069969177246, "learning_rate": 2.3414402008585886e-07, "logits/chosen": 0.7420355081558228, "logits/rejected": 0.7195169925689697, "logps/chosen": -190.8045654296875, "logps/ref_chosen": -46.17853546142578, "logps/ref_rejected": -57.756500244140625, "logps/rejected": -257.866455078125, "loss": 1.1684, "margin_dpo/margin_mean": 55.48392868041992, "margin_dpo/margin_std": 100.01844787597656, "step": 377 }, { "KL/chosen_KL_mean": -142.30294799804688, "KL/mean": -169.65269470214844, "KL/rejected_KL_mean": -197.00244140625, "KL/std": 103.02887725830078, "epoch": 0.5714285714285714, "fcm_dpo/beta": 0.006364922039210796, "fcm_dpo/delta": 0.057820893824100494, "fcm_dpo/margin": 54.6994743347168, "fcm_dpo/q_t": 0.4231072664260864, "grad_norm": 11.828926086425781, "learning_rate": 2.3282469092517977e-07, "logits/chosen": 0.8158556222915649, "logits/rejected": 0.7710492610931396, "logps/chosen": -201.52182006835938, "logps/ref_chosen": -59.21887969970703, "logps/ref_rejected": -71.24818420410156, "logps/rejected": -268.2506103515625, "loss": 1.1709, "margin_dpo/margin_mean": 54.69947814941406, "margin_dpo/margin_std": 101.46234130859375, "step": 378 }, { "KL/chosen_KL_mean": -144.59205627441406, "KL/mean": -184.87893676757812, "KL/rejected_KL_mean": -225.16583251953125, "KL/std": 108.66886138916016, "epoch": 0.5729402872260015, "fcm_dpo/beta": 0.0058668795973062515, "fcm_dpo/delta": -0.09539224952459335, "fcm_dpo/margin": 80.57376098632812, "fcm_dpo/q_t": 0.3924974203109741, "grad_norm": 14.25523567199707, "learning_rate": 2.3150584219481643e-07, "logits/chosen": 0.7613098621368408, "logits/rejected": 0.6914204955101013, "logps/chosen": -220.9086456298828, "logps/ref_chosen": -76.31658935546875, "logps/ref_rejected": -104.26200103759766, "logps/rejected": -329.4278259277344, "loss": 1.0754, "margin_dpo/margin_mean": 80.57376098632812, "margin_dpo/margin_std": 117.86245727539062, "step": 379 }, { "KL/chosen_KL_mean": -121.277099609375, "KL/mean": -164.54278564453125, "KL/rejected_KL_mean": -207.80848693847656, "KL/std": 95.66365051269531, "epoch": 0.5744520030234316, "fcm_dpo/beta": 0.005375675857067108, "fcm_dpo/delta": -0.08500650525093079, "fcm_dpo/margin": 86.53138732910156, "fcm_dpo/q_t": 0.38998135924339294, "grad_norm": 12.326929092407227, "learning_rate": 2.3018751078580283e-07, "logits/chosen": 0.7833170890808105, "logits/rejected": 0.7477537393569946, "logps/chosen": -182.56027221679688, "logps/ref_chosen": -61.283164978027344, "logps/ref_rejected": -72.38892364501953, "logps/rejected": -280.1973876953125, "loss": 1.0612, "margin_dpo/margin_mean": 86.5313949584961, "margin_dpo/margin_std": 113.20811462402344, "step": 380 }, { "KL/chosen_KL_mean": -149.11305236816406, "KL/mean": -165.54931640625, "KL/rejected_KL_mean": -181.98558044433594, "KL/std": 100.11683654785156, "epoch": 0.5759637188208617, "fcm_dpo/beta": 0.0054644690826535225, "fcm_dpo/delta": 0.07773777097463608, "fcm_dpo/margin": 32.87253189086914, "fcm_dpo/q_t": 0.4607895016670227, "grad_norm": 12.004046440124512, "learning_rate": 2.288697335747027e-07, "logits/chosen": 0.7426190376281738, "logits/rejected": 0.7275179624557495, "logps/chosen": -207.32704162597656, "logps/ref_chosen": -58.2139892578125, "logps/ref_rejected": -60.78669357299805, "logps/rejected": -242.77227783203125, "loss": 1.3172, "margin_dpo/margin_mean": 32.872535705566406, "margin_dpo/margin_std": 121.07296752929688, "step": 381 }, { "KL/chosen_KL_mean": -149.33331298828125, "KL/mean": -180.80596923828125, "KL/rejected_KL_mean": -212.27862548828125, "KL/std": 100.90338134765625, "epoch": 0.5774754346182918, "fcm_dpo/beta": 0.005938488524407148, "fcm_dpo/delta": 0.02896289899945259, "fcm_dpo/margin": 62.9453125, "fcm_dpo/q_t": 0.41630202531814575, "grad_norm": 14.264102935791016, "learning_rate": 2.2755254742257706e-07, "logits/chosen": 0.7569193243980408, "logits/rejected": 0.7074561715126038, "logps/chosen": -211.15863037109375, "logps/ref_chosen": -61.82532501220703, "logps/ref_rejected": -83.0452880859375, "logps/rejected": -295.32391357421875, "loss": 1.1309, "margin_dpo/margin_mean": 62.9453125, "margin_dpo/margin_std": 99.38772583007812, "step": 382 }, { "KL/chosen_KL_mean": -153.87950134277344, "KL/mean": -180.53121948242188, "KL/rejected_KL_mean": -207.18296813964844, "KL/std": 100.1922607421875, "epoch": 0.5789871504157218, "fcm_dpo/beta": 0.006136381067335606, "fcm_dpo/delta": 0.08745638281106949, "fcm_dpo/margin": 53.30347442626953, "fcm_dpo/q_t": 0.4308212995529175, "grad_norm": 16.983381271362305, "learning_rate": 2.2623598917395436e-07, "logits/chosen": 0.6574628353118896, "logits/rejected": 0.6930520534515381, "logps/chosen": -234.4427490234375, "logps/ref_chosen": -80.56326293945312, "logps/ref_rejected": -74.62922668457031, "logps/rejected": -281.81219482421875, "loss": 1.2261, "margin_dpo/margin_mean": 53.30347442626953, "margin_dpo/margin_std": 126.07339477539062, "step": 383 }, { "KL/chosen_KL_mean": -140.81640625, "KL/mean": -174.13668823242188, "KL/rejected_KL_mean": -207.45697021484375, "KL/std": 96.11679077148438, "epoch": 0.5804988662131519, "fcm_dpo/beta": 0.006302967667579651, "fcm_dpo/delta": -0.025382589548826218, "fcm_dpo/margin": 66.64056396484375, "fcm_dpo/q_t": 0.4019022583961487, "grad_norm": 16.735795974731445, "learning_rate": 2.2492009565579875e-07, "logits/chosen": 0.777447521686554, "logits/rejected": 0.7444401979446411, "logps/chosen": -206.2915496826172, "logps/ref_chosen": -65.47514343261719, "logps/ref_rejected": -79.67378234863281, "logps/rejected": -287.1307373046875, "loss": 1.1074, "margin_dpo/margin_mean": 66.64056396484375, "margin_dpo/margin_std": 102.94216918945312, "step": 384 }, { "KL/chosen_KL_mean": -142.68655395507812, "KL/mean": -185.57028198242188, "KL/rejected_KL_mean": -228.4540252685547, "KL/std": 101.47017669677734, "epoch": 0.582010582010582, "fcm_dpo/beta": 0.005779832135885954, "fcm_dpo/delta": -0.12640322744846344, "fcm_dpo/margin": 85.7674789428711, "fcm_dpo/q_t": 0.38280969858169556, "grad_norm": 12.516278266906738, "learning_rate": 2.2360490367648084e-07, "logits/chosen": 0.6712303161621094, "logits/rejected": 0.6324940919876099, "logps/chosen": -208.74307250976562, "logps/ref_chosen": -66.0565185546875, "logps/ref_rejected": -86.68023681640625, "logps/rejected": -315.13427734375, "loss": 1.038, "margin_dpo/margin_mean": 85.7674789428711, "margin_dpo/margin_std": 104.35530090332031, "step": 385 }, { "KL/chosen_KL_mean": -165.44119262695312, "KL/mean": -193.70352172851562, "KL/rejected_KL_mean": -221.9658203125, "KL/std": 107.32279968261719, "epoch": 0.5835222978080121, "fcm_dpo/beta": 0.0058027347549796104, "fcm_dpo/delta": 0.08648616075515747, "fcm_dpo/margin": 56.52465057373047, "fcm_dpo/q_t": 0.42700403928756714, "grad_norm": 13.528538703918457, "learning_rate": 2.2229045002474724e-07, "logits/chosen": 0.6971240043640137, "logits/rejected": 0.6468753814697266, "logps/chosen": -241.06484985351562, "logps/ref_chosen": -75.6236572265625, "logps/ref_rejected": -92.62330627441406, "logps/rejected": -314.5891418457031, "loss": 1.1801, "margin_dpo/margin_mean": 56.52465057373047, "margin_dpo/margin_std": 108.72563171386719, "step": 386 }, { "KL/chosen_KL_mean": -144.2526397705078, "KL/mean": -184.9021759033203, "KL/rejected_KL_mean": -225.5517120361328, "KL/std": 96.83357238769531, "epoch": 0.5850340136054422, "fcm_dpo/beta": 0.005610157269984484, "fcm_dpo/delta": -0.07386106252670288, "fcm_dpo/margin": 81.29905700683594, "fcm_dpo/q_t": 0.3932124376296997, "grad_norm": 14.18612003326416, "learning_rate": 2.209767714686924e-07, "logits/chosen": 0.7909537553787231, "logits/rejected": 0.6846780776977539, "logps/chosen": -191.47434997558594, "logps/ref_chosen": -47.22170639038086, "logps/ref_rejected": -87.338134765625, "logps/rejected": -312.88983154296875, "loss": 1.0565, "margin_dpo/margin_mean": 81.29905700683594, "margin_dpo/margin_std": 101.80584716796875, "step": 387 }, { "KL/chosen_KL_mean": -150.70068359375, "KL/mean": -175.45492553710938, "KL/rejected_KL_mean": -200.20916748046875, "KL/std": 103.48775482177734, "epoch": 0.5865457294028723, "fcm_dpo/beta": 0.005595776252448559, "fcm_dpo/delta": 0.008372929878532887, "fcm_dpo/margin": 49.50847244262695, "fcm_dpo/q_t": 0.43925899267196655, "grad_norm": 12.53011417388916, "learning_rate": 2.1966390475472954e-07, "logits/chosen": 0.7262797355651855, "logits/rejected": 0.7237277626991272, "logps/chosen": -225.2801513671875, "logps/ref_chosen": -74.5794677734375, "logps/ref_rejected": -79.92558288574219, "logps/rejected": -280.134765625, "loss": 1.2396, "margin_dpo/margin_mean": 49.50847244262695, "margin_dpo/margin_std": 122.52720642089844, "step": 388 }, { "KL/chosen_KL_mean": -141.07130432128906, "KL/mean": -181.15814208984375, "KL/rejected_KL_mean": -221.24497985839844, "KL/std": 97.66059112548828, "epoch": 0.5880574452003023, "fcm_dpo/beta": 0.005440755747258663, "fcm_dpo/delta": -0.04618635028600693, "fcm_dpo/margin": 80.17366027832031, "fcm_dpo/q_t": 0.39910855889320374, "grad_norm": 18.9296932220459, "learning_rate": 2.1835188660656265e-07, "logits/chosen": 0.7523651719093323, "logits/rejected": 0.7214106917381287, "logps/chosen": -202.6956787109375, "logps/ref_chosen": -61.624366760253906, "logps/ref_rejected": -76.50978088378906, "logps/rejected": -297.7547607421875, "loss": 1.077, "margin_dpo/margin_mean": 80.17366027832031, "margin_dpo/margin_std": 107.77185821533203, "step": 389 }, { "KL/chosen_KL_mean": -125.87141418457031, "KL/mean": -156.2007293701172, "KL/rejected_KL_mean": -186.530029296875, "KL/std": 93.57325744628906, "epoch": 0.5895691609977324, "fcm_dpo/beta": 0.005581664387136698, "fcm_dpo/delta": 0.07422532886266708, "fcm_dpo/margin": 60.65862274169922, "fcm_dpo/q_t": 0.42511534690856934, "grad_norm": 11.519083023071289, "learning_rate": 2.170407537241599e-07, "logits/chosen": 0.8347935676574707, "logits/rejected": 0.7659469246864319, "logps/chosen": -171.7432861328125, "logps/ref_chosen": -45.871864318847656, "logps/ref_rejected": -61.305999755859375, "logps/rejected": -247.83602905273438, "loss": 1.1511, "margin_dpo/margin_mean": 60.658626556396484, "margin_dpo/margin_std": 100.20620727539062, "step": 390 }, { "KL/chosen_KL_mean": -141.71585083007812, "KL/mean": -179.6748046875, "KL/rejected_KL_mean": -217.63375854492188, "KL/std": 97.35914611816406, "epoch": 0.5910808767951625, "fcm_dpo/beta": 0.00556203443557024, "fcm_dpo/delta": -0.02935415506362915, "fcm_dpo/margin": 75.91790771484375, "fcm_dpo/q_t": 0.4031655788421631, "grad_norm": 12.19450569152832, "learning_rate": 2.1573054278272636e-07, "logits/chosen": 0.7507619857788086, "logits/rejected": 0.6888763308525085, "logps/chosen": -199.90286254882812, "logps/ref_chosen": -58.18701171875, "logps/ref_rejected": -83.63442993164062, "logps/rejected": -301.2681884765625, "loss": 1.1082, "margin_dpo/margin_mean": 75.91790771484375, "margin_dpo/margin_std": 116.9903793334961, "step": 391 }, { "KL/chosen_KL_mean": -130.40484619140625, "KL/mean": -169.21307373046875, "KL/rejected_KL_mean": -208.0213165283203, "KL/std": 106.27191925048828, "epoch": 0.5925925925925926, "fcm_dpo/beta": 0.005578967742621899, "fcm_dpo/delta": -0.04480854421854019, "fcm_dpo/margin": 77.61647033691406, "fcm_dpo/q_t": 0.4019958972930908, "grad_norm": 12.193196296691895, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 0.8328498601913452, "logits/rejected": 0.7778192162513733, "logps/chosen": -200.1493682861328, "logps/ref_chosen": -69.7445297241211, "logps/ref_rejected": -94.05877685546875, "logps/rejected": -302.080078125, "loss": 1.1104, "margin_dpo/margin_mean": 77.61647033691406, "margin_dpo/margin_std": 123.52142333984375, "step": 392 }, { "KL/chosen_KL_mean": -145.96200561523438, "KL/mean": -187.2437286376953, "KL/rejected_KL_mean": -228.52545166015625, "KL/std": 102.85442352294922, "epoch": 0.5941043083900227, "fcm_dpo/beta": 0.005069409962743521, "fcm_dpo/delta": -0.02902321144938469, "fcm_dpo/margin": 82.56344604492188, "fcm_dpo/q_t": 0.40151870250701904, "grad_norm": 10.115791320800781, "learning_rate": 2.131130332936195e-07, "logits/chosen": 0.7979670763015747, "logits/rejected": 0.7620906829833984, "logps/chosen": -198.29690551757812, "logps/ref_chosen": -52.33489990234375, "logps/ref_rejected": -74.33809661865234, "logps/rejected": -302.8635559082031, "loss": 1.0764, "margin_dpo/margin_mean": 82.56344604492188, "margin_dpo/margin_std": 100.86908721923828, "step": 393 }, { "KL/chosen_KL_mean": -139.4735870361328, "KL/mean": -175.12521362304688, "KL/rejected_KL_mean": -210.77684020996094, "KL/std": 96.35044860839844, "epoch": 0.5956160241874527, "fcm_dpo/beta": 0.005368366837501526, "fcm_dpo/delta": 0.019182713702321053, "fcm_dpo/margin": 71.30326080322266, "fcm_dpo/q_t": 0.41120874881744385, "grad_norm": 11.45832347869873, "learning_rate": 2.1180580796331323e-07, "logits/chosen": 0.7915951013565063, "logits/rejected": 0.7640597224235535, "logps/chosen": -200.14971923828125, "logps/ref_chosen": -60.6761360168457, "logps/ref_rejected": -71.36074829101562, "logps/rejected": -282.1375732421875, "loss": 1.1046, "margin_dpo/margin_mean": 71.30325317382812, "margin_dpo/margin_std": 95.12825012207031, "step": 394 }, { "KL/chosen_KL_mean": -140.49183654785156, "KL/mean": -175.02513122558594, "KL/rejected_KL_mean": -209.55844116210938, "KL/std": 99.29302978515625, "epoch": 0.5971277399848829, "fcm_dpo/beta": 0.0054627228528261185, "fcm_dpo/delta": 0.026971735060214996, "fcm_dpo/margin": 69.06660461425781, "fcm_dpo/q_t": 0.4158324599266052, "grad_norm": 12.025148391723633, "learning_rate": 2.104996510066625e-07, "logits/chosen": 0.8082689046859741, "logits/rejected": 0.7192596197128296, "logps/chosen": -191.09616088867188, "logps/ref_chosen": -50.60432434082031, "logps/ref_rejected": -77.08731079101562, "logps/rejected": -286.645751953125, "loss": 1.1223, "margin_dpo/margin_mean": 69.06660461425781, "margin_dpo/margin_std": 106.44404602050781, "step": 395 }, { "KL/chosen_KL_mean": -136.07615661621094, "KL/mean": -170.98126220703125, "KL/rejected_KL_mean": -205.88636779785156, "KL/std": 107.50096130371094, "epoch": 0.5986394557823129, "fcm_dpo/beta": 0.005069066770374775, "fcm_dpo/delta": -0.06986290216445923, "fcm_dpo/margin": 69.81021118164062, "fcm_dpo/q_t": 0.41486620903015137, "grad_norm": 10.510855674743652, "learning_rate": 2.0919459895968517e-07, "logits/chosen": 0.7915246486663818, "logits/rejected": 0.7065383195877075, "logps/chosen": -187.43576049804688, "logps/ref_chosen": -51.35961151123047, "logps/ref_rejected": -79.89360046386719, "logps/rejected": -285.77996826171875, "loss": 1.1229, "margin_dpo/margin_mean": 69.81021118164062, "margin_dpo/margin_std": 94.96488952636719, "step": 396 }, { "KL/chosen_KL_mean": -157.47796630859375, "KL/mean": -177.4036102294922, "KL/rejected_KL_mean": -197.32923889160156, "KL/std": 99.39573669433594, "epoch": 0.600151171579743, "fcm_dpo/beta": 0.005069066770374775, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 39.851280212402344, "fcm_dpo/q_t": 0.4533647298812866, "grad_norm": 11.057273864746094, "learning_rate": 2.078906883274924e-07, "logits/chosen": 0.7363402247428894, "logits/rejected": 0.6861610412597656, "logps/chosen": -223.9342041015625, "logps/ref_chosen": -66.45622253417969, "logps/ref_rejected": -85.74736785888672, "logps/rejected": -283.07659912109375, "loss": 1.2732, "margin_dpo/margin_mean": 39.851280212402344, "margin_dpo/margin_std": 111.93646240234375, "step": 397 }, { "KL/chosen_KL_mean": -131.88458251953125, "KL/mean": -177.76821899414062, "KL/rejected_KL_mean": -223.65185546875, "KL/std": 103.11227416992188, "epoch": 0.6016628873771731, "fcm_dpo/beta": 0.00475926510989666, "fcm_dpo/delta": -0.05145275592803955, "fcm_dpo/margin": 91.76725006103516, "fcm_dpo/q_t": 0.3985298275947571, "grad_norm": 9.738566398620605, "learning_rate": 2.065879555832674e-07, "logits/chosen": 0.7597224712371826, "logits/rejected": 0.701553225517273, "logps/chosen": -181.1288299560547, "logps/ref_chosen": -49.244239807128906, "logps/ref_rejected": -75.18949127197266, "logps/rejected": -298.8413391113281, "loss": 1.0615, "margin_dpo/margin_mean": 91.76725006103516, "margin_dpo/margin_std": 111.49891662597656, "step": 398 }, { "KL/chosen_KL_mean": -156.69384765625, "KL/mean": -210.0038604736328, "KL/rejected_KL_mean": -263.3138732910156, "KL/std": 118.74224853515625, "epoch": 0.6031746031746031, "fcm_dpo/beta": 0.004380588419735432, "fcm_dpo/delta": -0.09351267665624619, "fcm_dpo/margin": 106.62001037597656, "fcm_dpo/q_t": 0.3906853199005127, "grad_norm": 9.792754173278809, "learning_rate": 2.052864371672457e-07, "logits/chosen": 0.7528105974197388, "logits/rejected": 0.6174201965332031, "logps/chosen": -225.00064086914062, "logps/ref_chosen": -68.30679321289062, "logps/ref_rejected": -113.2708511352539, "logps/rejected": -376.584716796875, "loss": 1.0513, "margin_dpo/margin_mean": 106.62001037597656, "margin_dpo/margin_std": 135.46578979492188, "step": 399 }, { "KL/chosen_KL_mean": -173.648681640625, "KL/mean": -203.43667602539062, "KL/rejected_KL_mean": -233.22470092773438, "KL/std": 106.72013854980469, "epoch": 0.6046863189720333, "fcm_dpo/beta": 0.004395931493490934, "fcm_dpo/delta": 0.002488115569576621, "fcm_dpo/margin": 59.576026916503906, "fcm_dpo/q_t": 0.4374983310699463, "grad_norm": 13.163859367370605, "learning_rate": 2.0398616948569493e-07, "logits/chosen": 0.7592464685440063, "logits/rejected": 0.703935980796814, "logps/chosen": -245.27517700195312, "logps/ref_chosen": -71.62649536132812, "logps/ref_rejected": -90.98765563964844, "logps/rejected": -324.21234130859375, "loss": 1.1938, "margin_dpo/margin_mean": 59.57603073120117, "margin_dpo/margin_std": 100.98155212402344, "step": 400 }, { "KL/chosen_KL_mean": -131.31045532226562, "KL/mean": -173.59060668945312, "KL/rejected_KL_mean": -215.87075805664062, "KL/std": 113.15923309326172, "epoch": 0.6061980347694633, "fcm_dpo/beta": 0.004426930099725723, "fcm_dpo/delta": 0.030951913446187973, "fcm_dpo/margin": 84.5603256225586, "fcm_dpo/q_t": 0.4145042300224304, "grad_norm": 7.4008402824401855, "learning_rate": 2.0268718890989752e-07, "logits/chosen": 0.7799571752548218, "logits/rejected": 0.7018869519233704, "logps/chosen": -185.035400390625, "logps/ref_chosen": -53.72495651245117, "logps/ref_rejected": -75.06304931640625, "logps/rejected": -290.9338073730469, "loss": 1.1048, "margin_dpo/margin_mean": 84.5603256225586, "margin_dpo/margin_std": 110.60939025878906, "step": 401 }, { "KL/chosen_KL_mean": -145.2092742919922, "KL/mean": -177.13595581054688, "KL/rejected_KL_mean": -209.06265258789062, "KL/std": 94.68934631347656, "epoch": 0.6077097505668935, "fcm_dpo/beta": 0.00457565626129508, "fcm_dpo/delta": 0.009118745103478432, "fcm_dpo/margin": 63.85340118408203, "fcm_dpo/q_t": 0.4322190284729004, "grad_norm": 11.485504150390625, "learning_rate": 2.013895317751323e-07, "logits/chosen": 0.7673013210296631, "logits/rejected": 0.7580888271331787, "logps/chosen": -207.08319091796875, "logps/ref_chosen": -61.873931884765625, "logps/ref_rejected": -66.15198516845703, "logps/rejected": -275.21466064453125, "loss": 1.1854, "margin_dpo/margin_mean": 63.85340118408203, "margin_dpo/margin_std": 115.97992706298828, "step": 402 }, { "KL/chosen_KL_mean": -151.53750610351562, "KL/mean": -193.7296142578125, "KL/rejected_KL_mean": -235.92172241210938, "KL/std": 111.84996032714844, "epoch": 0.6092214663643235, "fcm_dpo/beta": 0.004578583873808384, "fcm_dpo/delta": 0.01650870591402054, "fcm_dpo/margin": 84.38421630859375, "fcm_dpo/q_t": 0.412356436252594, "grad_norm": 8.309311866760254, "learning_rate": 2.0009323437965898e-07, "logits/chosen": 0.882099986076355, "logits/rejected": 0.7999458312988281, "logps/chosen": -202.8590087890625, "logps/ref_chosen": -51.321502685546875, "logps/ref_rejected": -86.54010772705078, "logps/rejected": -322.46185302734375, "loss": 1.1045, "margin_dpo/margin_mean": 84.38421630859375, "margin_dpo/margin_std": 116.1924819946289, "step": 403 }, { "KL/chosen_KL_mean": -145.2956085205078, "KL/mean": -189.438720703125, "KL/rejected_KL_mean": -233.58184814453125, "KL/std": 115.39171600341797, "epoch": 0.6107331821617535, "fcm_dpo/beta": 0.004511928651481867, "fcm_dpo/delta": -0.0019165575504302979, "fcm_dpo/margin": 88.28622436523438, "fcm_dpo/q_t": 0.40923064947128296, "grad_norm": 12.346050262451172, "learning_rate": 1.9879833298370237e-07, "logits/chosen": 0.7575935125350952, "logits/rejected": 0.6699234247207642, "logps/chosen": -207.55850219726562, "logps/ref_chosen": -62.26288604736328, "logps/ref_rejected": -95.19029998779297, "logps/rejected": -328.77215576171875, "loss": 1.1111, "margin_dpo/margin_mean": 88.2862319946289, "margin_dpo/margin_std": 128.89202880859375, "step": 404 }, { "KL/chosen_KL_mean": -147.33062744140625, "KL/mean": -181.20425415039062, "KL/rejected_KL_mean": -215.077880859375, "KL/std": 100.42963409423828, "epoch": 0.6122448979591837, "fcm_dpo/beta": 0.004548364318907261, "fcm_dpo/delta": -0.02057464048266411, "fcm_dpo/margin": 67.74724578857422, "fcm_dpo/q_t": 0.4274998903274536, "grad_norm": 9.59747314453125, "learning_rate": 1.975048638084379e-07, "logits/chosen": 0.868112325668335, "logits/rejected": 0.8267897963523865, "logps/chosen": -197.9149627685547, "logps/ref_chosen": -50.5843391418457, "logps/ref_rejected": -65.43156433105469, "logps/rejected": -280.50946044921875, "loss": 1.1562, "margin_dpo/margin_mean": 67.74724578857422, "margin_dpo/margin_std": 100.54032897949219, "step": 405 }, { "KL/chosen_KL_mean": -138.15542602539062, "KL/mean": -185.86033630371094, "KL/rejected_KL_mean": -233.56524658203125, "KL/std": 109.98925018310547, "epoch": 0.6137566137566137, "fcm_dpo/beta": 0.004500585608184338, "fcm_dpo/delta": -0.03803994506597519, "fcm_dpo/margin": 95.40980529785156, "fcm_dpo/q_t": 0.3991668224334717, "grad_norm": 9.847400665283203, "learning_rate": 1.9621286303497914e-07, "logits/chosen": 0.8270584940910339, "logits/rejected": 0.6823967099189758, "logps/chosen": -187.15103149414062, "logps/ref_chosen": -48.99560546875, "logps/ref_rejected": -92.47774505615234, "logps/rejected": -326.04296875, "loss": 1.0758, "margin_dpo/margin_mean": 95.40980529785156, "margin_dpo/margin_std": 121.5294189453125, "step": 406 }, { "KL/chosen_KL_mean": -182.1513671875, "KL/mean": -219.0489501953125, "KL/rejected_KL_mean": -255.94656372070312, "KL/std": 128.31683349609375, "epoch": 0.6152683295540439, "fcm_dpo/beta": 0.004660026635974646, "fcm_dpo/delta": 0.06589356809854507, "fcm_dpo/margin": 73.79518127441406, "fcm_dpo/q_t": 0.4223455786705017, "grad_norm": 11.783304214477539, "learning_rate": 1.9492236680336483e-07, "logits/chosen": 0.7038700580596924, "logits/rejected": 0.6494981050491333, "logps/chosen": -271.55194091796875, "logps/ref_chosen": -89.40056610107422, "logps/ref_rejected": -99.28775024414062, "logps/rejected": -355.23431396484375, "loss": 1.1571, "margin_dpo/margin_mean": 73.7951889038086, "margin_dpo/margin_std": 128.01223754882812, "step": 407 }, { "KL/chosen_KL_mean": -134.6771697998047, "KL/mean": -185.23574829101562, "KL/rejected_KL_mean": -235.79437255859375, "KL/std": 102.01171112060547, "epoch": 0.6167800453514739, "fcm_dpo/beta": 0.004532967694103718, "fcm_dpo/delta": -0.076311394572258, "fcm_dpo/margin": 101.1171875, "fcm_dpo/q_t": 0.38965433835983276, "grad_norm": 9.231172561645508, "learning_rate": 1.9363341121154895e-07, "logits/chosen": 0.7848796844482422, "logits/rejected": 0.7198034524917603, "logps/chosen": -189.38107299804688, "logps/ref_chosen": -54.70391845703125, "logps/ref_rejected": -73.98648834228516, "logps/rejected": -309.7808532714844, "loss": 1.0377, "margin_dpo/margin_mean": 101.11719512939453, "margin_dpo/margin_std": 105.65650939941406, "step": 408 }, { "KL/chosen_KL_mean": -167.1765594482422, "KL/mean": -189.65097045898438, "KL/rejected_KL_mean": -212.1253662109375, "KL/std": 87.67469787597656, "epoch": 0.618291761148904, "fcm_dpo/beta": 0.004333215299993753, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 44.94880676269531, "fcm_dpo/q_t": 0.4536062777042389, "grad_norm": 10.822347640991211, "learning_rate": 1.9234603231438994e-07, "logits/chosen": 0.7813749313354492, "logits/rejected": 0.791049599647522, "logps/chosen": -229.29478454589844, "logps/ref_chosen": -62.11822509765625, "logps/ref_rejected": -61.933509826660156, "logps/rejected": -274.05889892578125, "loss": 1.2448, "margin_dpo/margin_mean": 44.94880676269531, "margin_dpo/margin_std": 97.49263763427734, "step": 409 }, { "KL/chosen_KL_mean": -157.98037719726562, "KL/mean": -198.59495544433594, "KL/rejected_KL_mean": -239.2095184326172, "KL/std": 103.38927459716797, "epoch": 0.6198034769463341, "fcm_dpo/beta": 0.004367862828075886, "fcm_dpo/delta": 0.05229746922850609, "fcm_dpo/margin": 81.22914123535156, "fcm_dpo/q_t": 0.41858938336372375, "grad_norm": 9.7050199508667, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 0.7918972373008728, "logits/rejected": 0.7738727927207947, "logps/chosen": -219.78305053710938, "logps/ref_chosen": -61.80266189575195, "logps/ref_rejected": -76.60002136230469, "logps/rejected": -315.8095397949219, "loss": 1.1201, "margin_dpo/margin_mean": 81.22914123535156, "margin_dpo/margin_std": 105.19297790527344, "step": 410 }, { "KL/chosen_KL_mean": -164.25729370117188, "KL/mean": -204.526611328125, "KL/rejected_KL_mean": -244.7958984375, "KL/std": 108.70731353759766, "epoch": 0.6213151927437641, "fcm_dpo/beta": 0.004682523664087057, "fcm_dpo/delta": 0.027940556406974792, "fcm_dpo/margin": 80.53861999511719, "fcm_dpo/q_t": 0.4140225946903229, "grad_norm": 9.122180938720703, "learning_rate": 1.8977614860195296e-07, "logits/chosen": 0.8158813118934631, "logits/rejected": 0.7707309722900391, "logps/chosen": -218.70269775390625, "logps/ref_chosen": -54.44539260864258, "logps/ref_rejected": -74.5650863647461, "logps/rejected": -319.3609924316406, "loss": 1.1114, "margin_dpo/margin_mean": 80.53861999511719, "margin_dpo/margin_std": 114.59844970703125, "step": 411 }, { "KL/chosen_KL_mean": -167.36849975585938, "KL/mean": -203.03668212890625, "KL/rejected_KL_mean": -238.70486450195312, "KL/std": 91.77555847167969, "epoch": 0.6228269085411943, "fcm_dpo/beta": 0.004863695241510868, "fcm_dpo/delta": 0.06431097537279129, "fcm_dpo/margin": 71.33634185791016, "fcm_dpo/q_t": 0.42097651958465576, "grad_norm": 10.8851957321167, "learning_rate": 1.8849371567184662e-07, "logits/chosen": 0.8179016709327698, "logits/rejected": 0.7678436040878296, "logps/chosen": -222.6165771484375, "logps/ref_chosen": -55.248085021972656, "logps/ref_rejected": -68.96623229980469, "logps/rejected": -307.67108154296875, "loss": 1.124, "margin_dpo/margin_mean": 71.33634185791016, "margin_dpo/margin_std": 97.02810668945312, "step": 412 }, { "KL/chosen_KL_mean": -188.40090942382812, "KL/mean": -223.15206909179688, "KL/rejected_KL_mean": -257.90325927734375, "KL/std": 112.07737731933594, "epoch": 0.6243386243386243, "fcm_dpo/beta": 0.005236283876001835, "fcm_dpo/delta": 0.04264630749821663, "fcm_dpo/margin": 69.50237274169922, "fcm_dpo/q_t": 0.4206687808036804, "grad_norm": 14.27915096282959, "learning_rate": 1.872130032047302e-07, "logits/chosen": 0.6741722822189331, "logits/rejected": 0.6519167423248291, "logps/chosen": -257.12164306640625, "logps/ref_chosen": -68.72074890136719, "logps/ref_rejected": -78.76539611816406, "logps/rejected": -336.6686706542969, "loss": 1.1709, "margin_dpo/margin_mean": 69.50237274169922, "margin_dpo/margin_std": 134.83111572265625, "step": 413 }, { "KL/chosen_KL_mean": -165.21878051757812, "KL/mean": -206.52334594726562, "KL/rejected_KL_mean": -247.8279266357422, "KL/std": 119.95925903320312, "epoch": 0.6258503401360545, "fcm_dpo/beta": 0.005115990526974201, "fcm_dpo/delta": -0.028705045580863953, "fcm_dpo/margin": 82.6091537475586, "fcm_dpo/q_t": 0.4012109339237213, "grad_norm": 10.268512725830078, "learning_rate": 1.8593404702488436e-07, "logits/chosen": 0.8100833892822266, "logits/rejected": 0.7635753750801086, "logps/chosen": -219.35699462890625, "logps/ref_chosen": -54.138214111328125, "logps/ref_rejected": -74.65741729736328, "logps/rejected": -322.4853515625, "loss": 1.0844, "margin_dpo/margin_mean": 82.6091537475586, "margin_dpo/margin_std": 109.7034912109375, "step": 414 }, { "KL/chosen_KL_mean": -165.16592407226562, "KL/mean": -196.70492553710938, "KL/rejected_KL_mean": -228.24395751953125, "KL/std": 107.95329284667969, "epoch": 0.6273620559334845, "fcm_dpo/beta": 0.005346394143998623, "fcm_dpo/delta": 0.07570715248584747, "fcm_dpo/margin": 63.078006744384766, "fcm_dpo/q_t": 0.4259011745452881, "grad_norm": 12.313481330871582, "learning_rate": 1.846568829074628e-07, "logits/chosen": 0.8486864566802979, "logits/rejected": 0.8356572985649109, "logps/chosen": -221.08448791503906, "logps/ref_chosen": -55.91856002807617, "logps/ref_rejected": -61.747703552246094, "logps/rejected": -289.99163818359375, "loss": 1.1686, "margin_dpo/margin_mean": 63.078006744384766, "margin_dpo/margin_std": 116.46731567382812, "step": 415 }, { "KL/chosen_KL_mean": -174.42445373535156, "KL/mean": -204.85211181640625, "KL/rejected_KL_mean": -235.27975463867188, "KL/std": 115.35769653320312, "epoch": 0.6288737717309146, "fcm_dpo/beta": 0.005210232920944691, "fcm_dpo/delta": -0.10238505899906158, "fcm_dpo/margin": 60.85530090332031, "fcm_dpo/q_t": 0.4263876676559448, "grad_norm": 13.751988410949707, "learning_rate": 1.8338154657749128e-07, "logits/chosen": 0.8052107095718384, "logits/rejected": 0.7643084526062012, "logps/chosen": -229.1475372314453, "logps/ref_chosen": -54.72308349609375, "logps/ref_rejected": -69.17388916015625, "logps/rejected": -304.4536437988281, "loss": 1.1932, "margin_dpo/margin_mean": 60.85530090332031, "margin_dpo/margin_std": 109.29032897949219, "step": 416 }, { "KL/chosen_KL_mean": -182.31307983398438, "KL/mean": -219.28237915039062, "KL/rejected_KL_mean": -256.2516784667969, "KL/std": 106.955810546875, "epoch": 0.6303854875283447, "fcm_dpo/beta": 0.005158806219696999, "fcm_dpo/delta": 0.018698185682296753, "fcm_dpo/margin": 73.93860626220703, "fcm_dpo/q_t": 0.4111923575401306, "grad_norm": 12.965594291687012, "learning_rate": 1.8210807370886849e-07, "logits/chosen": 0.9041584134101868, "logits/rejected": 0.8518546223640442, "logps/chosen": -239.10433959960938, "logps/ref_chosen": -56.791259765625, "logps/ref_rejected": -68.7791748046875, "logps/rejected": -325.0308532714844, "loss": 1.1487, "margin_dpo/margin_mean": 73.93860626220703, "margin_dpo/margin_std": 130.03167724609375, "step": 417 }, { "KL/chosen_KL_mean": -196.0471954345703, "KL/mean": -226.67172241210938, "KL/rejected_KL_mean": -257.2962646484375, "KL/std": 123.40777587890625, "epoch": 0.6318972033257747, "fcm_dpo/beta": 0.004728157073259354, "fcm_dpo/delta": -0.06328493356704712, "fcm_dpo/margin": 61.249061584472656, "fcm_dpo/q_t": 0.43131130933761597, "grad_norm": 12.652046203613281, "learning_rate": 1.8083649992336825e-07, "logits/chosen": 0.8404183387756348, "logits/rejected": 0.8438146114349365, "logps/chosen": -265.1551818847656, "logps/ref_chosen": -69.10798645019531, "logps/ref_rejected": -75.09132385253906, "logps/rejected": -332.3875732421875, "loss": 1.2083, "margin_dpo/margin_mean": 61.249053955078125, "margin_dpo/margin_std": 124.80625915527344, "step": 418 }, { "KL/chosen_KL_mean": -157.66766357421875, "KL/mean": -203.49771118164062, "KL/rejected_KL_mean": -249.32774353027344, "KL/std": 108.96971130371094, "epoch": 0.6334089191232048, "fcm_dpo/beta": 0.004607297480106354, "fcm_dpo/delta": -0.02855597995221615, "fcm_dpo/margin": 91.66007995605469, "fcm_dpo/q_t": 0.40371498465538025, "grad_norm": 10.530730247497559, "learning_rate": 1.7956686078964255e-07, "logits/chosen": 0.743537962436676, "logits/rejected": 0.7084311246871948, "logps/chosen": -215.8394317626953, "logps/ref_chosen": -58.1717643737793, "logps/ref_rejected": -71.67066955566406, "logps/rejected": -320.9984130859375, "loss": 1.0856, "margin_dpo/margin_mean": 91.66007995605469, "margin_dpo/margin_std": 126.45355224609375, "step": 419 }, { "KL/chosen_KL_mean": -190.76968383789062, "KL/mean": -213.75607299804688, "KL/rejected_KL_mean": -236.74246215820312, "KL/std": 112.41749572753906, "epoch": 0.6349206349206349, "fcm_dpo/beta": 0.0049021318554878235, "fcm_dpo/delta": 0.06469061225652695, "fcm_dpo/margin": 45.97278594970703, "fcm_dpo/q_t": 0.45175454020500183, "grad_norm": 11.888525009155273, "learning_rate": 1.782991918222275e-07, "logits/chosen": 0.8070861101150513, "logits/rejected": 0.7711919546127319, "logps/chosen": -247.8231964111328, "logps/ref_chosen": -57.05351257324219, "logps/ref_rejected": -62.670982360839844, "logps/rejected": -299.4134521484375, "loss": 1.2724, "margin_dpo/margin_mean": 45.97278594970703, "margin_dpo/margin_std": 131.39532470703125, "step": 420 }, { "KL/chosen_KL_mean": -177.93927001953125, "KL/mean": -212.15362548828125, "KL/rejected_KL_mean": -246.36798095703125, "KL/std": 111.99041748046875, "epoch": 0.636432350718065, "fcm_dpo/beta": 0.00480629364028573, "fcm_dpo/delta": -0.01974395476281643, "fcm_dpo/margin": 68.42868041992188, "fcm_dpo/q_t": 0.4252956509590149, "grad_norm": 12.691519737243652, "learning_rate": 1.7703352848054887e-07, "logits/chosen": 0.7821997404098511, "logits/rejected": 0.7315264344215393, "logps/chosen": -235.2625274658203, "logps/ref_chosen": -57.32324981689453, "logps/ref_rejected": -75.33782958984375, "logps/rejected": -321.705810546875, "loss": 1.2077, "margin_dpo/margin_mean": 68.42868041992188, "margin_dpo/margin_std": 150.48504638671875, "step": 421 }, { "KL/chosen_KL_mean": -159.26858520507812, "KL/mean": -201.4471435546875, "KL/rejected_KL_mean": -243.6256866455078, "KL/std": 111.61254119873047, "epoch": 0.6379440665154951, "fcm_dpo/beta": 0.004849913530051708, "fcm_dpo/delta": -0.012699028477072716, "fcm_dpo/margin": 84.35710906982422, "fcm_dpo/q_t": 0.4054141044616699, "grad_norm": 12.056890487670898, "learning_rate": 1.7576990616793137e-07, "logits/chosen": 0.7797449827194214, "logits/rejected": 0.7793635129928589, "logps/chosen": -226.326171875, "logps/ref_chosen": -67.05757141113281, "logps/ref_rejected": -72.12803649902344, "logps/rejected": -315.75372314453125, "loss": 1.0885, "margin_dpo/margin_mean": 84.35710906982422, "margin_dpo/margin_std": 111.04570770263672, "step": 422 }, { "KL/chosen_KL_mean": -155.98098754882812, "KL/mean": -204.0811767578125, "KL/rejected_KL_mean": -252.18136596679688, "KL/std": 113.7120132446289, "epoch": 0.6394557823129252, "fcm_dpo/beta": 0.004519253503531218, "fcm_dpo/delta": -0.04623626172542572, "fcm_dpo/margin": 96.20036315917969, "fcm_dpo/q_t": 0.39895230531692505, "grad_norm": 11.132472038269043, "learning_rate": 1.745083602306071e-07, "logits/chosen": 0.8631881475448608, "logits/rejected": 0.8088440895080566, "logps/chosen": -210.04266357421875, "logps/ref_chosen": -54.06167221069336, "logps/ref_rejected": -76.64092254638672, "logps/rejected": -328.8222961425781, "loss": 1.0682, "margin_dpo/margin_mean": 96.20036315917969, "margin_dpo/margin_std": 121.70199584960938, "step": 423 }, { "KL/chosen_KL_mean": -170.24893188476562, "KL/mean": -215.80943298339844, "KL/rejected_KL_mean": -261.36993408203125, "KL/std": 104.85726928710938, "epoch": 0.6409674981103552, "fcm_dpo/beta": 0.004399476572871208, "fcm_dpo/delta": -0.004416920244693756, "fcm_dpo/margin": 91.12101745605469, "fcm_dpo/q_t": 0.40782856941223145, "grad_norm": 13.965441703796387, "learning_rate": 1.7324892595672804e-07, "logits/chosen": 0.7467511892318726, "logits/rejected": 0.7114660739898682, "logps/chosen": -223.85780334472656, "logps/ref_chosen": -53.60887145996094, "logps/ref_rejected": -79.2139892578125, "logps/rejected": -340.58392333984375, "loss": 1.0948, "margin_dpo/margin_mean": 91.12101745605469, "margin_dpo/margin_std": 122.10706329345703, "step": 424 }, { "KL/chosen_KL_mean": -168.24703979492188, "KL/mean": -202.56069946289062, "KL/rejected_KL_mean": -236.87437438964844, "KL/std": 100.82597351074219, "epoch": 0.6424792139077853, "fcm_dpo/beta": 0.004823003895580769, "fcm_dpo/delta": 0.08188430219888687, "fcm_dpo/margin": 68.62734985351562, "fcm_dpo/q_t": 0.42684072256088257, "grad_norm": 10.987018585205078, "learning_rate": 1.7199163857537824e-07, "logits/chosen": 0.8584731817245483, "logits/rejected": 0.8374955654144287, "logps/chosen": -226.66171264648438, "logps/ref_chosen": -58.41468048095703, "logps/ref_rejected": -66.59054565429688, "logps/rejected": -303.46490478515625, "loss": 1.1573, "margin_dpo/margin_mean": 68.62734985351562, "margin_dpo/margin_std": 115.73839569091797, "step": 425 }, { "KL/chosen_KL_mean": -202.30221557617188, "KL/mean": -222.89907836914062, "KL/rejected_KL_mean": -243.49595642089844, "KL/std": 112.37277221679688, "epoch": 0.6439909297052154, "fcm_dpo/beta": 0.004896195139735937, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 41.19374084472656, "fcm_dpo/q_t": 0.45397740602493286, "grad_norm": 16.15119171142578, "learning_rate": 1.7073653325558828e-07, "logits/chosen": 0.78400719165802, "logits/rejected": 0.7848511934280396, "logps/chosen": -274.01043701171875, "logps/ref_chosen": -71.70822143554688, "logps/ref_rejected": -73.57725524902344, "logps/rejected": -317.0732116699219, "loss": 1.3074, "margin_dpo/margin_mean": 41.19373321533203, "margin_dpo/margin_std": 142.9339599609375, "step": 426 }, { "KL/chosen_KL_mean": -190.27249145507812, "KL/mean": -228.701171875, "KL/rejected_KL_mean": -267.1298522949219, "KL/std": 124.13754272460938, "epoch": 0.6455026455026455, "fcm_dpo/beta": 0.004953271709382534, "fcm_dpo/delta": 0.02383248880505562, "fcm_dpo/margin": 76.85736083984375, "fcm_dpo/q_t": 0.4170149266719818, "grad_norm": 12.632922172546387, "learning_rate": 1.6948364510535218e-07, "logits/chosen": 0.8083971738815308, "logits/rejected": 0.7516036033630371, "logps/chosen": -248.91525268554688, "logps/ref_chosen": -58.64276885986328, "logps/ref_rejected": -86.25437927246094, "logps/rejected": -353.38421630859375, "loss": 1.1428, "margin_dpo/margin_mean": 76.85736083984375, "margin_dpo/margin_std": 134.9580078125, "step": 427 }, { "KL/chosen_KL_mean": -185.6763916015625, "KL/mean": -223.56564331054688, "KL/rejected_KL_mean": -261.45489501953125, "KL/std": 125.037353515625, "epoch": 0.6470143613000756, "fcm_dpo/beta": 0.005163976922631264, "fcm_dpo/delta": 0.008169233798980713, "fcm_dpo/margin": 75.77848815917969, "fcm_dpo/q_t": 0.4132087230682373, "grad_norm": 11.813305854797363, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 0.766205906867981, "logits/rejected": 0.736466646194458, "logps/chosen": -252.27244567871094, "logps/ref_chosen": -66.5960464477539, "logps/ref_rejected": -82.3941650390625, "logps/rejected": -343.84906005859375, "loss": 1.1464, "margin_dpo/margin_mean": 75.77848052978516, "margin_dpo/margin_std": 133.88519287109375, "step": 428 }, { "KL/chosen_KL_mean": -188.0496368408203, "KL/mean": -216.96371459960938, "KL/rejected_KL_mean": -245.87782287597656, "KL/std": 101.20547485351562, "epoch": 0.6485260770975056, "fcm_dpo/beta": 0.005533907096832991, "fcm_dpo/delta": 0.09032148122787476, "fcm_dpo/margin": 57.82816696166992, "fcm_dpo/q_t": 0.4274323582649231, "grad_norm": 13.291508674621582, "learning_rate": 1.669846604344412e-07, "logits/chosen": 0.8079313635826111, "logits/rejected": 0.8284199237823486, "logps/chosen": -245.059326171875, "logps/ref_chosen": -57.00970458984375, "logps/ref_rejected": -59.86549377441406, "logps/rejected": -305.7433166503906, "loss": 1.1943, "margin_dpo/margin_mean": 57.828163146972656, "margin_dpo/margin_std": 117.090576171875, "step": 429 }, { "KL/chosen_KL_mean": -166.90045166015625, "KL/mean": -215.31846618652344, "KL/rejected_KL_mean": -263.73651123046875, "KL/std": 102.39797973632812, "epoch": 0.6500377928949358, "fcm_dpo/beta": 0.005032334476709366, "fcm_dpo/delta": -0.11732571572065353, "fcm_dpo/margin": 96.83605194091797, "fcm_dpo/q_t": 0.38132691383361816, "grad_norm": 13.358454704284668, "learning_rate": 1.6573863381573954e-07, "logits/chosen": 0.724281370639801, "logits/rejected": 0.7237146496772766, "logps/chosen": -226.46363830566406, "logps/ref_chosen": -59.563194274902344, "logps/ref_rejected": -70.52289581298828, "logps/rejected": -334.2593994140625, "loss": 1.0456, "margin_dpo/margin_mean": 96.8360595703125, "margin_dpo/margin_std": 120.72003936767578, "step": 430 }, { "KL/chosen_KL_mean": -165.01373291015625, "KL/mean": -202.18789672851562, "KL/rejected_KL_mean": -239.36204528808594, "KL/std": 107.67405700683594, "epoch": 0.6515495086923658, "fcm_dpo/beta": 0.005004248581826687, "fcm_dpo/delta": 0.034310389310121536, "fcm_dpo/margin": 74.34831237792969, "fcm_dpo/q_t": 0.41700905561447144, "grad_norm": 12.656464576721191, "learning_rate": 1.6449496416858282e-07, "logits/chosen": 0.7730939388275146, "logits/rejected": 0.724521815776825, "logps/chosen": -215.21405029296875, "logps/ref_chosen": -50.20032501220703, "logps/ref_rejected": -77.81680297851562, "logps/rejected": -317.1788330078125, "loss": 1.1371, "margin_dpo/margin_mean": 74.34829711914062, "margin_dpo/margin_std": 122.35536193847656, "step": 431 }, { "KL/chosen_KL_mean": -174.32611083984375, "KL/mean": -212.357421875, "KL/rejected_KL_mean": -250.38873291015625, "KL/std": 107.8907241821289, "epoch": 0.6530612244897959, "fcm_dpo/beta": 0.005113082937896252, "fcm_dpo/delta": 0.013686059042811394, "fcm_dpo/margin": 76.06259155273438, "fcm_dpo/q_t": 0.4114718437194824, "grad_norm": 13.092035293579102, "learning_rate": 1.632536862810844e-07, "logits/chosen": 0.8555986881256104, "logits/rejected": 0.8065499067306519, "logps/chosen": -235.98887634277344, "logps/ref_chosen": -61.662757873535156, "logps/ref_rejected": -83.94496154785156, "logps/rejected": -334.33367919921875, "loss": 1.1371, "margin_dpo/margin_mean": 76.06259155273438, "margin_dpo/margin_std": 129.0126190185547, "step": 432 }, { "KL/chosen_KL_mean": -171.813232421875, "KL/mean": -214.63470458984375, "KL/rejected_KL_mean": -257.4562072753906, "KL/std": 100.15315246582031, "epoch": 0.654572940287226, "fcm_dpo/beta": 0.005063719116151333, "fcm_dpo/delta": -0.04309210181236267, "fcm_dpo/margin": 85.64299774169922, "fcm_dpo/q_t": 0.39825403690338135, "grad_norm": 13.218973159790039, "learning_rate": 1.6201483487445515e-07, "logits/chosen": 0.8633350133895874, "logits/rejected": 0.8686789870262146, "logps/chosen": -235.5424041748047, "logps/ref_chosen": -63.72917938232422, "logps/ref_rejected": -65.8391342163086, "logps/rejected": -323.29534912109375, "loss": 1.078, "margin_dpo/margin_mean": 85.64300537109375, "margin_dpo/margin_std": 113.72454071044922, "step": 433 }, { "KL/chosen_KL_mean": -146.77532958984375, "KL/mean": -195.40478515625, "KL/rejected_KL_mean": -244.0342559814453, "KL/std": 111.6845703125, "epoch": 0.656084656084656, "fcm_dpo/beta": 0.004525492433458567, "fcm_dpo/delta": -0.06290946900844574, "fcm_dpo/margin": 97.25892639160156, "fcm_dpo/q_t": 0.394904762506485, "grad_norm": 11.273813247680664, "learning_rate": 1.6077844460203204e-07, "logits/chosen": 0.9331784248352051, "logits/rejected": 0.8754293322563171, "logps/chosen": -194.74864196777344, "logps/ref_chosen": -47.97331619262695, "logps/ref_rejected": -72.51132202148438, "logps/rejected": -316.54559326171875, "loss": 1.0843, "margin_dpo/margin_mean": 97.25892639160156, "margin_dpo/margin_std": 132.83673095703125, "step": 434 }, { "KL/chosen_KL_mean": -178.30308532714844, "KL/mean": -213.94564819335938, "KL/rejected_KL_mean": -249.58822631835938, "KL/std": 107.2773208618164, "epoch": 0.6575963718820862, "fcm_dpo/beta": 0.004943176172673702, "fcm_dpo/delta": 0.054163962602615356, "fcm_dpo/margin": 71.28514099121094, "fcm_dpo/q_t": 0.4210607409477234, "grad_norm": 13.454949378967285, "learning_rate": 1.5954455004830878e-07, "logits/chosen": 0.8959484696388245, "logits/rejected": 0.8630224466323853, "logps/chosen": -235.3633270263672, "logps/ref_chosen": -57.06024932861328, "logps/ref_rejected": -71.69146728515625, "logps/rejected": -321.2796936035156, "loss": 1.1558, "margin_dpo/margin_mean": 71.28514099121094, "margin_dpo/margin_std": 124.44223022460938, "step": 435 }, { "KL/chosen_KL_mean": -178.91229248046875, "KL/mean": -209.08297729492188, "KL/rejected_KL_mean": -239.253662109375, "KL/std": 104.4656982421875, "epoch": 0.6591080876795162, "fcm_dpo/beta": 0.005222709849476814, "fcm_dpo/delta": 0.10130466520786285, "fcm_dpo/margin": 60.341373443603516, "fcm_dpo/q_t": 0.43232089281082153, "grad_norm": 14.217085838317871, "learning_rate": 1.5831318572796847e-07, "logits/chosen": 0.8186235427856445, "logits/rejected": 0.7705166339874268, "logps/chosen": -235.07034301757812, "logps/ref_chosen": -56.158050537109375, "logps/ref_rejected": -67.63787841796875, "logps/rejected": -306.89154052734375, "loss": 1.2038, "margin_dpo/margin_mean": 60.341373443603516, "margin_dpo/margin_std": 128.90338134765625, "step": 436 }, { "KL/chosen_KL_mean": -184.0316619873047, "KL/mean": -222.54013061523438, "KL/rejected_KL_mean": -261.0486145019531, "KL/std": 116.43235778808594, "epoch": 0.6606198034769464, "fcm_dpo/beta": 0.004938242956995964, "fcm_dpo/delta": -0.09390990436077118, "fcm_dpo/margin": 77.01693725585938, "fcm_dpo/q_t": 0.4115540087223053, "grad_norm": 15.683857917785645, "learning_rate": 1.5708438608491815e-07, "logits/chosen": 0.8214137554168701, "logits/rejected": 0.7116411924362183, "logps/chosen": -241.0174560546875, "logps/ref_chosen": -56.98578643798828, "logps/ref_rejected": -85.61524963378906, "logps/rejected": -346.66387939453125, "loss": 1.1721, "margin_dpo/margin_mean": 77.01693725585938, "margin_dpo/margin_std": 149.37376403808594, "step": 437 }, { "KL/chosen_KL_mean": -158.45684814453125, "KL/mean": -206.56228637695312, "KL/rejected_KL_mean": -254.66769409179688, "KL/std": 119.14268493652344, "epoch": 0.6621315192743764, "fcm_dpo/beta": 0.004753970541059971, "fcm_dpo/delta": -0.07428638637065887, "fcm_dpo/margin": 96.21085357666016, "fcm_dpo/q_t": 0.39683008193969727, "grad_norm": 12.5221529006958, "learning_rate": 1.558581854913253e-07, "logits/chosen": 0.8680797219276428, "logits/rejected": 0.8115641474723816, "logps/chosen": -199.73463439941406, "logps/ref_chosen": -41.27777862548828, "logps/ref_rejected": -65.33840942382812, "logps/rejected": -320.006103515625, "loss": 1.0695, "margin_dpo/margin_mean": 96.21085357666016, "margin_dpo/margin_std": 133.3427276611328, "step": 438 }, { "KL/chosen_KL_mean": -186.97760009765625, "KL/mean": -230.8473358154297, "KL/rejected_KL_mean": -274.71710205078125, "KL/std": 122.46157836914062, "epoch": 0.6636432350718064, "fcm_dpo/beta": 0.004707343876361847, "fcm_dpo/delta": -0.02342548966407776, "fcm_dpo/margin": 87.739501953125, "fcm_dpo/q_t": 0.4038691222667694, "grad_norm": 11.63642406463623, "learning_rate": 1.5463461824665658e-07, "logits/chosen": 0.7208713293075562, "logits/rejected": 0.6962095499038696, "logps/chosen": -268.395263671875, "logps/ref_chosen": -81.41764831542969, "logps/ref_rejected": -94.72309875488281, "logps/rejected": -369.440185546875, "loss": 1.1026, "margin_dpo/margin_mean": 87.739501953125, "margin_dpo/margin_std": 122.69038391113281, "step": 439 }, { "KL/chosen_KL_mean": -155.75674438476562, "KL/mean": -199.6446075439453, "KL/rejected_KL_mean": -243.532470703125, "KL/std": 107.34757232666016, "epoch": 0.6651549508692366, "fcm_dpo/beta": 0.004554657265543938, "fcm_dpo/delta": -0.00109158456325531, "fcm_dpo/margin": 87.77574157714844, "fcm_dpo/q_t": 0.4082149267196655, "grad_norm": 15.26758861541748, "learning_rate": 1.534137185767178e-07, "logits/chosen": 0.836760401725769, "logits/rejected": 0.7494317293167114, "logps/chosen": -198.294921875, "logps/ref_chosen": -42.538185119628906, "logps/ref_rejected": -69.78813934326172, "logps/rejected": -313.32061767578125, "loss": 1.1007, "margin_dpo/margin_mean": 87.77574157714844, "margin_dpo/margin_std": 121.38520812988281, "step": 440 }, { "KL/chosen_KL_mean": -164.15768432617188, "KL/mean": -210.33230590820312, "KL/rejected_KL_mean": -256.50689697265625, "KL/std": 118.62611389160156, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.004367801360785961, "fcm_dpo/delta": -0.006083235144615173, "fcm_dpo/margin": 92.34922790527344, "fcm_dpo/q_t": 0.405091255903244, "grad_norm": 11.903091430664062, "learning_rate": 1.521955206326976e-07, "logits/chosen": 0.7793187499046326, "logits/rejected": 0.7015305161476135, "logps/chosen": -221.7509002685547, "logps/ref_chosen": -57.593223571777344, "logps/ref_rejected": -84.82878875732422, "logps/rejected": -341.335693359375, "loss": 1.0786, "margin_dpo/margin_mean": 92.34922790527344, "margin_dpo/margin_std": 108.52428436279297, "step": 441 }, { "KL/chosen_KL_mean": -195.50466918945312, "KL/mean": -242.55752563476562, "KL/rejected_KL_mean": -289.61041259765625, "KL/std": 119.05079650878906, "epoch": 0.6681783824640968, "fcm_dpo/beta": 0.004326670430600643, "fcm_dpo/delta": -0.010936200618743896, "fcm_dpo/margin": 94.10575866699219, "fcm_dpo/q_t": 0.4057489037513733, "grad_norm": 14.003538131713867, "learning_rate": 1.5098005849021078e-07, "logits/chosen": 0.8129810094833374, "logits/rejected": 0.7782344818115234, "logps/chosen": -262.96588134765625, "logps/ref_chosen": -67.46121978759766, "logps/ref_rejected": -89.0693588256836, "logps/rejected": -378.67974853515625, "loss": 1.0866, "margin_dpo/margin_mean": 94.10575866699219, "margin_dpo/margin_std": 121.53174591064453, "step": 442 }, { "KL/chosen_KL_mean": -161.1103973388672, "KL/mean": -217.9424285888672, "KL/rejected_KL_mean": -274.77447509765625, "KL/std": 124.97842407226562, "epoch": 0.6696900982615268, "fcm_dpo/beta": 0.004096883349120617, "fcm_dpo/delta": -0.0867290198802948, "fcm_dpo/margin": 113.66409301757812, "fcm_dpo/q_t": 0.3905232548713684, "grad_norm": 11.163167953491211, "learning_rate": 1.4976736614834662e-07, "logits/chosen": 0.8272267580032349, "logits/rejected": 0.7736169099807739, "logps/chosen": -215.906494140625, "logps/ref_chosen": -54.79610061645508, "logps/ref_rejected": -77.80781555175781, "logps/rejected": -352.5823059082031, "loss": 1.0474, "margin_dpo/margin_mean": 113.66409301757812, "margin_dpo/margin_std": 140.31976318359375, "step": 443 }, { "KL/chosen_KL_mean": -192.77120971679688, "KL/mean": -222.12258911132812, "KL/rejected_KL_mean": -251.47393798828125, "KL/std": 128.48406982421875, "epoch": 0.671201814058957, "fcm_dpo/beta": 0.0042202952317893505, "fcm_dpo/delta": 0.045447517186403275, "fcm_dpo/margin": 58.70273208618164, "fcm_dpo/q_t": 0.44652265310287476, "grad_norm": 12.165427207946777, "learning_rate": 1.4855747752871654e-07, "logits/chosen": 0.8557889461517334, "logits/rejected": 0.779559850692749, "logps/chosen": -251.520263671875, "logps/ref_chosen": -58.749061584472656, "logps/ref_rejected": -86.87396240234375, "logps/rejected": -338.347900390625, "loss": 1.241, "margin_dpo/margin_mean": 58.702728271484375, "margin_dpo/margin_std": 142.91485595703125, "step": 444 }, { "KL/chosen_KL_mean": -184.20005798339844, "KL/mean": -232.47119140625, "KL/rejected_KL_mean": -280.7423095703125, "KL/std": 111.41131591796875, "epoch": 0.672713529856387, "fcm_dpo/beta": 0.004190181382000446, "fcm_dpo/delta": -0.005736490711569786, "fcm_dpo/margin": 96.54226684570312, "fcm_dpo/q_t": 0.4045560956001282, "grad_norm": 9.579313278198242, "learning_rate": 1.473504264745062e-07, "logits/chosen": 0.7703770399093628, "logits/rejected": 0.7707059383392334, "logps/chosen": -245.11749267578125, "logps/ref_chosen": -60.91743850708008, "logps/ref_rejected": -71.5637435913086, "logps/rejected": -352.30609130859375, "loss": 1.079, "margin_dpo/margin_mean": 96.54226684570312, "margin_dpo/margin_std": 116.30671691894531, "step": 445 }, { "KL/chosen_KL_mean": -165.2137451171875, "KL/mean": -213.72683715820312, "KL/rejected_KL_mean": -262.23992919921875, "KL/std": 113.98602294921875, "epoch": 0.674225245653817, "fcm_dpo/beta": 0.004296318627893925, "fcm_dpo/delta": -0.028332628309726715, "fcm_dpo/margin": 97.02619171142578, "fcm_dpo/q_t": 0.4015599489212036, "grad_norm": 11.402931213378906, "learning_rate": 1.461462467495284e-07, "logits/chosen": 0.8551122546195984, "logits/rejected": 0.7882475852966309, "logps/chosen": -214.0129852294922, "logps/ref_chosen": -48.79924774169922, "logps/ref_rejected": -71.8719482421875, "logps/rejected": -334.11187744140625, "loss": 1.0773, "margin_dpo/margin_mean": 97.02619171142578, "margin_dpo/margin_std": 112.55924987792969, "step": 446 }, { "KL/chosen_KL_mean": -161.5308837890625, "KL/mean": -218.6871337890625, "KL/rejected_KL_mean": -275.8433837890625, "KL/std": 113.90840148925781, "epoch": 0.6757369614512472, "fcm_dpo/beta": 0.0038491457235068083, "fcm_dpo/delta": -0.053994160145521164, "fcm_dpo/margin": 114.31251525878906, "fcm_dpo/q_t": 0.3937652111053467, "grad_norm": 9.615116119384766, "learning_rate": 1.4494497203727843e-07, "logits/chosen": 0.7905747294425964, "logits/rejected": 0.7021323442459106, "logps/chosen": -215.21359252929688, "logps/ref_chosen": -53.682716369628906, "logps/ref_rejected": -88.17315673828125, "logps/rejected": -364.01654052734375, "loss": 1.0531, "margin_dpo/margin_mean": 114.31251525878906, "margin_dpo/margin_std": 127.45619201660156, "step": 447 }, { "KL/chosen_KL_mean": -178.0913848876953, "KL/mean": -222.7434539794922, "KL/rejected_KL_mean": -267.3955383300781, "KL/std": 118.10449981689453, "epoch": 0.6772486772486772, "fcm_dpo/beta": 0.004021032713353634, "fcm_dpo/delta": 0.049615129828453064, "fcm_dpo/margin": 89.30415344238281, "fcm_dpo/q_t": 0.4181307256221771, "grad_norm": 8.591902732849121, "learning_rate": 1.4374663593999256e-07, "logits/chosen": 0.8444277048110962, "logits/rejected": 0.8045238256454468, "logps/chosen": -231.84263610839844, "logps/ref_chosen": -53.75125503540039, "logps/ref_rejected": -77.17623901367188, "logps/rejected": -344.57177734375, "loss": 1.1212, "margin_dpo/margin_mean": 89.30415344238281, "margin_dpo/margin_std": 128.15696716308594, "step": 448 }, { "KL/chosen_KL_mean": -212.66348266601562, "KL/mean": -235.39556884765625, "KL/rejected_KL_mean": -258.1276550292969, "KL/std": 124.3266372680664, "epoch": 0.6787603930461074, "fcm_dpo/beta": 0.0043518864549696445, "fcm_dpo/delta": 0.06915242969989777, "fcm_dpo/margin": 45.464168548583984, "fcm_dpo/q_t": 0.4554653763771057, "grad_norm": 18.149805068969727, "learning_rate": 1.4255127197770707e-07, "logits/chosen": 0.6996891498565674, "logits/rejected": 0.7006729245185852, "logps/chosen": -288.4908447265625, "logps/ref_chosen": -75.82737731933594, "logps/ref_rejected": -82.20687866210938, "logps/rejected": -340.33453369140625, "loss": 1.2685, "margin_dpo/margin_mean": 45.464168548583984, "margin_dpo/margin_std": 122.72837829589844, "step": 449 }, { "KL/chosen_KL_mean": -170.85467529296875, "KL/mean": -209.89654541015625, "KL/rejected_KL_mean": -248.93841552734375, "KL/std": 123.30638122558594, "epoch": 0.6802721088435374, "fcm_dpo/beta": 0.0044828737154603004, "fcm_dpo/delta": 0.060484807938337326, "fcm_dpo/margin": 78.08375549316406, "fcm_dpo/q_t": 0.42500025033950806, "grad_norm": 12.813301086425781, "learning_rate": 1.4135891358732205e-07, "logits/chosen": 0.9147292375564575, "logits/rejected": 0.8186343908309937, "logps/chosen": -217.97039794921875, "logps/ref_chosen": -47.11572265625, "logps/ref_rejected": -78.7546615600586, "logps/rejected": -327.6930847167969, "loss": 1.1614, "margin_dpo/margin_mean": 78.08374786376953, "margin_dpo/margin_std": 141.2745361328125, "step": 450 }, { "KL/chosen_KL_mean": -174.08990478515625, "KL/mean": -208.5985107421875, "KL/rejected_KL_mean": -243.10711669921875, "KL/std": 115.6333236694336, "epoch": 0.6817838246409675, "fcm_dpo/beta": 0.004921277053654194, "fcm_dpo/delta": 0.07145331054925919, "fcm_dpo/margin": 69.0172119140625, "fcm_dpo/q_t": 0.4246895909309387, "grad_norm": 13.280653953552246, "learning_rate": 1.4016959412166437e-07, "logits/chosen": 0.7936139106750488, "logits/rejected": 0.7563437223434448, "logps/chosen": -237.44033813476562, "logps/ref_chosen": -63.350440979003906, "logps/ref_rejected": -76.28530883789062, "logps/rejected": -319.3924255371094, "loss": 1.1558, "margin_dpo/margin_mean": 69.0172119140625, "margin_dpo/margin_std": 119.37599182128906, "step": 451 }, { "KL/chosen_KL_mean": -174.51087951660156, "KL/mean": -214.18081665039062, "KL/rejected_KL_mean": -253.8507843017578, "KL/std": 111.76406860351562, "epoch": 0.6832955404383976, "fcm_dpo/beta": 0.004958437755703926, "fcm_dpo/delta": 0.008032577112317085, "fcm_dpo/margin": 79.33988952636719, "fcm_dpo/q_t": 0.4116581082344055, "grad_norm": 13.893336296081543, "learning_rate": 1.3898334684855645e-07, "logits/chosen": 0.8049842119216919, "logits/rejected": 0.7350988388061523, "logps/chosen": -230.09671020507812, "logps/ref_chosen": -55.58583450317383, "logps/ref_rejected": -77.68738555908203, "logps/rejected": -331.53814697265625, "loss": 1.1346, "margin_dpo/margin_mean": 79.33989715576172, "margin_dpo/margin_std": 134.18746948242188, "step": 452 }, { "KL/chosen_KL_mean": -176.7831268310547, "KL/mean": -212.78689575195312, "KL/rejected_KL_mean": -248.79066467285156, "KL/std": 115.48379516601562, "epoch": 0.6848072562358276, "fcm_dpo/beta": 0.0051093436777591705, "fcm_dpo/delta": 0.03934450447559357, "fcm_dpo/margin": 72.00753784179688, "fcm_dpo/q_t": 0.418560266494751, "grad_norm": 16.030561447143555, "learning_rate": 1.3780020494988445e-07, "logits/chosen": 0.7819995880126953, "logits/rejected": 0.7671021223068237, "logps/chosen": -238.5613250732422, "logps/ref_chosen": -61.778202056884766, "logps/ref_rejected": -71.51403045654297, "logps/rejected": -320.3046875, "loss": 1.1529, "margin_dpo/margin_mean": 72.00753784179688, "margin_dpo/margin_std": 127.00164794921875, "step": 453 }, { "KL/chosen_KL_mean": -159.35975646972656, "KL/mean": -201.83590698242188, "KL/rejected_KL_mean": -244.31204223632812, "KL/std": 112.93978881835938, "epoch": 0.6863189720332578, "fcm_dpo/beta": 0.005041994154453278, "fcm_dpo/delta": -0.03642263263463974, "fcm_dpo/margin": 84.95228576660156, "fcm_dpo/q_t": 0.40106698870658875, "grad_norm": 11.94836139678955, "learning_rate": 1.366202015206706e-07, "logits/chosen": 0.8330647945404053, "logits/rejected": 0.80307936668396, "logps/chosen": -210.9549102783203, "logps/ref_chosen": -51.59515380859375, "logps/ref_rejected": -63.96732711791992, "logps/rejected": -308.27935791015625, "loss": 1.0955, "margin_dpo/margin_mean": 84.95228576660156, "margin_dpo/margin_std": 127.11490631103516, "step": 454 }, { "KL/chosen_KL_mean": -188.39620971679688, "KL/mean": -228.36636352539062, "KL/rejected_KL_mean": -268.3365173339844, "KL/std": 116.3778076171875, "epoch": 0.6878306878306878, "fcm_dpo/beta": 0.005019802134484053, "fcm_dpo/delta": -0.001610086765140295, "fcm_dpo/margin": 79.9402847290039, "fcm_dpo/q_t": 0.4116509258747101, "grad_norm": 14.03894329071045, "learning_rate": 1.354433695681474e-07, "logits/chosen": 0.7549115419387817, "logits/rejected": 0.7353414297103882, "logps/chosen": -259.04791259765625, "logps/ref_chosen": -70.65170288085938, "logps/ref_rejected": -77.44276428222656, "logps/rejected": -345.779296875, "loss": 1.1156, "margin_dpo/margin_mean": 79.9402847290039, "margin_dpo/margin_std": 126.06718444824219, "step": 455 }, { "KL/chosen_KL_mean": -179.34715270996094, "KL/mean": -218.92030334472656, "KL/rejected_KL_mean": -258.49346923828125, "KL/std": 115.43203735351562, "epoch": 0.6893424036281179, "fcm_dpo/beta": 0.004967734683305025, "fcm_dpo/delta": 0.00764651782810688, "fcm_dpo/margin": 79.14630126953125, "fcm_dpo/q_t": 0.41450706124305725, "grad_norm": 17.639022827148438, "learning_rate": 1.3426974201083439e-07, "logits/chosen": 0.7384557723999023, "logits/rejected": 0.6885507106781006, "logps/chosen": -235.7454376220703, "logps/ref_chosen": -56.398284912109375, "logps/ref_rejected": -82.61642456054688, "logps/rejected": -341.10986328125, "loss": 1.1349, "margin_dpo/margin_mean": 79.14630126953125, "margin_dpo/margin_std": 137.09796142578125, "step": 456 }, { "KL/chosen_KL_mean": -175.0521240234375, "KL/mean": -215.41928100585938, "KL/rejected_KL_mean": -255.78640747070312, "KL/std": 118.11895751953125, "epoch": 0.690854119425548, "fcm_dpo/beta": 0.004974281415343285, "fcm_dpo/delta": -0.0028288550674915314, "fcm_dpo/margin": 80.73428344726562, "fcm_dpo/q_t": 0.40718016028404236, "grad_norm": 14.620522499084473, "learning_rate": 1.3309935167761717e-07, "logits/chosen": 0.8967911005020142, "logits/rejected": 0.8270587921142578, "logps/chosen": -219.772705078125, "logps/ref_chosen": -44.72057342529297, "logps/ref_rejected": -68.1158676147461, "logps/rejected": -323.90228271484375, "loss": 1.0996, "margin_dpo/margin_mean": 80.73428344726562, "margin_dpo/margin_std": 113.923583984375, "step": 457 }, { "KL/chosen_KL_mean": -170.12730407714844, "KL/mean": -215.46165466308594, "KL/rejected_KL_mean": -260.7959899902344, "KL/std": 120.08485412597656, "epoch": 0.6923658352229781, "fcm_dpo/beta": 0.004880758002400398, "fcm_dpo/delta": -0.05446285754442215, "fcm_dpo/margin": 90.668701171875, "fcm_dpo/q_t": 0.3978913426399231, "grad_norm": 12.83903980255127, "learning_rate": 1.3193223130682936e-07, "logits/chosen": 0.8211711645126343, "logits/rejected": 0.7230532765388489, "logps/chosen": -220.13299560546875, "logps/ref_chosen": -50.00569152832031, "logps/ref_rejected": -87.50015258789062, "logps/rejected": -348.296142578125, "loss": 1.0924, "margin_dpo/margin_mean": 90.668701171875, "margin_dpo/margin_std": 135.3630828857422, "step": 458 }, { "KL/chosen_KL_mean": -168.60906982421875, "KL/mean": -220.11251831054688, "KL/rejected_KL_mean": -271.615966796875, "KL/std": 143.58975219726562, "epoch": 0.6938775510204082, "fcm_dpo/beta": 0.004629744682461023, "fcm_dpo/delta": -0.10486065596342087, "fcm_dpo/margin": 103.00687408447266, "fcm_dpo/q_t": 0.3852209448814392, "grad_norm": 11.949894905090332, "learning_rate": 1.3076841354533658e-07, "logits/chosen": 0.8680670261383057, "logits/rejected": 0.8436014652252197, "logps/chosen": -233.9870147705078, "logps/ref_chosen": -65.37794494628906, "logps/ref_rejected": -88.19244384765625, "logps/rejected": -359.80841064453125, "loss": 1.0705, "margin_dpo/margin_mean": 103.00686645507812, "margin_dpo/margin_std": 135.36453247070312, "step": 459 }, { "KL/chosen_KL_mean": -178.06622314453125, "KL/mean": -229.13934326171875, "KL/rejected_KL_mean": -280.21246337890625, "KL/std": 123.35194396972656, "epoch": 0.6953892668178382, "fcm_dpo/beta": 0.004125220701098442, "fcm_dpo/delta": -0.02931346371769905, "fcm_dpo/margin": 102.146240234375, "fcm_dpo/q_t": 0.4028850197792053, "grad_norm": 10.467647552490234, "learning_rate": 1.2960793094762345e-07, "logits/chosen": 0.8451239466667175, "logits/rejected": 0.7452915906906128, "logps/chosen": -242.62789916992188, "logps/ref_chosen": -64.5616683959961, "logps/ref_rejected": -88.67890167236328, "logps/rejected": -368.891357421875, "loss": 1.0814, "margin_dpo/margin_mean": 102.14624786376953, "margin_dpo/margin_std": 135.01199340820312, "step": 460 }, { "KL/chosen_KL_mean": -147.82064819335938, "KL/mean": -199.00198364257812, "KL/rejected_KL_mean": -250.18333435058594, "KL/std": 114.1442642211914, "epoch": 0.6969009826152683, "fcm_dpo/beta": 0.003954247571527958, "fcm_dpo/delta": -0.015024319291114807, "fcm_dpo/margin": 102.36270141601562, "fcm_dpo/q_t": 0.40458911657333374, "grad_norm": 10.667238235473633, "learning_rate": 1.2845081597488286e-07, "logits/chosen": 0.9264631867408752, "logits/rejected": 0.8578430414199829, "logps/chosen": -197.29855346679688, "logps/ref_chosen": -49.4779167175293, "logps/ref_rejected": -72.65262603759766, "logps/rejected": -322.8359680175781, "loss": 1.0864, "margin_dpo/margin_mean": 102.36270141601562, "margin_dpo/margin_std": 123.31861114501953, "step": 461 }, { "KL/chosen_KL_mean": -162.76797485351562, "KL/mean": -213.944580078125, "KL/rejected_KL_mean": -265.1211853027344, "KL/std": 106.37005615234375, "epoch": 0.6984126984126984, "fcm_dpo/beta": 0.004021008498966694, "fcm_dpo/delta": -0.015222661197185516, "fcm_dpo/margin": 102.35319519042969, "fcm_dpo/q_t": 0.402321994304657, "grad_norm": 10.825149536132812, "learning_rate": 1.27297100994108e-07, "logits/chosen": 0.8083179593086243, "logits/rejected": 0.7733680009841919, "logps/chosen": -223.26309204101562, "logps/ref_chosen": -60.4951171875, "logps/ref_rejected": -74.82136535644531, "logps/rejected": -339.94256591796875, "loss": 1.071, "margin_dpo/margin_mean": 102.35319519042969, "margin_dpo/margin_std": 118.42103576660156, "step": 462 }, { "KL/chosen_KL_mean": -198.61944580078125, "KL/mean": -227.19964599609375, "KL/rejected_KL_mean": -255.77984619140625, "KL/std": 110.61781311035156, "epoch": 0.6999244142101285, "fcm_dpo/beta": 0.004229356534779072, "fcm_dpo/delta": 0.04422418028116226, "fcm_dpo/margin": 57.160400390625, "fcm_dpo/q_t": 0.444000780582428, "grad_norm": 16.760709762573242, "learning_rate": 1.2614681827718695e-07, "logits/chosen": 0.8112308979034424, "logits/rejected": 0.807045578956604, "logps/chosen": -266.3045654296875, "logps/ref_chosen": -67.68511962890625, "logps/ref_rejected": -71.32196044921875, "logps/rejected": -327.101806640625, "loss": 1.2275, "margin_dpo/margin_mean": 57.160404205322266, "margin_dpo/margin_std": 124.53732299804688, "step": 463 }, { "KL/chosen_KL_mean": -178.37449645996094, "KL/mean": -226.03128051757812, "KL/rejected_KL_mean": -273.68804931640625, "KL/std": 128.52586364746094, "epoch": 0.7014361300075586, "fcm_dpo/beta": 0.004276517778635025, "fcm_dpo/delta": -0.010937310755252838, "fcm_dpo/margin": 95.31355285644531, "fcm_dpo/q_t": 0.40591591596603394, "grad_norm": 10.75070571899414, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 0.8277842998504639, "logits/rejected": 0.8054889440536499, "logps/chosen": -237.54013061523438, "logps/ref_chosen": -59.16564178466797, "logps/ref_rejected": -69.56146240234375, "logps/rejected": -343.24951171875, "loss": 1.1112, "margin_dpo/margin_mean": 95.31355285644531, "margin_dpo/margin_std": 142.11141967773438, "step": 464 }, { "KL/chosen_KL_mean": -185.91439819335938, "KL/mean": -228.27606201171875, "KL/rejected_KL_mean": -270.63775634765625, "KL/std": 119.63814544677734, "epoch": 0.7029478458049887, "fcm_dpo/beta": 0.0042515965178608894, "fcm_dpo/delta": 0.047829769551754, "fcm_dpo/margin": 84.72332000732422, "fcm_dpo/q_t": 0.41904404759407043, "grad_norm": 11.757485389709473, "learning_rate": 1.238566782415197e-07, "logits/chosen": 0.8830685615539551, "logits/rejected": 0.8336352109909058, "logps/chosen": -244.42807006835938, "logps/ref_chosen": -58.513671875, "logps/ref_rejected": -84.31745910644531, "logps/rejected": -354.9552001953125, "loss": 1.1376, "margin_dpo/margin_mean": 84.72331237792969, "margin_dpo/margin_std": 133.2949981689453, "step": 465 }, { "KL/chosen_KL_mean": -204.39483642578125, "KL/mean": -224.01669311523438, "KL/rejected_KL_mean": -243.63856506347656, "KL/std": 127.39469146728516, "epoch": 0.7044595616024187, "fcm_dpo/beta": 0.004723524209111929, "fcm_dpo/delta": 0.07361281663179398, "fcm_dpo/margin": 39.24374008178711, "fcm_dpo/q_t": 0.4571523070335388, "grad_norm": 17.618196487426758, "learning_rate": 1.2271688498291334e-07, "logits/chosen": 0.8117011189460754, "logits/rejected": 0.8161472082138062, "logps/chosen": -277.66064453125, "logps/ref_chosen": -73.26580810546875, "logps/ref_rejected": -74.83621215820312, "logps/rejected": -318.47479248046875, "loss": 1.3039, "margin_dpo/margin_mean": 39.24374008178711, "margin_dpo/margin_std": 132.60060119628906, "step": 466 }, { "KL/chosen_KL_mean": -177.04547119140625, "KL/mean": -218.3562774658203, "KL/rejected_KL_mean": -259.6671142578125, "KL/std": 126.77050018310547, "epoch": 0.7059712773998488, "fcm_dpo/beta": 0.004716761410236359, "fcm_dpo/delta": 0.012419500388205051, "fcm_dpo/margin": 82.62162780761719, "fcm_dpo/q_t": 0.4150483012199402, "grad_norm": 11.141740798950195, "learning_rate": 1.2158065210664848e-07, "logits/chosen": 0.872531533241272, "logits/rejected": 0.7571277022361755, "logps/chosen": -224.62493896484375, "logps/ref_chosen": -47.57947540283203, "logps/ref_rejected": -78.68522644042969, "logps/rejected": -338.3523254394531, "loss": 1.1276, "margin_dpo/margin_mean": 82.62162780761719, "margin_dpo/margin_std": 138.0587158203125, "step": 467 }, { "KL/chosen_KL_mean": -173.1776123046875, "KL/mean": -220.29592895507812, "KL/rejected_KL_mean": -267.41424560546875, "KL/std": 122.3834228515625, "epoch": 0.7074829931972789, "fcm_dpo/beta": 0.004614764358848333, "fcm_dpo/delta": -0.044785935431718826, "fcm_dpo/margin": 94.23664855957031, "fcm_dpo/q_t": 0.39811086654663086, "grad_norm": 13.203627586364746, "learning_rate": 1.204480113956011e-07, "logits/chosen": 0.8035502433776855, "logits/rejected": 0.7918655872344971, "logps/chosen": -237.10537719726562, "logps/ref_chosen": -63.92778778076172, "logps/ref_rejected": -76.51626586914062, "logps/rejected": -343.9305114746094, "loss": 1.1, "margin_dpo/margin_mean": 94.23664855957031, "margin_dpo/margin_std": 143.19485473632812, "step": 468 }, { "KL/chosen_KL_mean": -173.43836975097656, "KL/mean": -216.24713134765625, "KL/rejected_KL_mean": -259.055908203125, "KL/std": 119.78315734863281, "epoch": 0.708994708994709, "fcm_dpo/beta": 0.004494948778301477, "fcm_dpo/delta": 0.015453502535820007, "fcm_dpo/margin": 85.61750793457031, "fcm_dpo/q_t": 0.4122866988182068, "grad_norm": 13.859041213989258, "learning_rate": 1.1931899453216697e-07, "logits/chosen": 0.8977210521697998, "logits/rejected": 0.8815995454788208, "logps/chosen": -232.49655151367188, "logps/ref_chosen": -59.05818176269531, "logps/ref_rejected": -75.67672729492188, "logps/rejected": -334.73260498046875, "loss": 1.104, "margin_dpo/margin_mean": 85.61751556396484, "margin_dpo/margin_std": 115.52572631835938, "step": 469 }, { "KL/chosen_KL_mean": -161.53884887695312, "KL/mean": -206.596435546875, "KL/rejected_KL_mean": -251.65402221679688, "KL/std": 114.833740234375, "epoch": 0.7105064247921391, "fcm_dpo/beta": 0.004632364027202129, "fcm_dpo/delta": -0.022567128762602806, "fcm_dpo/margin": 90.11515808105469, "fcm_dpo/q_t": 0.4039248526096344, "grad_norm": 11.321891784667969, "learning_rate": 1.1819363309737438e-07, "logits/chosen": 0.8473949432373047, "logits/rejected": 0.7967231273651123, "logps/chosen": -209.40628051757812, "logps/ref_chosen": -47.86743927001953, "logps/ref_rejected": -65.96859741210938, "logps/rejected": -317.62261962890625, "loss": 1.0983, "margin_dpo/margin_mean": 90.11515808105469, "margin_dpo/margin_std": 130.05743408203125, "step": 470 }, { "KL/chosen_KL_mean": -158.923828125, "KL/mean": -207.42820739746094, "KL/rejected_KL_mean": -255.9325714111328, "KL/std": 111.91524505615234, "epoch": 0.7120181405895691, "fcm_dpo/beta": 0.004406137391924858, "fcm_dpo/delta": -0.03516135737299919, "fcm_dpo/margin": 97.00875854492188, "fcm_dpo/q_t": 0.3985697329044342, "grad_norm": 12.091167449951172, "learning_rate": 1.1707195857000215e-07, "logits/chosen": 0.8162208795547485, "logits/rejected": 0.772452712059021, "logps/chosen": -216.70169067382812, "logps/ref_chosen": -57.777854919433594, "logps/ref_rejected": -73.81172180175781, "logps/rejected": -329.7442932128906, "loss": 1.073, "margin_dpo/margin_mean": 97.00875854492188, "margin_dpo/margin_std": 121.57257080078125, "step": 471 }, { "KL/chosen_KL_mean": -159.90924072265625, "KL/mean": -197.77468872070312, "KL/rejected_KL_mean": -235.64015197753906, "KL/std": 118.32003784179688, "epoch": 0.7135298563869993, "fcm_dpo/beta": 0.004661578685045242, "fcm_dpo/delta": 0.05358702316880226, "fcm_dpo/margin": 75.73089599609375, "fcm_dpo/q_t": 0.4227460026741028, "grad_norm": 12.47477912902832, "learning_rate": 1.1595400232569768e-07, "logits/chosen": 0.8483719825744629, "logits/rejected": 0.8078272342681885, "logps/chosen": -215.81790161132812, "logps/ref_chosen": -55.908668518066406, "logps/ref_rejected": -74.70294189453125, "logps/rejected": -310.34307861328125, "loss": 1.1769, "margin_dpo/margin_mean": 75.73089599609375, "margin_dpo/margin_std": 150.27847290039062, "step": 472 }, { "KL/chosen_KL_mean": -164.11631774902344, "KL/mean": -217.00555419921875, "KL/rejected_KL_mean": -269.894775390625, "KL/std": 144.6307373046875, "epoch": 0.7150415721844293, "fcm_dpo/beta": 0.004342665895819664, "fcm_dpo/delta": -0.07786207646131516, "fcm_dpo/margin": 105.77847290039062, "fcm_dpo/q_t": 0.3978637456893921, "grad_norm": 13.936591148376465, "learning_rate": 1.1483979563610069e-07, "logits/chosen": 0.8966566324234009, "logits/rejected": 0.8024401068687439, "logps/chosen": -218.27719116210938, "logps/ref_chosen": -54.16088104248047, "logps/ref_rejected": -92.76789855957031, "logps/rejected": -362.66265869140625, "loss": 1.0969, "margin_dpo/margin_mean": 105.7784652709961, "margin_dpo/margin_std": 167.810302734375, "step": 473 }, { "KL/chosen_KL_mean": -165.61419677734375, "KL/mean": -208.4927520751953, "KL/rejected_KL_mean": -251.37130737304688, "KL/std": 123.07339477539062, "epoch": 0.7165532879818595, "fcm_dpo/beta": 0.004392385482788086, "fcm_dpo/delta": 0.028430059552192688, "fcm_dpo/margin": 85.75711822509766, "fcm_dpo/q_t": 0.41677016019821167, "grad_norm": 14.764763832092285, "learning_rate": 1.1372936966796709e-07, "logits/chosen": 0.8863992094993591, "logits/rejected": 0.8298487663269043, "logps/chosen": -212.29989624023438, "logps/ref_chosen": -46.685707092285156, "logps/ref_rejected": -71.44731903076172, "logps/rejected": -322.818603515625, "loss": 1.1344, "margin_dpo/margin_mean": 85.75711822509766, "margin_dpo/margin_std": 142.21832275390625, "step": 474 }, { "KL/chosen_KL_mean": -165.1353302001953, "KL/mean": -222.8227081298828, "KL/rejected_KL_mean": -280.51007080078125, "KL/std": 124.3143539428711, "epoch": 0.7180650037792895, "fcm_dpo/beta": 0.0040479195304214954, "fcm_dpo/delta": -0.09077291190624237, "fcm_dpo/margin": 115.37474060058594, "fcm_dpo/q_t": 0.38997000455856323, "grad_norm": 7.723613739013672, "learning_rate": 1.126227554822985e-07, "logits/chosen": 0.8512206077575684, "logits/rejected": 0.8066825866699219, "logps/chosen": -223.6226348876953, "logps/ref_chosen": -58.4873046875, "logps/ref_rejected": -87.00187683105469, "logps/rejected": -367.511962890625, "loss": 1.0486, "margin_dpo/margin_mean": 115.37474060058594, "margin_dpo/margin_std": 141.55166625976562, "step": 475 }, { "KL/chosen_KL_mean": -200.2625732421875, "KL/mean": -234.19955444335938, "KL/rejected_KL_mean": -268.13653564453125, "KL/std": 133.18336486816406, "epoch": 0.7195767195767195, "fcm_dpo/beta": 0.004081051331013441, "fcm_dpo/delta": 0.029893899336457253, "fcm_dpo/margin": 67.8740005493164, "fcm_dpo/q_t": 0.4379327893257141, "grad_norm": 12.724250793457031, "learning_rate": 1.1151998403347243e-07, "logits/chosen": 0.7401478290557861, "logits/rejected": 0.744006872177124, "logps/chosen": -275.6441955566406, "logps/ref_chosen": -75.38162231445312, "logps/ref_rejected": -76.99822235107422, "logps/rejected": -345.134765625, "loss": 1.2061, "margin_dpo/margin_mean": 67.87399291992188, "margin_dpo/margin_std": 140.630615234375, "step": 476 }, { "KL/chosen_KL_mean": -194.9725799560547, "KL/mean": -238.0779266357422, "KL/rejected_KL_mean": -281.1832580566406, "KL/std": 134.39028930664062, "epoch": 0.7210884353741497, "fcm_dpo/beta": 0.004254788625985384, "fcm_dpo/delta": 0.04072454944252968, "fcm_dpo/margin": 86.21067810058594, "fcm_dpo/q_t": 0.41996684670448303, "grad_norm": 12.63784122467041, "learning_rate": 1.1042108616837692e-07, "logits/chosen": 0.8575061559677124, "logits/rejected": 0.8164290189743042, "logps/chosen": -256.04595947265625, "logps/ref_chosen": -61.073387145996094, "logps/ref_rejected": -81.34375, "logps/rejected": -362.5270080566406, "loss": 1.1771, "margin_dpo/margin_mean": 86.21067810058594, "margin_dpo/margin_std": 171.40589904785156, "step": 477 }, { "KL/chosen_KL_mean": -172.208984375, "KL/mean": -203.42808532714844, "KL/rejected_KL_mean": -234.64720153808594, "KL/std": 107.170166015625, "epoch": 0.7226001511715797, "fcm_dpo/beta": 0.004686059430241585, "fcm_dpo/delta": 0.12666893005371094, "fcm_dpo/margin": 62.438201904296875, "fcm_dpo/q_t": 0.4363289177417755, "grad_norm": 15.702838897705078, "learning_rate": 1.0932609262554746e-07, "logits/chosen": 0.7970494031906128, "logits/rejected": 0.8083937168121338, "logps/chosen": -229.3762969970703, "logps/ref_chosen": -57.16731643676758, "logps/ref_rejected": -53.30917739868164, "logps/rejected": -287.95635986328125, "loss": 1.2065, "margin_dpo/margin_mean": 62.438201904296875, "margin_dpo/margin_std": 129.8063507080078, "step": 478 }, { "KL/chosen_KL_mean": -178.65333557128906, "KL/mean": -206.3043212890625, "KL/rejected_KL_mean": -233.95529174804688, "KL/std": 111.01861572265625, "epoch": 0.7241118669690099, "fcm_dpo/beta": 0.004972537979483604, "fcm_dpo/delta": 0.015346960164606571, "fcm_dpo/margin": 55.30194091796875, "fcm_dpo/q_t": 0.43899282813072205, "grad_norm": 15.627010345458984, "learning_rate": 1.0823503403430734e-07, "logits/chosen": 0.8058916926383972, "logits/rejected": 0.7788633108139038, "logps/chosen": -237.566650390625, "logps/ref_chosen": -58.91331481933594, "logps/ref_rejected": -63.7403450012207, "logps/rejected": -297.69561767578125, "loss": 1.2337, "margin_dpo/margin_mean": 55.30194091796875, "margin_dpo/margin_std": 132.65174865722656, "step": 479 }, { "KL/chosen_KL_mean": -186.09510803222656, "KL/mean": -228.88623046875, "KL/rejected_KL_mean": -271.6773376464844, "KL/std": 116.18428039550781, "epoch": 0.7256235827664399, "fcm_dpo/beta": 0.005056783556938171, "fcm_dpo/delta": -0.051207661628723145, "fcm_dpo/margin": 85.58222961425781, "fcm_dpo/q_t": 0.4012352526187897, "grad_norm": 15.959285736083984, "learning_rate": 1.0714794091391072e-07, "logits/chosen": 0.7871379852294922, "logits/rejected": 0.783062219619751, "logps/chosen": -248.89572143554688, "logps/ref_chosen": -62.80061340332031, "logps/ref_rejected": -67.58859252929688, "logps/rejected": -339.26593017578125, "loss": 1.1235, "margin_dpo/margin_mean": 85.58222961425781, "margin_dpo/margin_std": 135.55453491210938, "step": 480 }, { "KL/chosen_KL_mean": -183.6075439453125, "KL/mean": -224.15196228027344, "KL/rejected_KL_mean": -264.6963806152344, "KL/std": 122.73133087158203, "epoch": 0.72713529856387, "fcm_dpo/beta": 0.00472680851817131, "fcm_dpo/delta": 0.020053943619132042, "fcm_dpo/margin": 81.0888442993164, "fcm_dpo/q_t": 0.41520315408706665, "grad_norm": 12.672243118286133, "learning_rate": 1.0606484367268906e-07, "logits/chosen": 0.7809611558914185, "logits/rejected": 0.7874898314476013, "logps/chosen": -248.89404296875, "logps/ref_chosen": -65.28649139404297, "logps/ref_rejected": -70.78668212890625, "logps/rejected": -335.4830627441406, "loss": 1.1305, "margin_dpo/margin_mean": 81.0888442993164, "margin_dpo/margin_std": 134.0117645263672, "step": 481 }, { "KL/chosen_KL_mean": -202.88668823242188, "KL/mean": -247.89273071289062, "KL/rejected_KL_mean": -292.89874267578125, "KL/std": 136.02828979492188, "epoch": 0.7286470143613001, "fcm_dpo/beta": 0.004762349650263786, "fcm_dpo/delta": -0.03688174486160278, "fcm_dpo/margin": 90.01205444335938, "fcm_dpo/q_t": 0.4086783528327942, "grad_norm": 15.459307670593262, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 0.7737709879875183, "logits/rejected": 0.640209436416626, "logps/chosen": -263.7928771972656, "logps/ref_chosen": -60.906185150146484, "logps/ref_rejected": -103.44656372070312, "logps/rejected": -396.3453369140625, "loss": 1.1449, "margin_dpo/margin_mean": 90.01204681396484, "margin_dpo/margin_std": 167.98159790039062, "step": 482 }, { "KL/chosen_KL_mean": -170.41964721679688, "KL/mean": -221.99755859375, "KL/rejected_KL_mean": -273.57550048828125, "KL/std": 118.67953491210938, "epoch": 0.7301587301587301, "fcm_dpo/beta": 0.004441915079951286, "fcm_dpo/delta": -0.07519707083702087, "fcm_dpo/margin": 103.15583801269531, "fcm_dpo/q_t": 0.3927938640117645, "grad_norm": 13.764213562011719, "learning_rate": 1.0391075790138232e-07, "logits/chosen": 0.9073851704597473, "logits/rejected": 0.8138213157653809, "logps/chosen": -223.61166381835938, "logps/ref_chosen": -53.192012786865234, "logps/ref_rejected": -81.83927154541016, "logps/rejected": -355.4147644042969, "loss": 1.0646, "margin_dpo/margin_mean": 103.15583801269531, "margin_dpo/margin_std": 135.55938720703125, "step": 483 }, { "KL/chosen_KL_mean": -176.34378051757812, "KL/mean": -209.01864624023438, "KL/rejected_KL_mean": -241.69349670410156, "KL/std": 116.74180603027344, "epoch": 0.7316704459561603, "fcm_dpo/beta": 0.004746517166495323, "fcm_dpo/delta": 0.10179544240236282, "fcm_dpo/margin": 65.34970092773438, "fcm_dpo/q_t": 0.4296528100967407, "grad_norm": 16.811372756958008, "learning_rate": 1.0283982962570681e-07, "logits/chosen": 0.8936479687690735, "logits/rejected": 0.8692023754119873, "logps/chosen": -234.11325073242188, "logps/ref_chosen": -57.76945877075195, "logps/ref_rejected": -71.6829833984375, "logps/rejected": -313.37646484375, "loss": 1.1563, "margin_dpo/margin_mean": 65.34970092773438, "margin_dpo/margin_std": 101.48129272460938, "step": 484 }, { "KL/chosen_KL_mean": -182.8577117919922, "KL/mean": -219.1405029296875, "KL/rejected_KL_mean": -255.42327880859375, "KL/std": 115.46862030029297, "epoch": 0.7331821617535903, "fcm_dpo/beta": 0.004539255518466234, "fcm_dpo/delta": -0.049730200320482254, "fcm_dpo/margin": 72.56556701660156, "fcm_dpo/q_t": 0.4225718379020691, "grad_norm": 13.539854049682617, "learning_rate": 1.0177301773633992e-07, "logits/chosen": 0.8523825407028198, "logits/rejected": 0.838424801826477, "logps/chosen": -239.49356079101562, "logps/ref_chosen": -56.63584899902344, "logps/ref_rejected": -70.85614013671875, "logps/rejected": -326.2794189453125, "loss": 1.153, "margin_dpo/margin_mean": 72.56556701660156, "margin_dpo/margin_std": 115.97235107421875, "step": 485 }, { "KL/chosen_KL_mean": -208.73068237304688, "KL/mean": -245.28561401367188, "KL/rejected_KL_mean": -281.84051513671875, "KL/std": 138.67807006835938, "epoch": 0.7346938775510204, "fcm_dpo/beta": 0.004761700518429279, "fcm_dpo/delta": 0.06264105439186096, "fcm_dpo/margin": 73.10984802246094, "fcm_dpo/q_t": 0.4274640679359436, "grad_norm": 11.167536735534668, "learning_rate": 1.007103520743035e-07, "logits/chosen": 0.883628249168396, "logits/rejected": 0.7876710295677185, "logps/chosen": -265.07769775390625, "logps/ref_chosen": -56.347023010253906, "logps/ref_rejected": -85.97221374511719, "logps/rejected": -367.812744140625, "loss": 1.204, "margin_dpo/margin_mean": 73.10984802246094, "margin_dpo/margin_std": 163.16970825195312, "step": 486 }, { "KL/chosen_KL_mean": -188.4417724609375, "KL/mean": -229.6407470703125, "KL/rejected_KL_mean": -270.8397216796875, "KL/std": 127.54862213134766, "epoch": 0.7362055933484505, "fcm_dpo/beta": 0.00484395632520318, "fcm_dpo/delta": 0.0010717334225773811, "fcm_dpo/margin": 82.39796447753906, "fcm_dpo/q_t": 0.4158893823623657, "grad_norm": 14.043888092041016, "learning_rate": 9.965186236464046e-08, "logits/chosen": 0.9236916303634644, "logits/rejected": 0.8743011355400085, "logps/chosen": -249.05899047851562, "logps/ref_chosen": -60.617218017578125, "logps/ref_rejected": -82.50975036621094, "logps/rejected": -353.3494873046875, "loss": 1.1434, "margin_dpo/margin_mean": 82.39796447753906, "margin_dpo/margin_std": 153.81063842773438, "step": 487 }, { "KL/chosen_KL_mean": -175.81893920898438, "KL/mean": -222.56472778320312, "KL/rejected_KL_mean": -269.31048583984375, "KL/std": 118.71575164794922, "epoch": 0.7377173091458806, "fcm_dpo/beta": 0.00476812943816185, "fcm_dpo/delta": -0.06080658361315727, "fcm_dpo/margin": 93.4915771484375, "fcm_dpo/q_t": 0.3985556662082672, "grad_norm": 15.597735404968262, "learning_rate": 9.859757821558337e-08, "logits/chosen": 0.8415173888206482, "logits/rejected": 0.8006629347801208, "logps/chosen": -238.927978515625, "logps/ref_chosen": -63.10905075073242, "logps/ref_rejected": -82.49348449707031, "logps/rejected": -351.8039855957031, "loss": 1.0822, "margin_dpo/margin_mean": 93.49156951904297, "margin_dpo/margin_std": 130.8084716796875, "step": 488 }, { "KL/chosen_KL_mean": -205.70184326171875, "KL/mean": -234.09732055664062, "KL/rejected_KL_mean": -262.4927673339844, "KL/std": 130.96896362304688, "epoch": 0.7392290249433107, "fcm_dpo/beta": 0.005044779274612665, "fcm_dpo/delta": 0.13149571418762207, "fcm_dpo/margin": 56.7909049987793, "fcm_dpo/q_t": 0.4390004277229309, "grad_norm": 13.851383209228516, "learning_rate": 9.754752911772615e-08, "logits/chosen": 0.8348220586776733, "logits/rejected": 0.7962938547134399, "logps/chosen": -270.6907958984375, "logps/ref_chosen": -64.98896026611328, "logps/ref_rejected": -84.39607238769531, "logps/rejected": -346.88885498046875, "loss": 1.235, "margin_dpo/margin_mean": 56.79090881347656, "margin_dpo/margin_std": 138.2364501953125, "step": 489 }, { "KL/chosen_KL_mean": -185.4239501953125, "KL/mean": -219.5350341796875, "KL/rejected_KL_mean": -253.6461181640625, "KL/std": 125.25768280029297, "epoch": 0.7407407407407407, "fcm_dpo/beta": 0.005278023891150951, "fcm_dpo/delta": 0.047999098896980286, "fcm_dpo/margin": 68.22216796875, "fcm_dpo/q_t": 0.42334112524986267, "grad_norm": 12.744791984558105, "learning_rate": 9.650174444319956e-08, "logits/chosen": 0.8877488374710083, "logits/rejected": 0.8769302368164062, "logps/chosen": -247.33270263671875, "logps/ref_chosen": -61.90874481201172, "logps/ref_rejected": -70.58566284179688, "logps/rejected": -324.2317810058594, "loss": 1.2111, "margin_dpo/margin_mean": 68.22216796875, "margin_dpo/margin_std": 154.54383850097656, "step": 490 }, { "KL/chosen_KL_mean": -178.17283630371094, "KL/mean": -213.24208068847656, "KL/rejected_KL_mean": -248.31134033203125, "KL/std": 113.33309936523438, "epoch": 0.7422524565381708, "fcm_dpo/beta": 0.005064789205789566, "fcm_dpo/delta": -0.07285159081220627, "fcm_dpo/margin": 70.13851165771484, "fcm_dpo/q_t": 0.41720882058143616, "grad_norm": 10.481760025024414, "learning_rate": 9.546025344484868e-08, "logits/chosen": 0.8015914559364319, "logits/rejected": 0.7469338774681091, "logps/chosen": -233.64852905273438, "logps/ref_chosen": -55.47570037841797, "logps/ref_rejected": -78.70318603515625, "logps/rejected": -327.0145263671875, "loss": 1.1531, "margin_dpo/margin_mean": 70.13851165771484, "margin_dpo/margin_std": 121.06025695800781, "step": 491 }, { "KL/chosen_KL_mean": -211.74505615234375, "KL/mean": -244.18946838378906, "KL/rejected_KL_mean": -276.6338806152344, "KL/std": 131.94345092773438, "epoch": 0.7437641723356009, "fcm_dpo/beta": 0.005409009754657745, "fcm_dpo/delta": 0.05510532110929489, "fcm_dpo/margin": 64.88883209228516, "fcm_dpo/q_t": 0.4252876937389374, "grad_norm": 16.77579116821289, "learning_rate": 9.442308525541589e-08, "logits/chosen": 0.87005615234375, "logits/rejected": 0.8110706806182861, "logps/chosen": -279.03143310546875, "logps/ref_chosen": -67.28638458251953, "logps/ref_rejected": -82.78628540039062, "logps/rejected": -359.420166015625, "loss": 1.221, "margin_dpo/margin_mean": 64.88883209228516, "margin_dpo/margin_std": 148.9833984375, "step": 492 }, { "KL/chosen_KL_mean": -176.25186157226562, "KL/mean": -222.16436767578125, "KL/rejected_KL_mean": -268.0768737792969, "KL/std": 127.96397399902344, "epoch": 0.745275888133031, "fcm_dpo/beta": 0.005210395436733961, "fcm_dpo/delta": -0.10840007662773132, "fcm_dpo/margin": 91.82500457763672, "fcm_dpo/q_t": 0.3924015760421753, "grad_norm": 17.075223922729492, "learning_rate": 9.339026888672468e-08, "logits/chosen": 0.7961744666099548, "logits/rejected": 0.7387325763702393, "logps/chosen": -232.17938232421875, "logps/ref_chosen": -55.92750549316406, "logps/ref_rejected": -79.12149810791016, "logps/rejected": -347.1983642578125, "loss": 1.0952, "margin_dpo/margin_mean": 91.82499694824219, "margin_dpo/margin_std": 140.10226440429688, "step": 493 }, { "KL/chosen_KL_mean": -182.12063598632812, "KL/mean": -224.42507934570312, "KL/rejected_KL_mean": -266.72955322265625, "KL/std": 128.66799926757812, "epoch": 0.7467876039304611, "fcm_dpo/beta": 0.00474274717271328, "fcm_dpo/delta": -0.0021576210856437683, "fcm_dpo/margin": 84.60891723632812, "fcm_dpo/q_t": 0.4120602011680603, "grad_norm": 14.730215072631836, "learning_rate": 9.236183322886945e-08, "logits/chosen": 0.7500890493392944, "logits/rejected": 0.7123274207115173, "logps/chosen": -250.07473754882812, "logps/ref_chosen": -67.95410919189453, "logps/ref_rejected": -90.50865173339844, "logps/rejected": -357.23822021484375, "loss": 1.1551, "margin_dpo/margin_mean": 84.60891723632812, "margin_dpo/margin_std": 161.02288818359375, "step": 494 }, { "KL/chosen_KL_mean": -177.19039916992188, "KL/mean": -209.7973175048828, "KL/rejected_KL_mean": -242.4042205810547, "KL/std": 126.00395965576172, "epoch": 0.7482993197278912, "fcm_dpo/beta": 0.004787042737007141, "fcm_dpo/delta": -0.0018955229315906763, "fcm_dpo/margin": 65.21380615234375, "fcm_dpo/q_t": 0.43578070402145386, "grad_norm": 14.910195350646973, "learning_rate": 9.133780704940594e-08, "logits/chosen": 0.8767759799957275, "logits/rejected": 0.8284322023391724, "logps/chosen": -229.8158721923828, "logps/ref_chosen": -52.62546157836914, "logps/ref_rejected": -72.06781005859375, "logps/rejected": -314.4720458984375, "loss": 1.2215, "margin_dpo/margin_mean": 65.21380615234375, "margin_dpo/margin_std": 157.19482421875, "step": 495 }, { "KL/chosen_KL_mean": -191.9990234375, "KL/mean": -239.45957946777344, "KL/rejected_KL_mean": -286.9201354980469, "KL/std": 139.683349609375, "epoch": 0.7498110355253212, "fcm_dpo/beta": 0.004459347575902939, "fcm_dpo/delta": -0.03923889249563217, "fcm_dpo/margin": 94.92111206054688, "fcm_dpo/q_t": 0.41244807839393616, "grad_norm": 11.721460342407227, "learning_rate": 9.031821899254797e-08, "logits/chosen": 0.8311753869056702, "logits/rejected": 0.7325785160064697, "logps/chosen": -249.59634399414062, "logps/ref_chosen": -57.597320556640625, "logps/ref_rejected": -94.36127471923828, "logps/rejected": -381.28143310546875, "loss": 1.1432, "margin_dpo/margin_mean": 94.92111206054688, "margin_dpo/margin_std": 180.92669677734375, "step": 496 }, { "KL/chosen_KL_mean": -197.59085083007812, "KL/mean": -241.4178009033203, "KL/rejected_KL_mean": -285.2447509765625, "KL/std": 125.32099914550781, "epoch": 0.7513227513227513, "fcm_dpo/beta": 0.004555117338895798, "fcm_dpo/delta": 0.0004740636795759201, "fcm_dpo/margin": 87.65388488769531, "fcm_dpo/q_t": 0.4084148406982422, "grad_norm": 13.068496704101562, "learning_rate": 8.930309757836516e-08, "logits/chosen": 0.8443832993507385, "logits/rejected": 0.8150862455368042, "logps/chosen": -270.38079833984375, "logps/ref_chosen": -72.78994750976562, "logps/ref_rejected": -89.48483276367188, "logps/rejected": -374.7295837402344, "loss": 1.1185, "margin_dpo/margin_mean": 87.65388488769531, "margin_dpo/margin_std": 137.76541137695312, "step": 497 }, { "KL/chosen_KL_mean": -175.70782470703125, "KL/mean": -222.53150939941406, "KL/rejected_KL_mean": -269.3551940917969, "KL/std": 117.85083770751953, "epoch": 0.7528344671201814, "fcm_dpo/beta": 0.00448596803471446, "fcm_dpo/delta": -0.025814387947320938, "fcm_dpo/margin": 93.6473388671875, "fcm_dpo/q_t": 0.401056706905365, "grad_norm": 13.499424934387207, "learning_rate": 8.829247120198563e-08, "logits/chosen": 0.8166778087615967, "logits/rejected": 0.8045326471328735, "logps/chosen": -244.07354736328125, "logps/ref_chosen": -68.36572265625, "logps/ref_rejected": -71.28846740722656, "logps/rejected": -340.6436767578125, "loss": 1.0898, "margin_dpo/margin_mean": 93.6473388671875, "margin_dpo/margin_std": 130.13490295410156, "step": 498 }, { "KL/chosen_KL_mean": -179.5928497314453, "KL/mean": -229.11141967773438, "KL/rejected_KL_mean": -278.6299743652344, "KL/std": 132.99771118164062, "epoch": 0.7543461829176115, "fcm_dpo/beta": 0.004335935227572918, "fcm_dpo/delta": -0.03787044808268547, "fcm_dpo/margin": 99.03713989257812, "fcm_dpo/q_t": 0.40562453866004944, "grad_norm": 19.231489181518555, "learning_rate": 8.728636813280163e-08, "logits/chosen": 0.8330011367797852, "logits/rejected": 0.7787662148475647, "logps/chosen": -241.50167846679688, "logps/ref_chosen": -61.90882873535156, "logps/ref_rejected": -91.9411392211914, "logps/rejected": -370.57110595703125, "loss": 1.1363, "margin_dpo/margin_mean": 99.0371322631836, "margin_dpo/margin_std": 176.20849609375, "step": 499 }, { "KL/chosen_KL_mean": -188.66636657714844, "KL/mean": -225.65838623046875, "KL/rejected_KL_mean": -262.6504211425781, "KL/std": 115.54472351074219, "epoch": 0.7558578987150416, "fcm_dpo/beta": 0.004538041539490223, "fcm_dpo/delta": 0.07754088193178177, "fcm_dpo/margin": 73.98403930664062, "fcm_dpo/q_t": 0.42312851548194885, "grad_norm": 14.178508758544922, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.8060486316680908, "logits/rejected": 0.8183088302612305, "logps/chosen": -258.8922119140625, "logps/ref_chosen": -70.225830078125, "logps/ref_rejected": -71.72203063964844, "logps/rejected": -334.3724365234375, "loss": 1.1855, "margin_dpo/margin_mean": 73.98403930664062, "margin_dpo/margin_std": 147.24203491210938, "step": 500 }, { "KL/chosen_KL_mean": -187.6682891845703, "KL/mean": -222.70504760742188, "KL/rejected_KL_mean": -257.7417907714844, "KL/std": 122.72930908203125, "epoch": 0.7573696145124716, "fcm_dpo/beta": 0.004941802471876144, "fcm_dpo/delta": 0.06365126371383667, "fcm_dpo/margin": 70.07351684570312, "fcm_dpo/q_t": 0.42135995626449585, "grad_norm": 12.289847373962402, "learning_rate": 8.528784436016878e-08, "logits/chosen": 0.831879198551178, "logits/rejected": 0.8381158113479614, "logps/chosen": -252.26708984375, "logps/ref_chosen": -64.59880828857422, "logps/ref_rejected": -70.59329223632812, "logps/rejected": -328.3350830078125, "loss": 1.1276, "margin_dpo/margin_mean": 70.07351684570312, "margin_dpo/margin_std": 99.20466613769531, "step": 501 }, { "KL/chosen_KL_mean": -187.70001220703125, "KL/mean": -225.89833068847656, "KL/rejected_KL_mean": -264.09661865234375, "KL/std": 134.3505096435547, "epoch": 0.7588813303099018, "fcm_dpo/beta": 0.005120974965393543, "fcm_dpo/delta": 0.008247282356023788, "fcm_dpo/margin": 76.39662170410156, "fcm_dpo/q_t": 0.412162721157074, "grad_norm": 16.75588607788086, "learning_rate": 8.4295479559726e-08, "logits/chosen": 0.8271726369857788, "logits/rejected": 0.7866770029067993, "logps/chosen": -253.16665649414062, "logps/ref_chosen": -65.46662902832031, "logps/ref_rejected": -90.22233581542969, "logps/rejected": -354.3189697265625, "loss": 1.129, "margin_dpo/margin_mean": 76.39662170410156, "margin_dpo/margin_std": 124.2121353149414, "step": 502 }, { "KL/chosen_KL_mean": -165.67684936523438, "KL/mean": -204.37872314453125, "KL/rejected_KL_mean": -243.0806121826172, "KL/std": 110.60697174072266, "epoch": 0.7603930461073318, "fcm_dpo/beta": 0.00502370111644268, "fcm_dpo/delta": 0.013673887588083744, "fcm_dpo/margin": 77.40376281738281, "fcm_dpo/q_t": 0.4104214310646057, "grad_norm": 12.808483123779297, "learning_rate": 8.330774987092712e-08, "logits/chosen": 0.7944152355194092, "logits/rejected": 0.8020203709602356, "logps/chosen": -217.51161193847656, "logps/ref_chosen": -51.83476257324219, "logps/ref_rejected": -57.62522506713867, "logps/rejected": -300.7058410644531, "loss": 1.1297, "margin_dpo/margin_mean": 77.40376281738281, "margin_dpo/margin_std": 125.51486206054688, "step": 503 }, { "KL/chosen_KL_mean": -174.6210174560547, "KL/mean": -224.70675659179688, "KL/rejected_KL_mean": -274.79248046875, "KL/std": 113.20222473144531, "epoch": 0.7619047619047619, "fcm_dpo/beta": 0.004858715459704399, "fcm_dpo/delta": -0.11634810268878937, "fcm_dpo/margin": 100.17145538330078, "fcm_dpo/q_t": 0.38356009125709534, "grad_norm": 11.490617752075195, "learning_rate": 8.232468292269479e-08, "logits/chosen": 0.7988426089286804, "logits/rejected": 0.7896198034286499, "logps/chosen": -243.272216796875, "logps/ref_chosen": -68.65119934082031, "logps/ref_rejected": -77.91394805908203, "logps/rejected": -352.7064208984375, "loss": 1.0441, "margin_dpo/margin_mean": 100.17145538330078, "margin_dpo/margin_std": 116.76637268066406, "step": 504 }, { "KL/chosen_KL_mean": -184.5635986328125, "KL/mean": -219.55801391601562, "KL/rejected_KL_mean": -254.55239868164062, "KL/std": 134.74453735351562, "epoch": 0.763416477702192, "fcm_dpo/beta": 0.004393025301396847, "fcm_dpo/delta": -0.02926192618906498, "fcm_dpo/margin": 69.98880004882812, "fcm_dpo/q_t": 0.4304890036582947, "grad_norm": 11.797785758972168, "learning_rate": 8.134630621352483e-08, "logits/chosen": 0.8228546380996704, "logits/rejected": 0.7897151708602905, "logps/chosen": -244.5624542236328, "logps/ref_chosen": -59.99884796142578, "logps/ref_rejected": -76.88048553466797, "logps/rejected": -331.4328918457031, "loss": 1.2023, "margin_dpo/margin_mean": 69.98880004882812, "margin_dpo/margin_std": 146.3544921875, "step": 505 }, { "KL/chosen_KL_mean": -185.9619140625, "KL/mean": -222.78070068359375, "KL/rejected_KL_mean": -259.5994873046875, "KL/std": 118.3785171508789, "epoch": 0.764928193499622, "fcm_dpo/beta": 0.004599967040121555, "fcm_dpo/delta": 0.07407009601593018, "fcm_dpo/margin": 73.63758850097656, "fcm_dpo/q_t": 0.42431166768074036, "grad_norm": 12.498722076416016, "learning_rate": 8.037264711071698e-08, "logits/chosen": 0.7942636609077454, "logits/rejected": 0.785301685333252, "logps/chosen": -256.033203125, "logps/ref_chosen": -70.07130432128906, "logps/ref_rejected": -82.03775024414062, "logps/rejected": -341.63726806640625, "loss": 1.1882, "margin_dpo/margin_mean": 73.63758850097656, "margin_dpo/margin_std": 149.9478302001953, "step": 506 }, { "KL/chosen_KL_mean": -195.21092224121094, "KL/mean": -239.38064575195312, "KL/rejected_KL_mean": -283.55035400390625, "KL/std": 135.94154357910156, "epoch": 0.7664399092970522, "fcm_dpo/beta": 0.004573674872517586, "fcm_dpo/delta": -0.008770979940891266, "fcm_dpo/margin": 88.33943176269531, "fcm_dpo/q_t": 0.4142111539840698, "grad_norm": 13.64824390411377, "learning_rate": 7.940373284960933e-08, "logits/chosen": 0.7974521517753601, "logits/rejected": 0.7599077224731445, "logps/chosen": -267.21795654296875, "logps/ref_chosen": -72.00703430175781, "logps/ref_rejected": -93.94987487792969, "logps/rejected": -377.500244140625, "loss": 1.1482, "margin_dpo/margin_mean": 88.33943176269531, "margin_dpo/margin_std": 161.13748168945312, "step": 507 }, { "KL/chosen_KL_mean": -179.69149780273438, "KL/mean": -231.722900390625, "KL/rejected_KL_mean": -283.7542724609375, "KL/std": 142.07757568359375, "epoch": 0.7679516250944822, "fcm_dpo/beta": 0.004537220112979412, "fcm_dpo/delta": -0.09652116149663925, "fcm_dpo/margin": 104.06279754638672, "fcm_dpo/q_t": 0.3932833671569824, "grad_norm": 14.554759979248047, "learning_rate": 7.843959053281663e-08, "logits/chosen": 0.8035924434661865, "logits/rejected": 0.7036359310150146, "logps/chosen": -239.9114227294922, "logps/ref_chosen": -60.21992492675781, "logps/ref_rejected": -95.9200668334961, "logps/rejected": -379.67437744140625, "loss": 1.09, "margin_dpo/margin_mean": 104.06279754638672, "margin_dpo/margin_std": 155.7027130126953, "step": 508 }, { "KL/chosen_KL_mean": -190.60467529296875, "KL/mean": -227.2447967529297, "KL/rejected_KL_mean": -263.8849182128906, "KL/std": 124.68811798095703, "epoch": 0.7694633408919124, "fcm_dpo/beta": 0.004491401370614767, "fcm_dpo/delta": 0.08523663878440857, "fcm_dpo/margin": 73.28022003173828, "fcm_dpo/q_t": 0.4270988702774048, "grad_norm": 15.783040046691895, "learning_rate": 7.748024712947204e-08, "logits/chosen": 0.7905877232551575, "logits/rejected": 0.7737974524497986, "logps/chosen": -256.8748474121094, "logps/ref_chosen": -66.27017211914062, "logps/ref_rejected": -71.73065185546875, "logps/rejected": -335.61553955078125, "loss": 1.1929, "margin_dpo/margin_mean": 73.28022766113281, "margin_dpo/margin_std": 149.970458984375, "step": 509 }, { "KL/chosen_KL_mean": -189.751708984375, "KL/mean": -238.32249450683594, "KL/rejected_KL_mean": -286.8932800292969, "KL/std": 140.2471466064453, "epoch": 0.7709750566893424, "fcm_dpo/beta": 0.004407564178109169, "fcm_dpo/delta": -0.03904171660542488, "fcm_dpo/margin": 97.1415786743164, "fcm_dpo/q_t": 0.40892326831817627, "grad_norm": 13.543128967285156, "learning_rate": 7.652572947447272e-08, "logits/chosen": 0.8949325084686279, "logits/rejected": 0.8182798624038696, "logps/chosen": -243.2965850830078, "logps/ref_chosen": -53.54487609863281, "logps/ref_rejected": -91.36648559570312, "logps/rejected": -378.259765625, "loss": 1.1437, "margin_dpo/margin_mean": 97.14158630371094, "margin_dpo/margin_std": 181.95016479492188, "step": 510 }, { "KL/chosen_KL_mean": -178.1725616455078, "KL/mean": -232.1974334716797, "KL/rejected_KL_mean": -286.2223205566406, "KL/std": 122.31033325195312, "epoch": 0.7724867724867724, "fcm_dpo/beta": 0.0042327032424509525, "fcm_dpo/delta": -0.07427525520324707, "fcm_dpo/margin": 108.04974365234375, "fcm_dpo/q_t": 0.3912004828453064, "grad_norm": 15.069428443908691, "learning_rate": 7.557606426772961e-08, "logits/chosen": 0.8457983136177063, "logits/rejected": 0.7985345125198364, "logps/chosen": -234.01693725585938, "logps/ref_chosen": -55.844383239746094, "logps/ref_rejected": -86.49819946289062, "logps/rejected": -372.72052001953125, "loss": 1.0521, "margin_dpo/margin_mean": 108.04974365234375, "margin_dpo/margin_std": 128.53379821777344, "step": 511 }, { "KL/chosen_KL_mean": -185.82992553710938, "KL/mean": -217.70220947265625, "KL/rejected_KL_mean": -249.574462890625, "KL/std": 114.0875015258789, "epoch": 0.7739984882842026, "fcm_dpo/beta": 0.004275324288755655, "fcm_dpo/delta": 0.03214414417743683, "fcm_dpo/margin": 63.744571685791016, "fcm_dpo/q_t": 0.43680599331855774, "grad_norm": 19.423717498779297, "learning_rate": 7.463127807341966e-08, "logits/chosen": 0.7606216669082642, "logits/rejected": 0.7632473111152649, "logps/chosen": -247.4829559326172, "logps/ref_chosen": -61.653038024902344, "logps/ref_rejected": -72.83148193359375, "logps/rejected": -322.40594482421875, "loss": 1.2179, "margin_dpo/margin_mean": 63.74457550048828, "margin_dpo/margin_std": 138.28744506835938, "step": 512 }, { "KL/chosen_KL_mean": -166.70736694335938, "KL/mean": -207.901123046875, "KL/rejected_KL_mean": -249.09486389160156, "KL/std": 113.70684814453125, "epoch": 0.7755102040816326, "fcm_dpo/beta": 0.004403136670589447, "fcm_dpo/delta": 0.04559480771422386, "fcm_dpo/margin": 82.38749694824219, "fcm_dpo/q_t": 0.4186474084854126, "grad_norm": 10.126408576965332, "learning_rate": 7.369139731924401e-08, "logits/chosen": 0.9330278635025024, "logits/rejected": 0.8865524530410767, "logps/chosen": -217.5599365234375, "logps/ref_chosen": -50.85256576538086, "logps/ref_rejected": -69.21754455566406, "logps/rejected": -318.3124084472656, "loss": 1.1249, "margin_dpo/margin_mean": 82.38749694824219, "margin_dpo/margin_std": 122.80973052978516, "step": 513 }, { "KL/chosen_KL_mean": -187.76589965820312, "KL/mean": -235.5794677734375, "KL/rejected_KL_mean": -283.39300537109375, "KL/std": 126.61970520019531, "epoch": 0.7770219198790628, "fcm_dpo/beta": 0.004458842799067497, "fcm_dpo/delta": -0.034807316958904266, "fcm_dpo/margin": 95.62709045410156, "fcm_dpo/q_t": 0.4023844003677368, "grad_norm": 15.569659233093262, "learning_rate": 7.275644829568747e-08, "logits/chosen": 0.8444833755493164, "logits/rejected": 0.8213080763816833, "logps/chosen": -257.1508483886719, "logps/ref_chosen": -69.38493347167969, "logps/ref_rejected": -83.32447814941406, "logps/rejected": -366.71746826171875, "loss": 1.1018, "margin_dpo/margin_mean": 95.62709045410156, "margin_dpo/margin_std": 144.15618896484375, "step": 514 }, { "KL/chosen_KL_mean": -194.33404541015625, "KL/mean": -232.53646850585938, "KL/rejected_KL_mean": -270.7389221191406, "KL/std": 117.86701965332031, "epoch": 0.7785336356764928, "fcm_dpo/beta": 0.004516595974564552, "fcm_dpo/delta": 0.06656854599714279, "fcm_dpo/margin": 76.40487670898438, "fcm_dpo/q_t": 0.4221508502960205, "grad_norm": 12.275152206420898, "learning_rate": 7.182645715528435e-08, "logits/chosen": 0.8466659784317017, "logits/rejected": 0.7759132981300354, "logps/chosen": -248.02108764648438, "logps/ref_chosen": -53.687034606933594, "logps/ref_rejected": -83.59614562988281, "logps/rejected": -354.3350524902344, "loss": 1.1581, "margin_dpo/margin_mean": 76.40487670898438, "margin_dpo/margin_std": 133.73483276367188, "step": 515 }, { "KL/chosen_KL_mean": -168.5338897705078, "KL/mean": -205.44277954101562, "KL/rejected_KL_mean": -242.3516845703125, "KL/std": 116.71525573730469, "epoch": 0.780045351473923, "fcm_dpo/beta": 0.0048707169480621815, "fcm_dpo/delta": 0.04681536927819252, "fcm_dpo/margin": 73.81780242919922, "fcm_dpo/q_t": 0.41837072372436523, "grad_norm": 15.747931480407715, "learning_rate": 7.090144991188568e-08, "logits/chosen": 0.8174859285354614, "logits/rejected": 0.7878178358078003, "logps/chosen": -225.43560791015625, "logps/ref_chosen": -56.9017219543457, "logps/ref_rejected": -67.83477783203125, "logps/rejected": -310.18646240234375, "loss": 1.1655, "margin_dpo/margin_mean": 73.81779479980469, "margin_dpo/margin_std": 137.54354858398438, "step": 516 }, { "KL/chosen_KL_mean": -201.6470489501953, "KL/mean": -229.30369567871094, "KL/rejected_KL_mean": -256.9603271484375, "KL/std": 125.04222106933594, "epoch": 0.781557067271353, "fcm_dpo/beta": 0.004945510067045689, "fcm_dpo/delta": 0.042446672916412354, "fcm_dpo/margin": 55.31329345703125, "fcm_dpo/q_t": 0.44020599126815796, "grad_norm": 14.6834077835083, "learning_rate": 6.998145243993284e-08, "logits/chosen": 0.909896969795227, "logits/rejected": 0.9149903059005737, "logps/chosen": -263.42218017578125, "logps/ref_chosen": -61.775142669677734, "logps/ref_rejected": -62.88270950317383, "logps/rejected": -319.8430480957031, "loss": 1.225, "margin_dpo/margin_mean": 55.31329345703125, "margin_dpo/margin_std": 128.85104370117188, "step": 517 }, { "KL/chosen_KL_mean": -173.17193603515625, "KL/mean": -214.0082550048828, "KL/rejected_KL_mean": -254.8445587158203, "KL/std": 122.31529235839844, "epoch": 0.783068783068783, "fcm_dpo/beta": 0.005010384600609541, "fcm_dpo/delta": -0.011576948687434196, "fcm_dpo/margin": 81.67262268066406, "fcm_dpo/q_t": 0.4104600250720978, "grad_norm": 13.930161476135254, "learning_rate": 6.906649047373245e-08, "logits/chosen": 0.8191814422607422, "logits/rejected": 0.7862158417701721, "logps/chosen": -235.19717407226562, "logps/ref_chosen": -62.02523422241211, "logps/ref_rejected": -79.06085205078125, "logps/rejected": -333.9053955078125, "loss": 1.1273, "margin_dpo/margin_mean": 81.67261505126953, "margin_dpo/margin_std": 138.4313201904297, "step": 518 }, { "KL/chosen_KL_mean": -204.07974243164062, "KL/mean": -227.8219451904297, "KL/rejected_KL_mean": -251.5641326904297, "KL/std": 122.6117172241211, "epoch": 0.7845804988662132, "fcm_dpo/beta": 0.005166331306099892, "fcm_dpo/delta": 0.067388154566288, "fcm_dpo/margin": 47.48439025878906, "fcm_dpo/q_t": 0.4460979700088501, "grad_norm": 20.920434951782227, "learning_rate": 6.815658960673781e-08, "logits/chosen": 0.8258580565452576, "logits/rejected": 0.7891712188720703, "logps/chosen": -265.68609619140625, "logps/ref_chosen": -61.60636901855469, "logps/ref_rejected": -74.50727844238281, "logps/rejected": -326.0714111328125, "loss": 1.314, "margin_dpo/margin_mean": 47.48439025878906, "margin_dpo/margin_std": 160.26510620117188, "step": 519 }, { "KL/chosen_KL_mean": -189.1623992919922, "KL/mean": -226.52426147460938, "KL/rejected_KL_mean": -263.8861083984375, "KL/std": 124.83399963378906, "epoch": 0.7860922146636432, "fcm_dpo/beta": 0.005199735518544912, "fcm_dpo/delta": 0.009324587881565094, "fcm_dpo/margin": 74.72369384765625, "fcm_dpo/q_t": 0.4140581488609314, "grad_norm": 17.282926559448242, "learning_rate": 6.725177529083209e-08, "logits/chosen": 0.8755450248718262, "logits/rejected": 0.8296347856521606, "logps/chosen": -252.03582763671875, "logps/ref_chosen": -62.87343215942383, "logps/ref_rejected": -76.505615234375, "logps/rejected": -340.3917236328125, "loss": 1.141, "margin_dpo/margin_mean": 74.72369384765625, "margin_dpo/margin_std": 127.34550476074219, "step": 520 }, { "KL/chosen_KL_mean": -186.72055053710938, "KL/mean": -238.96612548828125, "KL/rejected_KL_mean": -291.2117004394531, "KL/std": 125.33415222167969, "epoch": 0.7876039304610734, "fcm_dpo/beta": 0.0048751672729849815, "fcm_dpo/delta": -0.14638295769691467, "fcm_dpo/margin": 104.49115753173828, "fcm_dpo/q_t": 0.3831254839897156, "grad_norm": 12.223865509033203, "learning_rate": 6.63520728356167e-08, "logits/chosen": 0.748036801815033, "logits/rejected": 0.6873229742050171, "logps/chosen": -250.92721557617188, "logps/ref_chosen": -64.20668029785156, "logps/ref_rejected": -92.28083038330078, "logps/rejected": -383.4925231933594, "loss": 1.0487, "margin_dpo/margin_mean": 104.49115753173828, "margin_dpo/margin_std": 144.06787109375, "step": 521 }, { "KL/chosen_KL_mean": -190.2344970703125, "KL/mean": -220.25045776367188, "KL/rejected_KL_mean": -250.2664337158203, "KL/std": 122.27368927001953, "epoch": 0.7891156462585034, "fcm_dpo/beta": 0.005026691593229771, "fcm_dpo/delta": 0.11638805270195007, "fcm_dpo/margin": 60.03194046020508, "fcm_dpo/q_t": 0.43588894605636597, "grad_norm": 15.411672592163086, "learning_rate": 6.545750740770336e-08, "logits/chosen": 0.8304263353347778, "logits/rejected": 0.823676586151123, "logps/chosen": -248.60421752929688, "logps/ref_chosen": -58.369720458984375, "logps/ref_rejected": -68.79248046875, "logps/rejected": -319.05889892578125, "loss": 1.2574, "margin_dpo/margin_mean": 60.03194046020508, "margin_dpo/margin_std": 158.7082061767578, "step": 522 }, { "KL/chosen_KL_mean": -195.03762817382812, "KL/mean": -234.07977294921875, "KL/rejected_KL_mean": -273.12188720703125, "KL/std": 126.46722412109375, "epoch": 0.7906273620559335, "fcm_dpo/beta": 0.005208607763051987, "fcm_dpo/delta": -0.0084223672747612, "fcm_dpo/margin": 78.08426666259766, "fcm_dpo/q_t": 0.40672677755355835, "grad_norm": 18.978595733642578, "learning_rate": 6.456810403001012e-08, "logits/chosen": 0.8376122713088989, "logits/rejected": 0.7323886156082153, "logps/chosen": -260.7508544921875, "logps/ref_chosen": -65.71324157714844, "logps/ref_rejected": -91.98896789550781, "logps/rejected": -365.11083984375, "loss": 1.1444, "margin_dpo/margin_mean": 78.08426666259766, "margin_dpo/margin_std": 140.59805297851562, "step": 523 }, { "KL/chosen_KL_mean": -172.65541076660156, "KL/mean": -211.43576049804688, "KL/rejected_KL_mean": -250.216064453125, "KL/std": 122.14218139648438, "epoch": 0.7921390778533636, "fcm_dpo/beta": 0.00523480586707592, "fcm_dpo/delta": -0.008444221690297127, "fcm_dpo/margin": 77.56066131591797, "fcm_dpo/q_t": 0.407756507396698, "grad_norm": 15.92771053314209, "learning_rate": 6.368388758106134e-08, "logits/chosen": 0.751681387424469, "logits/rejected": 0.7346346974372864, "logps/chosen": -249.00665283203125, "logps/ref_chosen": -76.35124969482422, "logps/ref_rejected": -89.96072387695312, "logps/rejected": -340.17681884765625, "loss": 1.1162, "margin_dpo/margin_mean": 77.56065368652344, "margin_dpo/margin_std": 122.21437072753906, "step": 524 }, { "KL/chosen_KL_mean": -193.3937225341797, "KL/mean": -225.09326171875, "KL/rejected_KL_mean": -256.79278564453125, "KL/std": 118.25725555419922, "epoch": 0.7936507936507936, "fcm_dpo/beta": 0.005362164229154587, "fcm_dpo/delta": 0.07258353382349014, "fcm_dpo/margin": 63.3990478515625, "fcm_dpo/q_t": 0.42902350425720215, "grad_norm": 20.461183547973633, "learning_rate": 6.280488279429185e-08, "logits/chosen": 0.6896591186523438, "logits/rejected": 0.6898474097251892, "logps/chosen": -268.8895263671875, "logps/ref_chosen": -75.49578857421875, "logps/ref_rejected": -84.04852294921875, "logps/rejected": -340.84130859375, "loss": 1.2048, "margin_dpo/margin_mean": 63.3990478515625, "margin_dpo/margin_std": 140.93850708007812, "step": 525 }, { "KL/chosen_KL_mean": -198.66851806640625, "KL/mean": -229.0810546875, "KL/rejected_KL_mean": -259.49359130859375, "KL/std": 123.21441650390625, "epoch": 0.7951625094482238, "fcm_dpo/beta": 0.005208217538893223, "fcm_dpo/delta": -0.06009836867451668, "fcm_dpo/margin": 60.825103759765625, "fcm_dpo/q_t": 0.4270055294036865, "grad_norm": 14.760220527648926, "learning_rate": 6.193111425735515e-08, "logits/chosen": 0.8644962310791016, "logits/rejected": 0.8045707941055298, "logps/chosen": -259.9609375, "logps/ref_chosen": -61.29241943359375, "logps/ref_rejected": -82.47763061523438, "logps/rejected": -341.9712219238281, "loss": 1.2005, "margin_dpo/margin_mean": 60.825103759765625, "margin_dpo/margin_std": 126.21774291992188, "step": 526 }, { "KL/chosen_KL_mean": -207.2027587890625, "KL/mean": -233.5564727783203, "KL/rejected_KL_mean": -259.9101867675781, "KL/std": 122.05842590332031, "epoch": 0.7966742252456538, "fcm_dpo/beta": 0.005240567959845066, "fcm_dpo/delta": 0.012346339412033558, "fcm_dpo/margin": 52.707401275634766, "fcm_dpo/q_t": 0.44000500440597534, "grad_norm": 15.85174560546875, "learning_rate": 6.106260641143546e-08, "logits/chosen": 0.8949396014213562, "logits/rejected": 0.8214942812919617, "logps/chosen": -268.6753845214844, "logps/ref_chosen": -61.472625732421875, "logps/ref_rejected": -90.52831268310547, "logps/rejected": -350.4385070800781, "loss": 1.2519, "margin_dpo/margin_mean": 52.70740509033203, "margin_dpo/margin_std": 137.97315979003906, "step": 527 }, { "KL/chosen_KL_mean": -183.61734008789062, "KL/mean": -214.14938354492188, "KL/rejected_KL_mean": -244.68141174316406, "KL/std": 114.77609252929688, "epoch": 0.7981859410430839, "fcm_dpo/beta": 0.005515486933290958, "fcm_dpo/delta": 0.07630209624767303, "fcm_dpo/margin": 61.0640869140625, "fcm_dpo/q_t": 0.4299367070198059, "grad_norm": 17.928499221801758, "learning_rate": 6.019938355056422e-08, "logits/chosen": 0.767498254776001, "logits/rejected": 0.709592878818512, "logps/chosen": -242.4093475341797, "logps/ref_chosen": -58.792015075683594, "logps/ref_rejected": -71.82516479492188, "logps/rejected": -316.506591796875, "loss": 1.237, "margin_dpo/margin_mean": 61.0640869140625, "margin_dpo/margin_std": 150.18453979492188, "step": 528 }, { "KL/chosen_KL_mean": -178.7403564453125, "KL/mean": -234.70297241210938, "KL/rejected_KL_mean": -290.6656494140625, "KL/std": 112.90904235839844, "epoch": 0.799697656840514, "fcm_dpo/beta": 0.004876470193266869, "fcm_dpo/delta": -0.2033100128173828, "fcm_dpo/margin": 111.92525482177734, "fcm_dpo/q_t": 0.366091251373291, "grad_norm": 15.41568660736084, "learning_rate": 5.934146982094049e-08, "logits/chosen": 0.7990100383758545, "logits/rejected": 0.7555786371231079, "logps/chosen": -233.81130981445312, "logps/ref_chosen": -55.070960998535156, "logps/ref_rejected": -75.44007873535156, "logps/rejected": -366.105712890625, "loss": 0.9968, "margin_dpo/margin_mean": 111.92524719238281, "margin_dpo/margin_std": 122.16747283935547, "step": 529 }, { "KL/chosen_KL_mean": -176.22964477539062, "KL/mean": -214.50759887695312, "KL/rejected_KL_mean": -252.78553771972656, "KL/std": 123.36885070800781, "epoch": 0.8012093726379441, "fcm_dpo/beta": 0.004759990144520998, "fcm_dpo/delta": 0.04360903054475784, "fcm_dpo/margin": 76.55589294433594, "fcm_dpo/q_t": 0.4207877516746521, "grad_norm": 16.144392013549805, "learning_rate": 5.848888922025552e-08, "logits/chosen": 0.8252302408218384, "logits/rejected": 0.78911954164505, "logps/chosen": -232.97344970703125, "logps/ref_chosen": -56.743812561035156, "logps/ref_rejected": -76.6692123413086, "logps/rejected": -329.4547424316406, "loss": 1.1438, "margin_dpo/margin_mean": 76.55589294433594, "margin_dpo/margin_std": 129.4632568359375, "step": 530 }, { "KL/chosen_KL_mean": -176.19247436523438, "KL/mean": -215.78732299804688, "KL/rejected_KL_mean": -255.38214111328125, "KL/std": 121.45870971679688, "epoch": 0.8027210884353742, "fcm_dpo/beta": 0.004897120874375105, "fcm_dpo/delta": 0.015145424753427505, "fcm_dpo/margin": 79.18968200683594, "fcm_dpo/q_t": 0.41359657049179077, "grad_norm": 13.057550430297852, "learning_rate": 5.7641665597021435e-08, "logits/chosen": 0.8119915723800659, "logits/rejected": 0.7415198087692261, "logps/chosen": -227.30892944335938, "logps/ref_chosen": -51.116455078125, "logps/ref_rejected": -79.52884674072266, "logps/rejected": -334.9110107421875, "loss": 1.1318, "margin_dpo/margin_mean": 79.18968200683594, "margin_dpo/margin_std": 131.63864135742188, "step": 531 }, { "KL/chosen_KL_mean": -206.1256103515625, "KL/mean": -245.78433227539062, "KL/rejected_KL_mean": -285.44305419921875, "KL/std": 119.93473815917969, "epoch": 0.8042328042328042, "fcm_dpo/beta": 0.004965721629559994, "fcm_dpo/delta": 0.007580554112792015, "fcm_dpo/margin": 79.31744384765625, "fcm_dpo/q_t": 0.4115769863128662, "grad_norm": 14.420048713684082, "learning_rate": 5.679982264990424e-08, "logits/chosen": 0.7928520441055298, "logits/rejected": 0.7500756978988647, "logps/chosen": -264.4055480957031, "logps/ref_chosen": -58.279945373535156, "logps/ref_rejected": -78.05426788330078, "logps/rejected": -363.497314453125, "loss": 1.1333, "margin_dpo/margin_mean": 79.31744384765625, "margin_dpo/margin_std": 133.19078063964844, "step": 532 }, { "KL/chosen_KL_mean": -148.75473022460938, "KL/mean": -195.34390258789062, "KL/rejected_KL_mean": -241.93309020996094, "KL/std": 129.021240234375, "epoch": 0.8057445200302343, "fcm_dpo/beta": 0.004798324778676033, "fcm_dpo/delta": -0.06049473583698273, "fcm_dpo/margin": 93.17835998535156, "fcm_dpo/q_t": 0.39857304096221924, "grad_norm": 14.216915130615234, "learning_rate": 5.596338392706076e-08, "logits/chosen": 0.9161920547485352, "logits/rejected": 0.865228533744812, "logps/chosen": -205.17274475097656, "logps/ref_chosen": -56.41801071166992, "logps/ref_rejected": -73.89324951171875, "logps/rejected": -315.82635498046875, "loss": 1.0874, "margin_dpo/margin_mean": 93.17835998535156, "margin_dpo/margin_std": 137.08950805664062, "step": 533 }, { "KL/chosen_KL_mean": -185.18804931640625, "KL/mean": -221.02896118164062, "KL/rejected_KL_mean": -256.86981201171875, "KL/std": 115.55854034423828, "epoch": 0.8072562358276644, "fcm_dpo/beta": 0.004827641882002354, "fcm_dpo/delta": 0.06521468609571457, "fcm_dpo/margin": 71.6817855834961, "fcm_dpo/q_t": 0.42473551630973816, "grad_norm": 13.707343101501465, "learning_rate": 5.513237282548033e-08, "logits/chosen": 0.7912318706512451, "logits/rejected": 0.7636306285858154, "logps/chosen": -245.93673706054688, "logps/ref_chosen": -60.748687744140625, "logps/ref_rejected": -73.8623046875, "logps/rejected": -330.73211669921875, "loss": 1.1807, "margin_dpo/margin_mean": 71.68177795410156, "margin_dpo/margin_std": 141.45501708984375, "step": 534 }, { "KL/chosen_KL_mean": -195.66860961914062, "KL/mean": -230.18612670898438, "KL/rejected_KL_mean": -264.70361328125, "KL/std": 125.1104507446289, "epoch": 0.8087679516250945, "fcm_dpo/beta": 0.005231472663581371, "fcm_dpo/delta": 0.045737870037555695, "fcm_dpo/margin": 69.0350112915039, "fcm_dpo/q_t": 0.4225993752479553, "grad_norm": 16.398719787597656, "learning_rate": 5.430681259032957e-08, "logits/chosen": 0.7519247531890869, "logits/rejected": 0.7029865980148315, "logps/chosen": -257.3060302734375, "logps/ref_chosen": -61.637413024902344, "logps/ref_rejected": -80.93138885498047, "logps/rejected": -345.635009765625, "loss": 1.1829, "margin_dpo/margin_mean": 69.0350112915039, "margin_dpo/margin_std": 139.1922607421875, "step": 535 }, { "KL/chosen_KL_mean": -169.75575256347656, "KL/mean": -222.75271606445312, "KL/rejected_KL_mean": -275.74969482421875, "KL/std": 124.12066650390625, "epoch": 0.8102796674225246, "fcm_dpo/beta": 0.00466238334774971, "fcm_dpo/delta": -0.1296248584985733, "fcm_dpo/margin": 105.99392700195312, "fcm_dpo/q_t": 0.3823320269584656, "grad_norm": 10.686798095703125, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 0.8615167140960693, "logits/rejected": 0.7933706045150757, "logps/chosen": -221.6447296142578, "logps/ref_chosen": -51.88897705078125, "logps/ref_rejected": -73.34864044189453, "logps/rejected": -349.09832763671875, "loss": 1.0272, "margin_dpo/margin_mean": 105.99392700195312, "margin_dpo/margin_std": 123.41902160644531, "step": 536 }, { "KL/chosen_KL_mean": -192.2476043701172, "KL/mean": -238.9248046875, "KL/rejected_KL_mean": -285.60198974609375, "KL/std": 134.6256866455078, "epoch": 0.8117913832199547, "fcm_dpo/beta": 0.004458250477910042, "fcm_dpo/delta": -0.021435163915157318, "fcm_dpo/margin": 93.35438537597656, "fcm_dpo/q_t": 0.4066166579723358, "grad_norm": 13.311497688293457, "learning_rate": 5.267213693697695e-08, "logits/chosen": 0.889856219291687, "logits/rejected": 0.8121524453163147, "logps/chosen": -246.4962158203125, "logps/ref_chosen": -54.248619079589844, "logps/ref_rejected": -94.94343566894531, "logps/rejected": -380.5454406738281, "loss": 1.1145, "margin_dpo/margin_mean": 93.35438537597656, "margin_dpo/margin_std": 148.28921508789062, "step": 537 }, { "KL/chosen_KL_mean": -196.03402709960938, "KL/mean": -239.52902221679688, "KL/rejected_KL_mean": -283.0240173339844, "KL/std": 132.6907958984375, "epoch": 0.8133030990173847, "fcm_dpo/beta": 0.004526033066213131, "fcm_dpo/delta": 0.00775851309299469, "fcm_dpo/margin": 86.99000549316406, "fcm_dpo/q_t": 0.41095876693725586, "grad_norm": 12.53797721862793, "learning_rate": 5.1863067244167144e-08, "logits/chosen": 0.8347610831260681, "logits/rejected": 0.8167983293533325, "logps/chosen": -266.1275634765625, "logps/ref_chosen": -70.09353637695312, "logps/ref_rejected": -79.49833679199219, "logps/rejected": -362.5223388671875, "loss": 1.1161, "margin_dpo/margin_mean": 86.99000549316406, "margin_dpo/margin_std": 134.7762451171875, "step": 538 }, { "KL/chosen_KL_mean": -199.68386840820312, "KL/mean": -234.05479431152344, "KL/rejected_KL_mean": -268.42572021484375, "KL/std": 121.34420776367188, "epoch": 0.8148148148148148, "fcm_dpo/beta": 0.004799469839781523, "fcm_dpo/delta": 0.084069162607193, "fcm_dpo/margin": 68.74186706542969, "fcm_dpo/q_t": 0.4309224486351013, "grad_norm": 14.797113418579102, "learning_rate": 5.105953986729195e-08, "logits/chosen": 0.7780267596244812, "logits/rejected": 0.7070616483688354, "logps/chosen": -261.61553955078125, "logps/ref_chosen": -61.93169403076172, "logps/ref_rejected": -84.08946228027344, "logps/rejected": -352.51519775390625, "loss": 1.1768, "margin_dpo/margin_mean": 68.74186706542969, "margin_dpo/margin_std": 133.76632690429688, "step": 539 }, { "KL/chosen_KL_mean": -183.94613647460938, "KL/mean": -238.01919555664062, "KL/rejected_KL_mean": -292.09228515625, "KL/std": 140.93951416015625, "epoch": 0.8163265306122449, "fcm_dpo/beta": 0.004561346955597401, "fcm_dpo/delta": -0.1230163723230362, "fcm_dpo/margin": 108.14613342285156, "fcm_dpo/q_t": 0.3840181827545166, "grad_norm": 12.530052185058594, "learning_rate": 5.026157728273966e-08, "logits/chosen": 0.8541229367256165, "logits/rejected": 0.7800333499908447, "logps/chosen": -246.650390625, "logps/ref_chosen": -62.704254150390625, "logps/ref_rejected": -95.63597106933594, "logps/rejected": -387.7282409667969, "loss": 1.0443, "margin_dpo/margin_mean": 108.14613342285156, "margin_dpo/margin_std": 136.21054077148438, "step": 540 }, { "KL/chosen_KL_mean": -178.83322143554688, "KL/mean": -221.52305603027344, "KL/rejected_KL_mean": -264.212890625, "KL/std": 120.34114837646484, "epoch": 0.817838246409675, "fcm_dpo/beta": 0.004383554682135582, "fcm_dpo/delta": 0.031111612915992737, "fcm_dpo/margin": 85.37966918945312, "fcm_dpo/q_t": 0.41337648034095764, "grad_norm": 11.763753890991211, "learning_rate": 4.9469201811239035e-08, "logits/chosen": 0.8471169471740723, "logits/rejected": 0.8764835596084595, "logps/chosen": -241.31405639648438, "logps/ref_chosen": -62.48084259033203, "logps/ref_rejected": -57.55541229248047, "logps/rejected": -321.768310546875, "loss": 1.1119, "margin_dpo/margin_mean": 85.37966918945312, "margin_dpo/margin_std": 117.1261978149414, "step": 541 }, { "KL/chosen_KL_mean": -156.00730895996094, "KL/mean": -203.59226989746094, "KL/rejected_KL_mean": -251.17723083496094, "KL/std": 113.75357055664062, "epoch": 0.8193499622071051, "fcm_dpo/beta": 0.00437610037624836, "fcm_dpo/delta": -0.02143116295337677, "fcm_dpo/margin": 95.16992950439453, "fcm_dpo/q_t": 0.4032912254333496, "grad_norm": 12.278472900390625, "learning_rate": 4.868243561723534e-08, "logits/chosen": 0.8880411386489868, "logits/rejected": 0.8495569229125977, "logps/chosen": -205.46218872070312, "logps/ref_chosen": -49.454891204833984, "logps/ref_rejected": -65.33275604248047, "logps/rejected": -316.510009765625, "loss": 1.0985, "margin_dpo/margin_mean": 95.16992950439453, "margin_dpo/margin_std": 138.26119995117188, "step": 542 }, { "KL/chosen_KL_mean": -170.74090576171875, "KL/mean": -218.884033203125, "KL/rejected_KL_mean": -267.02716064453125, "KL/std": 121.83112335205078, "epoch": 0.8208616780045351, "fcm_dpo/beta": 0.004359533078968525, "fcm_dpo/delta": -0.025253912433981895, "fcm_dpo/margin": 96.28624725341797, "fcm_dpo/q_t": 0.40311557054519653, "grad_norm": 10.389751434326172, "learning_rate": 4.790130070827028e-08, "logits/chosen": 0.8509294986724854, "logits/rejected": 0.7792444825172424, "logps/chosen": -221.84176635742188, "logps/ref_chosen": -51.100860595703125, "logps/ref_rejected": -76.06130981445312, "logps/rejected": -343.0884704589844, "loss": 1.093, "margin_dpo/margin_mean": 96.2862548828125, "margin_dpo/margin_std": 134.68243408203125, "step": 543 }, { "KL/chosen_KL_mean": -182.93453979492188, "KL/mean": -237.28018188476562, "KL/rejected_KL_mean": -291.62579345703125, "KL/std": 133.7188720703125, "epoch": 0.8223733938019653, "fcm_dpo/beta": 0.004085186868906021, "fcm_dpo/delta": -0.0571129247546196, "fcm_dpo/margin": 108.6912841796875, "fcm_dpo/q_t": 0.3980041742324829, "grad_norm": 13.596083641052246, "learning_rate": 4.7125818934366454e-08, "logits/chosen": 0.8448378443717957, "logits/rejected": 0.7801798582077026, "logps/chosen": -243.21177673339844, "logps/ref_chosen": -60.2772331237793, "logps/ref_rejected": -88.40553283691406, "logps/rejected": -380.0313720703125, "loss": 1.0862, "margin_dpo/margin_mean": 108.6912841796875, "margin_dpo/margin_std": 157.70164489746094, "step": 544 }, { "KL/chosen_KL_mean": -198.3109130859375, "KL/mean": -228.24383544921875, "KL/rejected_KL_mean": -258.1767883300781, "KL/std": 123.42090606689453, "epoch": 0.8238851095993953, "fcm_dpo/beta": 0.004550996236503124, "fcm_dpo/delta": 0.1455247700214386, "fcm_dpo/margin": 59.86585998535156, "fcm_dpo/q_t": 0.44071030616760254, "grad_norm": 14.440302848815918, "learning_rate": 4.635601198741607e-08, "logits/chosen": 0.8114050030708313, "logits/rejected": 0.7570880055427551, "logps/chosen": -259.9261474609375, "logps/ref_chosen": -61.61524963378906, "logps/ref_rejected": -78.71266174316406, "logps/rejected": -336.8894348144531, "loss": 1.2182, "margin_dpo/margin_mean": 59.86585998535156, "margin_dpo/margin_std": 132.5935821533203, "step": 545 }, { "KL/chosen_KL_mean": -189.25192260742188, "KL/mean": -222.0496826171875, "KL/rejected_KL_mean": -254.84744262695312, "KL/std": 116.97176361083984, "epoch": 0.8253968253968254, "fcm_dpo/beta": 0.0049902526661753654, "fcm_dpo/delta": 0.08677574247121811, "fcm_dpo/margin": 65.59550476074219, "fcm_dpo/q_t": 0.4274669885635376, "grad_norm": 19.589603424072266, "learning_rate": 4.559190140057428e-08, "logits/chosen": 0.9217461347579956, "logits/rejected": 0.9196021556854248, "logps/chosen": -248.565185546875, "logps/ref_chosen": -59.313262939453125, "logps/ref_rejected": -64.73631286621094, "logps/rejected": -319.583740234375, "loss": 1.2077, "margin_dpo/margin_mean": 65.59550476074219, "margin_dpo/margin_std": 144.1741180419922, "step": 546 }, { "KL/chosen_KL_mean": -165.40274047851562, "KL/mean": -215.89859008789062, "KL/rejected_KL_mean": -266.3944396972656, "KL/std": 122.97581481933594, "epoch": 0.8269085411942555, "fcm_dpo/beta": 0.004632370080798864, "fcm_dpo/delta": -0.09499009698629379, "fcm_dpo/margin": 100.99168395996094, "fcm_dpo/q_t": 0.389495313167572, "grad_norm": 12.160846710205078, "learning_rate": 4.483350854765672e-08, "logits/chosen": 0.7838672995567322, "logits/rejected": 0.7346804141998291, "logps/chosen": -220.37948608398438, "logps/ref_chosen": -54.97674560546875, "logps/ref_rejected": -75.35922241210938, "logps/rejected": -341.753662109375, "loss": 1.0705, "margin_dpo/margin_mean": 100.99168395996094, "margin_dpo/margin_std": 140.6912841796875, "step": 547 }, { "KL/chosen_KL_mean": -196.61190795898438, "KL/mean": -225.8825225830078, "KL/rejected_KL_mean": -255.1531524658203, "KL/std": 121.33120727539062, "epoch": 0.8284202569916855, "fcm_dpo/beta": 0.005131861194968224, "fcm_dpo/delta": 0.11382049322128296, "fcm_dpo/margin": 58.541263580322266, "fcm_dpo/q_t": 0.43540245294570923, "grad_norm": 17.081180572509766, "learning_rate": 4.4080854642541826e-08, "logits/chosen": 0.7621930837631226, "logits/rejected": 0.7174137830734253, "logps/chosen": -259.82257080078125, "logps/ref_chosen": -63.21067428588867, "logps/ref_rejected": -81.23347473144531, "logps/rejected": -336.3866271972656, "loss": 1.2171, "margin_dpo/margin_mean": 58.54126739501953, "margin_dpo/margin_std": 132.77597045898438, "step": 548 }, { "KL/chosen_KL_mean": -194.5125732421875, "KL/mean": -231.58607482910156, "KL/rejected_KL_mean": -268.6595458984375, "KL/std": 135.0965576171875, "epoch": 0.8299319727891157, "fcm_dpo/beta": 0.005150582641363144, "fcm_dpo/delta": 0.02017725259065628, "fcm_dpo/margin": 74.14700317382812, "fcm_dpo/q_t": 0.4193522334098816, "grad_norm": 20.222238540649414, "learning_rate": 4.333396073857723e-08, "logits/chosen": 0.9132688045501709, "logits/rejected": 0.8558034896850586, "logps/chosen": -258.78607177734375, "logps/ref_chosen": -64.27351379394531, "logps/ref_rejected": -92.31663513183594, "logps/rejected": -360.9761962890625, "loss": 1.195, "margin_dpo/margin_mean": 74.14698791503906, "margin_dpo/margin_std": 160.41629028320312, "step": 549 }, { "KL/chosen_KL_mean": -193.46360778808594, "KL/mean": -221.16964721679688, "KL/rejected_KL_mean": -248.87567138671875, "KL/std": 118.34536743164062, "epoch": 0.8314436885865457, "fcm_dpo/beta": 0.0052723027765750885, "fcm_dpo/delta": -0.006274649873375893, "fcm_dpo/margin": 55.412071228027344, "fcm_dpo/q_t": 0.4349931478500366, "grad_norm": 15.994507789611816, "learning_rate": 4.259284772799099e-08, "logits/chosen": 0.8577879667282104, "logits/rejected": 0.8388744592666626, "logps/chosen": -249.6940460205078, "logps/ref_chosen": -56.230438232421875, "logps/ref_rejected": -62.59788513183594, "logps/rejected": -311.47357177734375, "loss": 1.2279, "margin_dpo/margin_mean": 55.41207504272461, "margin_dpo/margin_std": 129.4113006591797, "step": 550 }, { "KL/chosen_KL_mean": -202.26455688476562, "KL/mean": -236.17559814453125, "KL/rejected_KL_mean": -270.086669921875, "KL/std": 128.12054443359375, "epoch": 0.8329554043839759, "fcm_dpo/beta": 0.005462060682475567, "fcm_dpo/delta": 0.03551365062594414, "fcm_dpo/margin": 67.82211303710938, "fcm_dpo/q_t": 0.4226077198982239, "grad_norm": 15.54771614074707, "learning_rate": 4.1857536341307176e-08, "logits/chosen": 0.857745885848999, "logits/rejected": 0.8316745758056641, "logps/chosen": -270.01177978515625, "logps/ref_chosen": -67.74720764160156, "logps/ref_rejected": -87.04285430908203, "logps/rejected": -357.1295166015625, "loss": 1.1536, "margin_dpo/margin_mean": 67.82211303710938, "margin_dpo/margin_std": 127.63706970214844, "step": 551 }, { "KL/chosen_KL_mean": -185.70755004882812, "KL/mean": -225.22647094726562, "KL/rejected_KL_mean": -264.74542236328125, "KL/std": 131.25448608398438, "epoch": 0.8344671201814059, "fcm_dpo/beta": 0.005472094286233187, "fcm_dpo/delta": -0.044923026114702225, "fcm_dpo/margin": 79.03784942626953, "fcm_dpo/q_t": 0.40045636892318726, "grad_norm": 20.222753524780273, "learning_rate": 4.112804714676593e-08, "logits/chosen": 0.8084758520126343, "logits/rejected": 0.7642360329627991, "logps/chosen": -248.63381958007812, "logps/ref_chosen": -62.92625427246094, "logps/ref_rejected": -82.98365783691406, "logps/rejected": -347.72906494140625, "loss": 1.1101, "margin_dpo/margin_mean": 79.037841796875, "margin_dpo/margin_std": 123.42013549804688, "step": 552 }, { "KL/chosen_KL_mean": -201.88070678710938, "KL/mean": -240.699462890625, "KL/rejected_KL_mean": -279.5182189941406, "KL/std": 127.67655944824219, "epoch": 0.8359788359788359, "fcm_dpo/beta": 0.005206805653870106, "fcm_dpo/delta": -0.005316682159900665, "fcm_dpo/margin": 77.63748168945312, "fcm_dpo/q_t": 0.4139271080493927, "grad_norm": 14.778176307678223, "learning_rate": 4.0404400549748144e-08, "logits/chosen": 0.8216131329536438, "logits/rejected": 0.7376811504364014, "logps/chosen": -257.919189453125, "logps/ref_chosen": -56.038490295410156, "logps/ref_rejected": -84.48454284667969, "logps/rejected": -364.00274658203125, "loss": 1.1602, "margin_dpo/margin_mean": 77.63748168945312, "margin_dpo/margin_std": 151.02700805664062, "step": 553 }, { "KL/chosen_KL_mean": -181.76394653320312, "KL/mean": -227.24557495117188, "KL/rejected_KL_mean": -272.7271728515625, "KL/std": 120.14936065673828, "epoch": 0.8374905517762661, "fcm_dpo/beta": 0.004935280419886112, "fcm_dpo/delta": -0.06360197812318802, "fcm_dpo/margin": 90.9632568359375, "fcm_dpo/q_t": 0.3936072587966919, "grad_norm": 14.283166885375977, "learning_rate": 3.968661679220467e-08, "logits/chosen": 0.7989473342895508, "logits/rejected": 0.7961627244949341, "logps/chosen": -246.2945556640625, "logps/ref_chosen": -64.53059387207031, "logps/ref_rejected": -71.2155990600586, "logps/rejected": -343.94281005859375, "loss": 1.0841, "margin_dpo/margin_mean": 90.9632568359375, "margin_dpo/margin_std": 130.098876953125, "step": 554 }, { "KL/chosen_KL_mean": -197.27484130859375, "KL/mean": -231.05706787109375, "KL/rejected_KL_mean": -264.83929443359375, "KL/std": 118.86752319335938, "epoch": 0.8390022675736961, "fcm_dpo/beta": 0.004804479889571667, "fcm_dpo/delta": -0.02939688041806221, "fcm_dpo/margin": 67.564453125, "fcm_dpo/q_t": 0.4228440821170807, "grad_norm": 14.867000579833984, "learning_rate": 3.89747159520904e-08, "logits/chosen": 0.8469399213790894, "logits/rejected": 0.8320943117141724, "logps/chosen": -263.9267578125, "logps/ref_chosen": -66.65191650390625, "logps/ref_rejected": -68.6667251586914, "logps/rejected": -333.5060119628906, "loss": 1.2053, "margin_dpo/margin_mean": 67.564453125, "margin_dpo/margin_std": 141.87362670898438, "step": 555 }, { "KL/chosen_KL_mean": -193.35748291015625, "KL/mean": -228.408935546875, "KL/rejected_KL_mean": -263.46038818359375, "KL/std": 123.95942687988281, "epoch": 0.8405139833711263, "fcm_dpo/beta": 0.004928875714540482, "fcm_dpo/delta": 0.06609125435352325, "fcm_dpo/margin": 70.10289001464844, "fcm_dpo/q_t": 0.4252849817276001, "grad_norm": 13.865388870239258, "learning_rate": 3.826871794280192e-08, "logits/chosen": 0.8748590350151062, "logits/rejected": 0.8386209607124329, "logps/chosen": -246.18984985351562, "logps/ref_chosen": -52.832366943359375, "logps/ref_rejected": -64.49044036865234, "logps/rejected": -327.9508056640625, "loss": 1.1806, "margin_dpo/margin_mean": 70.10289764404297, "margin_dpo/margin_std": 136.9292755126953, "step": 556 }, { "KL/chosen_KL_mean": -192.00454711914062, "KL/mean": -241.59774780273438, "KL/rejected_KL_mean": -291.19091796875, "KL/std": 123.56182861328125, "epoch": 0.8420256991685563, "fcm_dpo/beta": 0.004650956951081753, "fcm_dpo/delta": -0.08415950834751129, "fcm_dpo/margin": 99.18640899658203, "fcm_dpo/q_t": 0.3913089632987976, "grad_norm": 11.975605964660645, "learning_rate": 3.756864251262143e-08, "logits/chosen": 0.9212883710861206, "logits/rejected": 0.8599925637245178, "logps/chosen": -247.04052734375, "logps/ref_chosen": -55.03598403930664, "logps/ref_rejected": -75.80644989013672, "logps/rejected": -366.99737548828125, "loss": 1.0543, "margin_dpo/margin_mean": 99.18640899658203, "margin_dpo/margin_std": 121.38153076171875, "step": 557 }, { "KL/chosen_KL_mean": -181.1847686767578, "KL/mean": -234.77792358398438, "KL/rejected_KL_mean": -288.3710632324219, "KL/std": 131.50604248046875, "epoch": 0.8435374149659864, "fcm_dpo/beta": 0.004277751315385103, "fcm_dpo/delta": -0.08086490631103516, "fcm_dpo/margin": 107.18629455566406, "fcm_dpo/q_t": 0.39286717772483826, "grad_norm": 10.80577564239502, "learning_rate": 3.687450924416341e-08, "logits/chosen": 0.8621159195899963, "logits/rejected": 0.8185891509056091, "logps/chosen": -244.41111755371094, "logps/ref_chosen": -63.226348876953125, "logps/ref_rejected": -91.46881866455078, "logps/rejected": -379.8398742675781, "loss": 1.0614, "margin_dpo/margin_mean": 107.18629455566406, "margin_dpo/margin_std": 138.95535278320312, "step": 558 }, { "KL/chosen_KL_mean": -184.31796264648438, "KL/mean": -231.82647705078125, "KL/rejected_KL_mean": -279.3349609375, "KL/std": 131.37059020996094, "epoch": 0.8450491307634165, "fcm_dpo/beta": 0.0041785649955272675, "fcm_dpo/delta": 0.0005370378494262695, "fcm_dpo/margin": 95.01700592041016, "fcm_dpo/q_t": 0.4134005606174469, "grad_norm": 10.951775550842285, "learning_rate": 3.6186337553827743e-08, "logits/chosen": 0.8456987738609314, "logits/rejected": 0.7955515384674072, "logps/chosen": -245.839599609375, "logps/ref_chosen": -61.521644592285156, "logps/ref_rejected": -82.83859252929688, "logps/rejected": -362.1735534667969, "loss": 1.1311, "margin_dpo/margin_mean": 95.01700592041016, "margin_dpo/margin_std": 159.78627014160156, "step": 559 }, { "KL/chosen_KL_mean": -200.00225830078125, "KL/mean": -245.2239990234375, "KL/rejected_KL_mean": -290.44573974609375, "KL/std": 124.63345336914062, "epoch": 0.8465608465608465, "fcm_dpo/beta": 0.004382754676043987, "fcm_dpo/delta": 0.002800390124320984, "fcm_dpo/margin": 90.44346618652344, "fcm_dpo/q_t": 0.4082622528076172, "grad_norm": 15.912379264831543, "learning_rate": 3.550414669125573e-08, "logits/chosen": 0.8386662006378174, "logits/rejected": 0.8095963001251221, "logps/chosen": -260.64349365234375, "logps/ref_chosen": -60.64122009277344, "logps/ref_rejected": -78.75474548339844, "logps/rejected": -369.20050048828125, "loss": 1.0935, "margin_dpo/margin_mean": 90.4434814453125, "margin_dpo/margin_std": 116.68689727783203, "step": 560 }, { "KL/chosen_KL_mean": -186.88943481445312, "KL/mean": -225.20635986328125, "KL/rejected_KL_mean": -263.52325439453125, "KL/std": 120.019287109375, "epoch": 0.8480725623582767, "fcm_dpo/beta": 0.004479461815208197, "fcm_dpo/delta": 0.0687173530459404, "fcm_dpo/margin": 76.63383483886719, "fcm_dpo/q_t": 0.42475682497024536, "grad_norm": 11.804110527038574, "learning_rate": 3.482795573879241e-08, "logits/chosen": 0.7993382215499878, "logits/rejected": 0.7716021537780762, "logps/chosen": -249.38803100585938, "logps/ref_chosen": -62.49859619140625, "logps/ref_rejected": -78.72064208984375, "logps/rejected": -342.243896484375, "loss": 1.1584, "margin_dpo/margin_mean": 76.63383483886719, "margin_dpo/margin_std": 132.6756591796875, "step": 561 }, { "KL/chosen_KL_mean": -198.23243713378906, "KL/mean": -250.70225524902344, "KL/rejected_KL_mean": -303.17205810546875, "KL/std": 146.47659301757812, "epoch": 0.8495842781557067, "fcm_dpo/beta": 0.00422039907425642, "fcm_dpo/delta": -0.06546664237976074, "fcm_dpo/margin": 104.93963623046875, "fcm_dpo/q_t": 0.3977462649345398, "grad_norm": 15.125443458557129, "learning_rate": 3.415778361095226e-08, "logits/chosen": 0.8647781610488892, "logits/rejected": 0.8419674038887024, "logps/chosen": -273.01416015625, "logps/ref_chosen": -74.78173828125, "logps/ref_rejected": -92.63499450683594, "logps/rejected": -395.80706787109375, "loss": 1.0757, "margin_dpo/margin_mean": 104.93963623046875, "margin_dpo/margin_std": 140.125244140625, "step": 562 }, { "KL/chosen_KL_mean": -163.90536499023438, "KL/mean": -207.61956787109375, "KL/rejected_KL_mean": -251.3337860107422, "KL/std": 107.05549621582031, "epoch": 0.8510959939531368, "fcm_dpo/beta": 0.004395781550556421, "fcm_dpo/delta": 0.019247282296419144, "fcm_dpo/margin": 87.4284439086914, "fcm_dpo/q_t": 0.41145721077919006, "grad_norm": 18.718957901000977, "learning_rate": 3.349364905389032e-08, "logits/chosen": 0.9080963730812073, "logits/rejected": 0.862755537033081, "logps/chosen": -214.10385131835938, "logps/ref_chosen": -50.19850158691406, "logps/ref_rejected": -66.76687622070312, "logps/rejected": -318.10064697265625, "loss": 1.124, "margin_dpo/margin_mean": 87.4284439086914, "margin_dpo/margin_std": 136.39706420898438, "step": 563 }, { "KL/chosen_KL_mean": -165.5902099609375, "KL/mean": -220.1693115234375, "KL/rejected_KL_mean": -274.7484130859375, "KL/std": 116.85183715820312, "epoch": 0.8526077097505669, "fcm_dpo/beta": 0.004158479161560535, "fcm_dpo/delta": -0.07036039233207703, "fcm_dpo/margin": 109.1581802368164, "fcm_dpo/q_t": 0.39226311445236206, "grad_norm": 11.553539276123047, "learning_rate": 3.283557064487785e-08, "logits/chosen": 0.8010308146476746, "logits/rejected": 0.7797284126281738, "logps/chosen": -221.33106994628906, "logps/ref_chosen": -55.7408447265625, "logps/ref_rejected": -74.82323455810547, "logps/rejected": -349.5716552734375, "loss": 1.0657, "margin_dpo/margin_mean": 109.1581802368164, "margin_dpo/margin_std": 142.97979736328125, "step": 564 }, { "KL/chosen_KL_mean": -198.5447998046875, "KL/mean": -242.15261840820312, "KL/rejected_KL_mean": -285.76043701171875, "KL/std": 124.1990966796875, "epoch": 0.854119425547997, "fcm_dpo/beta": 0.004263964481651783, "fcm_dpo/delta": 0.033044081181287766, "fcm_dpo/margin": 87.21564483642578, "fcm_dpo/q_t": 0.41419297456741333, "grad_norm": 13.659796714782715, "learning_rate": 3.218356679178252e-08, "logits/chosen": 0.8753294944763184, "logits/rejected": 0.833126962184906, "logps/chosen": -256.8821716308594, "logps/ref_chosen": -58.33738327026367, "logps/ref_rejected": -78.31776428222656, "logps/rejected": -364.07818603515625, "loss": 1.1152, "margin_dpo/margin_mean": 87.21563720703125, "margin_dpo/margin_std": 123.91053009033203, "step": 565 }, { "KL/chosen_KL_mean": -190.3877716064453, "KL/mean": -228.84381103515625, "KL/rejected_KL_mean": -267.29986572265625, "KL/std": 131.32406616210938, "epoch": 0.8556311413454271, "fcm_dpo/beta": 0.00454290583729744, "fcm_dpo/delta": 0.05613939091563225, "fcm_dpo/margin": 76.91210174560547, "fcm_dpo/q_t": 0.42436790466308594, "grad_norm": 12.740806579589844, "learning_rate": 3.1537655732553764e-08, "logits/chosen": 0.8327311277389526, "logits/rejected": 0.830025851726532, "logps/chosen": -261.61151123046875, "logps/ref_chosen": -71.22373962402344, "logps/ref_rejected": -71.11601257324219, "logps/rejected": -338.4158935546875, "loss": 1.1979, "margin_dpo/margin_mean": 76.912109375, "margin_dpo/margin_std": 165.12557983398438, "step": 566 }, { "KL/chosen_KL_mean": -179.38357543945312, "KL/mean": -228.2648468017578, "KL/rejected_KL_mean": -277.1461181640625, "KL/std": 120.70500946044922, "epoch": 0.8571428571428571, "fcm_dpo/beta": 0.004256717395037413, "fcm_dpo/delta": -0.02629101276397705, "fcm_dpo/margin": 97.76256561279297, "fcm_dpo/q_t": 0.4018706977367401, "grad_norm": 10.185369491577148, "learning_rate": 3.089785553471233e-08, "logits/chosen": 0.8462409973144531, "logits/rejected": 0.7748077511787415, "logps/chosen": -232.05284118652344, "logps/ref_chosen": -52.669273376464844, "logps/ref_rejected": -74.34785461425781, "logps/rejected": -351.4939880371094, "loss": 1.093, "margin_dpo/margin_mean": 97.76256561279297, "margin_dpo/margin_std": 132.65936279296875, "step": 567 }, { "KL/chosen_KL_mean": -167.775634765625, "KL/mean": -226.44500732421875, "KL/rejected_KL_mean": -285.1143798828125, "KL/std": 128.91400146484375, "epoch": 0.8586545729402872, "fcm_dpo/beta": 0.004162575583904982, "fcm_dpo/delta": -0.11834442615509033, "fcm_dpo/margin": 117.33872985839844, "fcm_dpo/q_t": 0.3848383128643036, "grad_norm": 13.649627685546875, "learning_rate": 3.026418409484513e-08, "logits/chosen": 0.8517801761627197, "logits/rejected": 0.7852275371551514, "logps/chosen": -219.95364379882812, "logps/ref_chosen": -52.178001403808594, "logps/ref_rejected": -85.8277587890625, "logps/rejected": -370.942138671875, "loss": 1.0358, "margin_dpo/margin_mean": 117.3387222290039, "margin_dpo/margin_std": 133.35511779785156, "step": 568 }, { "KL/chosen_KL_mean": -196.99539184570312, "KL/mean": -225.05218505859375, "KL/rejected_KL_mean": -253.1090087890625, "KL/std": 130.43826293945312, "epoch": 0.8601662887377173, "fcm_dpo/beta": 0.0039661955088377, "fcm_dpo/delta": 0.02197522297501564, "fcm_dpo/margin": 56.11360549926758, "fcm_dpo/q_t": 0.4494395852088928, "grad_norm": 13.287290573120117, "learning_rate": 2.963665913810451e-08, "logits/chosen": 0.7764110565185547, "logits/rejected": 0.7512776851654053, "logps/chosen": -259.6446533203125, "logps/ref_chosen": -62.649261474609375, "logps/ref_rejected": -75.4298324584961, "logps/rejected": -328.538818359375, "loss": 1.2453, "margin_dpo/margin_mean": 56.11360549926758, "margin_dpo/margin_std": 131.75128173828125, "step": 569 }, { "KL/chosen_KL_mean": -173.2542266845703, "KL/mean": -227.6485595703125, "KL/rejected_KL_mean": -282.04290771484375, "KL/std": 120.7261962890625, "epoch": 0.8616780045351474, "fcm_dpo/beta": 0.003945560660213232, "fcm_dpo/delta": -0.03856857866048813, "fcm_dpo/margin": 108.78868103027344, "fcm_dpo/q_t": 0.40049558877944946, "grad_norm": 11.730306625366211, "learning_rate": 2.9015298217712453e-08, "logits/chosen": 0.8236995935440063, "logits/rejected": 0.7503319978713989, "logps/chosen": -223.2960205078125, "logps/ref_chosen": -50.04179382324219, "logps/ref_rejected": -78.27146911621094, "logps/rejected": -360.3143615722656, "loss": 1.0746, "margin_dpo/margin_mean": 108.7886734008789, "margin_dpo/margin_std": 139.9506378173828, "step": 570 }, { "KL/chosen_KL_mean": -183.8597412109375, "KL/mean": -215.78030395507812, "KL/rejected_KL_mean": -247.7008514404297, "KL/std": 117.1596908569336, "epoch": 0.8631897203325775, "fcm_dpo/beta": 0.003919776063412428, "fcm_dpo/delta": 0.05289241671562195, "fcm_dpo/margin": 63.841087341308594, "fcm_dpo/q_t": 0.44315797090530396, "grad_norm": 10.650339126586914, "learning_rate": 2.840011871446962e-08, "logits/chosen": 0.8218934535980225, "logits/rejected": 0.8082360625267029, "logps/chosen": -237.51657104492188, "logps/ref_chosen": -53.65681457519531, "logps/ref_rejected": -66.13298034667969, "logps/rejected": -313.8338317871094, "loss": 1.2213, "margin_dpo/margin_mean": 63.841087341308594, "margin_dpo/margin_std": 135.2704315185547, "step": 571 }, { "KL/chosen_KL_mean": -192.3538818359375, "KL/mean": -225.6499786376953, "KL/rejected_KL_mean": -258.9460754394531, "KL/std": 122.84170532226562, "epoch": 0.8647014361300076, "fcm_dpo/beta": 0.0040804725140333176, "fcm_dpo/delta": 0.02796788141131401, "fcm_dpo/margin": 66.59219360351562, "fcm_dpo/q_t": 0.43682506680488586, "grad_norm": 12.89725399017334, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 0.824752926826477, "logits/rejected": 0.863124668598175, "logps/chosen": -267.17181396484375, "logps/ref_chosen": -74.81792449951172, "logps/ref_rejected": -65.88681030273438, "logps/rejected": -324.8328857421875, "loss": 1.1976, "margin_dpo/margin_mean": 66.59219360351562, "margin_dpo/margin_std": 126.39369201660156, "step": 572 }, { "KL/chosen_KL_mean": -205.2719268798828, "KL/mean": -247.1676788330078, "KL/rejected_KL_mean": -289.06341552734375, "KL/std": 138.2989501953125, "epoch": 0.8662131519274376, "fcm_dpo/beta": 0.0043049827218055725, "fcm_dpo/delta": 0.0475587323307991, "fcm_dpo/margin": 83.79151916503906, "fcm_dpo/q_t": 0.423142671585083, "grad_norm": 12.913630485534668, "learning_rate": 2.718837261761528e-08, "logits/chosen": 0.838950514793396, "logits/rejected": 0.8056519031524658, "logps/chosen": -273.99755859375, "logps/ref_chosen": -68.72564697265625, "logps/ref_rejected": -88.16201782226562, "logps/rejected": -377.2254638671875, "loss": 1.1755, "margin_dpo/margin_mean": 83.79151916503906, "margin_dpo/margin_std": 166.9637451171875, "step": 573 }, { "KL/chosen_KL_mean": -182.779296875, "KL/mean": -236.46792602539062, "KL/rejected_KL_mean": -290.156494140625, "KL/std": 128.50814819335938, "epoch": 0.8677248677248677, "fcm_dpo/beta": 0.004241702146828175, "fcm_dpo/delta": -0.07349122315645218, "fcm_dpo/margin": 107.37722778320312, "fcm_dpo/q_t": 0.3932126462459564, "grad_norm": 11.176490783691406, "learning_rate": 2.659183991914696e-08, "logits/chosen": 0.8853078484535217, "logits/rejected": 0.8360748887062073, "logps/chosen": -239.09271240234375, "logps/ref_chosen": -56.31340026855469, "logps/ref_rejected": -83.91553497314453, "logps/rejected": -374.0720520019531, "loss": 1.0484, "margin_dpo/margin_mean": 107.37722778320312, "margin_dpo/margin_std": 123.63481140136719, "step": 574 }, { "KL/chosen_KL_mean": -188.70425415039062, "KL/mean": -226.75677490234375, "KL/rejected_KL_mean": -264.809326171875, "KL/std": 128.05438232421875, "epoch": 0.8692365835222978, "fcm_dpo/beta": 0.003928046207875013, "fcm_dpo/delta": -0.026024334132671356, "fcm_dpo/margin": 76.10505676269531, "fcm_dpo/q_t": 0.4327458143234253, "grad_norm": 11.480947494506836, "learning_rate": 2.600155642716606e-08, "logits/chosen": 0.8551039099693298, "logits/rejected": 0.7906278371810913, "logps/chosen": -253.2883758544922, "logps/ref_chosen": -64.5841293334961, "logps/ref_rejected": -93.47034454345703, "logps/rejected": -358.2796630859375, "loss": 1.2022, "margin_dpo/margin_mean": 76.10505676269531, "margin_dpo/margin_std": 155.01919555664062, "step": 575 }, { "KL/chosen_KL_mean": -165.97982788085938, "KL/mean": -223.73573303222656, "KL/rejected_KL_mean": -281.49163818359375, "KL/std": 128.92742919921875, "epoch": 0.8707482993197279, "fcm_dpo/beta": 0.0036893547512590885, "fcm_dpo/delta": -0.03991346433758736, "fcm_dpo/margin": 115.5118179321289, "fcm_dpo/q_t": 0.39912840723991394, "grad_norm": 10.718217849731445, "learning_rate": 2.5417538653170754e-08, "logits/chosen": 0.8531392216682434, "logits/rejected": 0.7734321355819702, "logps/chosen": -219.26034545898438, "logps/ref_chosen": -53.28052520751953, "logps/ref_rejected": -84.2000503540039, "logps/rejected": -365.69171142578125, "loss": 1.0733, "margin_dpo/margin_mean": 115.5118179321289, "margin_dpo/margin_std": 141.59327697753906, "step": 576 }, { "KL/chosen_KL_mean": -186.1846923828125, "KL/mean": -221.00384521484375, "KL/rejected_KL_mean": -255.822998046875, "KL/std": 124.04796600341797, "epoch": 0.872260015117158, "fcm_dpo/beta": 0.0041536302305758, "fcm_dpo/delta": 0.12937919795513153, "fcm_dpo/margin": 69.6383056640625, "fcm_dpo/q_t": 0.43577146530151367, "grad_norm": 13.241288185119629, "learning_rate": 2.4839802933393607e-08, "logits/chosen": 0.8255484104156494, "logits/rejected": 0.8245355486869812, "logps/chosen": -248.50938415527344, "logps/ref_chosen": -62.32468795776367, "logps/ref_rejected": -67.300537109375, "logps/rejected": -323.12353515625, "loss": 1.1965, "margin_dpo/margin_mean": 69.6383056640625, "margin_dpo/margin_std": 131.79922485351562, "step": 577 }, { "KL/chosen_KL_mean": -177.88160705566406, "KL/mean": -212.32455444335938, "KL/rejected_KL_mean": -246.7675018310547, "KL/std": 131.18441772460938, "epoch": 0.873771730914588, "fcm_dpo/beta": 0.004623678512871265, "fcm_dpo/delta": 0.09660382568836212, "fcm_dpo/margin": 68.88590240478516, "fcm_dpo/q_t": 0.4327906668186188, "grad_norm": 14.404627799987793, "learning_rate": 2.4268365428344733e-08, "logits/chosen": 0.8864824771881104, "logits/rejected": 0.8790782690048218, "logps/chosen": -234.5371856689453, "logps/ref_chosen": -56.65557861328125, "logps/ref_rejected": -68.21835327148438, "logps/rejected": -314.98583984375, "loss": 1.2075, "margin_dpo/margin_mean": 68.88590240478516, "margin_dpo/margin_std": 151.61154174804688, "step": 578 }, { "KL/chosen_KL_mean": -182.9407501220703, "KL/mean": -233.8430938720703, "KL/rejected_KL_mean": -284.74542236328125, "KL/std": 119.40805053710938, "epoch": 0.8752834467120182, "fcm_dpo/beta": 0.004438809119164944, "fcm_dpo/delta": -0.06895958632230759, "fcm_dpo/margin": 101.8046875, "fcm_dpo/q_t": 0.3916490375995636, "grad_norm": 11.517224311828613, "learning_rate": 2.3703242122359357e-08, "logits/chosen": 0.7991275191307068, "logits/rejected": 0.7866017818450928, "logps/chosen": -239.7504119873047, "logps/ref_chosen": -56.809661865234375, "logps/ref_rejected": -68.09613037109375, "logps/rejected": -352.841552734375, "loss": 1.0552, "margin_dpo/margin_mean": 101.8046875, "margin_dpo/margin_std": 123.41009521484375, "step": 579 }, { "KL/chosen_KL_mean": -186.92544555664062, "KL/mean": -233.40525817871094, "KL/rejected_KL_mean": -279.88507080078125, "KL/std": 144.23040771484375, "epoch": 0.8767951625094482, "fcm_dpo/beta": 0.004471241496503353, "fcm_dpo/delta": -0.022698376327753067, "fcm_dpo/margin": 92.95962524414062, "fcm_dpo/q_t": 0.40974441170692444, "grad_norm": 14.038193702697754, "learning_rate": 2.3144448823151392e-08, "logits/chosen": 0.8470299243927002, "logits/rejected": 0.8106606602668762, "logps/chosen": -244.6255645751953, "logps/ref_chosen": -57.70011520385742, "logps/ref_rejected": -77.90664672851562, "logps/rejected": -357.7917175292969, "loss": 1.1332, "margin_dpo/margin_mean": 92.95962524414062, "margin_dpo/margin_std": 161.46102905273438, "step": 580 }, { "KL/chosen_KL_mean": -203.38998413085938, "KL/mean": -243.64874267578125, "KL/rejected_KL_mean": -283.90753173828125, "KL/std": 124.84356689453125, "epoch": 0.8783068783068783, "fcm_dpo/beta": 0.004432911053299904, "fcm_dpo/delta": 0.05242428183555603, "fcm_dpo/margin": 80.51752471923828, "fcm_dpo/q_t": 0.4223071336746216, "grad_norm": 12.285096168518066, "learning_rate": 2.259200116137039e-08, "logits/chosen": 0.8738093376159668, "logits/rejected": 0.8231921792030334, "logps/chosen": -262.72235107421875, "logps/ref_chosen": -59.332359313964844, "logps/ref_rejected": -83.64482116699219, "logps/rejected": -367.5523376464844, "loss": 1.168, "margin_dpo/margin_mean": 80.51752471923828, "margin_dpo/margin_std": 151.82763671875, "step": 581 }, { "KL/chosen_KL_mean": -186.85243225097656, "KL/mean": -224.37152099609375, "KL/rejected_KL_mean": -261.890625, "KL/std": 117.22724151611328, "epoch": 0.8798185941043084, "fcm_dpo/beta": 0.004786387085914612, "fcm_dpo/delta": 0.047857534140348434, "fcm_dpo/margin": 75.03822326660156, "fcm_dpo/q_t": 0.4219922423362732, "grad_norm": 12.014780044555664, "learning_rate": 2.204591459016525e-08, "logits/chosen": 0.8366580605506897, "logits/rejected": 0.8666825294494629, "logps/chosen": -251.01528930664062, "logps/ref_chosen": -64.16285705566406, "logps/ref_rejected": -58.632896423339844, "logps/rejected": -320.5235595703125, "loss": 1.1613, "margin_dpo/margin_mean": 75.03822326660156, "margin_dpo/margin_std": 137.0283966064453, "step": 582 }, { "KL/chosen_KL_mean": -183.95529174804688, "KL/mean": -235.9126739501953, "KL/rejected_KL_mean": -287.87005615234375, "KL/std": 138.58448791503906, "epoch": 0.8813303099017384, "fcm_dpo/beta": 0.004435844719409943, "fcm_dpo/delta": -0.08139852434396744, "fcm_dpo/margin": 103.91476440429688, "fcm_dpo/q_t": 0.3945728540420532, "grad_norm": 13.930092811584473, "learning_rate": 2.1506204384751064e-08, "logits/chosen": 0.9213510751724243, "logits/rejected": 0.8438308238983154, "logps/chosen": -235.82769775390625, "logps/ref_chosen": -51.87239456176758, "logps/ref_rejected": -83.86331176757812, "logps/rejected": -371.7333679199219, "loss": 1.0912, "margin_dpo/margin_mean": 103.91476440429688, "margin_dpo/margin_std": 158.84449768066406, "step": 583 }, { "KL/chosen_KL_mean": -168.98516845703125, "KL/mean": -212.2644805908203, "KL/rejected_KL_mean": -255.54379272460938, "KL/std": 121.94096374511719, "epoch": 0.8828420256991686, "fcm_dpo/beta": 0.004407728090882301, "fcm_dpo/delta": 0.02222277782857418, "fcm_dpo/margin": 86.5586166381836, "fcm_dpo/q_t": 0.4167934060096741, "grad_norm": 14.670916557312012, "learning_rate": 2.09728856419826e-08, "logits/chosen": 0.9364026784896851, "logits/rejected": 0.8511247634887695, "logps/chosen": -215.55654907226562, "logps/ref_chosen": -46.571388244628906, "logps/ref_rejected": -80.67969512939453, "logps/rejected": -336.2234802246094, "loss": 1.1498, "margin_dpo/margin_mean": 86.5586166381836, "margin_dpo/margin_std": 155.74969482421875, "step": 584 }, { "KL/chosen_KL_mean": -198.3673095703125, "KL/mean": -229.12481689453125, "KL/rejected_KL_mean": -259.88232421875, "KL/std": 135.75291442871094, "epoch": 0.8843537414965986, "fcm_dpo/beta": 0.004529936239123344, "fcm_dpo/delta": 0.014046979136765003, "fcm_dpo/margin": 61.515010833740234, "fcm_dpo/q_t": 0.43704885244369507, "grad_norm": 13.787379264831543, "learning_rate": 2.044597327993153e-08, "logits/chosen": 0.8455356359481812, "logits/rejected": 0.7982293367385864, "logps/chosen": -256.4918518066406, "logps/ref_chosen": -58.124534606933594, "logps/ref_rejected": -79.00538635253906, "logps/rejected": -338.8876953125, "loss": 1.2115, "margin_dpo/margin_mean": 61.5150146484375, "margin_dpo/margin_std": 131.28334045410156, "step": 585 }, { "KL/chosen_KL_mean": -179.8886260986328, "KL/mean": -218.91390991210938, "KL/rejected_KL_mean": -257.939208984375, "KL/std": 107.77200317382812, "epoch": 0.8858654572940288, "fcm_dpo/beta": 0.004673574585467577, "fcm_dpo/delta": 0.04316072538495064, "fcm_dpo/margin": 78.0505599975586, "fcm_dpo/q_t": 0.4169018268585205, "grad_norm": 11.847397804260254, "learning_rate": 1.9925482037469187e-08, "logits/chosen": 0.8758537769317627, "logits/rejected": 0.8435441255569458, "logps/chosen": -233.99026489257812, "logps/ref_chosen": -54.10163879394531, "logps/ref_rejected": -63.72113037109375, "logps/rejected": -321.66033935546875, "loss": 1.108, "margin_dpo/margin_mean": 78.0505599975586, "margin_dpo/margin_std": 101.43817138671875, "step": 586 }, { "KL/chosen_KL_mean": -194.5895538330078, "KL/mean": -232.21359252929688, "KL/rejected_KL_mean": -269.837646484375, "KL/std": 132.40415954589844, "epoch": 0.8873771730914588, "fcm_dpo/beta": 0.0048727430403232574, "fcm_dpo/delta": 0.040885716676712036, "fcm_dpo/margin": 75.24810791015625, "fcm_dpo/q_t": 0.41261202096939087, "grad_norm": 17.464710235595703, "learning_rate": 1.9411426473854687e-08, "logits/chosen": 0.9062135815620422, "logits/rejected": 0.9054749011993408, "logps/chosen": -258.0067443847656, "logps/ref_chosen": -63.41719436645508, "logps/ref_rejected": -63.47003936767578, "logps/rejected": -333.30767822265625, "loss": 1.2118, "margin_dpo/margin_mean": 75.24810791015625, "margin_dpo/margin_std": 178.36077880859375, "step": 587 }, { "KL/chosen_KL_mean": -198.12240600585938, "KL/mean": -241.58908081054688, "KL/rejected_KL_mean": -285.05572509765625, "KL/std": 132.34690856933594, "epoch": 0.8888888888888888, "fcm_dpo/beta": 0.004769774153828621, "fcm_dpo/delta": -0.021612241864204407, "fcm_dpo/margin": 86.93331909179688, "fcm_dpo/q_t": 0.4080020487308502, "grad_norm": 18.997106552124023, "learning_rate": 1.890382096832699e-08, "logits/chosen": 0.9018181562423706, "logits/rejected": 0.8665111064910889, "logps/chosen": -260.32342529296875, "logps/ref_chosen": -62.20103454589844, "logps/ref_rejected": -82.10249328613281, "logps/rejected": -367.1582336425781, "loss": 1.1557, "margin_dpo/margin_mean": 86.93331146240234, "margin_dpo/margin_std": 165.61778259277344, "step": 588 }, { "KL/chosen_KL_mean": -183.2729034423828, "KL/mean": -228.86248779296875, "KL/rejected_KL_mean": -274.45208740234375, "KL/std": 124.7643051147461, "epoch": 0.890400604686319, "fcm_dpo/beta": 0.004779135808348656, "fcm_dpo/delta": -0.04623799026012421, "fcm_dpo/margin": 91.17916870117188, "fcm_dpo/q_t": 0.39753201603889465, "grad_norm": 11.72799015045166, "learning_rate": 1.840267971970344e-08, "logits/chosen": 0.8504548668861389, "logits/rejected": 0.8245229721069336, "logps/chosen": -239.98651123046875, "logps/ref_chosen": -56.71361541748047, "logps/ref_rejected": -76.7366943359375, "logps/rejected": -351.1887512207031, "loss": 1.0765, "margin_dpo/margin_mean": 91.17916870117188, "margin_dpo/margin_std": 120.0302734375, "step": 589 }, { "KL/chosen_KL_mean": -199.25860595703125, "KL/mean": -239.832275390625, "KL/rejected_KL_mean": -280.40594482421875, "KL/std": 113.57868957519531, "epoch": 0.891912320483749, "fcm_dpo/beta": 0.00477856770157814, "fcm_dpo/delta": 0.012983638793230057, "fcm_dpo/margin": 81.1473388671875, "fcm_dpo/q_t": 0.4115731716156006, "grad_norm": 17.637571334838867, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 0.7954062819480896, "logits/rejected": 0.7640959620475769, "logps/chosen": -265.7724304199219, "logps/ref_chosen": -66.5138168334961, "logps/ref_rejected": -85.70820617675781, "logps/rejected": -366.1141357421875, "loss": 1.1162, "margin_dpo/margin_mean": 81.1473388671875, "margin_dpo/margin_std": 120.72219848632812, "step": 590 }, { "KL/chosen_KL_mean": -169.27191162109375, "KL/mean": -223.61370849609375, "KL/rejected_KL_mean": -277.95550537109375, "KL/std": 137.65127563476562, "epoch": 0.8934240362811792, "fcm_dpo/beta": 0.004203197546303272, "fcm_dpo/delta": -0.08832882344722748, "fcm_dpo/margin": 108.68359375, "fcm_dpo/q_t": 0.39565524458885193, "grad_norm": 12.918521881103516, "learning_rate": 1.7419845883949098e-08, "logits/chosen": 0.9233525991439819, "logits/rejected": 0.8787599802017212, "logps/chosen": -229.96908569335938, "logps/ref_chosen": -60.697181701660156, "logps/ref_rejected": -86.12278747558594, "logps/rejected": -364.07830810546875, "loss": 1.1002, "margin_dpo/margin_mean": 108.68359375, "margin_dpo/margin_std": 169.64715576171875, "step": 591 }, { "KL/chosen_KL_mean": -185.65309143066406, "KL/mean": -226.5780029296875, "KL/rejected_KL_mean": -267.5029296875, "KL/std": 115.59219360351562, "epoch": 0.8949357520786092, "fcm_dpo/beta": 0.004358598962426186, "fcm_dpo/delta": 0.05125339329242706, "fcm_dpo/margin": 81.84980773925781, "fcm_dpo/q_t": 0.42204612493515015, "grad_norm": 13.795918464660645, "learning_rate": 1.6938180788793556e-08, "logits/chosen": 0.8985134363174438, "logits/rejected": 0.8028838634490967, "logps/chosen": -236.89041137695312, "logps/ref_chosen": -51.237327575683594, "logps/ref_rejected": -81.60242462158203, "logps/rejected": -349.1053466796875, "loss": 1.1411, "margin_dpo/margin_mean": 81.84980010986328, "margin_dpo/margin_std": 132.00807189941406, "step": 592 }, { "KL/chosen_KL_mean": -152.5126953125, "KL/mean": -199.21270751953125, "KL/rejected_KL_mean": -245.91268920898438, "KL/std": 117.2364501953125, "epoch": 0.8964474678760394, "fcm_dpo/beta": 0.004420733079314232, "fcm_dpo/delta": -0.017015092074871063, "fcm_dpo/margin": 93.39999389648438, "fcm_dpo/q_t": 0.40301674604415894, "grad_norm": 15.768633842468262, "learning_rate": 1.6463034933723336e-08, "logits/chosen": 0.8698031902313232, "logits/rejected": 0.7922099232673645, "logps/chosen": -194.59271240234375, "logps/ref_chosen": -42.08000183105469, "logps/ref_rejected": -68.47499084472656, "logps/rejected": -314.3876953125, "loss": 1.0992, "margin_dpo/margin_mean": 93.39999389648438, "margin_dpo/margin_std": 134.39059448242188, "step": 593 }, { "KL/chosen_KL_mean": -196.26040649414062, "KL/mean": -233.34219360351562, "KL/rejected_KL_mean": -270.42401123046875, "KL/std": 118.11385345458984, "epoch": 0.8979591836734694, "fcm_dpo/beta": 0.004671279340982437, "fcm_dpo/delta": 0.06467173248529434, "fcm_dpo/margin": 74.16361999511719, "fcm_dpo/q_t": 0.42175400257110596, "grad_norm": 13.117592811584473, "learning_rate": 1.5994421609589385e-08, "logits/chosen": 0.8108823299407959, "logits/rejected": 0.8017544746398926, "logps/chosen": -259.9190673828125, "logps/ref_chosen": -63.658668518066406, "logps/ref_rejected": -70.35597229003906, "logps/rejected": -340.77996826171875, "loss": 1.1458, "margin_dpo/margin_mean": 74.16361999511719, "margin_dpo/margin_std": 120.93412780761719, "step": 594 }, { "KL/chosen_KL_mean": -179.6163330078125, "KL/mean": -230.7767791748047, "KL/rejected_KL_mean": -281.937255859375, "KL/std": 127.30757141113281, "epoch": 0.8994708994708994, "fcm_dpo/beta": 0.004537756554782391, "fcm_dpo/delta": -0.08347880840301514, "fcm_dpo/margin": 102.32089233398438, "fcm_dpo/q_t": 0.3941081464290619, "grad_norm": 10.797310829162598, "learning_rate": 1.553235392451377e-08, "logits/chosen": 0.892518162727356, "logits/rejected": 0.8190580606460571, "logps/chosen": -235.8350830078125, "logps/ref_chosen": -56.21875762939453, "logps/ref_rejected": -83.95773315429688, "logps/rejected": -365.8949890136719, "loss": 1.09, "margin_dpo/margin_mean": 102.32089233398438, "margin_dpo/margin_std": 156.1725616455078, "step": 595 }, { "KL/chosen_KL_mean": -209.66781616210938, "KL/mean": -228.1407470703125, "KL/rejected_KL_mean": -246.61367797851562, "KL/std": 114.44010925292969, "epoch": 0.9009826152683296, "fcm_dpo/beta": 0.004751176573336124, "fcm_dpo/delta": 0.08440345525741577, "fcm_dpo/margin": 36.94586181640625, "fcm_dpo/q_t": 0.46237796545028687, "grad_norm": 15.789997100830078, "learning_rate": 1.507684480352292e-08, "logits/chosen": 0.7922879457473755, "logits/rejected": 0.8138083219528198, "logps/chosen": -278.148681640625, "logps/ref_chosen": -68.48088073730469, "logps/ref_rejected": -61.732967376708984, "logps/rejected": -308.34661865234375, "loss": 1.2976, "margin_dpo/margin_mean": 36.945858001708984, "margin_dpo/margin_std": 119.91374206542969, "step": 596 }, { "KL/chosen_KL_mean": -160.72445678710938, "KL/mean": -199.198486328125, "KL/rejected_KL_mean": -237.67250061035156, "KL/std": 101.29132080078125, "epoch": 0.9024943310657596, "fcm_dpo/beta": 0.004925125278532505, "fcm_dpo/delta": 0.023953083902597427, "fcm_dpo/margin": 76.94803619384766, "fcm_dpo/q_t": 0.41399139165878296, "grad_norm": 11.786072731018066, "learning_rate": 1.4627906988186111e-08, "logits/chosen": 0.8552061319351196, "logits/rejected": 0.8432618975639343, "logps/chosen": -209.58197021484375, "logps/ref_chosen": -48.85750961303711, "logps/ref_rejected": -55.068084716796875, "logps/rejected": -292.7406005859375, "loss": 1.1202, "margin_dpo/margin_mean": 76.94802856445312, "margin_dpo/margin_std": 115.46106719970703, "step": 597 }, { "KL/chosen_KL_mean": -208.59335327148438, "KL/mean": -235.1393280029297, "KL/rejected_KL_mean": -261.685302734375, "KL/std": 123.59971618652344, "epoch": 0.9040060468631897, "fcm_dpo/beta": 0.004996468313038349, "fcm_dpo/delta": 0.052093617618083954, "fcm_dpo/margin": 53.09196853637695, "fcm_dpo/q_t": 0.4427511692047119, "grad_norm": 13.579572677612305, "learning_rate": 1.4185553036259095e-08, "logits/chosen": 0.856338620185852, "logits/rejected": 0.7920749187469482, "logps/chosen": -267.4804992675781, "logps/ref_chosen": -58.88715362548828, "logps/ref_rejected": -81.43145751953125, "logps/rejected": -343.11676025390625, "loss": 1.2382, "margin_dpo/margin_mean": 53.09196472167969, "margin_dpo/margin_std": 130.64337158203125, "step": 598 }, { "KL/chosen_KL_mean": -209.20242309570312, "KL/mean": -238.07550048828125, "KL/rejected_KL_mean": -266.94854736328125, "KL/std": 117.22828674316406, "epoch": 0.9055177626606198, "fcm_dpo/beta": 0.005158504005521536, "fcm_dpo/delta": 0.012377151288092136, "fcm_dpo/margin": 57.74616241455078, "fcm_dpo/q_t": 0.43443089723587036, "grad_norm": 17.706785202026367, "learning_rate": 1.3749795321332885e-08, "logits/chosen": 0.9016659259796143, "logits/rejected": 0.869124174118042, "logps/chosen": -266.80963134765625, "logps/ref_chosen": -57.60719299316406, "logps/ref_rejected": -71.80469512939453, "logps/rejected": -338.75323486328125, "loss": 1.2257, "margin_dpo/margin_mean": 57.74616241455078, "margin_dpo/margin_std": 135.96075439453125, "step": 599 }, { "KL/chosen_KL_mean": -190.96435546875, "KL/mean": -227.06735229492188, "KL/rejected_KL_mean": -263.1703186035156, "KL/std": 130.8008575439453, "epoch": 0.9070294784580499, "fcm_dpo/beta": 0.005027216859161854, "fcm_dpo/delta": -0.06495367735624313, "fcm_dpo/margin": 72.2059555053711, "fcm_dpo/q_t": 0.4207063317298889, "grad_norm": 18.09197998046875, "learning_rate": 1.3320646032487393e-08, "logits/chosen": 0.8972000479698181, "logits/rejected": 0.8484115600585938, "logps/chosen": -249.40667724609375, "logps/ref_chosen": -58.44231414794922, "logps/ref_rejected": -83.64639282226562, "logps/rejected": -346.81671142578125, "loss": 1.1778, "margin_dpo/margin_mean": 72.2059555053711, "margin_dpo/margin_std": 141.4042205810547, "step": 600 }, { "KL/chosen_KL_mean": -171.49896240234375, "KL/mean": -221.0233154296875, "KL/rejected_KL_mean": -270.5477294921875, "KL/std": 137.1717529296875, "epoch": 0.90854119425548, "fcm_dpo/beta": 0.004522332921624184, "fcm_dpo/delta": -0.06632857024669647, "fcm_dpo/margin": 99.04876708984375, "fcm_dpo/q_t": 0.3979038596153259, "grad_norm": 11.509042739868164, "learning_rate": 1.2898117173950868e-08, "logits/chosen": 0.8533494472503662, "logits/rejected": 0.7871492505073547, "logps/chosen": -227.09327697753906, "logps/ref_chosen": -55.59432601928711, "logps/ref_rejected": -83.68630981445312, "logps/rejected": -354.2340393066406, "loss": 1.1023, "margin_dpo/margin_mean": 99.04876708984375, "margin_dpo/margin_std": 156.123291015625, "step": 601 }, { "KL/chosen_KL_mean": -161.41921997070312, "KL/mean": -205.10360717773438, "KL/rejected_KL_mean": -248.7879638671875, "KL/std": 117.01727294921875, "epoch": 0.91005291005291, "fcm_dpo/beta": 0.00461677648127079, "fcm_dpo/delta": -0.00556892529129982, "fcm_dpo/margin": 87.36872863769531, "fcm_dpo/q_t": 0.4065745770931244, "grad_norm": 13.123578071594238, "learning_rate": 1.2482220564763667e-08, "logits/chosen": 0.8886693716049194, "logits/rejected": 0.8638613224029541, "logps/chosen": -217.76841735839844, "logps/ref_chosen": -56.349185943603516, "logps/ref_rejected": -71.9959716796875, "logps/rejected": -320.783935546875, "loss": 1.1009, "margin_dpo/margin_mean": 87.36872863769531, "margin_dpo/margin_std": 123.98216247558594, "step": 602 }, { "KL/chosen_KL_mean": -178.2960205078125, "KL/mean": -222.55340576171875, "KL/rejected_KL_mean": -266.810791015625, "KL/std": 118.53578186035156, "epoch": 0.9115646258503401, "fcm_dpo/beta": 0.004479277413338423, "fcm_dpo/delta": 0.00364796444773674, "fcm_dpo/margin": 88.51475524902344, "fcm_dpo/q_t": 0.41039514541625977, "grad_norm": 11.956192970275879, "learning_rate": 1.2072967838448051e-08, "logits/chosen": 0.8445330858230591, "logits/rejected": 0.8046483993530273, "logps/chosen": -231.46441650390625, "logps/ref_chosen": -53.16838836669922, "logps/ref_rejected": -73.8604736328125, "logps/rejected": -340.6712646484375, "loss": 1.1182, "margin_dpo/margin_mean": 88.51475524902344, "margin_dpo/margin_std": 137.9913787841797, "step": 603 }, { "KL/chosen_KL_mean": -175.42764282226562, "KL/mean": -211.24090576171875, "KL/rejected_KL_mean": -247.05416870117188, "KL/std": 111.7984848022461, "epoch": 0.9130763416477702, "fcm_dpo/beta": 0.004825977608561516, "fcm_dpo/delta": 0.0639285072684288, "fcm_dpo/margin": 71.62651062011719, "fcm_dpo/q_t": 0.4246947765350342, "grad_norm": 13.927318572998047, "learning_rate": 1.1670370442682459e-08, "logits/chosen": 0.8277266025543213, "logits/rejected": 0.8371890783309937, "logps/chosen": -248.0770721435547, "logps/ref_chosen": -72.64942169189453, "logps/ref_rejected": -69.8792724609375, "logps/rejected": -316.9334411621094, "loss": 1.1873, "margin_dpo/margin_mean": 71.62651062011719, "margin_dpo/margin_std": 147.93710327148438, "step": 604 }, { "KL/chosen_KL_mean": -195.2484588623047, "KL/mean": -235.98434448242188, "KL/rejected_KL_mean": -276.7202453613281, "KL/std": 116.7174301147461, "epoch": 0.9145880574452003, "fcm_dpo/beta": 0.0048104869201779366, "fcm_dpo/delta": 0.009344127029180527, "fcm_dpo/margin": 81.47175598144531, "fcm_dpo/q_t": 0.41186654567718506, "grad_norm": 15.041853904724121, "learning_rate": 1.1274439638981532e-08, "logits/chosen": 0.9091506004333496, "logits/rejected": 0.8648948669433594, "logps/chosen": -256.8612976074219, "logps/ref_chosen": -61.61284637451172, "logps/ref_rejected": -79.34398651123047, "logps/rejected": -356.064208984375, "loss": 1.1374, "margin_dpo/margin_mean": 81.47175598144531, "margin_dpo/margin_std": 138.40643310546875, "step": 605 }, { "KL/chosen_KL_mean": -171.4249725341797, "KL/mean": -221.96023559570312, "KL/rejected_KL_mean": -272.4954833984375, "KL/std": 122.88982391357422, "epoch": 0.9160997732426304, "fcm_dpo/beta": 0.004657290410250425, "fcm_dpo/delta": -0.09221285581588745, "fcm_dpo/margin": 101.07051086425781, "fcm_dpo/q_t": 0.3912612795829773, "grad_norm": 14.664449691772461, "learning_rate": 1.0885186502381016e-08, "logits/chosen": 0.8313355445861816, "logits/rejected": 0.7794297933578491, "logps/chosen": -225.8892059326172, "logps/ref_chosen": -54.46424102783203, "logps/ref_rejected": -79.62708282470703, "logps/rejected": -352.12255859375, "loss": 1.0671, "margin_dpo/margin_mean": 101.07051086425781, "margin_dpo/margin_std": 137.8955078125, "step": 606 }, { "KL/chosen_KL_mean": -189.69970703125, "KL/mean": -232.14910888671875, "KL/rejected_KL_mean": -274.5985107421875, "KL/std": 117.85604858398438, "epoch": 0.9176114890400605, "fcm_dpo/beta": 0.0040831370279192924, "fcm_dpo/delta": -0.0878070592880249, "fcm_dpo/margin": 84.89881896972656, "fcm_dpo/q_t": 0.41531041264533997, "grad_norm": 10.932661056518555, "learning_rate": 1.0502621921127774e-08, "logits/chosen": 0.8385425806045532, "logits/rejected": 0.8218169212341309, "logps/chosen": -252.56057739257812, "logps/ref_chosen": -62.86086654663086, "logps/ref_rejected": -72.5501937866211, "logps/rejected": -347.1487121582031, "loss": 1.1455, "margin_dpo/margin_mean": 84.89881896972656, "margin_dpo/margin_std": 132.09307861328125, "step": 607 }, { "KL/chosen_KL_mean": -192.0813446044922, "KL/mean": -239.98777770996094, "KL/rejected_KL_mean": -287.8941955566406, "KL/std": 133.00662231445312, "epoch": 0.9191232048374905, "fcm_dpo/beta": 0.004131226800382137, "fcm_dpo/delta": 0.004865831695497036, "fcm_dpo/margin": 95.81285858154297, "fcm_dpo/q_t": 0.4098547101020813, "grad_norm": 11.770204544067383, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 0.8204215168952942, "logits/rejected": 0.756758451461792, "logps/chosen": -255.26206970214844, "logps/ref_chosen": -63.18071746826172, "logps/ref_rejected": -99.15888214111328, "logps/rejected": -387.0531005859375, "loss": 1.1027, "margin_dpo/margin_mean": 95.81285095214844, "margin_dpo/margin_std": 135.89065551757812, "step": 608 }, { "KL/chosen_KL_mean": -171.8243408203125, "KL/mean": -219.75531005859375, "KL/rejected_KL_mean": -267.686279296875, "KL/std": 114.76544952392578, "epoch": 0.9206349206349206, "fcm_dpo/beta": 0.0040517146699130535, "fcm_dpo/delta": 0.01250215619802475, "fcm_dpo/margin": 95.86190795898438, "fcm_dpo/q_t": 0.40848666429519653, "grad_norm": 11.491697311401367, "learning_rate": 9.757601041885694e-09, "logits/chosen": 0.9150640368461609, "logits/rejected": 0.8839371800422668, "logps/chosen": -220.44757080078125, "logps/ref_chosen": -48.62322235107422, "logps/ref_rejected": -68.28271484375, "logps/rejected": -335.968994140625, "loss": 1.0908, "margin_dpo/margin_mean": 95.8619155883789, "margin_dpo/margin_std": 112.57235717773438, "step": 609 }, { "KL/chosen_KL_mean": -191.83123779296875, "KL/mean": -234.95062255859375, "KL/rejected_KL_mean": -278.0699768066406, "KL/std": 130.08535766601562, "epoch": 0.9221466364323507, "fcm_dpo/beta": 0.004214122891426086, "fcm_dpo/delta": 0.04403061419725418, "fcm_dpo/margin": 86.23873138427734, "fcm_dpo/q_t": 0.41813790798187256, "grad_norm": 13.131653785705566, "learning_rate": 9.395165583732379e-09, "logits/chosen": 0.8536557555198669, "logits/rejected": 0.8505412340164185, "logps/chosen": -264.4963684082031, "logps/ref_chosen": -72.66513061523438, "logps/ref_rejected": -87.15310668945312, "logps/rejected": -365.22308349609375, "loss": 1.1557, "margin_dpo/margin_mean": 86.23873138427734, "margin_dpo/margin_std": 151.37991333007812, "step": 610 }, { "KL/chosen_KL_mean": -172.49746704101562, "KL/mean": -208.9520721435547, "KL/rejected_KL_mean": -245.40664672851562, "KL/std": 112.63088989257812, "epoch": 0.9236583522297808, "fcm_dpo/beta": 0.004568580538034439, "fcm_dpo/delta": 0.08063990622758865, "fcm_dpo/margin": 72.90918731689453, "fcm_dpo/q_t": 0.4252277612686157, "grad_norm": 12.904760360717773, "learning_rate": 9.03946036001449e-09, "logits/chosen": 0.9144739508628845, "logits/rejected": 0.8739849925041199, "logps/chosen": -220.80606079101562, "logps/ref_chosen": -48.30857849121094, "logps/ref_rejected": -70.6141128540039, "logps/rejected": -316.020751953125, "loss": 1.1462, "margin_dpo/margin_mean": 72.90919494628906, "margin_dpo/margin_std": 113.95925903320312, "step": 611 }, { "KL/chosen_KL_mean": -189.24465942382812, "KL/mean": -245.61618041992188, "KL/rejected_KL_mean": -301.9877014160156, "KL/std": 120.45782470703125, "epoch": 0.9251700680272109, "fcm_dpo/beta": 0.004316345788538456, "fcm_dpo/delta": -0.11510652303695679, "fcm_dpo/margin": 112.74305725097656, "fcm_dpo/q_t": 0.3831419050693512, "grad_norm": 11.220308303833008, "learning_rate": 8.690495320571839e-09, "logits/chosen": 0.7996529936790466, "logits/rejected": 0.7398805618286133, "logps/chosen": -250.47622680664062, "logps/ref_chosen": -61.23155975341797, "logps/ref_rejected": -94.37979888916016, "logps/rejected": -396.36749267578125, "loss": 1.0325, "margin_dpo/margin_mean": 112.74304962158203, "margin_dpo/margin_std": 131.59378051757812, "step": 612 }, { "KL/chosen_KL_mean": -167.00177001953125, "KL/mean": -214.05615234375, "KL/rejected_KL_mean": -261.11053466796875, "KL/std": 132.7792510986328, "epoch": 0.926681783824641, "fcm_dpo/beta": 0.004251118283718824, "fcm_dpo/delta": -0.0009491164237260818, "fcm_dpo/margin": 94.1087646484375, "fcm_dpo/q_t": 0.4064924120903015, "grad_norm": 10.253765106201172, "learning_rate": 8.348280226706722e-09, "logits/chosen": 0.8172672390937805, "logits/rejected": 0.8157082796096802, "logps/chosen": -220.98487854003906, "logps/ref_chosen": -53.98310852050781, "logps/ref_rejected": -58.32208251953125, "logps/rejected": -319.4326171875, "loss": 1.1017, "margin_dpo/margin_mean": 94.1087646484375, "margin_dpo/margin_std": 132.22064208984375, "step": 613 }, { "KL/chosen_KL_mean": -189.78424072265625, "KL/mean": -234.03488159179688, "KL/rejected_KL_mean": -278.2854919433594, "KL/std": 111.17167663574219, "epoch": 0.9281934996220711, "fcm_dpo/beta": 0.004374989774078131, "fcm_dpo/delta": 0.009936317801475525, "fcm_dpo/margin": 88.50125122070312, "fcm_dpo/q_t": 0.40952184796333313, "grad_norm": 13.926337242126465, "learning_rate": 8.012824650910937e-09, "logits/chosen": 0.8528718948364258, "logits/rejected": 0.848002016544342, "logps/chosen": -250.02728271484375, "logps/ref_chosen": -60.24303436279297, "logps/ref_rejected": -72.26258850097656, "logps/rejected": -350.548095703125, "loss": 1.1145, "margin_dpo/margin_mean": 88.50125885009766, "margin_dpo/margin_std": 126.64340209960938, "step": 614 }, { "KL/chosen_KL_mean": -195.78854370117188, "KL/mean": -248.58724975585938, "KL/rejected_KL_mean": -301.38592529296875, "KL/std": 149.01504516601562, "epoch": 0.9297052154195011, "fcm_dpo/beta": 0.0039672525599598885, "fcm_dpo/delta": -0.03305935114622116, "fcm_dpo/margin": 105.5973892211914, "fcm_dpo/q_t": 0.4037814438343048, "grad_norm": 11.752750396728516, "learning_rate": 7.684137976598088e-09, "logits/chosen": 0.8259449005126953, "logits/rejected": 0.775432825088501, "logps/chosen": -267.8832092285156, "logps/ref_chosen": -72.09467315673828, "logps/ref_rejected": -104.02980041503906, "logps/rejected": -405.41571044921875, "loss": 1.1085, "margin_dpo/margin_mean": 105.59738159179688, "margin_dpo/margin_std": 159.66317749023438, "step": 615 }, { "KL/chosen_KL_mean": -188.25526428222656, "KL/mean": -227.6978759765625, "KL/rejected_KL_mean": -267.1404724121094, "KL/std": 137.1472930908203, "epoch": 0.9312169312169312, "fcm_dpo/beta": 0.00431628804653883, "fcm_dpo/delta": 0.0716884434223175, "fcm_dpo/margin": 78.88520812988281, "fcm_dpo/q_t": 0.42356163263320923, "grad_norm": 11.668061256408691, "learning_rate": 7.36222939784098e-09, "logits/chosen": 0.8835878968238831, "logits/rejected": 0.8267635107040405, "logps/chosen": -246.78598022460938, "logps/ref_chosen": -58.530723571777344, "logps/ref_rejected": -75.48025512695312, "logps/rejected": -342.6207275390625, "loss": 1.1449, "margin_dpo/margin_mean": 78.88520812988281, "margin_dpo/margin_std": 126.27625274658203, "step": 616 }, { "KL/chosen_KL_mean": -196.17718505859375, "KL/mean": -233.40261840820312, "KL/rejected_KL_mean": -270.6280517578125, "KL/std": 124.0031967163086, "epoch": 0.9327286470143613, "fcm_dpo/beta": 0.0042549436911940575, "fcm_dpo/delta": -0.03333226591348648, "fcm_dpo/margin": 74.45088195800781, "fcm_dpo/q_t": 0.4275980591773987, "grad_norm": 15.777302742004395, "learning_rate": 7.047107919114586e-09, "logits/chosen": 0.8467553853988647, "logits/rejected": 0.8062554597854614, "logps/chosen": -253.78585815429688, "logps/ref_chosen": -57.608673095703125, "logps/ref_rejected": -81.22109985351562, "logps/rejected": -351.8491516113281, "loss": 1.1771, "margin_dpo/margin_mean": 74.45087432861328, "margin_dpo/margin_std": 135.0968475341797, "step": 617 }, { "KL/chosen_KL_mean": -183.11309814453125, "KL/mean": -228.11277770996094, "KL/rejected_KL_mean": -273.1124572753906, "KL/std": 136.9832763671875, "epoch": 0.9342403628117913, "fcm_dpo/beta": 0.0042600324377417564, "fcm_dpo/delta": 0.019995192065835, "fcm_dpo/margin": 89.99934387207031, "fcm_dpo/q_t": 0.4153464436531067, "grad_norm": 16.51608657836914, "learning_rate": 6.738782355044048e-09, "logits/chosen": 0.8677645921707153, "logits/rejected": 0.7911471128463745, "logps/chosen": -239.80905151367188, "logps/ref_chosen": -56.69594192504883, "logps/ref_rejected": -85.92362976074219, "logps/rejected": -359.03607177734375, "loss": 1.1202, "margin_dpo/margin_mean": 89.99934387207031, "margin_dpo/margin_std": 139.83460998535156, "step": 618 }, { "KL/chosen_KL_mean": -174.80682373046875, "KL/mean": -222.34530639648438, "KL/rejected_KL_mean": -269.8837890625, "KL/std": 126.5418701171875, "epoch": 0.9357520786092215, "fcm_dpo/beta": 0.004261237569153309, "fcm_dpo/delta": -0.0073447078466415405, "fcm_dpo/margin": 95.07696533203125, "fcm_dpo/q_t": 0.40738731622695923, "grad_norm": 11.838799476623535, "learning_rate": 6.437261330158206e-09, "logits/chosen": 0.9209197759628296, "logits/rejected": 0.8575242757797241, "logps/chosen": -228.865234375, "logps/ref_chosen": -54.05841827392578, "logps/ref_rejected": -83.55493927001953, "logps/rejected": -353.438720703125, "loss": 1.103, "margin_dpo/margin_mean": 95.07696533203125, "margin_dpo/margin_std": 138.84814453125, "step": 619 }, { "KL/chosen_KL_mean": -183.2464599609375, "KL/mean": -217.73406982421875, "KL/rejected_KL_mean": -252.22164916992188, "KL/std": 117.54043579101562, "epoch": 0.9372637944066515, "fcm_dpo/beta": 0.004206355661153793, "fcm_dpo/delta": -0.04886791855096817, "fcm_dpo/margin": 68.97517395019531, "fcm_dpo/q_t": 0.4305534362792969, "grad_norm": 12.827408790588379, "learning_rate": 6.142553278648238e-09, "logits/chosen": 0.8646059036254883, "logits/rejected": 0.8682705163955688, "logps/chosen": -246.61618041992188, "logps/ref_chosen": -63.36971664428711, "logps/ref_rejected": -65.68269348144531, "logps/rejected": -317.9043273925781, "loss": 1.194, "margin_dpo/margin_mean": 68.97517395019531, "margin_dpo/margin_std": 122.68234252929688, "step": 620 }, { "KL/chosen_KL_mean": -176.3372344970703, "KL/mean": -221.6025848388672, "KL/rejected_KL_mean": -266.867919921875, "KL/std": 131.4630584716797, "epoch": 0.9387755102040817, "fcm_dpo/beta": 0.004246983677148819, "fcm_dpo/delta": 0.016644608229398727, "fcm_dpo/margin": 90.53071594238281, "fcm_dpo/q_t": 0.4165423810482025, "grad_norm": 12.35801887512207, "learning_rate": 5.854666444131934e-09, "logits/chosen": 0.8844429850578308, "logits/rejected": 0.7948249578475952, "logps/chosen": -228.658447265625, "logps/ref_chosen": -52.321224212646484, "logps/ref_rejected": -88.09001159667969, "logps/rejected": -354.95794677734375, "loss": 1.1435, "margin_dpo/margin_mean": 90.53071594238281, "margin_dpo/margin_std": 158.0870361328125, "step": 621 }, { "KL/chosen_KL_mean": -194.33999633789062, "KL/mean": -236.56668090820312, "KL/rejected_KL_mean": -278.7933349609375, "KL/std": 134.55697631835938, "epoch": 0.9402872260015117, "fcm_dpo/beta": 0.004374613985419273, "fcm_dpo/delta": 0.034500494599342346, "fcm_dpo/margin": 84.45333862304688, "fcm_dpo/q_t": 0.41781359910964966, "grad_norm": 14.24582576751709, "learning_rate": 5.573608879422875e-09, "logits/chosen": 0.8124125003814697, "logits/rejected": 0.7737932205200195, "logps/chosen": -254.20545959472656, "logps/ref_chosen": -59.86545944213867, "logps/ref_rejected": -81.86668395996094, "logps/rejected": -360.6600341796875, "loss": 1.1388, "margin_dpo/margin_mean": 84.45333862304688, "margin_dpo/margin_std": 139.4012451171875, "step": 622 }, { "KL/chosen_KL_mean": -206.7667236328125, "KL/mean": -251.3427276611328, "KL/rejected_KL_mean": -295.9187316894531, "KL/std": 141.90228271484375, "epoch": 0.9417989417989417, "fcm_dpo/beta": 0.004265275783836842, "fcm_dpo/delta": 0.02134121209383011, "fcm_dpo/margin": 89.15202331542969, "fcm_dpo/q_t": 0.41413989663124084, "grad_norm": 13.33872127532959, "learning_rate": 5.299388446305342e-09, "logits/chosen": 0.8636614680290222, "logits/rejected": 0.8233832120895386, "logps/chosen": -274.13519287109375, "logps/ref_chosen": -67.36846160888672, "logps/ref_rejected": -82.02733612060547, "logps/rejected": -377.9460754394531, "loss": 1.1384, "margin_dpo/margin_mean": 89.15202331542969, "margin_dpo/margin_std": 147.57623291015625, "step": 623 }, { "KL/chosen_KL_mean": -176.5563201904297, "KL/mean": -221.74380493164062, "KL/rejected_KL_mean": -266.9313049316406, "KL/std": 130.1385498046875, "epoch": 0.9433106575963719, "fcm_dpo/beta": 0.004365057684481144, "fcm_dpo/delta": 0.005921470001339912, "fcm_dpo/margin": 90.37495422363281, "fcm_dpo/q_t": 0.41168880462646484, "grad_norm": 15.640230178833008, "learning_rate": 5.03201281531429e-09, "logits/chosen": 0.8767238855361938, "logits/rejected": 0.7973580360412598, "logps/chosen": -227.58287048339844, "logps/ref_chosen": -51.02655029296875, "logps/ref_rejected": -76.49203491210938, "logps/rejected": -343.42333984375, "loss": 1.1173, "margin_dpo/margin_mean": 90.37496185302734, "margin_dpo/margin_std": 139.69216918945312, "step": 624 }, { "KL/chosen_KL_mean": -175.95306396484375, "KL/mean": -212.92311096191406, "KL/rejected_KL_mean": -249.89315795898438, "KL/std": 124.17606353759766, "epoch": 0.9448223733938019, "fcm_dpo/beta": 0.00467075128108263, "fcm_dpo/delta": 0.06579786539077759, "fcm_dpo/margin": 73.94007873535156, "fcm_dpo/q_t": 0.4268485903739929, "grad_norm": 13.2764310836792, "learning_rate": 4.7714894655209174e-09, "logits/chosen": 0.9225313663482666, "logits/rejected": 0.8460739850997925, "logps/chosen": -230.1606903076172, "logps/ref_chosen": -54.20761489868164, "logps/ref_rejected": -84.93669128417969, "logps/rejected": -334.829833984375, "loss": 1.1812, "margin_dpo/margin_mean": 73.94007873535156, "margin_dpo/margin_std": 149.19314575195312, "step": 625 }, { "KL/chosen_KL_mean": -166.25491333007812, "KL/mean": -217.4595184326172, "KL/rejected_KL_mean": -268.66412353515625, "KL/std": 134.90798950195312, "epoch": 0.9463340891912321, "fcm_dpo/beta": 0.004452715627849102, "fcm_dpo/delta": -0.07368428260087967, "fcm_dpo/margin": 102.40919494628906, "fcm_dpo/q_t": 0.3990449607372284, "grad_norm": 14.270200729370117, "learning_rate": 4.517825684323323e-09, "logits/chosen": 0.941727876663208, "logits/rejected": 0.8290482759475708, "logps/chosen": -211.3169403076172, "logps/ref_chosen": -45.06201934814453, "logps/ref_rejected": -89.66368103027344, "logps/rejected": -358.3277893066406, "loss": 1.0958, "margin_dpo/margin_mean": 102.40919494628906, "margin_dpo/margin_std": 164.34068298339844, "step": 626 }, { "KL/chosen_KL_mean": -190.93905639648438, "KL/mean": -245.65646362304688, "KL/rejected_KL_mean": -300.3738708496094, "KL/std": 140.89553833007812, "epoch": 0.9478458049886621, "fcm_dpo/beta": 0.004127138294279575, "fcm_dpo/delta": -0.0688527449965477, "fcm_dpo/margin": 109.43482208251953, "fcm_dpo/q_t": 0.3921513855457306, "grad_norm": 11.92895221710205, "learning_rate": 4.271028567242818e-09, "logits/chosen": 0.8075392246246338, "logits/rejected": 0.7088248133659363, "logps/chosen": -249.7301025390625, "logps/ref_chosen": -58.791053771972656, "logps/ref_rejected": -94.90802001953125, "logps/rejected": -395.2818908691406, "loss": 1.0566, "margin_dpo/margin_mean": 109.43482208251953, "margin_dpo/margin_std": 133.880126953125, "step": 627 }, { "KL/chosen_KL_mean": -176.84475708007812, "KL/mean": -227.326416015625, "KL/rejected_KL_mean": -277.8080749511719, "KL/std": 130.1131591796875, "epoch": 0.9493575207860923, "fcm_dpo/beta": 0.0041605280712246895, "fcm_dpo/delta": -0.029174987226724625, "fcm_dpo/margin": 100.96332550048828, "fcm_dpo/q_t": 0.4007333219051361, "grad_norm": 13.848322868347168, "learning_rate": 4.0311050177251895e-09, "logits/chosen": 0.8667212128639221, "logits/rejected": 0.8345490097999573, "logps/chosen": -229.64833068847656, "logps/ref_chosen": -52.80357360839844, "logps/ref_rejected": -76.49468994140625, "logps/rejected": -354.3027648925781, "loss": 1.1171, "margin_dpo/margin_mean": 100.96331787109375, "margin_dpo/margin_std": 148.15411376953125, "step": 628 }, { "KL/chosen_KL_mean": -190.57797241210938, "KL/mean": -226.43057250976562, "KL/rejected_KL_mean": -262.28314208984375, "KL/std": 119.66000366210938, "epoch": 0.9508692365835223, "fcm_dpo/beta": 0.004353722557425499, "fcm_dpo/delta": 0.10218150913715363, "fcm_dpo/margin": 71.70518493652344, "fcm_dpo/q_t": 0.43269526958465576, "grad_norm": 12.406708717346191, "learning_rate": 3.798061746947995e-09, "logits/chosen": 0.8476235270500183, "logits/rejected": 0.8516208529472351, "logps/chosen": -261.29547119140625, "logps/ref_chosen": -70.71749877929688, "logps/ref_rejected": -78.96273803710938, "logps/rejected": -341.24591064453125, "loss": 1.174, "margin_dpo/margin_mean": 71.70518493652344, "margin_dpo/margin_std": 130.66549682617188, "step": 629 }, { "KL/chosen_KL_mean": -175.83795166015625, "KL/mean": -226.13722229003906, "KL/rejected_KL_mean": -276.4364929199219, "KL/std": 127.85799407958984, "epoch": 0.9523809523809523, "fcm_dpo/beta": 0.0043222480453550816, "fcm_dpo/delta": -0.04444940388202667, "fcm_dpo/margin": 100.59855651855469, "fcm_dpo/q_t": 0.3986924886703491, "grad_norm": 11.189255714416504, "learning_rate": 3.5719052736323806e-09, "logits/chosen": 0.8098629713058472, "logits/rejected": 0.7818838357925415, "logps/chosen": -232.0393524169922, "logps/ref_chosen": -56.201412200927734, "logps/ref_rejected": -74.69807434082031, "logps/rejected": -351.13458251953125, "loss": 1.0859, "margin_dpo/margin_mean": 100.59855651855469, "margin_dpo/margin_std": 141.52880859375, "step": 630 }, { "KL/chosen_KL_mean": -167.0948028564453, "KL/mean": -227.14959716796875, "KL/rejected_KL_mean": -287.20440673828125, "KL/std": 138.20928955078125, "epoch": 0.9538926681783825, "fcm_dpo/beta": 0.0038028829731047153, "fcm_dpo/delta": -0.08643369376659393, "fcm_dpo/margin": 120.109619140625, "fcm_dpo/q_t": 0.3914579153060913, "grad_norm": 11.006467819213867, "learning_rate": 3.352641923861144e-09, "logits/chosen": 0.9120550155639648, "logits/rejected": 0.8212620615959167, "logps/chosen": -225.9154052734375, "logps/ref_chosen": -58.82059860229492, "logps/ref_rejected": -96.51437377929688, "logps/rejected": -383.71881103515625, "loss": 1.0626, "margin_dpo/margin_mean": 120.109619140625, "margin_dpo/margin_std": 151.4070281982422, "step": 631 }, { "KL/chosen_KL_mean": -169.6227264404297, "KL/mean": -222.89845275878906, "KL/rejected_KL_mean": -276.1741638183594, "KL/std": 119.03538513183594, "epoch": 0.9554043839758125, "fcm_dpo/beta": 0.0038762884214520454, "fcm_dpo/delta": -0.016792653128504753, "fcm_dpo/margin": 106.55143737792969, "fcm_dpo/q_t": 0.40256255865097046, "grad_norm": 12.226197242736816, "learning_rate": 3.140277830901428e-09, "logits/chosen": 0.8677657842636108, "logits/rejected": 0.8573723435401917, "logps/chosen": -228.40878295898438, "logps/ref_chosen": -58.786048889160156, "logps/ref_rejected": -67.21923828125, "logps/rejected": -343.3934020996094, "loss": 1.0755, "margin_dpo/margin_mean": 106.55143737792969, "margin_dpo/margin_std": 128.8999786376953, "step": 632 }, { "KL/chosen_KL_mean": -166.22283935546875, "KL/mean": -209.18299865722656, "KL/rejected_KL_mean": -252.14317321777344, "KL/std": 116.75459289550781, "epoch": 0.9569160997732427, "fcm_dpo/beta": 0.004068909212946892, "fcm_dpo/delta": 0.05746911093592644, "fcm_dpo/margin": 85.92031860351562, "fcm_dpo/q_t": 0.42180460691452026, "grad_norm": 12.814493179321289, "learning_rate": 2.9348189350335007e-09, "logits/chosen": 0.8555501699447632, "logits/rejected": 0.8156156539916992, "logps/chosen": -218.35304260253906, "logps/ref_chosen": -52.13019561767578, "logps/ref_rejected": -67.23016357421875, "logps/rejected": -319.3733215332031, "loss": 1.1366, "margin_dpo/margin_mean": 85.92031860351562, "margin_dpo/margin_std": 131.57418823242188, "step": 633 }, { "KL/chosen_KL_mean": -199.20892333984375, "KL/mean": -214.2444610595703, "KL/rejected_KL_mean": -229.27999877929688, "KL/std": 122.4713363647461, "epoch": 0.9584278155706727, "fcm_dpo/beta": 0.0040403553284704685, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 30.071069717407227, "fcm_dpo/q_t": 0.4731084406375885, "grad_norm": 14.749358177185059, "learning_rate": 2.736270983384276e-09, "logits/chosen": 0.917083203792572, "logits/rejected": 0.9344348907470703, "logps/chosen": -260.188720703125, "logps/ref_chosen": -60.97979736328125, "logps/ref_rejected": -58.50825119018555, "logps/rejected": -287.7882385253906, "loss": 1.3462, "margin_dpo/margin_mean": 30.07107162475586, "margin_dpo/margin_std": 140.06642150878906, "step": 634 }, { "KL/chosen_KL_mean": -216.4016876220703, "KL/mean": -251.16989135742188, "KL/rejected_KL_mean": -285.9380798339844, "KL/std": 117.64241027832031, "epoch": 0.9599395313681028, "fcm_dpo/beta": 0.0044487882405519485, "fcm_dpo/delta": 0.10310401022434235, "fcm_dpo/margin": 69.5363998413086, "fcm_dpo/q_t": 0.43169939517974854, "grad_norm": 12.148490905761719, "learning_rate": 2.5446395297668287e-09, "logits/chosen": 0.7875484228134155, "logits/rejected": 0.7464572787284851, "logps/chosen": -282.37469482421875, "logps/ref_chosen": -65.9730224609375, "logps/ref_rejected": -85.61317443847656, "logps/rejected": -371.55126953125, "loss": 1.1925, "margin_dpo/margin_mean": 69.5363998413086, "margin_dpo/margin_std": 138.6806640625, "step": 635 }, { "KL/chosen_KL_mean": -168.04318237304688, "KL/mean": -216.12437438964844, "KL/rejected_KL_mean": -264.20556640625, "KL/std": 121.20172882080078, "epoch": 0.9614512471655329, "fcm_dpo/beta": 0.0043456945568323135, "fcm_dpo/delta": -0.02371588721871376, "fcm_dpo/margin": 96.16236877441406, "fcm_dpo/q_t": 0.40238264203071594, "grad_norm": 9.903681755065918, "learning_rate": 2.359929934524829e-09, "logits/chosen": 0.885582685470581, "logits/rejected": 0.8005257844924927, "logps/chosen": -217.18336486816406, "logps/ref_chosen": -49.140167236328125, "logps/ref_rejected": -81.26971435546875, "logps/rejected": -345.47528076171875, "loss": 1.0856, "margin_dpo/margin_mean": 96.16236877441406, "margin_dpo/margin_std": 128.5138702392578, "step": 636 }, { "KL/chosen_KL_mean": -227.61679077148438, "KL/mean": -262.9964599609375, "KL/rejected_KL_mean": -298.3760986328125, "KL/std": 125.6603775024414, "epoch": 0.9629629629629629, "fcm_dpo/beta": 0.004716330207884312, "fcm_dpo/delta": 0.07630355656147003, "fcm_dpo/margin": 70.7593002319336, "fcm_dpo/q_t": 0.42755448818206787, "grad_norm": 14.731769561767578, "learning_rate": 2.1821473643827137e-09, "logits/chosen": 0.8204032182693481, "logits/rejected": 0.7771281599998474, "logps/chosen": -301.3133850097656, "logps/ref_chosen": -73.69658660888672, "logps/ref_rejected": -83.01487731933594, "logps/rejected": -381.3909912109375, "loss": 1.1985, "margin_dpo/margin_mean": 70.7593002319336, "margin_dpo/margin_std": 152.50254821777344, "step": 637 }, { "KL/chosen_KL_mean": -196.76817321777344, "KL/mean": -239.29661560058594, "KL/rejected_KL_mean": -281.8250732421875, "KL/std": 121.583740234375, "epoch": 0.9644746787603931, "fcm_dpo/beta": 0.004684511572122574, "fcm_dpo/delta": 0.0015891995280981064, "fcm_dpo/margin": 85.056884765625, "fcm_dpo/q_t": 0.4112497568130493, "grad_norm": 13.911808967590332, "learning_rate": 2.0112967923011646e-09, "logits/chosen": 0.8388936519622803, "logits/rejected": 0.8055061101913452, "logps/chosen": -259.54974365234375, "logps/ref_chosen": -62.78158187866211, "logps/ref_rejected": -85.40478515625, "logps/rejected": -367.2298583984375, "loss": 1.12, "margin_dpo/margin_mean": 85.05690002441406, "margin_dpo/margin_std": 135.8700714111328, "step": 638 }, { "KL/chosen_KL_mean": -174.95285034179688, "KL/mean": -225.14132690429688, "KL/rejected_KL_mean": -275.3298034667969, "KL/std": 131.77854919433594, "epoch": 0.9659863945578231, "fcm_dpo/beta": 0.004515929147601128, "fcm_dpo/delta": -0.06874261051416397, "fcm_dpo/margin": 100.37693786621094, "fcm_dpo/q_t": 0.3964504301548004, "grad_norm": 15.656349182128906, "learning_rate": 1.847382997337943e-09, "logits/chosen": 0.8310115337371826, "logits/rejected": 0.7619854807853699, "logps/chosen": -228.7194366455078, "logps/ref_chosen": -53.76658630371094, "logps/ref_rejected": -72.30009460449219, "logps/rejected": -347.6298828125, "loss": 1.0799, "margin_dpo/margin_mean": 100.37693786621094, "margin_dpo/margin_std": 143.62814331054688, "step": 639 }, { "KL/chosen_KL_mean": -183.02471923828125, "KL/mean": -226.49563598632812, "KL/rejected_KL_mean": -269.966552734375, "KL/std": 124.19609069824219, "epoch": 0.9674981103552532, "fcm_dpo/beta": 0.004421434365212917, "fcm_dpo/delta": 0.018865177407860756, "fcm_dpo/margin": 86.94181823730469, "fcm_dpo/q_t": 0.413200318813324, "grad_norm": 13.584686279296875, "learning_rate": 1.690410564514244e-09, "logits/chosen": 0.9080848693847656, "logits/rejected": 0.8571711778640747, "logps/chosen": -234.4425048828125, "logps/ref_chosen": -51.41777801513672, "logps/ref_rejected": -77.27879333496094, "logps/rejected": -347.245361328125, "loss": 1.1242, "margin_dpo/margin_mean": 86.94182586669922, "margin_dpo/margin_std": 136.48196411132812, "step": 640 }, { "KL/chosen_KL_mean": -191.9429931640625, "KL/mean": -231.77511596679688, "KL/rejected_KL_mean": -271.60723876953125, "KL/std": 114.53807067871094, "epoch": 0.9690098261526833, "fcm_dpo/beta": 0.004588712938129902, "fcm_dpo/delta": 0.04207627847790718, "fcm_dpo/margin": 79.66427612304688, "fcm_dpo/q_t": 0.416546106338501, "grad_norm": 13.669086456298828, "learning_rate": 1.5403838846864692e-09, "logits/chosen": 0.8353901505470276, "logits/rejected": 0.8291752338409424, "logps/chosen": -262.9976501464844, "logps/ref_chosen": -71.0546646118164, "logps/ref_rejected": -82.2440185546875, "logps/rejected": -353.85125732421875, "loss": 1.1129, "margin_dpo/margin_mean": 79.66426849365234, "margin_dpo/margin_std": 108.70352172851562, "step": 641 }, { "KL/chosen_KL_mean": -200.2201690673828, "KL/mean": -229.66928100585938, "KL/rejected_KL_mean": -259.118408203125, "KL/std": 112.77413940429688, "epoch": 0.9705215419501134, "fcm_dpo/beta": 0.004786022938787937, "fcm_dpo/delta": 0.019887451082468033, "fcm_dpo/margin": 58.898216247558594, "fcm_dpo/q_t": 0.43794071674346924, "grad_norm": 16.15554428100586, "learning_rate": 1.3973071544233218e-09, "logits/chosen": 0.785338282585144, "logits/rejected": 0.8072504997253418, "logps/chosen": -269.1494445800781, "logps/ref_chosen": -68.92927551269531, "logps/ref_rejected": -70.85682678222656, "logps/rejected": -329.9752197265625, "loss": 1.2236, "margin_dpo/margin_mean": 58.898216247558594, "margin_dpo/margin_std": 135.51150512695312, "step": 642 }, { "KL/chosen_KL_mean": -182.64639282226562, "KL/mean": -230.23406982421875, "KL/rejected_KL_mean": -277.82177734375, "KL/std": 124.7582015991211, "epoch": 0.9720332577475435, "fcm_dpo/beta": 0.004608517978340387, "fcm_dpo/delta": -0.04959561303257942, "fcm_dpo/margin": 95.17536926269531, "fcm_dpo/q_t": 0.40014487504959106, "grad_norm": 18.453994750976562, "learning_rate": 1.261184375888541e-09, "logits/chosen": 0.8454148769378662, "logits/rejected": 0.7771642208099365, "logps/chosen": -247.95541381835938, "logps/ref_chosen": -65.30903625488281, "logps/ref_rejected": -83.61613464355469, "logps/rejected": -361.43792724609375, "loss": 1.0966, "margin_dpo/margin_mean": 95.17538452148438, "margin_dpo/margin_std": 143.47547912597656, "step": 643 }, { "KL/chosen_KL_mean": -173.703125, "KL/mean": -203.21925354003906, "KL/rejected_KL_mean": -232.7353973388672, "KL/std": 107.22509765625, "epoch": 0.9735449735449735, "fcm_dpo/beta": 0.0045800781808793545, "fcm_dpo/delta": 0.01119247917085886, "fcm_dpo/margin": 59.03228759765625, "fcm_dpo/q_t": 0.43719398975372314, "grad_norm": 12.038887977600098, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 0.9204496145248413, "logits/rejected": 0.8933486342430115, "logps/chosen": -224.70571899414062, "logps/ref_chosen": -51.002601623535156, "logps/ref_rejected": -64.46372985839844, "logps/rejected": -297.1991271972656, "loss": 1.2189, "margin_dpo/margin_mean": 59.032283782958984, "margin_dpo/margin_std": 127.94892883300781, "step": 644 }, { "KL/chosen_KL_mean": -190.76394653320312, "KL/mean": -227.95298767089844, "KL/rejected_KL_mean": -265.14202880859375, "KL/std": 117.67291259765625, "epoch": 0.9750566893424036, "fcm_dpo/beta": 0.004811743274331093, "fcm_dpo/delta": 0.05072278529405594, "fcm_dpo/margin": 74.37810516357422, "fcm_dpo/q_t": 0.41881194710731506, "grad_norm": 16.8271541595459, "learning_rate": 1.0098157099674987e-09, "logits/chosen": 0.831100344657898, "logits/rejected": 0.8188281059265137, "logps/chosen": -251.72735595703125, "logps/ref_chosen": -60.963409423828125, "logps/ref_rejected": -69.73353576660156, "logps/rejected": -334.8755798339844, "loss": 1.1414, "margin_dpo/margin_mean": 74.37811279296875, "margin_dpo/margin_std": 121.78832244873047, "step": 645 }, { "KL/chosen_KL_mean": -207.34078979492188, "KL/mean": -244.42474365234375, "KL/rejected_KL_mean": -281.5086975097656, "KL/std": 142.02871704101562, "epoch": 0.9765684051398337, "fcm_dpo/beta": 0.004996123258024454, "fcm_dpo/delta": 0.03588254749774933, "fcm_dpo/margin": 74.16790771484375, "fcm_dpo/q_t": 0.4193630516529083, "grad_norm": 13.366632461547852, "learning_rate": 8.945768539031783e-10, "logits/chosen": 0.8661332130432129, "logits/rejected": 0.8158410787582397, "logps/chosen": -269.630859375, "logps/ref_chosen": -62.290069580078125, "logps/ref_rejected": -85.54812622070312, "logps/rejected": -367.05682373046875, "loss": 1.1618, "margin_dpo/margin_mean": 74.16790771484375, "margin_dpo/margin_std": 139.99423217773438, "step": 646 }, { "KL/chosen_KL_mean": -203.80487060546875, "KL/mean": -262.35650634765625, "KL/rejected_KL_mean": -320.9080810546875, "KL/std": 138.29147338867188, "epoch": 0.9780801209372638, "fcm_dpo/beta": 0.004455167334526777, "fcm_dpo/delta": -0.16519597172737122, "fcm_dpo/margin": 117.10321044921875, "fcm_dpo/q_t": 0.37394261360168457, "grad_norm": 14.46107006072998, "learning_rate": 7.863060120144316e-10, "logits/chosen": 0.8677129745483398, "logits/rejected": 0.7881165742874146, "logps/chosen": -271.32073974609375, "logps/ref_chosen": -67.515869140625, "logps/ref_rejected": -101.50871276855469, "logps/rejected": -422.41680908203125, "loss": 1.016, "margin_dpo/margin_mean": 117.10321044921875, "margin_dpo/margin_std": 135.55255126953125, "step": 647 }, { "KL/chosen_KL_mean": -205.17529296875, "KL/mean": -244.43492126464844, "KL/rejected_KL_mean": -283.6945495605469, "KL/std": 120.4975357055664, "epoch": 0.9795918367346939, "fcm_dpo/beta": 0.0043856678530573845, "fcm_dpo/delta": 0.06681666523218155, "fcm_dpo/margin": 78.51927185058594, "fcm_dpo/q_t": 0.4253219962120056, "grad_norm": 12.654325485229492, "learning_rate": 6.850062128694045e-10, "logits/chosen": 0.8319413661956787, "logits/rejected": 0.7821545004844666, "logps/chosen": -269.771240234375, "logps/ref_chosen": -64.59593963623047, "logps/ref_rejected": -83.384033203125, "logps/rejected": -367.07861328125, "loss": 1.189, "margin_dpo/margin_mean": 78.51927185058594, "margin_dpo/margin_std": 157.42373657226562, "step": 648 }, { "KL/chosen_KL_mean": -184.55213928222656, "KL/mean": -225.38185119628906, "KL/rejected_KL_mean": -266.2115783691406, "KL/std": 116.33067321777344, "epoch": 0.981103552532124, "fcm_dpo/beta": 0.0045933909714221954, "fcm_dpo/delta": 0.030288267880678177, "fcm_dpo/margin": 81.659423828125, "fcm_dpo/q_t": 0.41458964347839355, "grad_norm": 17.702682495117188, "learning_rate": 5.906802900412788e-10, "logits/chosen": 0.8983275890350342, "logits/rejected": 0.8456778526306152, "logps/chosen": -233.86178588867188, "logps/ref_chosen": -49.30964660644531, "logps/ref_rejected": -73.73710632324219, "logps/rejected": -339.94866943359375, "loss": 1.1419, "margin_dpo/margin_mean": 81.659423828125, "margin_dpo/margin_std": 137.75778198242188, "step": 649 }, { "KL/chosen_KL_mean": -179.00872802734375, "KL/mean": -223.29705810546875, "KL/rejected_KL_mean": -267.58538818359375, "KL/std": 121.32177734375, "epoch": 0.982615268329554, "fcm_dpo/beta": 0.0045938557013869286, "fcm_dpo/delta": -0.009712686762213707, "fcm_dpo/margin": 88.57667541503906, "fcm_dpo/q_t": 0.408522367477417, "grad_norm": 14.306544303894043, "learning_rate": 5.033308820289184e-10, "logits/chosen": 0.912330150604248, "logits/rejected": 0.8635398149490356, "logps/chosen": -234.07199096679688, "logps/ref_chosen": -55.06325912475586, "logps/ref_rejected": -77.39610290527344, "logps/rejected": -344.98150634765625, "loss": 1.1298, "margin_dpo/margin_mean": 88.57667541503906, "margin_dpo/margin_std": 149.25350952148438, "step": 650 }, { "KL/chosen_KL_mean": -176.9525146484375, "KL/mean": -215.21630859375, "KL/rejected_KL_mean": -253.48007202148438, "KL/std": 118.9674301147461, "epoch": 0.9841269841269841, "fcm_dpo/beta": 0.004845967050641775, "fcm_dpo/delta": 0.03382531553506851, "fcm_dpo/margin": 76.52754211425781, "fcm_dpo/q_t": 0.4186941683292389, "grad_norm": 13.464821815490723, "learning_rate": 4.2296043218295606e-10, "logits/chosen": 0.8908314108848572, "logits/rejected": 0.8330064415931702, "logps/chosen": -231.01768493652344, "logps/ref_chosen": -54.065162658691406, "logps/ref_rejected": -77.79080200195312, "logps/rejected": -331.2708740234375, "loss": 1.1333, "margin_dpo/margin_mean": 76.52754974365234, "margin_dpo/margin_std": 123.83651733398438, "step": 651 }, { "KL/chosen_KL_mean": -209.56051635742188, "KL/mean": -243.226806640625, "KL/rejected_KL_mean": -276.89312744140625, "KL/std": 132.98809814453125, "epoch": 0.9856386999244142, "fcm_dpo/beta": 0.004979660268872976, "fcm_dpo/delta": 0.07733257114887238, "fcm_dpo/margin": 67.33261108398438, "fcm_dpo/q_t": 0.4313938617706299, "grad_norm": 16.682828903198242, "learning_rate": 3.4957118863768176e-10, "logits/chosen": 0.8507139682769775, "logits/rejected": 0.8146477937698364, "logps/chosen": -273.2008056640625, "logps/ref_chosen": -63.64030456542969, "logps/ref_rejected": -78.86882019042969, "logps/rejected": -355.761962890625, "loss": 1.2273, "margin_dpo/margin_mean": 67.33261108398438, "margin_dpo/margin_std": 161.7130889892578, "step": 652 }, { "KL/chosen_KL_mean": -185.29013061523438, "KL/mean": -227.1661376953125, "KL/rejected_KL_mean": -269.04217529296875, "KL/std": 115.51123046875, "epoch": 0.9871504157218443, "fcm_dpo/beta": 0.005149069242179394, "fcm_dpo/delta": -0.04090055823326111, "fcm_dpo/margin": 83.75203704833984, "fcm_dpo/q_t": 0.4029849171638489, "grad_norm": 16.87303352355957, "learning_rate": 2.831652042480093e-10, "logits/chosen": 0.8277177810668945, "logits/rejected": 0.7922142148017883, "logps/chosen": -246.95849609375, "logps/ref_chosen": -61.668373107910156, "logps/ref_rejected": -73.83012390136719, "logps/rejected": -342.872314453125, "loss": 1.1128, "margin_dpo/margin_mean": 83.75202941894531, "margin_dpo/margin_std": 134.10134887695312, "step": 653 }, { "KL/chosen_KL_mean": -193.46804809570312, "KL/mean": -234.66799926757812, "KL/rejected_KL_mean": -275.8679504394531, "KL/std": 139.91705322265625, "epoch": 0.9886621315192744, "fcm_dpo/beta": 0.004517903085798025, "fcm_dpo/delta": -0.09830736368894577, "fcm_dpo/margin": 82.39989471435547, "fcm_dpo/q_t": 0.4111466705799103, "grad_norm": 13.85952377319336, "learning_rate": 2.2374433653205016e-10, "logits/chosen": 0.8260544538497925, "logits/rejected": 0.7493513226509094, "logps/chosen": -251.03631591796875, "logps/ref_chosen": -57.568267822265625, "logps/ref_rejected": -87.74789428710938, "logps/rejected": -363.6158447265625, "loss": 1.1364, "margin_dpo/margin_mean": 82.39990234375, "margin_dpo/margin_std": 130.38128662109375, "step": 654 }, { "KL/chosen_KL_mean": -152.67041015625, "KL/mean": -211.55709838867188, "KL/rejected_KL_mean": -270.4437561035156, "KL/std": 120.74604797363281, "epoch": 0.9901738473167044, "fcm_dpo/beta": 0.0039674765430390835, "fcm_dpo/delta": -0.10667331516742706, "fcm_dpo/margin": 117.77336120605469, "fcm_dpo/q_t": 0.3849404454231262, "grad_norm": 10.699645042419434, "learning_rate": 1.7131024761923852e-10, "logits/chosen": 0.8190682530403137, "logits/rejected": 0.7526806592941284, "logps/chosen": -204.81756591796875, "logps/ref_chosen": -52.14714813232422, "logps/ref_rejected": -80.85014343261719, "logps/rejected": -351.29388427734375, "loss": 1.0359, "margin_dpo/margin_mean": 117.77336120605469, "margin_dpo/margin_std": 123.65576171875, "step": 655 }, { "KL/chosen_KL_mean": -205.341796875, "KL/mean": -246.5940704345703, "KL/rejected_KL_mean": -287.84637451171875, "KL/std": 123.82130432128906, "epoch": 0.9916855631141346, "fcm_dpo/beta": 0.004260948859155178, "fcm_dpo/delta": 0.05830314755439758, "fcm_dpo/margin": 82.5046157836914, "fcm_dpo/q_t": 0.42144423723220825, "grad_norm": 11.265812873840332, "learning_rate": 1.2586440420372934e-10, "logits/chosen": 0.7942889928817749, "logits/rejected": 0.7635731101036072, "logps/chosen": -278.5985107421875, "logps/ref_chosen": -73.25672912597656, "logps/ref_rejected": -85.35127258300781, "logps/rejected": -373.1976318359375, "loss": 1.1449, "margin_dpo/margin_mean": 82.50460815429688, "margin_dpo/margin_std": 136.50827026367188, "step": 656 }, { "KL/chosen_KL_mean": -177.6498565673828, "KL/mean": -230.7796173095703, "KL/rejected_KL_mean": -283.90936279296875, "KL/std": 136.02264404296875, "epoch": 0.9931972789115646, "fcm_dpo/beta": 0.004170597996562719, "fcm_dpo/delta": -0.055298324674367905, "fcm_dpo/margin": 106.25950622558594, "fcm_dpo/q_t": 0.3979775309562683, "grad_norm": 10.715606689453125, "learning_rate": 8.740807750345913e-11, "logits/chosen": 0.9494369029998779, "logits/rejected": 0.8862440586090088, "logps/chosen": -227.37326049804688, "logps/ref_chosen": -49.72339630126953, "logps/ref_rejected": -75.1568603515625, "logps/rejected": -359.06622314453125, "loss": 1.0828, "margin_dpo/margin_mean": 106.25951385498047, "margin_dpo/margin_std": 150.11639404296875, "step": 657 }, { "KL/chosen_KL_mean": -190.97988891601562, "KL/mean": -234.16209411621094, "KL/rejected_KL_mean": -277.34429931640625, "KL/std": 144.20794677734375, "epoch": 0.9947089947089947, "fcm_dpo/beta": 0.00430716760456562, "fcm_dpo/delta": 0.027860961854457855, "fcm_dpo/margin": 86.36438751220703, "fcm_dpo/q_t": 0.4168539345264435, "grad_norm": 10.777701377868652, "learning_rate": 5.594234322453539e-11, "logits/chosen": 0.8744544982910156, "logits/rejected": 0.8389475345611572, "logps/chosen": -254.02622985839844, "logps/ref_chosen": -63.04634094238281, "logps/ref_rejected": -83.44963073730469, "logps/rejected": -360.7939147949219, "loss": 1.1717, "margin_dpo/margin_mean": 86.3643798828125, "margin_dpo/margin_std": 166.66415405273438, "step": 658 }, { "KL/chosen_KL_mean": -192.81484985351562, "KL/mean": -224.96310424804688, "KL/rejected_KL_mean": -257.111328125, "KL/std": 123.95234680175781, "epoch": 0.9962207105064248, "fcm_dpo/beta": 0.004288731142878532, "fcm_dpo/delta": 0.0237504281103611, "fcm_dpo/margin": 64.29650115966797, "fcm_dpo/q_t": 0.4373660981655121, "grad_norm": 13.88487720489502, "learning_rate": 3.146808153123293e-11, "logits/chosen": 0.9137390851974487, "logits/rejected": 0.8617255687713623, "logps/chosen": -247.89505004882812, "logps/ref_chosen": -55.0802001953125, "logps/ref_rejected": -71.91049194335938, "logps/rejected": -329.0218505859375, "loss": 1.2184, "margin_dpo/margin_mean": 64.2965087890625, "margin_dpo/margin_std": 141.48397827148438, "step": 659 }, { "KL/chosen_KL_mean": -181.04736328125, "KL/mean": -235.65093994140625, "KL/rejected_KL_mean": -290.2545166015625, "KL/std": 134.4461669921875, "epoch": 0.9977324263038548, "fcm_dpo/beta": 0.004096397198736668, "fcm_dpo/delta": -0.06114449352025986, "fcm_dpo/margin": 109.20713806152344, "fcm_dpo/q_t": 0.3970072269439697, "grad_norm": 12.943835258483887, "learning_rate": 1.3985977021235829e-11, "logits/chosen": 0.947117269039154, "logits/rejected": 0.8982895612716675, "logps/chosen": -235.57327270507812, "logps/ref_chosen": -54.525917053222656, "logps/ref_rejected": -81.23604583740234, "logps/rejected": -371.49053955078125, "loss": 1.0676, "margin_dpo/margin_mean": 109.20713806152344, "margin_dpo/margin_std": 146.55783081054688, "step": 660 }, { "KL/chosen_KL_mean": -211.49526977539062, "KL/mean": -244.50070190429688, "KL/rejected_KL_mean": -277.506103515625, "KL/std": 132.15182495117188, "epoch": 0.999244142101285, "fcm_dpo/beta": 0.004478364251554012, "fcm_dpo/delta": 0.11909957975149155, "fcm_dpo/margin": 66.0108413696289, "fcm_dpo/q_t": 0.4358712136745453, "grad_norm": 12.0364990234375, "learning_rate": 3.4965187065971735e-12, "logits/chosen": 0.8396966457366943, "logits/rejected": 0.7795485258102417, "logps/chosen": -271.867919921875, "logps/ref_chosen": -60.37263870239258, "logps/ref_rejected": -77.42874145507812, "logps/rejected": -354.9348449707031, "loss": 1.2199, "margin_dpo/margin_mean": 66.0108413696289, "margin_dpo/margin_std": 152.02621459960938, "step": 661 }, { "epoch": 0.999244142101285, "step": 661, "total_flos": 0.0, "train_loss": 1.1707610893718414, "train_runtime": 1649.6542, "train_samples_per_second": 25.664, "train_steps_per_second": 0.401 } ], "logging_steps": 1, "max_steps": 661, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }