{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.52816009521484, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.20802307128906, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": 0.016103744506835938, "KL/mean": 0.019720733165740967, "KL/rejected_KL_mean": 0.023336410522460938, "KL/std": 0.25345832109451294, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.007235169410705566, "fcm_dpo/q_t": 0.5001815557479858, "grad_norm": 70.94044494628906, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.48162591457366943, "logits/rejected": -0.4422367215156555, "logps/chosen": -60.965492248535156, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.64926147460938, "loss": 1.3874, "margin_dpo/margin_mean": -0.007234543561935425, "margin_dpo/margin_std": 0.3761873245239258, "step": 3 }, { "KL/chosen_KL_mean": 0.007844924926757812, "KL/mean": 0.011015832424163818, "KL/rejected_KL_mean": 0.014186859130859375, "KL/std": 0.27283304929733276, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.006340086460113525, "fcm_dpo/q_t": 0.5001586675643921, "grad_norm": 71.89739990234375, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.4688315987586975, "logits/rejected": -0.4412044882774353, "logps/chosen": -56.759864807128906, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.63291931152344, "loss": 1.3873, "margin_dpo/margin_mean": -0.00634080171585083, "margin_dpo/margin_std": 0.4030381143093109, "step": 4 }, { "KL/chosen_KL_mean": 0.0037784576416015625, "KL/mean": 0.03290402889251709, "KL/rejected_KL_mean": 0.06203460693359375, "KL/std": 0.3190087080001831, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.05825453996658325, "fcm_dpo/q_t": 0.5014559030532837, "grad_norm": 90.05491638183594, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.5145796537399292, "logits/rejected": -0.47089338302612305, "logps/chosen": -53.85559844970703, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.08715057373047, "loss": 1.3926, "margin_dpo/margin_mean": -0.05825403332710266, "margin_dpo/margin_std": 0.4133654236793518, "step": 5 }, { "KL/chosen_KL_mean": -0.008159637451171875, "KL/mean": -0.009616628289222717, "KL/rejected_KL_mean": -0.011074066162109375, "KL/std": 0.2630811631679535, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0029135048389434814, "fcm_dpo/q_t": 0.49992650747299194, "grad_norm": 91.91436004638672, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.5270358920097351, "logits/rejected": -0.48671644926071167, "logps/chosen": -63.01564407348633, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.65641784667969, "loss": 1.3864, "margin_dpo/margin_mean": 0.002912759780883789, "margin_dpo/margin_std": 0.37860625982284546, "step": 6 }, { "KL/chosen_KL_mean": 0.018707275390625, "KL/mean": 0.022922277450561523, "KL/rejected_KL_mean": 0.027141571044921875, "KL/std": 0.28703397512435913, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.008430540561676025, "fcm_dpo/q_t": 0.5002114176750183, "grad_norm": 82.6740493774414, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.498199999332428, "logits/rejected": -0.4643540382385254, "logps/chosen": -57.75611114501953, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.89344787597656, "loss": 1.3876, "margin_dpo/margin_mean": -0.008429855108261108, "margin_dpo/margin_std": 0.42599087953567505, "step": 7 }, { "KL/chosen_KL_mean": -0.012296676635742188, "KL/mean": 0.0035399198532104492, "KL/rejected_KL_mean": 0.019378662109375, "KL/std": 0.2982986569404602, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.031677067279815674, "fcm_dpo/q_t": 0.5007911920547485, "grad_norm": 79.09713745117188, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.5000207424163818, "logits/rejected": -0.4732367694377899, "logps/chosen": -58.72833251953125, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.29204559326172, "loss": 1.3899, "margin_dpo/margin_mean": -0.03167632222175598, "margin_dpo/margin_std": 0.404108464717865, "step": 8 }, { "KL/chosen_KL_mean": 0.021236419677734375, "KL/mean": 0.0014689713716506958, "KL/rejected_KL_mean": -0.018299102783203125, "KL/std": 0.270100474357605, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.039538174867630005, "fcm_dpo/q_t": 0.4990116357803345, "grad_norm": 84.92678833007812, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.4937548339366913, "logits/rejected": -0.4469431936740875, "logps/chosen": -69.84561157226562, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.62096405029297, "loss": 1.3827, "margin_dpo/margin_mean": 0.03953742980957031, "margin_dpo/margin_std": 0.35995495319366455, "step": 9 }, { "KL/chosen_KL_mean": 0.003574371337890625, "KL/mean": -0.0032436102628707886, "KL/rejected_KL_mean": -0.010059356689453125, "KL/std": 0.25196051597595215, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013635605573654175, "fcm_dpo/q_t": 0.49965864419937134, "grad_norm": 70.92430114746094, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.5009649991989136, "logits/rejected": -0.4579939842224121, "logps/chosen": -48.35411071777344, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.38212585449219, "loss": 1.3853, "margin_dpo/margin_mean": 0.013636231422424316, "margin_dpo/margin_std": 0.36122623085975647, "step": 10 }, { "KL/chosen_KL_mean": 0.005367279052734375, "KL/mean": -0.006761014461517334, "KL/rejected_KL_mean": -0.018890380859375, "KL/std": 0.24701017141342163, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.024251699447631836, "fcm_dpo/q_t": 0.49939417839050293, "grad_norm": 68.93864440917969, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.45688319206237793, "logits/rejected": -0.4317648708820343, "logps/chosen": -53.01148986816406, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.79927062988281, "loss": 1.3842, "margin_dpo/margin_mean": 0.024251878261566162, "margin_dpo/margin_std": 0.3476247489452362, "step": 11 }, { "KL/chosen_KL_mean": -0.05808830261230469, "KL/mean": -0.053139448165893555, "KL/rejected_KL_mean": -0.048183441162109375, "KL/std": 0.2798803448677063, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.009897112846374512, "fcm_dpo/q_t": 0.5002454519271851, "grad_norm": 89.550048828125, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5406717658042908, "logits/rejected": -0.5045890808105469, "logps/chosen": -61.863525390625, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.90645599365234, "loss": 1.3877, "margin_dpo/margin_mean": -0.009897887706756592, "margin_dpo/margin_std": 0.40574026107788086, "step": 12 }, { "KL/chosen_KL_mean": 0.030796051025390625, "KL/mean": 0.014434963464736938, "KL/rejected_KL_mean": -0.00193023681640625, "KL/std": 0.294820100069046, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.032728374004364014, "fcm_dpo/q_t": 0.49918168783187866, "grad_norm": 79.24481201171875, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.4753603935241699, "logits/rejected": -0.44688206911087036, "logps/chosen": -64.22956085205078, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.20501708984375, "loss": 1.3835, "margin_dpo/margin_mean": 0.03272801637649536, "margin_dpo/margin_std": 0.4066898226737976, "step": 13 }, { "KL/chosen_KL_mean": -0.05864906311035156, "KL/mean": -0.018393635749816895, "KL/rejected_KL_mean": 0.021862030029296875, "KL/std": 0.29221126437187195, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0805087685585022, "fcm_dpo/q_t": 0.5020102858543396, "grad_norm": 85.43807983398438, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.4688841998577118, "logits/rejected": -0.4292323887348175, "logps/chosen": -58.168861389160156, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.02522277832031, "loss": 1.3949, "margin_dpo/margin_mean": -0.08050811290740967, "margin_dpo/margin_std": 0.4493297040462494, "step": 14 }, { "KL/chosen_KL_mean": 0.013751983642578125, "KL/mean": -0.004344776272773743, "KL/rejected_KL_mean": -0.022441864013671875, "KL/std": 0.2262694537639618, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.036195188760757446, "fcm_dpo/q_t": 0.49909502267837524, "grad_norm": 64.15103912353516, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.5063309669494629, "logits/rejected": -0.48831403255462646, "logps/chosen": -56.953155517578125, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.83108520507812, "loss": 1.3829, "margin_dpo/margin_mean": 0.036195307970047, "margin_dpo/margin_std": 0.31367939710617065, "step": 15 }, { "KL/chosen_KL_mean": -0.02686309814453125, "KL/mean": -0.023277834057807922, "KL/rejected_KL_mean": -0.01969146728515625, "KL/std": 0.28893446922302246, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.007169216871261597, "fcm_dpo/q_t": 0.5001786947250366, "grad_norm": 83.99118041992188, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.5097619891166687, "logits/rejected": -0.46720778942108154, "logps/chosen": -61.766754150390625, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.38916015625, "loss": 1.3874, "margin_dpo/margin_mean": -0.007169753313064575, "margin_dpo/margin_std": 0.4101220369338989, "step": 16 }, { "KL/chosen_KL_mean": 0.011844635009765625, "KL/mean": -0.012153714895248413, "KL/rejected_KL_mean": -0.03615570068359375, "KL/std": 0.26074981689453125, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.047999799251556396, "fcm_dpo/q_t": 0.4987994134426117, "grad_norm": 78.72090148925781, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.5130709409713745, "logits/rejected": -0.47745299339294434, "logps/chosen": -67.698486328125, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.41481018066406, "loss": 1.3818, "margin_dpo/margin_mean": 0.04799976944923401, "margin_dpo/margin_std": 0.34855252504348755, "step": 17 }, { "KL/chosen_KL_mean": -0.01177215576171875, "KL/mean": -0.058126211166381836, "KL/rejected_KL_mean": -0.10448455810546875, "KL/std": 0.24388915300369263, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.092709481716156, "fcm_dpo/q_t": 0.49768269062042236, "grad_norm": 81.854248046875, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.49614471197128296, "logits/rejected": -0.4409538507461548, "logps/chosen": -47.75126266479492, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.57676696777344, "loss": 1.3773, "margin_dpo/margin_mean": 0.09270939230918884, "margin_dpo/margin_std": 0.3233555257320404, "step": 18 }, { "KL/chosen_KL_mean": 0.007350921630859375, "KL/mean": -0.03270593285560608, "KL/rejected_KL_mean": -0.07275772094726562, "KL/std": 0.22124189138412476, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08010363578796387, "fcm_dpo/q_t": 0.4979983866214752, "grad_norm": 73.39251708984375, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.5021491050720215, "logits/rejected": -0.4532221555709839, "logps/chosen": -70.19801330566406, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.83035278320312, "loss": 1.3786, "margin_dpo/margin_mean": 0.08010467886924744, "margin_dpo/margin_std": 0.34056466817855835, "step": 19 }, { "KL/chosen_KL_mean": 0.010915756225585938, "KL/mean": -0.03122629225254059, "KL/rejected_KL_mean": -0.07336044311523438, "KL/std": 0.26908406615257263, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08428379893302917, "fcm_dpo/q_t": 0.4978940486907959, "grad_norm": 73.79589080810547, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5578715801239014, "logits/rejected": -0.5020963549613953, "logps/chosen": -50.792327880859375, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.89671325683594, "loss": 1.3783, "margin_dpo/margin_mean": 0.08428427577018738, "margin_dpo/margin_std": 0.3846767842769623, "step": 20 }, { "KL/chosen_KL_mean": 0.016414642333984375, "KL/mean": -0.05401710420846939, "KL/rejected_KL_mean": -0.12445068359375, "KL/std": 0.31130141019821167, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14086376130580902, "fcm_dpo/q_t": 0.49647945165634155, "grad_norm": 77.40738677978516, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.4897395968437195, "logits/rejected": -0.4655163288116455, "logps/chosen": -50.046600341796875, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.99324035644531, "loss": 1.3726, "margin_dpo/margin_mean": 0.14086447656154633, "margin_dpo/margin_std": 0.3929217755794525, "step": 21 }, { "KL/chosen_KL_mean": 0.07324600219726562, "KL/mean": -0.054158031940460205, "KL/rejected_KL_mean": -0.18156051635742188, "KL/std": 0.3565206825733185, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2548089623451233, "fcm_dpo/q_t": 0.4936355948448181, "grad_norm": 83.94691467285156, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.4774931073188782, "logits/rejected": -0.43337398767471313, "logps/chosen": -58.98439025878906, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.68623352050781, "loss": 1.3617, "margin_dpo/margin_mean": 0.2548080384731293, "margin_dpo/margin_std": 0.523194432258606, "step": 22 }, { "KL/chosen_KL_mean": 0.026395797729492188, "KL/mean": -0.0561884343624115, "KL/rejected_KL_mean": -0.13876724243164062, "KL/std": 0.30823904275894165, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1651589274406433, "fcm_dpo/q_t": 0.49587440490722656, "grad_norm": 80.6090087890625, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.5116769671440125, "logits/rejected": -0.4903678894042969, "logps/chosen": -60.051300048828125, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.2783203125, "loss": 1.3703, "margin_dpo/margin_mean": 0.16515937447547913, "margin_dpo/margin_std": 0.4328920841217041, "step": 23 }, { "KL/chosen_KL_mean": 0.053745269775390625, "KL/mean": -0.060146935284137726, "KL/rejected_KL_mean": -0.174041748046875, "KL/std": 0.277465283870697, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22778569161891937, "fcm_dpo/q_t": 0.49430739879608154, "grad_norm": 82.69357299804688, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5428138375282288, "logits/rejected": -0.525646448135376, "logps/chosen": -44.23728942871094, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.29925537109375, "loss": 1.364, "margin_dpo/margin_mean": 0.22778579592704773, "margin_dpo/margin_std": 0.36717647314071655, "step": 24 }, { "KL/chosen_KL_mean": 0.017560958862304688, "KL/mean": -0.10330188274383545, "KL/rejected_KL_mean": -0.22416305541992188, "KL/std": 0.36914387345314026, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24172252416610718, "fcm_dpo/q_t": 0.4939602315425873, "grad_norm": 73.93270111083984, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.5116393566131592, "logits/rejected": -0.48214203119277954, "logps/chosen": -52.519493103027344, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.56635284423828, "loss": 1.3628, "margin_dpo/margin_mean": 0.24172285199165344, "margin_dpo/margin_std": 0.4416242837905884, "step": 25 }, { "KL/chosen_KL_mean": 0.13255691528320312, "KL/mean": -0.08234043419361115, "KL/rejected_KL_mean": -0.2972412109375, "KL/std": 0.4482305645942688, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.42980238795280457, "fcm_dpo/q_t": 0.48926833271980286, "grad_norm": 86.75181579589844, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5427736043930054, "logits/rejected": -0.5113131999969482, "logps/chosen": -53.790252685546875, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.65695190429688, "loss": 1.3445, "margin_dpo/margin_mean": 0.42980268597602844, "margin_dpo/margin_std": 0.5426281690597534, "step": 26 }, { "KL/chosen_KL_mean": 0.12249946594238281, "KL/mean": -0.1331784576177597, "KL/rejected_KL_mean": -0.3888587951660156, "KL/std": 0.4570690989494324, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5113544464111328, "fcm_dpo/q_t": 0.4872331917285919, "grad_norm": 93.51718139648438, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.5056776404380798, "logits/rejected": -0.4691791832447052, "logps/chosen": -42.776031494140625, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.1130599975586, "loss": 1.3366, "margin_dpo/margin_mean": 0.5113544464111328, "margin_dpo/margin_std": 0.5590527057647705, "step": 27 }, { "KL/chosen_KL_mean": 0.03354072570800781, "KL/mean": -0.11374713480472565, "KL/rejected_KL_mean": -0.2610359191894531, "KL/std": 0.41249018907546997, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.29457351565361023, "fcm_dpo/q_t": 0.49264103174209595, "grad_norm": 75.92487335205078, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.49452316761016846, "logits/rejected": -0.4381081163883209, "logps/chosen": -60.5229606628418, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.66215515136719, "loss": 1.3579, "margin_dpo/margin_mean": 0.29457390308380127, "margin_dpo/margin_std": 0.5741066336631775, "step": 28 }, { "KL/chosen_KL_mean": 0.06393051147460938, "KL/mean": -0.18335245549678802, "KL/rejected_KL_mean": -0.430633544921875, "KL/std": 0.4628360867500305, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4945622384548187, "fcm_dpo/q_t": 0.4876513183116913, "grad_norm": 91.02857971191406, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.5597679615020752, "logits/rejected": -0.5140695571899414, "logps/chosen": -57.74385452270508, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.82498168945312, "loss": 1.3383, "margin_dpo/margin_mean": 0.4945625960826874, "margin_dpo/margin_std": 0.5654176473617554, "step": 29 }, { "KL/chosen_KL_mean": 0.1397686004638672, "KL/mean": -0.20081256330013275, "KL/rejected_KL_mean": -0.5413970947265625, "KL/std": 0.5910484194755554, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6811658143997192, "fcm_dpo/q_t": 0.48299551010131836, "grad_norm": 87.69357299804688, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.4750695824623108, "logits/rejected": -0.4435637593269348, "logps/chosen": -52.43760299682617, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -99.03060150146484, "loss": 1.3203, "margin_dpo/margin_mean": 0.6811659336090088, "margin_dpo/margin_std": 0.6095134019851685, "step": 30 }, { "KL/chosen_KL_mean": 0.1102294921875, "KL/mean": -0.13631178438663483, "KL/rejected_KL_mean": -0.3828544616699219, "KL/std": 0.5659123063087463, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.49308571219444275, "fcm_dpo/q_t": 0.4877015948295593, "grad_norm": 67.67135620117188, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.5170990228652954, "logits/rejected": -0.4736338257789612, "logps/chosen": -63.696693420410156, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.27685546875, "loss": 1.3388, "margin_dpo/margin_mean": 0.49308544397354126, "margin_dpo/margin_std": 0.6761853694915771, "step": 31 }, { "KL/chosen_KL_mean": 0.21260833740234375, "KL/mean": -0.19022028148174286, "KL/rejected_KL_mean": -0.5930519104003906, "KL/std": 0.7161545753479004, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8056564331054688, "fcm_dpo/q_t": 0.47991523146629333, "grad_norm": 83.33721160888672, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.504935085773468, "logits/rejected": -0.46319282054901123, "logps/chosen": -62.52691650390625, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.91055297851562, "loss": 1.3091, "margin_dpo/margin_mean": 0.8056566715240479, "margin_dpo/margin_std": 0.8240557909011841, "step": 32 }, { "KL/chosen_KL_mean": 0.09860420227050781, "KL/mean": -0.18287688493728638, "KL/rejected_KL_mean": -0.4643592834472656, "KL/std": 0.5310133099555969, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5629608035087585, "fcm_dpo/q_t": 0.48594412207603455, "grad_norm": 72.46797180175781, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.5104891657829285, "logits/rejected": -0.4849121570587158, "logps/chosen": -53.16236877441406, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.34949493408203, "loss": 1.3317, "margin_dpo/margin_mean": 0.5629609823226929, "margin_dpo/margin_std": 0.5864159464836121, "step": 33 }, { "KL/chosen_KL_mean": 0.04570198059082031, "KL/mean": -0.32518571615219116, "KL/rejected_KL_mean": -0.696075439453125, "KL/std": 0.7097070813179016, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7417768239974976, "fcm_dpo/q_t": 0.48150891065597534, "grad_norm": 77.95159912109375, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.4825457036495209, "logits/rejected": -0.4650344252586365, "logps/chosen": -50.77162551879883, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.617919921875, "loss": 1.3155, "margin_dpo/margin_mean": 0.7417754530906677, "margin_dpo/margin_std": 0.8860844969749451, "step": 34 }, { "KL/chosen_KL_mean": 0.15726661682128906, "KL/mean": -0.44504478573799133, "KL/rejected_KL_mean": -1.0473518371582031, "KL/std": 0.9632636308670044, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2046229839324951, "fcm_dpo/q_t": 0.47004756331443787, "grad_norm": 83.60721588134766, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.5268377065658569, "logits/rejected": -0.49007338285446167, "logps/chosen": -50.867225646972656, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.87178039550781, "loss": 1.2721, "margin_dpo/margin_mean": 1.204622745513916, "margin_dpo/margin_std": 1.0357599258422852, "step": 35 }, { "KL/chosen_KL_mean": 0.02988433837890625, "KL/mean": -0.5372347831726074, "KL/rejected_KL_mean": -1.1043548583984375, "KL/std": 1.094662070274353, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1342408657073975, "fcm_dpo/q_t": 0.47180622816085815, "grad_norm": 72.68156433105469, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5595717430114746, "logits/rejected": -0.523262619972229, "logps/chosen": -51.96160888671875, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.14498138427734, "loss": 1.2802, "margin_dpo/margin_mean": 1.1342405080795288, "margin_dpo/margin_std": 1.2309517860412598, "step": 36 }, { "KL/chosen_KL_mean": 0.023458480834960938, "KL/mean": -0.4883181154727936, "KL/rejected_KL_mean": -1.0000991821289062, "KL/std": 1.0986820459365845, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0235590934753418, "fcm_dpo/q_t": 0.4746357798576355, "grad_norm": 61.56761932373047, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.5261737108230591, "logits/rejected": -0.4837297797203064, "logps/chosen": -62.78364944458008, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.89517211914062, "loss": 1.2918, "margin_dpo/margin_mean": 1.0235590934753418, "margin_dpo/margin_std": 1.4123997688293457, "step": 37 }, { "KL/chosen_KL_mean": 0.1512908935546875, "KL/mean": -0.5362459421157837, "KL/rejected_KL_mean": -1.2237815856933594, "KL/std": 1.3535866737365723, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3750734329223633, "fcm_dpo/q_t": 0.4660550653934479, "grad_norm": 69.93470001220703, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5330358743667603, "logits/rejected": -0.5002619028091431, "logps/chosen": -48.239227294921875, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.13622283935547, "loss": 1.2601, "margin_dpo/margin_mean": 1.3750736713409424, "margin_dpo/margin_std": 1.628260850906372, "step": 38 }, { "KL/chosen_KL_mean": 0.08315658569335938, "KL/mean": -0.7190086841583252, "KL/rejected_KL_mean": -1.5211677551269531, "KL/std": 1.2815454006195068, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6043219566345215, "fcm_dpo/q_t": 0.46017903089523315, "grad_norm": 74.16421508789062, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.5571820735931396, "logits/rejected": -0.5172927379608154, "logps/chosen": -50.66731643676758, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.09068298339844, "loss": 1.2371, "margin_dpo/margin_mean": 1.6043217182159424, "margin_dpo/margin_std": 1.3714519739151, "step": 39 }, { "KL/chosen_KL_mean": 0.18222999572753906, "KL/mean": -0.5860625505447388, "KL/rejected_KL_mean": -1.3543624877929688, "KL/std": 1.4266393184661865, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5365939140319824, "fcm_dpo/q_t": 0.46209070086479187, "grad_norm": 60.399505615234375, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.5290929079055786, "logits/rejected": -0.4998553693294525, "logps/chosen": -57.80283737182617, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.65443420410156, "loss": 1.246, "margin_dpo/margin_mean": 1.5365946292877197, "margin_dpo/margin_std": 1.694861650466919, "step": 40 }, { "KL/chosen_KL_mean": 0.00180816650390625, "KL/mean": -0.9451515078544617, "KL/rejected_KL_mean": -1.8921089172363281, "KL/std": 1.8964862823486328, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8939173221588135, "fcm_dpo/q_t": 0.4534837007522583, "grad_norm": 68.17810821533203, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.5442007184028625, "logits/rejected": -0.5080921649932861, "logps/chosen": -62.694007873535156, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.91563415527344, "loss": 1.217, "margin_dpo/margin_mean": 1.8939182758331299, "margin_dpo/margin_std": 2.07293701171875, "step": 41 }, { "KL/chosen_KL_mean": 0.20568084716796875, "KL/mean": -1.066096544265747, "KL/rejected_KL_mean": -2.337871551513672, "KL/std": 2.129526138305664, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.12230241298675537, "fcm_dpo/delta": 0.12432946264743805, "fcm_dpo/margin": 2.5435502529144287, "fcm_dpo/q_t": 0.43291333317756653, "grad_norm": 92.2998046875, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.5354119539260864, "logits/rejected": -0.4888347387313843, "logps/chosen": -58.760746002197266, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.24624633789062, "loss": 1.119, "margin_dpo/margin_mean": 2.5435495376586914, "margin_dpo/margin_std": 2.3769054412841797, "step": 42 }, { "KL/chosen_KL_mean": 0.5492343902587891, "KL/mean": -0.6893892288208008, "KL/rejected_KL_mean": -1.9280166625976562, "KL/std": 1.8261648416519165, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.14197269082069397, "fcm_dpo/delta": 0.07470498234033585, "fcm_dpo/margin": 2.477250099182129, "fcm_dpo/q_t": 0.42087799310684204, "grad_norm": 93.16883087158203, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.5261520147323608, "logits/rejected": -0.5005497932434082, "logps/chosen": -53.60676193237305, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.408203125, "loss": 1.0844, "margin_dpo/margin_mean": 2.477250099182129, "margin_dpo/margin_std": 1.9302922487258911, "step": 43 }, { "KL/chosen_KL_mean": 0.22637557983398438, "KL/mean": -1.1899750232696533, "KL/rejected_KL_mean": -2.6063270568847656, "KL/std": 2.267411231994629, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.14421430230140686, "fcm_dpo/delta": -0.01492190733551979, "fcm_dpo/margin": 2.8327009677886963, "fcm_dpo/q_t": 0.40067583322525024, "grad_norm": 101.48235321044922, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.4636678099632263, "logits/rejected": -0.44394707679748535, "logps/chosen": -49.85212326049805, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.39009094238281, "loss": 1.0429, "margin_dpo/margin_mean": 2.8327014446258545, "margin_dpo/margin_std": 2.2093987464904785, "step": 44 }, { "KL/chosen_KL_mean": 0.19671249389648438, "KL/mean": -0.9757533073425293, "KL/rejected_KL_mean": -2.1482200622558594, "KL/std": 2.1102242469787598, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.15825220942497253, "fcm_dpo/delta": 0.041776977479457855, "fcm_dpo/margin": 2.344935417175293, "fcm_dpo/q_t": 0.4162585139274597, "grad_norm": 85.96018981933594, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.5120048522949219, "logits/rejected": -0.5001665353775024, "logps/chosen": -48.21821594238281, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.08464813232422, "loss": 1.0898, "margin_dpo/margin_mean": 2.344935417175293, "margin_dpo/margin_std": 2.62362003326416, "step": 45 }, { "KL/chosen_KL_mean": 0.17397499084472656, "KL/mean": -1.2946666479110718, "KL/rejected_KL_mean": -2.7633094787597656, "KL/std": 2.6588382720947266, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.14704476296901703, "fcm_dpo/delta": -0.05587401241064072, "fcm_dpo/margin": 2.9372873306274414, "fcm_dpo/q_t": 0.3960358500480652, "grad_norm": 88.10020446777344, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.517956018447876, "logits/rejected": -0.46634984016418457, "logps/chosen": -55.8254508972168, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.4158935546875, "loss": 1.0542, "margin_dpo/margin_mean": 2.9372870922088623, "margin_dpo/margin_std": 3.269395351409912, "step": 46 }, { "KL/chosen_KL_mean": 0.4055042266845703, "KL/mean": -1.0942176580429077, "KL/rejected_KL_mean": -2.5939369201660156, "KL/std": 2.6453235149383545, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.13928335905075073, "fcm_dpo/delta": -0.032578371465206146, "fcm_dpo/margin": 2.9994425773620605, "fcm_dpo/q_t": 0.3979858160018921, "grad_norm": 82.3074951171875, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5774295330047607, "logits/rejected": -0.5251759886741638, "logps/chosen": -57.52057647705078, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.27314758300781, "loss": 1.0483, "margin_dpo/margin_mean": 2.999443769454956, "margin_dpo/margin_std": 2.69224214553833, "step": 47 }, { "KL/chosen_KL_mean": 0.05373954772949219, "KL/mean": -1.51072359085083, "KL/rejected_KL_mean": -3.075183868408203, "KL/std": 2.700470447540283, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.12641848623752594, "fcm_dpo/delta": 0.0021915659308433533, "fcm_dpo/margin": 3.1289291381835938, "fcm_dpo/q_t": 0.406181663274765, "grad_norm": 87.43419647216797, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.5810247659683228, "logits/rejected": -0.5221822261810303, "logps/chosen": -57.134334564208984, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.09178161621094, "loss": 1.0689, "margin_dpo/margin_mean": 3.1289291381835938, "margin_dpo/margin_std": 2.9688820838928223, "step": 48 }, { "KL/chosen_KL_mean": 0.3045005798339844, "KL/mean": -1.5968005657196045, "KL/rejected_KL_mean": -3.498096466064453, "KL/std": 3.415726661682129, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.11936050653457642, "fcm_dpo/delta": -0.0978364571928978, "fcm_dpo/margin": 3.8025965690612793, "fcm_dpo/q_t": 0.38723063468933105, "grad_norm": 69.9267578125, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.5347216725349426, "logits/rejected": -0.4745851755142212, "logps/chosen": -61.380767822265625, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.26557159423828, "loss": 1.0376, "margin_dpo/margin_mean": 3.8025965690612793, "margin_dpo/margin_std": 3.9496779441833496, "step": 49 }, { "KL/chosen_KL_mean": -0.15890121459960938, "KL/mean": -2.2356748580932617, "KL/rejected_KL_mean": -4.31243896484375, "KL/std": 3.5314018726348877, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10124868154525757, "fcm_dpo/delta": -0.03730197250843048, "fcm_dpo/margin": 4.153538227081299, "fcm_dpo/q_t": 0.3976140022277832, "grad_norm": 63.207149505615234, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5480292439460754, "logits/rejected": -0.512259304523468, "logps/chosen": -58.88304138183594, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.67058563232422, "loss": 1.0489, "margin_dpo/margin_mean": 4.153538703918457, "margin_dpo/margin_std": 4.076920986175537, "step": 50 }, { "KL/chosen_KL_mean": -0.2769756317138672, "KL/mean": -2.2801334857940674, "KL/rejected_KL_mean": -4.283294677734375, "KL/std": 4.116324424743652, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.09770938754081726, "fcm_dpo/delta": 0.00890747457742691, "fcm_dpo/margin": 4.006317138671875, "fcm_dpo/q_t": 0.4107528030872345, "grad_norm": 52.39238357543945, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.539941132068634, "logits/rejected": -0.5076016187667847, "logps/chosen": -61.65064239501953, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.28529357910156, "loss": 1.0914, "margin_dpo/margin_mean": 4.006316661834717, "margin_dpo/margin_std": 5.059848308563232, "step": 51 }, { "KL/chosen_KL_mean": 0.4057025909423828, "KL/mean": -2.6870739459991455, "KL/rejected_KL_mean": -5.779850006103516, "KL/std": 4.8188886642456055, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.077643021941185, "fcm_dpo/delta": -0.17084355652332306, "fcm_dpo/margin": 6.185553073883057, "fcm_dpo/q_t": 0.3687407672405243, "grad_norm": 52.488014221191406, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.5524188876152039, "logits/rejected": -0.4968685805797577, "logps/chosen": -51.93165588378906, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.75376892089844, "loss": 0.9997, "margin_dpo/margin_mean": 6.185553073883057, "margin_dpo/margin_std": 5.159676551818848, "step": 52 }, { "KL/chosen_KL_mean": -0.19318580627441406, "KL/mean": -3.297855854034424, "KL/rejected_KL_mean": -6.402523040771484, "KL/std": 5.35381555557251, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.07053211331367493, "fcm_dpo/delta": -0.07264071702957153, "fcm_dpo/margin": 6.209340572357178, "fcm_dpo/q_t": 0.3935110867023468, "grad_norm": 48.63420104980469, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.6061801910400391, "logits/rejected": -0.5843982696533203, "logps/chosen": -53.507835388183594, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.18611907958984, "loss": 1.0485, "margin_dpo/margin_mean": 6.209340572357178, "margin_dpo/margin_std": 6.154953479766846, "step": 53 }, { "KL/chosen_KL_mean": -0.3580665588378906, "KL/mean": -3.091987133026123, "KL/rejected_KL_mean": -5.825908660888672, "KL/std": 5.11324405670166, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.06830751895904541, "fcm_dpo/delta": 0.039879053831100464, "fcm_dpo/margin": 5.467842102050781, "fcm_dpo/q_t": 0.41404271125793457, "grad_norm": 49.40464782714844, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.5767498016357422, "logits/rejected": -0.5225586295127869, "logps/chosen": -51.046730041503906, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.54130554199219, "loss": 1.0757, "margin_dpo/margin_mean": 5.467841148376465, "margin_dpo/margin_std": 5.10695743560791, "step": 54 }, { "KL/chosen_KL_mean": -0.9405612945556641, "KL/mean": -4.179176330566406, "KL/rejected_KL_mean": -7.417789459228516, "KL/std": 6.182194232940674, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.06581026315689087, "fcm_dpo/delta": -0.04618923366069794, "fcm_dpo/margin": 6.477225303649902, "fcm_dpo/q_t": 0.3977408707141876, "grad_norm": 41.99592971801758, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6314051151275635, "logits/rejected": -0.5677323341369629, "logps/chosen": -63.55579376220703, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.41129302978516, "loss": 1.0627, "margin_dpo/margin_mean": 6.4772257804870605, "margin_dpo/margin_std": 7.436443328857422, "step": 55 }, { "KL/chosen_KL_mean": -0.7300090789794922, "KL/mean": -3.821871280670166, "KL/rejected_KL_mean": -6.913738250732422, "KL/std": 6.0643815994262695, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.06379474699497223, "fcm_dpo/delta": 0.00866011157631874, "fcm_dpo/margin": 6.183729648590088, "fcm_dpo/q_t": 0.4088730216026306, "grad_norm": 37.987911224365234, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.5912456512451172, "logits/rejected": -0.5477725267410278, "logps/chosen": -58.66273498535156, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.08818054199219, "loss": 1.0816, "margin_dpo/margin_mean": 6.183730602264404, "margin_dpo/margin_std": 7.2952046394348145, "step": 56 }, { "KL/chosen_KL_mean": -0.7435302734375, "KL/mean": -4.197587966918945, "KL/rejected_KL_mean": -7.651645660400391, "KL/std": 5.97504997253418, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.061574921011924744, "fcm_dpo/delta": -0.04527352750301361, "fcm_dpo/margin": 6.908114910125732, "fcm_dpo/q_t": 0.3953145742416382, "grad_norm": 41.76232147216797, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.5989680290222168, "logits/rejected": -0.571702241897583, "logps/chosen": -71.23881530761719, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.21710205078125, "loss": 1.0479, "margin_dpo/margin_mean": 6.908115386962891, "margin_dpo/margin_std": 6.625408172607422, "step": 57 }, { "KL/chosen_KL_mean": -1.0145072937011719, "KL/mean": -4.882469177246094, "KL/rejected_KL_mean": -8.75042724609375, "KL/std": 6.831453323364258, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.0555315725505352, "fcm_dpo/delta": -0.05244975909590721, "fcm_dpo/margin": 7.735917568206787, "fcm_dpo/q_t": 0.3955305814743042, "grad_norm": 40.50571060180664, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.5726555585861206, "logits/rejected": -0.49422115087509155, "logps/chosen": -63.14744567871094, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.36772155761719, "loss": 1.0532, "margin_dpo/margin_mean": 7.7359185218811035, "margin_dpo/margin_std": 8.176453590393066, "step": 58 }, { "KL/chosen_KL_mean": -1.488332748413086, "KL/mean": -5.7610859870910645, "KL/rejected_KL_mean": -10.033843994140625, "KL/std": 7.584994792938232, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.047585196793079376, "fcm_dpo/delta": -0.022728845477104187, "fcm_dpo/margin": 8.545510292053223, "fcm_dpo/q_t": 0.40144288539886475, "grad_norm": 37.327362060546875, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.6131638288497925, "logits/rejected": -0.572055459022522, "logps/chosen": -53.420860290527344, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.9190444946289, "loss": 1.0661, "margin_dpo/margin_mean": 8.545509338378906, "margin_dpo/margin_std": 8.80247688293457, "step": 59 }, { "KL/chosen_KL_mean": -2.7225379943847656, "KL/mean": -6.128490447998047, "KL/rejected_KL_mean": -9.534439086914062, "KL/std": 6.866521835327148, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.053369902074337006, "fcm_dpo/delta": 0.05756625533103943, "fcm_dpo/margin": 6.811899185180664, "fcm_dpo/q_t": 0.4179559648036957, "grad_norm": 40.32707214355469, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.6027133464813232, "logits/rejected": -0.5420011878013611, "logps/chosen": -63.66472625732422, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.92784118652344, "loss": 1.0996, "margin_dpo/margin_mean": 6.811899185180664, "margin_dpo/margin_std": 7.766169548034668, "step": 60 }, { "KL/chosen_KL_mean": -1.580484390258789, "KL/mean": -5.808887481689453, "KL/rejected_KL_mean": -10.03729248046875, "KL/std": 9.263717651367188, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.05006622523069382, "fcm_dpo/delta": -0.04818345606327057, "fcm_dpo/margin": 8.456808090209961, "fcm_dpo/q_t": 0.40202924609184265, "grad_norm": 35.04801940917969, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.6034557819366455, "logits/rejected": -0.5694935917854309, "logps/chosen": -62.21400833129883, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.88978576660156, "loss": 1.0795, "margin_dpo/margin_mean": 8.456808090209961, "margin_dpo/margin_std": 11.565231323242188, "step": 61 }, { "KL/chosen_KL_mean": -1.7315292358398438, "KL/mean": -4.907068729400635, "KL/rejected_KL_mean": -8.08260726928711, "KL/std": 6.785726070404053, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.057009097188711166, "fcm_dpo/delta": 0.05251770839095116, "fcm_dpo/margin": 6.351075649261475, "fcm_dpo/q_t": 0.4195292592048645, "grad_norm": 36.52075958251953, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.6032343506813049, "logits/rejected": -0.5685884952545166, "logps/chosen": -57.882301330566406, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.6488037109375, "loss": 1.1108, "margin_dpo/margin_mean": 6.351076126098633, "margin_dpo/margin_std": 8.331364631652832, "step": 62 }, { "KL/chosen_KL_mean": -2.8989200592041016, "KL/mean": -7.218653678894043, "KL/rejected_KL_mean": -11.538387298583984, "KL/std": 8.756481170654297, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.04904904216527939, "fcm_dpo/delta": -0.0558142215013504, "fcm_dpo/margin": 8.639472961425781, "fcm_dpo/q_t": 0.3941209316253662, "grad_norm": 36.34617614746094, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.608661949634552, "logits/rejected": -0.5645633935928345, "logps/chosen": -76.04631042480469, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -109.14845275878906, "loss": 1.0578, "margin_dpo/margin_mean": 8.639472007751465, "margin_dpo/margin_std": 9.21734619140625, "step": 63 }, { "KL/chosen_KL_mean": -1.1949291229248047, "KL/mean": -6.278130054473877, "KL/rejected_KL_mean": -11.361335754394531, "KL/std": 9.37601089477539, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.04156487435102463, "fcm_dpo/delta": -0.07142579555511475, "fcm_dpo/margin": 10.166400909423828, "fcm_dpo/q_t": 0.39335623383522034, "grad_norm": 31.952661514282227, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.6194932460784912, "logits/rejected": -0.5888575911521912, "logps/chosen": -55.19352722167969, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -104.89154052734375, "loss": 1.0654, "margin_dpo/margin_mean": 10.166400909423828, "margin_dpo/margin_std": 11.463658332824707, "step": 64 }, { "KL/chosen_KL_mean": -3.5524654388427734, "KL/mean": -8.784185409545898, "KL/rejected_KL_mean": -14.01590347290039, "KL/std": 10.731781005859375, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.03978603705763817, "fcm_dpo/delta": -0.03537318855524063, "fcm_dpo/margin": 10.463440895080566, "fcm_dpo/q_t": 0.39846062660217285, "grad_norm": 32.09275436401367, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6814702153205872, "logits/rejected": -0.6694004535675049, "logps/chosen": -68.38846588134766, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -123.96236419677734, "loss": 1.0664, "margin_dpo/margin_mean": 10.46343994140625, "margin_dpo/margin_std": 11.84306812286377, "step": 65 }, { "KL/chosen_KL_mean": -3.371671676635742, "KL/mean": -8.132331848144531, "KL/rejected_KL_mean": -12.892993927001953, "KL/std": 9.663976669311523, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.04018988832831383, "fcm_dpo/delta": 0.02319152280688286, "fcm_dpo/margin": 9.521313667297363, "fcm_dpo/q_t": 0.4129948616027832, "grad_norm": 28.723054885864258, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.6573896408081055, "logits/rejected": -0.6258237957954407, "logps/chosen": -54.8151969909668, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -88.529296875, "loss": 1.0948, "margin_dpo/margin_mean": 9.521312713623047, "margin_dpo/margin_std": 11.660235404968262, "step": 66 }, { "KL/chosen_KL_mean": -3.1228866577148438, "KL/mean": -8.022209167480469, "KL/rejected_KL_mean": -12.92153549194336, "KL/std": 10.03524398803711, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.042240217328071594, "fcm_dpo/delta": -0.024514786899089813, "fcm_dpo/margin": 9.798646926879883, "fcm_dpo/q_t": 0.40358948707580566, "grad_norm": 30.853675842285156, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.6088730096817017, "logits/rejected": -0.5662700533866882, "logps/chosen": -62.46369171142578, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -85.70881652832031, "loss": 1.0751, "margin_dpo/margin_mean": 9.798646926879883, "margin_dpo/margin_std": 12.186027526855469, "step": 67 }, { "KL/chosen_KL_mean": -3.8463668823242188, "KL/mean": -8.590458869934082, "KL/rejected_KL_mean": -13.334552764892578, "KL/std": 9.031107902526855, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.041834283620119095, "fcm_dpo/delta": 0.003476101905107498, "fcm_dpo/margin": 9.48818588256836, "fcm_dpo/q_t": 0.4069967567920685, "grad_norm": 29.675251007080078, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6760765314102173, "logits/rejected": -0.6185558438301086, "logps/chosen": -69.05220031738281, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -90.54180145263672, "loss": 1.0714, "margin_dpo/margin_mean": 9.48818588256836, "margin_dpo/margin_std": 10.152603149414062, "step": 68 }, { "KL/chosen_KL_mean": -4.753917694091797, "KL/mean": -10.989311218261719, "KL/rejected_KL_mean": -17.224708557128906, "KL/std": 11.462624549865723, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.034877799451351166, "fcm_dpo/delta": -0.07517345249652863, "fcm_dpo/margin": 12.470787048339844, "fcm_dpo/q_t": 0.39176517724990845, "grad_norm": 30.214656829833984, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.6015123724937439, "logits/rejected": -0.5792900323867798, "logps/chosen": -64.57315826416016, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -120.61357116699219, "loss": 1.0442, "margin_dpo/margin_mean": 12.470787048339844, "margin_dpo/margin_std": 12.772706985473633, "step": 69 }, { "KL/chosen_KL_mean": -7.221284866333008, "KL/mean": -13.940620422363281, "KL/rejected_KL_mean": -20.659957885742188, "KL/std": 14.507774353027344, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.0326962023973465, "fcm_dpo/delta": -0.06987478584051132, "fcm_dpo/margin": 13.438671112060547, "fcm_dpo/q_t": 0.39314454793930054, "grad_norm": 29.549423217773438, "learning_rate": 5e-07, "logits/chosen": -0.6086961627006531, "logits/rejected": -0.5738969445228577, "logps/chosen": -69.15192413330078, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -111.72074127197266, "loss": 1.0564, "margin_dpo/margin_mean": 13.438671112060547, "margin_dpo/margin_std": 15.399250984191895, "step": 70 }, { "KL/chosen_KL_mean": -7.312736511230469, "KL/mean": -15.442537307739258, "KL/rejected_KL_mean": -23.572338104248047, "KL/std": 15.629602432250977, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.026657406240701675, "fcm_dpo/delta": -0.06790647655725479, "fcm_dpo/margin": 16.259611129760742, "fcm_dpo/q_t": 0.3945091962814331, "grad_norm": 26.020174026489258, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.6621614694595337, "logits/rejected": -0.6253249645233154, "logps/chosen": -69.06307220458984, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -120.9089584350586, "loss": 1.0485, "margin_dpo/margin_mean": 16.259613037109375, "margin_dpo/margin_std": 17.08768653869629, "step": 71 }, { "KL/chosen_KL_mean": -8.572813034057617, "KL/mean": -16.923320770263672, "KL/rejected_KL_mean": -25.273826599121094, "KL/std": 16.826129913330078, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.025071250274777412, "fcm_dpo/delta": -0.03367580845952034, "fcm_dpo/margin": 16.70101547241211, "fcm_dpo/q_t": 0.4001430869102478, "grad_norm": 24.218080520629883, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6549113988876343, "logits/rejected": -0.617749810218811, "logps/chosen": -74.626220703125, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -120.5608139038086, "loss": 1.0698, "margin_dpo/margin_mean": 16.70101547241211, "margin_dpo/margin_std": 20.266984939575195, "step": 72 }, { "KL/chosen_KL_mean": -10.759519577026367, "KL/mean": -19.278858184814453, "KL/rejected_KL_mean": -27.798202514648438, "KL/std": 21.811187744140625, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.02377324365079403, "fcm_dpo/delta": -0.011964067816734314, "fcm_dpo/margin": 17.038679122924805, "fcm_dpo/q_t": 0.40983206033706665, "grad_norm": 23.957334518432617, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.6897900104522705, "logits/rejected": -0.6500684022903442, "logps/chosen": -77.01579284667969, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -118.25433349609375, "loss": 1.1199, "margin_dpo/margin_mean": 17.038681030273438, "margin_dpo/margin_std": 27.641098022460938, "step": 73 }, { "KL/chosen_KL_mean": -11.2384033203125, "KL/mean": -20.760555267333984, "KL/rejected_KL_mean": -30.28270721435547, "KL/std": 24.14501190185547, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.022344253957271576, "fcm_dpo/delta": -0.0470757819712162, "fcm_dpo/margin": 19.044301986694336, "fcm_dpo/q_t": 0.40436846017837524, "grad_norm": 24.303987503051758, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.6677048206329346, "logits/rejected": -0.6622331142425537, "logps/chosen": -64.66328430175781, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -126.22964477539062, "loss": 1.0957, "margin_dpo/margin_mean": 19.044300079345703, "margin_dpo/margin_std": 30.3118896484375, "step": 74 }, { "KL/chosen_KL_mean": -11.227849960327148, "KL/mean": -24.636085510253906, "KL/rejected_KL_mean": -38.04431915283203, "KL/std": 24.73340606689453, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.018182795494794846, "fcm_dpo/delta": -0.16822758316993713, "fcm_dpo/margin": 26.81647491455078, "fcm_dpo/q_t": 0.3744775354862213, "grad_norm": 22.551315307617188, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.7105200290679932, "logits/rejected": -0.70478755235672, "logps/chosen": -63.08951187133789, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -149.29830932617188, "loss": 1.0253, "margin_dpo/margin_mean": 26.81647491455078, "margin_dpo/margin_std": 28.454429626464844, "step": 75 }, { "KL/chosen_KL_mean": -13.570169448852539, "KL/mean": -21.888824462890625, "KL/rejected_KL_mean": -30.207481384277344, "KL/std": 20.21454620361328, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.019159602001309395, "fcm_dpo/delta": 0.1141112893819809, "fcm_dpo/margin": 16.637313842773438, "fcm_dpo/q_t": 0.43109238147735596, "grad_norm": 21.629785537719727, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.7143449783325195, "logits/rejected": -0.6758897304534912, "logps/chosen": -66.83621215820312, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -108.42411041259766, "loss": 1.1366, "margin_dpo/margin_mean": 16.63731575012207, "margin_dpo/margin_std": 22.22722816467285, "step": 76 }, { "KL/chosen_KL_mean": -12.503726959228516, "KL/mean": -26.28228759765625, "KL/rejected_KL_mean": -40.06085205078125, "KL/std": 27.661182403564453, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.016155380755662918, "fcm_dpo/delta": -0.10240481793880463, "fcm_dpo/margin": 27.55712127685547, "fcm_dpo/q_t": 0.38546115159988403, "grad_norm": 21.97159767150879, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.6718529462814331, "logits/rejected": -0.635248601436615, "logps/chosen": -70.60040283203125, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -133.83447265625, "loss": 1.0496, "margin_dpo/margin_mean": 27.55712127685547, "margin_dpo/margin_std": 31.21552085876465, "step": 77 }, { "KL/chosen_KL_mean": -12.715435028076172, "KL/mean": -22.89756202697754, "KL/rejected_KL_mean": -33.07969284057617, "KL/std": 22.80880355834961, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.0179641991853714, "fcm_dpo/delta": 0.04957953840494156, "fcm_dpo/margin": 20.364255905151367, "fcm_dpo/q_t": 0.41824695467948914, "grad_norm": 20.99672508239746, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.6376138925552368, "logits/rejected": -0.6148805618286133, "logps/chosen": -68.3292236328125, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -118.01405334472656, "loss": 1.1079, "margin_dpo/margin_mean": 20.364255905151367, "margin_dpo/margin_std": 26.547138214111328, "step": 78 }, { "KL/chosen_KL_mean": -13.37647819519043, "KL/mean": -24.498859405517578, "KL/rejected_KL_mean": -35.62124252319336, "KL/std": 25.48963737487793, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.01794726401567459, "fcm_dpo/delta": 0.0012769072782248259, "fcm_dpo/margin": 22.24475860595703, "fcm_dpo/q_t": 0.41002583503723145, "grad_norm": 19.79173469543457, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6492550373077393, "logits/rejected": -0.6207257509231567, "logps/chosen": -68.82696533203125, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -123.26880645751953, "loss": 1.0989, "margin_dpo/margin_mean": 22.24475860595703, "margin_dpo/margin_std": 31.258174896240234, "step": 79 }, { "KL/chosen_KL_mean": -17.12893295288086, "KL/mean": -28.049373626708984, "KL/rejected_KL_mean": -38.969810485839844, "KL/std": 28.13406753540039, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.017532825469970703, "fcm_dpo/delta": 0.020619504153728485, "fcm_dpo/margin": 21.840877532958984, "fcm_dpo/q_t": 0.41682252287864685, "grad_norm": 21.230684280395508, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7177095413208008, "logits/rejected": -0.6798745393753052, "logps/chosen": -75.64822387695312, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -126.5173110961914, "loss": 1.1204, "margin_dpo/margin_mean": 21.840877532958984, "margin_dpo/margin_std": 32.85547637939453, "step": 80 }, { "KL/chosen_KL_mean": -16.53759765625, "KL/mean": -31.449586868286133, "KL/rejected_KL_mean": -46.361572265625, "KL/std": 33.21380615234375, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.015006015077233315, "fcm_dpo/delta": -0.10901469737291336, "fcm_dpo/margin": 29.823978424072266, "fcm_dpo/q_t": 0.3896971344947815, "grad_norm": 20.577526092529297, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.7138460874557495, "logits/rejected": -0.6892383694648743, "logps/chosen": -82.98646545410156, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -176.02427673339844, "loss": 1.0665, "margin_dpo/margin_mean": 29.823976516723633, "margin_dpo/margin_std": 39.704833984375, "step": 81 }, { "KL/chosen_KL_mean": -19.28495216369629, "KL/mean": -31.137420654296875, "KL/rejected_KL_mean": -42.98988723754883, "KL/std": 30.5316104888916, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.016485687345266342, "fcm_dpo/delta": -0.008920453488826752, "fcm_dpo/margin": 23.70493507385254, "fcm_dpo/q_t": 0.4126429855823517, "grad_norm": 23.881196975708008, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6434165239334106, "logits/rejected": -0.6364510655403137, "logps/chosen": -71.517333984375, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -133.73313903808594, "loss": 1.1577, "margin_dpo/margin_mean": 23.704936981201172, "margin_dpo/margin_std": 42.49738311767578, "step": 82 }, { "KL/chosen_KL_mean": -19.908788681030273, "KL/mean": -34.38926696777344, "KL/rejected_KL_mean": -48.869747161865234, "KL/std": 31.844987869262695, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.014069490134716034, "fcm_dpo/delta": -0.0153765007853508, "fcm_dpo/margin": 28.960956573486328, "fcm_dpo/q_t": 0.4053932726383209, "grad_norm": 19.797786712646484, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6841074228286743, "logits/rejected": -0.657940149307251, "logps/chosen": -75.73617553710938, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -152.5856475830078, "loss": 1.0873, "margin_dpo/margin_mean": 28.96095848083496, "margin_dpo/margin_std": 38.147422790527344, "step": 83 }, { "KL/chosen_KL_mean": -19.783796310424805, "KL/mean": -31.47045135498047, "KL/rejected_KL_mean": -43.157108306884766, "KL/std": 27.439373016357422, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.015063371509313583, "fcm_dpo/delta": 0.07021048665046692, "fcm_dpo/margin": 23.373310089111328, "fcm_dpo/q_t": 0.4214816689491272, "grad_norm": 20.031166076660156, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6388710737228394, "logits/rejected": -0.5998183488845825, "logps/chosen": -86.9599609375, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -130.45571899414062, "loss": 1.1181, "margin_dpo/margin_mean": 23.373310089111328, "margin_dpo/margin_std": 27.697301864624023, "step": 84 }, { "KL/chosen_KL_mean": -19.78733253479004, "KL/mean": -32.103843688964844, "KL/rejected_KL_mean": -44.42035675048828, "KL/std": 27.902463912963867, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.016904689371585846, "fcm_dpo/delta": -0.03945329040288925, "fcm_dpo/margin": 24.633026123046875, "fcm_dpo/q_t": 0.4018494188785553, "grad_norm": 21.946027755737305, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6458739638328552, "logits/rejected": -0.6141970157623291, "logps/chosen": -78.19395446777344, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -123.05915832519531, "loss": 1.099, "margin_dpo/margin_mean": 24.633026123046875, "margin_dpo/margin_std": 33.796836853027344, "step": 85 }, { "KL/chosen_KL_mean": -26.077333450317383, "KL/mean": -39.15126037597656, "KL/rejected_KL_mean": -52.225189208984375, "KL/std": 39.593040466308594, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.015605635941028595, "fcm_dpo/delta": -0.01844988390803337, "fcm_dpo/margin": 26.14786148071289, "fcm_dpo/q_t": 0.4177693724632263, "grad_norm": 25.58841896057129, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.6518966555595398, "logits/rejected": -0.6431847810745239, "logps/chosen": -82.21479797363281, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -140.34683227539062, "loss": 1.1793, "margin_dpo/margin_mean": 26.147863388061523, "margin_dpo/margin_std": 55.436424255371094, "step": 86 }, { "KL/chosen_KL_mean": -25.545059204101562, "KL/mean": -41.57789611816406, "KL/rejected_KL_mean": -57.61074447631836, "KL/std": 39.62342071533203, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.013056030496954918, "fcm_dpo/delta": -0.044095173478126526, "fcm_dpo/margin": 32.0656852722168, "fcm_dpo/q_t": 0.40036922693252563, "grad_norm": 19.81938362121582, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.6956198215484619, "logits/rejected": -0.6780788898468018, "logps/chosen": -81.18115234375, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -153.0783233642578, "loss": 1.1109, "margin_dpo/margin_mean": 32.0656852722168, "margin_dpo/margin_std": 48.967384338378906, "step": 87 }, { "KL/chosen_KL_mean": -32.21333694458008, "KL/mean": -44.31005096435547, "KL/rejected_KL_mean": -56.40675735473633, "KL/std": 42.025611877441406, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.014491596259176731, "fcm_dpo/delta": 0.07550811022520065, "fcm_dpo/margin": 24.193424224853516, "fcm_dpo/q_t": 0.4285997748374939, "grad_norm": 21.64423179626465, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.6556172370910645, "logits/rejected": -0.6542757749557495, "logps/chosen": -105.88449096679688, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -163.11526489257812, "loss": 1.1734, "margin_dpo/margin_mean": 24.193424224853516, "margin_dpo/margin_std": 45.42322540283203, "step": 88 }, { "KL/chosen_KL_mean": -20.79347801208496, "KL/mean": -34.439876556396484, "KL/rejected_KL_mean": -48.08627700805664, "KL/std": 35.92295837402344, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.01547418162226677, "fcm_dpo/delta": -0.04180596396327019, "fcm_dpo/margin": 27.29279136657715, "fcm_dpo/q_t": 0.4021642208099365, "grad_norm": 21.672611236572266, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7387075424194336, "logits/rejected": -0.7136551141738892, "logps/chosen": -81.41839599609375, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -130.16983032226562, "loss": 1.1057, "margin_dpo/margin_mean": 27.29279327392578, "margin_dpo/margin_std": 42.178958892822266, "step": 89 }, { "KL/chosen_KL_mean": -25.24016761779785, "KL/mean": -44.94635772705078, "KL/rejected_KL_mean": -64.65254211425781, "KL/std": 48.8939208984375, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.012419058009982109, "fcm_dpo/delta": -0.1753031611442566, "fcm_dpo/margin": 39.412376403808594, "fcm_dpo/q_t": 0.38558393716812134, "grad_norm": 20.073680877685547, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6283840537071228, "logits/rejected": -0.6491606831550598, "logps/chosen": -78.52548217773438, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -176.197265625, "loss": 1.1068, "margin_dpo/margin_mean": 39.41238021850586, "margin_dpo/margin_std": 61.327945709228516, "step": 90 }, { "KL/chosen_KL_mean": -27.13432502746582, "KL/mean": -41.85686111450195, "KL/rejected_KL_mean": -56.57939910888672, "KL/std": 37.675636291503906, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.011744784191250801, "fcm_dpo/delta": 0.07527971267700195, "fcm_dpo/margin": 29.445079803466797, "fcm_dpo/q_t": 0.42634034156799316, "grad_norm": 17.86397361755371, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6645762920379639, "logits/rejected": -0.635959267616272, "logps/chosen": -88.93728637695312, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -144.45335388183594, "loss": 1.151, "margin_dpo/margin_mean": 29.445079803466797, "margin_dpo/margin_std": 50.49188995361328, "step": 91 }, { "KL/chosen_KL_mean": -23.93476676940918, "KL/mean": -39.31444549560547, "KL/rejected_KL_mean": -54.69412612915039, "KL/std": 39.209224700927734, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.012650266289710999, "fcm_dpo/delta": 0.008795984089374542, "fcm_dpo/margin": 30.75935935974121, "fcm_dpo/q_t": 0.41261714696884155, "grad_norm": 18.466337203979492, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.6946055889129639, "logits/rejected": -0.6752103567123413, "logps/chosen": -75.57553100585938, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -132.5753173828125, "loss": 1.1221, "margin_dpo/margin_mean": 30.759361267089844, "margin_dpo/margin_std": 47.41279602050781, "step": 92 }, { "KL/chosen_KL_mean": -27.03689193725586, "KL/mean": -42.25169372558594, "KL/rejected_KL_mean": -57.466495513916016, "KL/std": 36.04038619995117, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.012704822234809399, "fcm_dpo/delta": 0.01873387210071087, "fcm_dpo/margin": 30.429601669311523, "fcm_dpo/q_t": 0.4136204719543457, "grad_norm": 19.59035873413086, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.7035015821456909, "logits/rejected": -0.6627704501152039, "logps/chosen": -79.56613159179688, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -134.6272430419922, "loss": 1.103, "margin_dpo/margin_mean": 30.42959976196289, "margin_dpo/margin_std": 42.63709259033203, "step": 93 }, { "KL/chosen_KL_mean": -28.926584243774414, "KL/mean": -47.09135818481445, "KL/rejected_KL_mean": -65.25613403320312, "KL/std": 42.653038024902344, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.011295975185930729, "fcm_dpo/delta": -0.028756409883499146, "fcm_dpo/margin": 36.32954788208008, "fcm_dpo/q_t": 0.40284472703933716, "grad_norm": 17.930683135986328, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.626760721206665, "logits/rejected": -0.618344783782959, "logps/chosen": -90.14920043945312, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -164.85516357421875, "loss": 1.0816, "margin_dpo/margin_mean": 36.32954788208008, "margin_dpo/margin_std": 45.59572219848633, "step": 94 }, { "KL/chosen_KL_mean": -28.432294845581055, "KL/mean": -43.03887176513672, "KL/rejected_KL_mean": -57.64545440673828, "KL/std": 43.11521530151367, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.012412654235959053, "fcm_dpo/delta": 0.05812495946884155, "fcm_dpo/margin": 29.213159561157227, "fcm_dpo/q_t": 0.42024633288383484, "grad_norm": 19.021087646484375, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.694536030292511, "logits/rejected": -0.673601508140564, "logps/chosen": -80.95594024658203, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -133.5258026123047, "loss": 1.133, "margin_dpo/margin_mean": 29.213159561157227, "margin_dpo/margin_std": 45.21534729003906, "step": 95 }, { "KL/chosen_KL_mean": -28.583940505981445, "KL/mean": -48.43400573730469, "KL/rejected_KL_mean": -68.28407287597656, "KL/std": 46.205230712890625, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.010704778134822845, "fcm_dpo/delta": -0.0798785462975502, "fcm_dpo/margin": 39.70012664794922, "fcm_dpo/q_t": 0.3951551914215088, "grad_norm": 17.751604080200195, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.6716827154159546, "logits/rejected": -0.6496946811676025, "logps/chosen": -90.74091339111328, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -164.88009643554688, "loss": 1.0813, "margin_dpo/margin_mean": 39.70012664794922, "margin_dpo/margin_std": 52.09516906738281, "step": 96 }, { "KL/chosen_KL_mean": -29.381603240966797, "KL/mean": -44.338138580322266, "KL/rejected_KL_mean": -59.294677734375, "KL/std": 39.46461486816406, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.010876108892261982, "fcm_dpo/delta": -0.03550150245428085, "fcm_dpo/margin": 29.913070678710938, "fcm_dpo/q_t": 0.4207790791988373, "grad_norm": 18.363679885864258, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.5955668091773987, "logits/rejected": -0.5594468116760254, "logps/chosen": -84.02796936035156, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -136.25942993164062, "loss": 1.1386, "margin_dpo/margin_mean": 29.913070678710938, "margin_dpo/margin_std": 39.46156311035156, "step": 97 }, { "KL/chosen_KL_mean": -37.65998458862305, "KL/mean": -54.85460662841797, "KL/rejected_KL_mean": -72.04922485351562, "KL/std": 45.98478698730469, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.0109539944678545, "fcm_dpo/delta": 0.03750079125165939, "fcm_dpo/margin": 34.38923645019531, "fcm_dpo/q_t": 0.4147164225578308, "grad_norm": 20.02430534362793, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.652593731880188, "logits/rejected": -0.6433310508728027, "logps/chosen": -102.91860961914062, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -168.57672119140625, "loss": 1.1337, "margin_dpo/margin_mean": 34.38923645019531, "margin_dpo/margin_std": 55.96516799926758, "step": 98 }, { "KL/chosen_KL_mean": -28.500194549560547, "KL/mean": -50.64104461669922, "KL/rejected_KL_mean": -72.78189849853516, "KL/std": 48.286502838134766, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.00977346021682024, "fcm_dpo/delta": -0.06876382231712341, "fcm_dpo/margin": 44.28170394897461, "fcm_dpo/q_t": 0.3981579542160034, "grad_norm": 16.403064727783203, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.6286486387252808, "logits/rejected": -0.6337354183197021, "logps/chosen": -74.13867950439453, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -159.21983337402344, "loss": 1.0777, "margin_dpo/margin_mean": 44.28170394897461, "margin_dpo/margin_std": 60.85698699951172, "step": 99 }, { "KL/chosen_KL_mean": -32.810997009277344, "KL/mean": -44.14629364013672, "KL/rejected_KL_mean": -55.4815788269043, "KL/std": 39.670440673828125, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.010382324457168579, "fcm_dpo/delta": 0.05089900270104408, "fcm_dpo/margin": 22.670581817626953, "fcm_dpo/q_t": 0.4473540782928467, "grad_norm": 18.2591495513916, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6274121999740601, "logits/rejected": -0.5915518999099731, "logps/chosen": -90.40498352050781, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -129.5417938232422, "loss": 1.23, "margin_dpo/margin_mean": 22.670583724975586, "margin_dpo/margin_std": 50.92362976074219, "step": 100 }, { "KL/chosen_KL_mean": -40.26070022583008, "KL/mean": -54.877384185791016, "KL/rejected_KL_mean": -69.49407958984375, "KL/std": 49.381065368652344, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.010775712318718433, "fcm_dpo/delta": -0.0062071713618934155, "fcm_dpo/margin": 29.23336410522461, "fcm_dpo/q_t": 0.43012934923171997, "grad_norm": 19.166645050048828, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6265345811843872, "logits/rejected": -0.6024205684661865, "logps/chosen": -101.90955352783203, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -152.6837615966797, "loss": 1.1904, "margin_dpo/margin_mean": 29.23336410522461, "margin_dpo/margin_std": 58.819854736328125, "step": 101 }, { "KL/chosen_KL_mean": -42.29863739013672, "KL/mean": -52.66799545288086, "KL/rejected_KL_mean": -63.037353515625, "KL/std": 45.68564224243164, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.011057091876864433, "fcm_dpo/delta": 0.02545342966914177, "fcm_dpo/margin": 20.73870849609375, "fcm_dpo/q_t": 0.4484948515892029, "grad_norm": 22.39993667602539, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.6203207969665527, "logits/rejected": -0.580852210521698, "logps/chosen": -106.37751770019531, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -131.22442626953125, "loss": 1.2607, "margin_dpo/margin_mean": 20.73870849609375, "margin_dpo/margin_std": 55.18003463745117, "step": 102 }, { "KL/chosen_KL_mean": -35.84786605834961, "KL/mean": -58.045082092285156, "KL/rejected_KL_mean": -80.24229431152344, "KL/std": 51.89337921142578, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.009846476837992668, "fcm_dpo/delta": -0.07356812059879303, "fcm_dpo/margin": 44.39442443847656, "fcm_dpo/q_t": 0.3977162837982178, "grad_norm": 18.495681762695312, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6326062679290771, "logits/rejected": -0.603610634803772, "logps/chosen": -97.14714050292969, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -173.81500244140625, "loss": 1.0847, "margin_dpo/margin_mean": 44.39442443847656, "margin_dpo/margin_std": 64.85426330566406, "step": 103 }, { "KL/chosen_KL_mean": -36.992637634277344, "KL/mean": -58.22779846191406, "KL/rejected_KL_mean": -79.46296691894531, "KL/std": 55.633026123046875, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.00963391549885273, "fcm_dpo/delta": -0.015489146113395691, "fcm_dpo/margin": 42.47032928466797, "fcm_dpo/q_t": 0.410830020904541, "grad_norm": 17.655324935913086, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.6385079622268677, "logits/rejected": -0.6165839433670044, "logps/chosen": -91.36540985107422, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -169.02767944335938, "loss": 1.1177, "margin_dpo/margin_mean": 42.47032928466797, "margin_dpo/margin_std": 69.24513244628906, "step": 104 }, { "KL/chosen_KL_mean": -35.22704315185547, "KL/mean": -65.53314208984375, "KL/rejected_KL_mean": -95.83924865722656, "KL/std": 52.481048583984375, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.007748795207589865, "fcm_dpo/delta": -0.13560038805007935, "fcm_dpo/margin": 60.61220932006836, "fcm_dpo/q_t": 0.37679198384284973, "grad_norm": 16.846649169921875, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.5944653749465942, "logits/rejected": -0.5763468742370605, "logps/chosen": -89.86598205566406, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -193.81277465820312, "loss": 1.0206, "margin_dpo/margin_mean": 60.612213134765625, "margin_dpo/margin_std": 59.409732818603516, "step": 105 }, { "KL/chosen_KL_mean": -40.11956024169922, "KL/mean": -58.95640563964844, "KL/rejected_KL_mean": -77.79325866699219, "KL/std": 47.83055114746094, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.0085395947098732, "fcm_dpo/delta": 0.11511291563510895, "fcm_dpo/margin": 37.6736946105957, "fcm_dpo/q_t": 0.43202945590019226, "grad_norm": 16.822023391723633, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.5823087096214294, "logits/rejected": -0.56536865234375, "logps/chosen": -94.95245361328125, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -163.01788330078125, "loss": 1.1419, "margin_dpo/margin_mean": 37.67369079589844, "margin_dpo/margin_std": 51.76994323730469, "step": 106 }, { "KL/chosen_KL_mean": -47.187808990478516, "KL/mean": -68.33793640136719, "KL/rejected_KL_mean": -89.4880599975586, "KL/std": 58.32842254638672, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.0090809166431427, "fcm_dpo/delta": 0.023154791444540024, "fcm_dpo/margin": 42.300254821777344, "fcm_dpo/q_t": 0.41329509019851685, "grad_norm": 18.076017379760742, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.6128166913986206, "logits/rejected": -0.5746489763259888, "logps/chosen": -116.89562225341797, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -184.22756958007812, "loss": 1.1189, "margin_dpo/margin_mean": 42.300254821777344, "margin_dpo/margin_std": 62.243568420410156, "step": 107 }, { "KL/chosen_KL_mean": -40.908172607421875, "KL/mean": -67.95404052734375, "KL/rejected_KL_mean": -94.99990844726562, "KL/std": 65.11196899414062, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.008320159278810024, "fcm_dpo/delta": -0.09254781901836395, "fcm_dpo/margin": 54.09173583984375, "fcm_dpo/q_t": 0.3973488211631775, "grad_norm": 16.829160690307617, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.5930562019348145, "logits/rejected": -0.5809206962585449, "logps/chosen": -96.91806030273438, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -190.79592895507812, "loss": 1.0985, "margin_dpo/margin_mean": 54.09173583984375, "margin_dpo/margin_std": 84.54571533203125, "step": 108 }, { "KL/chosen_KL_mean": -39.185829162597656, "KL/mean": -66.94438171386719, "KL/rejected_KL_mean": -94.70292663574219, "KL/std": 61.541465759277344, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.007376614026725292, "fcm_dpo/delta": -0.021881788969039917, "fcm_dpo/margin": 55.51709747314453, "fcm_dpo/q_t": 0.4035341143608093, "grad_norm": 17.845430374145508, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5359349250793457, "logits/rejected": -0.5162516832351685, "logps/chosen": -102.07131958007812, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -193.388671875, "loss": 1.0947, "margin_dpo/margin_mean": 55.51709747314453, "margin_dpo/margin_std": 77.26396179199219, "step": 109 }, { "KL/chosen_KL_mean": -41.10560989379883, "KL/mean": -66.01383972167969, "KL/rejected_KL_mean": -90.92208099365234, "KL/std": 61.80369567871094, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.006370611023157835, "fcm_dpo/delta": -0.08743350207805634, "fcm_dpo/margin": 49.81646728515625, "fcm_dpo/q_t": 0.41799625754356384, "grad_norm": 12.691235542297363, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.5346049666404724, "logits/rejected": -0.5050790309906006, "logps/chosen": -99.85929870605469, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -170.67208862304688, "loss": 1.1549, "margin_dpo/margin_mean": 49.81646728515625, "margin_dpo/margin_std": 75.78208923339844, "step": 110 }, { "KL/chosen_KL_mean": -45.89485168457031, "KL/mean": -75.09423828125, "KL/rejected_KL_mean": -104.29363250732422, "KL/std": 70.21835327148438, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.006470158696174622, "fcm_dpo/delta": 0.03387834131717682, "fcm_dpo/margin": 58.398780822753906, "fcm_dpo/q_t": 0.41569167375564575, "grad_norm": 14.799457550048828, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.5766105055809021, "logits/rejected": -0.5566138029098511, "logps/chosen": -114.51895904541016, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -202.72250366210938, "loss": 1.1264, "margin_dpo/margin_mean": 58.39877700805664, "margin_dpo/margin_std": 89.97933959960938, "step": 111 }, { "KL/chosen_KL_mean": -42.42522048950195, "KL/mean": -60.012123107910156, "KL/rejected_KL_mean": -77.59901428222656, "KL/std": 46.559471130371094, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.00751154450699687, "fcm_dpo/delta": 0.04849451780319214, "fcm_dpo/margin": 35.173797607421875, "fcm_dpo/q_t": 0.4408394694328308, "grad_norm": 15.346757888793945, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.53926682472229, "logits/rejected": -0.5091655254364014, "logps/chosen": -92.67486572265625, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -142.37344360351562, "loss": 1.1874, "margin_dpo/margin_mean": 35.173797607421875, "margin_dpo/margin_std": 58.55726623535156, "step": 112 }, { "KL/chosen_KL_mean": -55.43156433105469, "KL/mean": -73.22611999511719, "KL/rejected_KL_mean": -91.02066802978516, "KL/std": 52.921966552734375, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.009013749659061432, "fcm_dpo/delta": 0.11262636631727219, "fcm_dpo/margin": 35.58909606933594, "fcm_dpo/q_t": 0.4331721365451813, "grad_norm": 19.263153076171875, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.5204916596412659, "logits/rejected": -0.483734130859375, "logps/chosen": -122.14451599121094, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -168.9893798828125, "loss": 1.1549, "margin_dpo/margin_mean": 35.58909606933594, "margin_dpo/margin_std": 58.14237976074219, "step": 113 }, { "KL/chosen_KL_mean": -49.490875244140625, "KL/mean": -79.34724426269531, "KL/rejected_KL_mean": -109.20361328125, "KL/std": 74.02787780761719, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.007360486313700676, "fcm_dpo/delta": -0.10784411430358887, "fcm_dpo/margin": 59.712745666503906, "fcm_dpo/q_t": 0.3967573046684265, "grad_norm": 18.57329750061035, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.5290780067443848, "logits/rejected": -0.5046583414077759, "logps/chosen": -107.27595520019531, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -196.3132781982422, "loss": 1.0889, "margin_dpo/margin_mean": 59.712745666503906, "margin_dpo/margin_std": 90.68304443359375, "step": 114 }, { "KL/chosen_KL_mean": -62.15449523925781, "KL/mean": -82.36180114746094, "KL/rejected_KL_mean": -102.5691146850586, "KL/std": 71.60066223144531, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.008717365562915802, "fcm_dpo/delta": 0.06397520750761032, "fcm_dpo/margin": 40.414615631103516, "fcm_dpo/q_t": 0.4292001724243164, "grad_norm": 26.535457611083984, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.5466749668121338, "logits/rejected": -0.5373319387435913, "logps/chosen": -127.73713684082031, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -201.1346435546875, "loss": 1.2197, "margin_dpo/margin_mean": 40.41461944580078, "margin_dpo/margin_std": 96.31269073486328, "step": 115 }, { "KL/chosen_KL_mean": -45.30133819580078, "KL/mean": -68.48527526855469, "KL/rejected_KL_mean": -91.66920471191406, "KL/std": 58.61573028564453, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.008865797892212868, "fcm_dpo/delta": -0.027695897966623306, "fcm_dpo/margin": 46.367855072021484, "fcm_dpo/q_t": 0.4044142961502075, "grad_norm": 19.043041229248047, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.5231291055679321, "logits/rejected": -0.5219400525093079, "logps/chosen": -96.70165252685547, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -172.19105529785156, "loss": 1.1011, "margin_dpo/margin_mean": 46.367855072021484, "margin_dpo/margin_std": 64.93318939208984, "step": 116 }, { "KL/chosen_KL_mean": -58.65241241455078, "KL/mean": -76.01322937011719, "KL/rejected_KL_mean": -93.37403869628906, "KL/std": 61.22138977050781, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.009560597129166126, "fcm_dpo/delta": 0.09886346757411957, "fcm_dpo/margin": 34.72163009643555, "fcm_dpo/q_t": 0.4316639304161072, "grad_norm": 31.035114288330078, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.49749547243118286, "logits/rejected": -0.46514734625816345, "logps/chosen": -127.95082092285156, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -159.95803833007812, "loss": 1.1874, "margin_dpo/margin_mean": 34.72163391113281, "margin_dpo/margin_std": 71.40594482421875, "step": 117 }, { "KL/chosen_KL_mean": -44.07014465332031, "KL/mean": -66.22931671142578, "KL/rejected_KL_mean": -88.38848114013672, "KL/std": 55.144588470458984, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.009519684128463268, "fcm_dpo/delta": -0.037742018699645996, "fcm_dpo/margin": 44.318336486816406, "fcm_dpo/q_t": 0.40347346663475037, "grad_norm": 20.083301544189453, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.563313364982605, "logits/rejected": -0.5480994582176208, "logps/chosen": -99.71112060546875, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -164.0575408935547, "loss": 1.0928, "margin_dpo/margin_mean": 44.318336486816406, "margin_dpo/margin_std": 63.96471405029297, "step": 118 }, { "KL/chosen_KL_mean": -60.53836441040039, "KL/mean": -82.12613677978516, "KL/rejected_KL_mean": -103.71391296386719, "KL/std": 61.16850280761719, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.00799720548093319, "fcm_dpo/delta": -0.07085751742124557, "fcm_dpo/margin": 43.175559997558594, "fcm_dpo/q_t": 0.41732871532440186, "grad_norm": 19.45294952392578, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.553020715713501, "logits/rejected": -0.545320987701416, "logps/chosen": -134.04855346679688, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -206.6912078857422, "loss": 1.1596, "margin_dpo/margin_mean": 43.175559997558594, "margin_dpo/margin_std": 76.69107055664062, "step": 119 }, { "KL/chosen_KL_mean": -60.02840805053711, "KL/mean": -90.28742980957031, "KL/rejected_KL_mean": -120.54644775390625, "KL/std": 74.63396453857422, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.007171543315052986, "fcm_dpo/delta": -0.06302839517593384, "fcm_dpo/margin": 60.51802444458008, "fcm_dpo/q_t": 0.3962337374687195, "grad_norm": 22.441465377807617, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.5601464509963989, "logits/rejected": -0.5344448089599609, "logps/chosen": -136.80923461914062, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -228.5701904296875, "loss": 1.0734, "margin_dpo/margin_mean": 60.51802062988281, "margin_dpo/margin_std": 79.95449829101562, "step": 120 }, { "KL/chosen_KL_mean": -56.952545166015625, "KL/mean": -91.06138610839844, "KL/rejected_KL_mean": -125.17024230957031, "KL/std": 72.70287322998047, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.0062125981785357, "fcm_dpo/delta": -0.05240712687373161, "fcm_dpo/margin": 68.21770477294922, "fcm_dpo/q_t": 0.39837193489074707, "grad_norm": 17.03141212463379, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.5124868154525757, "logits/rejected": -0.511413037776947, "logps/chosen": -118.74243927001953, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -235.16481018066406, "loss": 1.0811, "margin_dpo/margin_mean": 68.21769714355469, "margin_dpo/margin_std": 90.72044372558594, "step": 121 }, { "KL/chosen_KL_mean": -51.569435119628906, "KL/mean": -93.51173400878906, "KL/rejected_KL_mean": -135.4540252685547, "KL/std": 81.56539916992188, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.005438469350337982, "fcm_dpo/delta": -0.10527084767818451, "fcm_dpo/margin": 83.88459777832031, "fcm_dpo/q_t": 0.3847019076347351, "grad_norm": 17.03737449645996, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.4612678587436676, "logits/rejected": -0.4794490337371826, "logps/chosen": -98.47164916992188, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -242.168212890625, "loss": 1.0341, "margin_dpo/margin_mean": 83.88459777832031, "margin_dpo/margin_std": 86.80702209472656, "step": 122 }, { "KL/chosen_KL_mean": -59.32091522216797, "KL/mean": -86.37959289550781, "KL/rejected_KL_mean": -113.43827819824219, "KL/std": 70.35043334960938, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.005368525628000498, "fcm_dpo/delta": 0.021436911076307297, "fcm_dpo/margin": 54.11735534667969, "fcm_dpo/q_t": 0.4345107674598694, "grad_norm": 15.537219047546387, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.4674427807331085, "logits/rejected": -0.4554457664489746, "logps/chosen": -120.65955352783203, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -201.2136688232422, "loss": 1.1808, "margin_dpo/margin_mean": 54.11735534667969, "margin_dpo/margin_std": 95.79205322265625, "step": 123 }, { "KL/chosen_KL_mean": -70.40459442138672, "KL/mean": -111.80335998535156, "KL/rejected_KL_mean": -153.20213317871094, "KL/std": 99.7587890625, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.00502355583012104, "fcm_dpo/delta": -0.029820293188095093, "fcm_dpo/margin": 82.79753112792969, "fcm_dpo/q_t": 0.40601056814193726, "grad_norm": 14.011857986450195, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.4481953978538513, "logits/rejected": -0.4538896083831787, "logps/chosen": -141.85293579101562, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -270.78271484375, "loss": 1.1052, "margin_dpo/margin_mean": 82.79753875732422, "margin_dpo/margin_std": 125.91355895996094, "step": 124 }, { "KL/chosen_KL_mean": -57.097442626953125, "KL/mean": -92.41481018066406, "KL/rejected_KL_mean": -127.73218536376953, "KL/std": 83.84275817871094, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.005386849865317345, "fcm_dpo/delta": 0.029525037854909897, "fcm_dpo/margin": 70.63475036621094, "fcm_dpo/q_t": 0.4158494174480438, "grad_norm": 14.315393447875977, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.4188230633735657, "logits/rejected": -0.42580023407936096, "logps/chosen": -107.23438262939453, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -211.72079467773438, "loss": 1.1187, "margin_dpo/margin_mean": 70.63475036621094, "margin_dpo/margin_std": 106.31303405761719, "step": 125 }, { "KL/chosen_KL_mean": -62.320526123046875, "KL/mean": -96.66703796386719, "KL/rejected_KL_mean": -131.0135498046875, "KL/std": 78.08503723144531, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.005546340718865395, "fcm_dpo/delta": 0.030749067664146423, "fcm_dpo/margin": 68.69302368164062, "fcm_dpo/q_t": 0.4163479208946228, "grad_norm": 15.782130241394043, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.42107057571411133, "logits/rejected": -0.42162832617759705, "logps/chosen": -117.98759460449219, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -229.143310546875, "loss": 1.1145, "margin_dpo/margin_mean": 68.69302368164062, "margin_dpo/margin_std": 100.53376007080078, "step": 126 }, { "KL/chosen_KL_mean": -61.44911193847656, "KL/mean": -84.82119750976562, "KL/rejected_KL_mean": -108.19326782226562, "KL/std": 68.35600280761719, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.006859184708446264, "fcm_dpo/delta": 0.11228857934474945, "fcm_dpo/margin": 46.74416732788086, "fcm_dpo/q_t": 0.43170320987701416, "grad_norm": 19.79861831665039, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.45557230710983276, "logits/rejected": -0.44303327798843384, "logps/chosen": -118.00379180908203, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -184.98904418945312, "loss": 1.1732, "margin_dpo/margin_mean": 46.744171142578125, "margin_dpo/margin_std": 86.41580200195312, "step": 127 }, { "KL/chosen_KL_mean": -64.04533386230469, "KL/mean": -89.68724060058594, "KL/rejected_KL_mean": -115.32914733886719, "KL/std": 80.13230895996094, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.007653497625142336, "fcm_dpo/delta": 0.0008252784609794617, "fcm_dpo/margin": 51.28382110595703, "fcm_dpo/q_t": 0.4136595129966736, "grad_norm": 32.553951263427734, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.439397931098938, "logits/rejected": -0.42928874492645264, "logps/chosen": -122.16629028320312, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -191.76809692382812, "loss": 1.1786, "margin_dpo/margin_mean": 51.283817291259766, "margin_dpo/margin_std": 102.103515625, "step": 128 }, { "KL/chosen_KL_mean": -78.18240356445312, "KL/mean": -104.9556884765625, "KL/rejected_KL_mean": -131.72897338867188, "KL/std": 91.75530242919922, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.0073263440281152725, "fcm_dpo/delta": 0.01247208472341299, "fcm_dpo/margin": 53.54655075073242, "fcm_dpo/q_t": 0.4221315085887909, "grad_norm": 23.202054977416992, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.4406889081001282, "logits/rejected": -0.4349208474159241, "logps/chosen": -145.09878540039062, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -228.37118530273438, "loss": 1.1973, "margin_dpo/margin_mean": 53.54655456542969, "margin_dpo/margin_std": 122.08485412597656, "step": 129 }, { "KL/chosen_KL_mean": -56.172401428222656, "KL/mean": -96.15986633300781, "KL/rejected_KL_mean": -136.1473388671875, "KL/std": 85.45478820800781, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.006229862105101347, "fcm_dpo/delta": -0.19545677304267883, "fcm_dpo/margin": 79.97492980957031, "fcm_dpo/q_t": 0.37694329023361206, "grad_norm": 17.230426788330078, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.42614853382110596, "logits/rejected": -0.4212262034416199, "logps/chosen": -100.83924865722656, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -218.92898559570312, "loss": 1.0744, "margin_dpo/margin_mean": 79.97492980957031, "margin_dpo/margin_std": 107.18331146240234, "step": 130 }, { "KL/chosen_KL_mean": -52.116981506347656, "KL/mean": -93.28705596923828, "KL/rejected_KL_mean": -134.45712280273438, "KL/std": 90.56842041015625, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.004888694733381271, "fcm_dpo/delta": -0.004482886288315058, "fcm_dpo/margin": 82.34013366699219, "fcm_dpo/q_t": 0.40506911277770996, "grad_norm": 26.67673683166504, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.42915642261505127, "logits/rejected": -0.4353182315826416, "logps/chosen": -97.04157257080078, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -222.90113830566406, "loss": 1.0791, "margin_dpo/margin_mean": 82.34013366699219, "margin_dpo/margin_std": 98.78244018554688, "step": 131 }, { "KL/chosen_KL_mean": -67.49314880371094, "KL/mean": -99.89106750488281, "KL/rejected_KL_mean": -132.28900146484375, "KL/std": 87.30780029296875, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.005424443632364273, "fcm_dpo/delta": 0.07522791624069214, "fcm_dpo/margin": 64.79584503173828, "fcm_dpo/q_t": 0.42780178785324097, "grad_norm": 16.56679916381836, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.46568989753723145, "logits/rejected": -0.4727591872215271, "logps/chosen": -126.49423217773438, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -220.18115234375, "loss": 1.1552, "margin_dpo/margin_mean": 64.79584503173828, "margin_dpo/margin_std": 114.17736053466797, "step": 132 }, { "KL/chosen_KL_mean": -78.84221649169922, "KL/mean": -106.69076538085938, "KL/rejected_KL_mean": -134.53932189941406, "KL/std": 77.18131256103516, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.006308811716735363, "fcm_dpo/delta": 0.0753559023141861, "fcm_dpo/margin": 55.69710159301758, "fcm_dpo/q_t": 0.4273163676261902, "grad_norm": 21.23187255859375, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.46139687299728394, "logits/rejected": -0.4528288245201111, "logps/chosen": -145.44671630859375, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -230.8728790283203, "loss": 1.157, "margin_dpo/margin_mean": 55.69709777832031, "margin_dpo/margin_std": 99.82786560058594, "step": 133 }, { "KL/chosen_KL_mean": -59.535091400146484, "KL/mean": -90.11434173583984, "KL/rejected_KL_mean": -120.69359588623047, "KL/std": 71.90062713623047, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.006723651196807623, "fcm_dpo/delta": -0.022656653076410294, "fcm_dpo/margin": 61.15849304199219, "fcm_dpo/q_t": 0.40606844425201416, "grad_norm": 17.838544845581055, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.4176589846611023, "logits/rejected": -0.4087156057357788, "logps/chosen": -111.6043472290039, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -208.34811401367188, "loss": 1.0923, "margin_dpo/margin_mean": 61.15850067138672, "margin_dpo/margin_std": 85.12577056884766, "step": 134 }, { "KL/chosen_KL_mean": -64.89012145996094, "KL/mean": -110.44607543945312, "KL/rejected_KL_mean": -156.00204467773438, "KL/std": 98.89341735839844, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.005103720352053642, "fcm_dpo/delta": -0.13181009888648987, "fcm_dpo/margin": 91.11190795898438, "fcm_dpo/q_t": 0.3853245973587036, "grad_norm": 24.731225967407227, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.41161245107650757, "logits/rejected": -0.4464326500892639, "logps/chosen": -115.24398803710938, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -271.9817810058594, "loss": 1.061, "margin_dpo/margin_mean": 91.11190795898438, "margin_dpo/margin_std": 121.44206237792969, "step": 135 }, { "KL/chosen_KL_mean": -77.75067138671875, "KL/mean": -107.7598876953125, "KL/rejected_KL_mean": -137.76910400390625, "KL/std": 92.06062316894531, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.005009594839066267, "fcm_dpo/delta": 0.013183820061385632, "fcm_dpo/margin": 60.01844024658203, "fcm_dpo/q_t": 0.4368482232093811, "grad_norm": 20.24871063232422, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.41852521896362305, "logits/rejected": -0.41355031728744507, "logps/chosen": -142.82318115234375, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -234.09033203125, "loss": 1.1918, "margin_dpo/margin_mean": 60.01844024658203, "margin_dpo/margin_std": 120.7431640625, "step": 136 }, { "KL/chosen_KL_mean": -70.75765991210938, "KL/mean": -118.09391784667969, "KL/rejected_KL_mean": -165.43017578125, "KL/std": 117.8000717163086, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.0045467158779501915, "fcm_dpo/delta": -0.05582471936941147, "fcm_dpo/margin": 94.67250061035156, "fcm_dpo/q_t": 0.402421772480011, "grad_norm": 18.368188858032227, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.40962690114974976, "logits/rejected": -0.43391185998916626, "logps/chosen": -119.51677703857422, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -279.2939453125, "loss": 1.096, "margin_dpo/margin_mean": 94.67250061035156, "margin_dpo/margin_std": 143.3657989501953, "step": 137 }, { "KL/chosen_KL_mean": -75.74452209472656, "KL/mean": -111.02808380126953, "KL/rejected_KL_mean": -146.31163024902344, "KL/std": 89.46757507324219, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.005052464548498392, "fcm_dpo/delta": 0.06267927587032318, "fcm_dpo/margin": 70.56710815429688, "fcm_dpo/q_t": 0.42063024640083313, "grad_norm": 20.598432540893555, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.40584802627563477, "logits/rejected": -0.3970094919204712, "logps/chosen": -136.26417541503906, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -239.50857543945312, "loss": 1.1109, "margin_dpo/margin_mean": 70.56710815429688, "margin_dpo/margin_std": 90.8817367553711, "step": 138 }, { "KL/chosen_KL_mean": -64.63111114501953, "KL/mean": -105.27629089355469, "KL/rejected_KL_mean": -145.92144775390625, "KL/std": 83.68637084960938, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.005053876433521509, "fcm_dpo/delta": -0.019144825637340546, "fcm_dpo/margin": 81.29034423828125, "fcm_dpo/q_t": 0.4056248664855957, "grad_norm": 15.781937599182129, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.38525593280792236, "logits/rejected": -0.37835609912872314, "logps/chosen": -111.52249145507812, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -225.64944458007812, "loss": 1.0888, "margin_dpo/margin_mean": 81.29034423828125, "margin_dpo/margin_std": 111.37606811523438, "step": 139 }, { "KL/chosen_KL_mean": -77.94660949707031, "KL/mean": -114.49009704589844, "KL/rejected_KL_mean": -151.03359985351562, "KL/std": 91.72400665283203, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.0051695131696760654, "fcm_dpo/delta": 0.03525704890489578, "fcm_dpo/margin": 73.08699035644531, "fcm_dpo/q_t": 0.4163084924221039, "grad_norm": 16.439830780029297, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.3429441452026367, "logits/rejected": -0.3306434154510498, "logps/chosen": -136.92132568359375, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -234.3177032470703, "loss": 1.1224, "margin_dpo/margin_mean": 73.08699035644531, "margin_dpo/margin_std": 112.87854766845703, "step": 140 }, { "KL/chosen_KL_mean": -88.09237670898438, "KL/mean": -122.84248352050781, "KL/rejected_KL_mean": -157.59259033203125, "KL/std": 105.68617248535156, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.005516390781849623, "fcm_dpo/delta": 0.02600482851266861, "fcm_dpo/margin": 69.5002212524414, "fcm_dpo/q_t": 0.4152142405509949, "grad_norm": 25.1342716217041, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.3684391975402832, "logits/rejected": -0.35982775688171387, "logps/chosen": -163.16802978515625, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -255.78485107421875, "loss": 1.1451, "margin_dpo/margin_mean": 69.50021362304688, "margin_dpo/margin_std": 123.44959259033203, "step": 141 }, { "KL/chosen_KL_mean": -85.69429016113281, "KL/mean": -125.2984848022461, "KL/rejected_KL_mean": -164.9027099609375, "KL/std": 109.44830322265625, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.0053951190784573555, "fcm_dpo/delta": -0.05040793865919113, "fcm_dpo/margin": 79.20841217041016, "fcm_dpo/q_t": 0.4017929434776306, "grad_norm": 22.805734634399414, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.3611358404159546, "logits/rejected": -0.36291757225990295, "logps/chosen": -143.7222137451172, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -259.48492431640625, "loss": 1.1263, "margin_dpo/margin_mean": 79.20841217041016, "margin_dpo/margin_std": 128.5096893310547, "step": 142 }, { "KL/chosen_KL_mean": -88.5622329711914, "KL/mean": -114.20449829101562, "KL/rejected_KL_mean": -139.84677124023438, "KL/std": 94.44708251953125, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.005363130010664463, "fcm_dpo/delta": 0.035532694309949875, "fcm_dpo/margin": 51.28453063964844, "fcm_dpo/q_t": 0.4414982795715332, "grad_norm": 20.507484436035156, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.3294772803783417, "logits/rejected": -0.3267552852630615, "logps/chosen": -146.15869140625, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -218.84634399414062, "loss": 1.2134, "margin_dpo/margin_mean": 51.28453063964844, "margin_dpo/margin_std": 110.63523864746094, "step": 143 }, { "KL/chosen_KL_mean": -80.96118927001953, "KL/mean": -111.06156921386719, "KL/rejected_KL_mean": -141.16195678710938, "KL/std": 81.7334213256836, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.005931549705564976, "fcm_dpo/delta": 0.0660046860575676, "fcm_dpo/margin": 60.200782775878906, "fcm_dpo/q_t": 0.4232863783836365, "grad_norm": 20.33156394958496, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.3503091037273407, "logits/rejected": -0.3469555974006653, "logps/chosen": -140.8675537109375, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -223.1622314453125, "loss": 1.1353, "margin_dpo/margin_mean": 60.200782775878906, "margin_dpo/margin_std": 95.25154113769531, "step": 144 }, { "KL/chosen_KL_mean": -77.64210510253906, "KL/mean": -109.43412780761719, "KL/rejected_KL_mean": -141.22616577148438, "KL/std": 79.6153564453125, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.006183978170156479, "fcm_dpo/delta": 0.011190323159098625, "fcm_dpo/margin": 63.584068298339844, "fcm_dpo/q_t": 0.41196322441101074, "grad_norm": 25.684057235717773, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.376176118850708, "logits/rejected": -0.3664652109146118, "logps/chosen": -134.24276733398438, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -219.0924835205078, "loss": 1.1163, "margin_dpo/margin_mean": 63.584068298339844, "margin_dpo/margin_std": 97.96441650390625, "step": 145 }, { "KL/chosen_KL_mean": -102.16140747070312, "KL/mean": -126.88875579833984, "KL/rejected_KL_mean": -151.61610412597656, "KL/std": 86.07960510253906, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.006922571919858456, "fcm_dpo/delta": 0.08818544447422028, "fcm_dpo/margin": 49.45469665527344, "fcm_dpo/q_t": 0.42923611402511597, "grad_norm": 36.29349136352539, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.372928261756897, "logits/rejected": -0.35708069801330566, "logps/chosen": -168.161865234375, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -233.3188934326172, "loss": 1.2057, "margin_dpo/margin_mean": 49.45469665527344, "margin_dpo/margin_std": 103.38996887207031, "step": 146 }, { "KL/chosen_KL_mean": -74.98482513427734, "KL/mean": -108.55328369140625, "KL/rejected_KL_mean": -142.12173461914062, "KL/std": 86.59007263183594, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.006506077013909817, "fcm_dpo/delta": -0.07235047966241837, "fcm_dpo/margin": 67.13691711425781, "fcm_dpo/q_t": 0.3988075852394104, "grad_norm": 22.362390518188477, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.37692469358444214, "logits/rejected": -0.3635402321815491, "logps/chosen": -128.39031982421875, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -213.51235961914062, "loss": 1.1083, "margin_dpo/margin_mean": 67.13691711425781, "margin_dpo/margin_std": 108.45335388183594, "step": 147 }, { "KL/chosen_KL_mean": -73.58265686035156, "KL/mean": -103.37672424316406, "KL/rejected_KL_mean": -133.17080688476562, "KL/std": 88.8348388671875, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.005902706179767847, "fcm_dpo/delta": -0.09467141330242157, "fcm_dpo/margin": 59.5881462097168, "fcm_dpo/q_t": 0.4176294803619385, "grad_norm": 16.640806198120117, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.4213970899581909, "logits/rejected": -0.4161149859428406, "logps/chosen": -138.5197296142578, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -236.2646484375, "loss": 1.1526, "margin_dpo/margin_mean": 59.5881462097168, "margin_dpo/margin_std": 94.07696533203125, "step": 148 }, { "KL/chosen_KL_mean": -68.221435546875, "KL/mean": -104.93063354492188, "KL/rejected_KL_mean": -141.63983154296875, "KL/std": 77.2359619140625, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.005145592149347067, "fcm_dpo/delta": 0.023085758090019226, "fcm_dpo/margin": 73.41839599609375, "fcm_dpo/q_t": 0.412888765335083, "grad_norm": 15.609312057495117, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.43985214829444885, "logits/rejected": -0.4274490475654602, "logps/chosen": -126.69520568847656, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -240.95458984375, "loss": 1.1062, "margin_dpo/margin_mean": 73.41839599609375, "margin_dpo/margin_std": 91.98696899414062, "step": 149 }, { "KL/chosen_KL_mean": -59.703102111816406, "KL/mean": -95.98683166503906, "KL/rejected_KL_mean": -132.27056884765625, "KL/std": 89.55494689941406, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.005316386930644512, "fcm_dpo/delta": 0.011053726077079773, "fcm_dpo/margin": 72.56747436523438, "fcm_dpo/q_t": 0.4142092168331146, "grad_norm": 16.736658096313477, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.41275283694267273, "logits/rejected": -0.41830599308013916, "logps/chosen": -105.4089126586914, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -215.61817932128906, "loss": 1.1153, "margin_dpo/margin_mean": 72.56747436523438, "margin_dpo/margin_std": 106.04312133789062, "step": 150 }, { "KL/chosen_KL_mean": -77.2176513671875, "KL/mean": -112.65522766113281, "KL/rejected_KL_mean": -148.09280395507812, "KL/std": 87.22770690917969, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.005660551600158215, "fcm_dpo/delta": -0.002181790303438902, "fcm_dpo/margin": 70.87516784667969, "fcm_dpo/q_t": 0.40790289640426636, "grad_norm": 21.483774185180664, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.4776611328125, "logits/rejected": -0.4529585838317871, "logps/chosen": -147.78848266601562, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -248.55662536621094, "loss": 1.0927, "margin_dpo/margin_mean": 70.87516784667969, "margin_dpo/margin_std": 95.74323272705078, "step": 151 }, { "KL/chosen_KL_mean": -65.65343475341797, "KL/mean": -108.78428649902344, "KL/rejected_KL_mean": -151.91513061523438, "KL/std": 86.67942810058594, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.005145312752574682, "fcm_dpo/delta": -0.07856889814138412, "fcm_dpo/margin": 86.26168823242188, "fcm_dpo/q_t": 0.39314860105514526, "grad_norm": 18.017948150634766, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.4266291558742523, "logits/rejected": -0.42508143186569214, "logps/chosen": -125.81782531738281, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -258.05560302734375, "loss": 1.063, "margin_dpo/margin_mean": 86.26168823242188, "margin_dpo/margin_std": 107.51152038574219, "step": 152 }, { "KL/chosen_KL_mean": -65.02685546875, "KL/mean": -102.04881286621094, "KL/rejected_KL_mean": -139.07077026367188, "KL/std": 95.22334289550781, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.005174952559173107, "fcm_dpo/delta": 0.024773182347416878, "fcm_dpo/margin": 74.04390716552734, "fcm_dpo/q_t": 0.4167134463787079, "grad_norm": 14.81849193572998, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.44491156935691833, "logits/rejected": -0.44107282161712646, "logps/chosen": -121.34213256835938, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -224.72659301757812, "loss": 1.1225, "margin_dpo/margin_mean": 74.04390716552734, "margin_dpo/margin_std": 117.32939147949219, "step": 153 }, { "KL/chosen_KL_mean": -78.82939910888672, "KL/mean": -113.46821594238281, "KL/rejected_KL_mean": -148.10702514648438, "KL/std": 93.49275207519531, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.005523172207176685, "fcm_dpo/delta": 0.02156968042254448, "fcm_dpo/margin": 69.27762603759766, "fcm_dpo/q_t": 0.41571739315986633, "grad_norm": 18.01680564880371, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.4389742910861969, "logits/rejected": -0.44377291202545166, "logps/chosen": -141.57196044921875, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -252.35122680664062, "loss": 1.1476, "margin_dpo/margin_mean": 69.27761840820312, "margin_dpo/margin_std": 122.24600982666016, "step": 154 }, { "KL/chosen_KL_mean": -71.37049102783203, "KL/mean": -105.14212036132812, "KL/rejected_KL_mean": -138.9137420654297, "KL/std": 85.17216491699219, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.0056947702541947365, "fcm_dpo/delta": 0.021101046353578568, "fcm_dpo/margin": 67.54327392578125, "fcm_dpo/q_t": 0.41405290365219116, "grad_norm": 18.612998962402344, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.45382189750671387, "logits/rejected": -0.43611928820610046, "logps/chosen": -132.023681640625, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -216.40594482421875, "loss": 1.1265, "margin_dpo/margin_mean": 67.54327392578125, "margin_dpo/margin_std": 108.4144287109375, "step": 155 }, { "KL/chosen_KL_mean": -96.85684204101562, "KL/mean": -116.4049301147461, "KL/rejected_KL_mean": -135.95301818847656, "KL/std": 89.89227294921875, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.006321663036942482, "fcm_dpo/delta": 0.06302600353956223, "fcm_dpo/margin": 39.09617614746094, "fcm_dpo/q_t": 0.44758695363998413, "grad_norm": 30.659738540649414, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.4037202298641205, "logits/rejected": -0.3993948698043823, "logps/chosen": -166.34872436523438, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -213.122314453125, "loss": 1.2862, "margin_dpo/margin_mean": 39.09617614746094, "margin_dpo/margin_std": 119.15692138671875, "step": 156 }, { "KL/chosen_KL_mean": -85.81370544433594, "KL/mean": -130.08242797851562, "KL/rejected_KL_mean": -174.35113525390625, "KL/std": 98.00643920898438, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.004979247227311134, "fcm_dpo/delta": -0.10685013234615326, "fcm_dpo/margin": 88.53744506835938, "fcm_dpo/q_t": 0.38473406434059143, "grad_norm": 23.637866973876953, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.37596505880355835, "logits/rejected": -0.38393884897232056, "logps/chosen": -147.18214416503906, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -281.99749755859375, "loss": 1.073, "margin_dpo/margin_mean": 88.53744506835938, "margin_dpo/margin_std": 110.94694519042969, "step": 157 }, { "KL/chosen_KL_mean": -86.83956909179688, "KL/mean": -135.0098876953125, "KL/rejected_KL_mean": -183.18020629882812, "KL/std": 121.85169982910156, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.004572378005832434, "fcm_dpo/delta": -0.07351740449666977, "fcm_dpo/margin": 96.34066009521484, "fcm_dpo/q_t": 0.3994863033294678, "grad_norm": 16.999359130859375, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.37230780720710754, "logits/rejected": -0.391128271818161, "logps/chosen": -144.45248413085938, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -296.8748779296875, "loss": 1.0944, "margin_dpo/margin_mean": 96.34065246582031, "margin_dpo/margin_std": 147.7144775390625, "step": 158 }, { "KL/chosen_KL_mean": -100.27496337890625, "KL/mean": -131.74691772460938, "KL/rejected_KL_mean": -163.2188720703125, "KL/std": 107.50895690917969, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.004534922074526548, "fcm_dpo/delta": 0.02816414088010788, "fcm_dpo/margin": 62.94389343261719, "fcm_dpo/q_t": 0.435373455286026, "grad_norm": 18.136377334594727, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.3773287534713745, "logits/rejected": -0.34688425064086914, "logps/chosen": -181.83531188964844, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -252.11758422851562, "loss": 1.1876, "margin_dpo/margin_mean": 62.94389343261719, "margin_dpo/margin_std": 117.23402404785156, "step": 159 }, { "KL/chosen_KL_mean": -103.02062225341797, "KL/mean": -146.32418823242188, "KL/rejected_KL_mean": -189.62774658203125, "KL/std": 114.74168395996094, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.0045206244103610516, "fcm_dpo/delta": 0.009057410061359406, "fcm_dpo/margin": 86.60713958740234, "fcm_dpo/q_t": 0.4141198396682739, "grad_norm": 18.559383392333984, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.3365239202976227, "logits/rejected": -0.3285549283027649, "logps/chosen": -168.75149536132812, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -286.8455810546875, "loss": 1.1313, "margin_dpo/margin_mean": 86.60713958740234, "margin_dpo/margin_std": 143.93826293945312, "step": 160 }, { "KL/chosen_KL_mean": -87.73458862304688, "KL/mean": -126.02169799804688, "KL/rejected_KL_mean": -164.30880737304688, "KL/std": 91.14512634277344, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.004974995274096727, "fcm_dpo/delta": 0.030578553676605225, "fcm_dpo/margin": 76.57421875, "fcm_dpo/q_t": 0.41616570949554443, "grad_norm": 17.85219383239746, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.34718695282936096, "logits/rejected": -0.34764522314071655, "logps/chosen": -140.17105102539062, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -247.73976135253906, "loss": 1.1167, "margin_dpo/margin_mean": 76.57421875, "margin_dpo/margin_std": 114.59629821777344, "step": 161 }, { "KL/chosen_KL_mean": -87.06214904785156, "KL/mean": -124.53192901611328, "KL/rejected_KL_mean": -162.001708984375, "KL/std": 101.81707763671875, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.004378783982247114, "fcm_dpo/delta": -0.07103817164897919, "fcm_dpo/margin": 74.93955993652344, "fcm_dpo/q_t": 0.41890764236450195, "grad_norm": 17.84354019165039, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.40842747688293457, "logits/rejected": -0.3954179883003235, "logps/chosen": -149.6727294921875, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -251.3922882080078, "loss": 1.1551, "margin_dpo/margin_mean": 74.93955993652344, "margin_dpo/margin_std": 121.97283935546875, "step": 162 }, { "KL/chosen_KL_mean": -97.13140869140625, "KL/mean": -133.43283081054688, "KL/rejected_KL_mean": -169.73428344726562, "KL/std": 102.79348754882812, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.004893934354186058, "fcm_dpo/delta": 0.06739577651023865, "fcm_dpo/margin": 72.60286712646484, "fcm_dpo/q_t": 0.4250032901763916, "grad_norm": 18.780895233154297, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.2808549404144287, "logits/rejected": -0.2610132694244385, "logps/chosen": -152.15304565429688, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -245.15249633789062, "loss": 1.1469, "margin_dpo/margin_mean": 72.60285949707031, "margin_dpo/margin_std": 123.78816223144531, "step": 163 }, { "KL/chosen_KL_mean": -90.34315490722656, "KL/mean": -135.48098754882812, "KL/rejected_KL_mean": -180.6188201904297, "KL/std": 100.77694702148438, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.004474318120628595, "fcm_dpo/delta": -0.025637470185756683, "fcm_dpo/margin": 90.27565002441406, "fcm_dpo/q_t": 0.40300068259239197, "grad_norm": 18.992538452148438, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.33539408445358276, "logits/rejected": -0.32457149028778076, "logps/chosen": -145.98382568359375, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -260.283447265625, "loss": 1.0914, "margin_dpo/margin_mean": 90.27565002441406, "margin_dpo/margin_std": 116.16989135742188, "step": 164 }, { "KL/chosen_KL_mean": -97.24948120117188, "KL/mean": -127.26203155517578, "KL/rejected_KL_mean": -157.2745819091797, "KL/std": 88.08245849609375, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.005466942209750414, "fcm_dpo/delta": 0.10794621706008911, "fcm_dpo/margin": 60.025089263916016, "fcm_dpo/q_t": 0.43372809886932373, "grad_norm": 26.800586700439453, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.3530671000480652, "logits/rejected": -0.3305925726890564, "logps/chosen": -158.5601806640625, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -230.9451904296875, "loss": 1.1889, "margin_dpo/margin_mean": 60.025089263916016, "margin_dpo/margin_std": 118.67210388183594, "step": 165 }, { "KL/chosen_KL_mean": -87.03776550292969, "KL/mean": -144.3040771484375, "KL/rejected_KL_mean": -201.57040405273438, "KL/std": 130.58055114746094, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.004105499014258385, "fcm_dpo/delta": -0.187312051653862, "fcm_dpo/margin": 114.53263854980469, "fcm_dpo/q_t": 0.3861845135688782, "grad_norm": 15.165700912475586, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.2735704779624939, "logits/rejected": -0.2837375998497009, "logps/chosen": -138.0213623046875, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -299.66552734375, "loss": 1.0692, "margin_dpo/margin_mean": 114.53263854980469, "margin_dpo/margin_std": 165.4405059814453, "step": 166 }, { "KL/chosen_KL_mean": -88.03388977050781, "KL/mean": -147.37371826171875, "KL/rejected_KL_mean": -206.71356201171875, "KL/std": 116.15589904785156, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.0036684987135231495, "fcm_dpo/delta": -0.06418722122907639, "fcm_dpo/margin": 118.67967987060547, "fcm_dpo/q_t": 0.3941141664981842, "grad_norm": 17.225709915161133, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.2889915704727173, "logits/rejected": -0.29361122846603394, "logps/chosen": -138.45797729492188, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -302.7439880371094, "loss": 1.0561, "margin_dpo/margin_mean": 118.6796875, "margin_dpo/margin_std": 136.93655395507812, "step": 167 }, { "KL/chosen_KL_mean": -97.47393798828125, "KL/mean": -140.19537353515625, "KL/rejected_KL_mean": -182.9167938232422, "KL/std": 109.24185180664062, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.004024019930511713, "fcm_dpo/delta": 0.08574581891298294, "fcm_dpo/margin": 85.44287109375, "fcm_dpo/q_t": 0.4270554780960083, "grad_norm": 18.420507431030273, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.29539740085601807, "logits/rejected": -0.2888868451118469, "logps/chosen": -146.936767578125, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -258.225341796875, "loss": 1.1323, "margin_dpo/margin_mean": 85.44287109375, "margin_dpo/margin_std": 124.32752990722656, "step": 168 }, { "KL/chosen_KL_mean": -101.9376449584961, "KL/mean": -139.52291870117188, "KL/rejected_KL_mean": -177.1082000732422, "KL/std": 107.82769012451172, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.0039043950382620096, "fcm_dpo/delta": -0.04128513112664223, "fcm_dpo/margin": 75.17054748535156, "fcm_dpo/q_t": 0.4314765930175781, "grad_norm": 17.273651123046875, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.3079976439476013, "logits/rejected": -0.2939789295196533, "logps/chosen": -161.7410888671875, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -260.4539489746094, "loss": 1.1953, "margin_dpo/margin_mean": 75.17054748535156, "margin_dpo/margin_std": 148.5015869140625, "step": 169 }, { "KL/chosen_KL_mean": -92.11761474609375, "KL/mean": -135.65200805664062, "KL/rejected_KL_mean": -179.18641662597656, "KL/std": 106.36629486083984, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.004083374515175819, "fcm_dpo/delta": 0.06564062833786011, "fcm_dpo/margin": 87.06880187988281, "fcm_dpo/q_t": 0.422102689743042, "grad_norm": 16.470184326171875, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.2703856825828552, "logits/rejected": -0.26214104890823364, "logps/chosen": -141.58938598632812, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -255.103759765625, "loss": 1.1365, "margin_dpo/margin_mean": 87.06879425048828, "margin_dpo/margin_std": 128.26217651367188, "step": 170 }, { "KL/chosen_KL_mean": -134.2930908203125, "KL/mean": -169.67434692382812, "KL/rejected_KL_mean": -205.0555877685547, "KL/std": 124.128662109375, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.00497001688927412, "fcm_dpo/delta": 0.07393293082714081, "fcm_dpo/margin": 70.76249694824219, "fcm_dpo/q_t": 0.4315279722213745, "grad_norm": 27.584213256835938, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.29333311319351196, "logits/rejected": -0.2810017764568329, "logps/chosen": -218.7924041748047, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -314.43768310546875, "loss": 1.2014, "margin_dpo/margin_mean": 70.76250457763672, "margin_dpo/margin_std": 159.743896484375, "step": 171 }, { "KL/chosen_KL_mean": -115.03340148925781, "KL/mean": -152.57827758789062, "KL/rejected_KL_mean": -190.12313842773438, "KL/std": 118.62234497070312, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.0052231112495064735, "fcm_dpo/delta": 0.01281255204230547, "fcm_dpo/margin": 75.0897445678711, "fcm_dpo/q_t": 0.4145969748497009, "grad_norm": 22.732559204101562, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.3138948976993561, "logits/rejected": -0.29758453369140625, "logps/chosen": -183.68731689453125, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -275.559814453125, "loss": 1.1405, "margin_dpo/margin_mean": 75.08975219726562, "margin_dpo/margin_std": 133.15069580078125, "step": 172 }, { "KL/chosen_KL_mean": -104.79826354980469, "KL/mean": -142.23779296875, "KL/rejected_KL_mean": -179.67730712890625, "KL/std": 111.67139434814453, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.00543089909479022, "fcm_dpo/delta": -0.016082536429166794, "fcm_dpo/margin": 74.8790512084961, "fcm_dpo/q_t": 0.40886810421943665, "grad_norm": 25.01607322692871, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.377377450466156, "logits/rejected": -0.3528056740760803, "logps/chosen": -167.84915161132812, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -258.3612365722656, "loss": 1.1119, "margin_dpo/margin_mean": 74.8790512084961, "margin_dpo/margin_std": 115.37376403808594, "step": 173 }, { "KL/chosen_KL_mean": -96.8199462890625, "KL/mean": -141.00047302246094, "KL/rejected_KL_mean": -185.18099975585938, "KL/std": 111.30195617675781, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.0049737137742340565, "fcm_dpo/delta": -0.0776623785495758, "fcm_dpo/margin": 88.36105346679688, "fcm_dpo/q_t": 0.3986249268054962, "grad_norm": 24.379945755004883, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.30099165439605713, "logits/rejected": -0.3199496269226074, "logps/chosen": -150.18292236328125, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -287.09222412109375, "loss": 1.1075, "margin_dpo/margin_mean": 88.36104583740234, "margin_dpo/margin_std": 135.80799865722656, "step": 174 }, { "KL/chosen_KL_mean": -88.39326477050781, "KL/mean": -147.26588439941406, "KL/rejected_KL_mean": -206.13851928710938, "KL/std": 116.80426788330078, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.0037730231415480375, "fcm_dpo/delta": -0.0859874039888382, "fcm_dpo/margin": 117.74525451660156, "fcm_dpo/q_t": 0.3884362578392029, "grad_norm": 26.734390258789062, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.3031741976737976, "logits/rejected": -0.28989243507385254, "logps/chosen": -133.81103515625, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -295.6443176269531, "loss": 1.0409, "margin_dpo/margin_mean": 117.74525451660156, "margin_dpo/margin_std": 123.38618469238281, "step": 175 }, { "KL/chosen_KL_mean": -90.31321716308594, "KL/mean": -141.83433532714844, "KL/rejected_KL_mean": -193.35546875, "KL/std": 116.11126708984375, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.0038136134389787912, "fcm_dpo/delta": 0.010705736465752125, "fcm_dpo/margin": 103.042236328125, "fcm_dpo/q_t": 0.4118039608001709, "grad_norm": 16.391658782958984, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.309362530708313, "logits/rejected": -0.325575590133667, "logps/chosen": -140.76605224609375, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -288.9144287109375, "loss": 1.0923, "margin_dpo/margin_mean": 103.042236328125, "margin_dpo/margin_std": 134.9136199951172, "step": 176 }, { "KL/chosen_KL_mean": -105.74380493164062, "KL/mean": -156.05882263183594, "KL/rejected_KL_mean": -206.3738250732422, "KL/std": 128.35159301757812, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.003937173634767532, "fcm_dpo/delta": 0.002443891018629074, "fcm_dpo/margin": 100.63003540039062, "fcm_dpo/q_t": 0.41030067205429077, "grad_norm": 20.429180145263672, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.3264981508255005, "logits/rejected": -0.31806203722953796, "logps/chosen": -166.96026611328125, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -302.2676086425781, "loss": 1.0932, "margin_dpo/margin_mean": 100.63003540039062, "margin_dpo/margin_std": 134.44273376464844, "step": 177 }, { "KL/chosen_KL_mean": -116.06280517578125, "KL/mean": -179.25152587890625, "KL/rejected_KL_mean": -242.44021606445312, "KL/std": 147.4817657470703, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.0033705285750329494, "fcm_dpo/delta": -0.053363341838121414, "fcm_dpo/margin": 126.37743377685547, "fcm_dpo/q_t": 0.39890462160110474, "grad_norm": 19.848587036132812, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.22271186113357544, "logits/rejected": -0.21497488021850586, "logps/chosen": -174.3275909423828, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -347.8055419921875, "loss": 1.0721, "margin_dpo/margin_mean": 126.37743377685547, "margin_dpo/margin_std": 161.1014862060547, "step": 178 }, { "KL/chosen_KL_mean": -124.49735260009766, "KL/mean": -168.00872802734375, "KL/rejected_KL_mean": -211.52008056640625, "KL/std": 130.20875549316406, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.003844078164547682, "fcm_dpo/delta": 0.0992623046040535, "fcm_dpo/margin": 87.0227279663086, "fcm_dpo/q_t": 0.43123751878738403, "grad_norm": 34.565582275390625, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.27690017223358154, "logits/rejected": -0.2880879044532776, "logps/chosen": -185.55567932128906, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -302.04791259765625, "loss": 1.1778, "margin_dpo/margin_mean": 87.0227279663086, "margin_dpo/margin_std": 163.81394958496094, "step": 179 }, { "KL/chosen_KL_mean": -107.85954284667969, "KL/mean": -166.43894958496094, "KL/rejected_KL_mean": -225.0183563232422, "KL/std": 115.33984375, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.0037114559672772884, "fcm_dpo/delta": -0.06508003920316696, "fcm_dpo/margin": 117.1588134765625, "fcm_dpo/q_t": 0.39406827092170715, "grad_norm": 16.350337982177734, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.23366385698318481, "logits/rejected": -0.22989457845687866, "logps/chosen": -162.2022705078125, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -323.2301940917969, "loss": 1.0637, "margin_dpo/margin_mean": 117.1588134765625, "margin_dpo/margin_std": 143.82162475585938, "step": 180 }, { "KL/chosen_KL_mean": -94.5392074584961, "KL/mean": -129.8724365234375, "KL/rejected_KL_mean": -165.2056884765625, "KL/std": 107.95272827148438, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.004456082358956337, "fcm_dpo/delta": 0.11765934526920319, "fcm_dpo/margin": 70.6664810180664, "fcm_dpo/q_t": 0.4351644515991211, "grad_norm": 19.006668090820312, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.2771533727645874, "logits/rejected": -0.2554428279399872, "logps/chosen": -149.5396728515625, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -226.86184692382812, "loss": 1.1826, "margin_dpo/margin_mean": 70.6664810180664, "margin_dpo/margin_std": 136.99830627441406, "step": 181 }, { "KL/chosen_KL_mean": -89.14537811279297, "KL/mean": -153.301513671875, "KL/rejected_KL_mean": -217.45767211914062, "KL/std": 129.44326782226562, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.0035601062700152397, "fcm_dpo/delta": -0.1331714242696762, "fcm_dpo/margin": 128.31228637695312, "fcm_dpo/q_t": 0.38162392377853394, "grad_norm": 15.877484321594238, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.24869897961616516, "logits/rejected": -0.259911447763443, "logps/chosen": -130.25323486328125, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -306.979248046875, "loss": 1.054, "margin_dpo/margin_mean": 128.31228637695312, "margin_dpo/margin_std": 157.36795043945312, "step": 182 }, { "KL/chosen_KL_mean": -132.1239776611328, "KL/mean": -164.78997802734375, "KL/rejected_KL_mean": -197.4559783935547, "KL/std": 107.94178771972656, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.003544195555150509, "fcm_dpo/delta": 0.000493988860398531, "fcm_dpo/margin": 65.33200073242188, "fcm_dpo/q_t": 0.4455084502696991, "grad_norm": 17.170263290405273, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.2581177055835724, "logits/rejected": -0.2515699863433838, "logps/chosen": -189.64854431152344, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -273.43170166015625, "loss": 1.2172, "margin_dpo/margin_mean": 65.33200073242188, "margin_dpo/margin_std": 118.64833068847656, "step": 183 }, { "KL/chosen_KL_mean": -111.37728881835938, "KL/mean": -146.77032470703125, "KL/rejected_KL_mean": -182.16336059570312, "KL/std": 101.87884521484375, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.003663355251774192, "fcm_dpo/delta": 0.03253895044326782, "fcm_dpo/margin": 70.78607177734375, "fcm_dpo/q_t": 0.4417157769203186, "grad_norm": 16.201236724853516, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.28559648990631104, "logits/rejected": -0.27648642659187317, "logps/chosen": -169.9222412109375, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -258.79742431640625, "loss": 1.1955, "margin_dpo/margin_mean": 70.78607177734375, "margin_dpo/margin_std": 128.29718017578125, "step": 184 }, { "KL/chosen_KL_mean": -122.75660705566406, "KL/mean": -149.69287109375, "KL/rejected_KL_mean": -176.62911987304688, "KL/std": 117.6004638671875, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.003782514948397875, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 53.87251663208008, "fcm_dpo/q_t": 0.4551408886909485, "grad_norm": 18.46157455444336, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.2715925872325897, "logits/rejected": -0.25268396735191345, "logps/chosen": -184.78245544433594, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -250.3917236328125, "loss": 1.2595, "margin_dpo/margin_mean": 53.872520446777344, "margin_dpo/margin_std": 141.81597900390625, "step": 185 }, { "KL/chosen_KL_mean": -115.03028869628906, "KL/mean": -167.48983764648438, "KL/rejected_KL_mean": -219.94937133789062, "KL/std": 118.58721160888672, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.0037781421560794115, "fcm_dpo/delta": 0.005793063435703516, "fcm_dpo/margin": 104.91907501220703, "fcm_dpo/q_t": 0.40997469425201416, "grad_norm": 19.549184799194336, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.3371797502040863, "logits/rejected": -0.3270975947380066, "logps/chosen": -184.38375854492188, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -308.0218200683594, "loss": 1.0931, "margin_dpo/margin_mean": 104.9190673828125, "margin_dpo/margin_std": 139.93943786621094, "step": 186 }, { "KL/chosen_KL_mean": -102.89697265625, "KL/mean": -148.2735137939453, "KL/rejected_KL_mean": -193.65003967285156, "KL/std": 112.91288757324219, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.00417533703148365, "fcm_dpo/delta": 0.025836054235696793, "fcm_dpo/margin": 90.75306701660156, "fcm_dpo/q_t": 0.4136863350868225, "grad_norm": 21.79146385192871, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.2833201587200165, "logits/rejected": -0.27770209312438965, "logps/chosen": -155.6534423828125, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -275.619140625, "loss": 1.1064, "margin_dpo/margin_mean": 90.75306701660156, "margin_dpo/margin_std": 121.31842041015625, "step": 187 }, { "KL/chosen_KL_mean": -96.85295104980469, "KL/mean": -152.17840576171875, "KL/rejected_KL_mean": -207.50384521484375, "KL/std": 124.83462524414062, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.003823988139629364, "fcm_dpo/delta": -0.04001890495419502, "fcm_dpo/margin": 110.6509017944336, "fcm_dpo/q_t": 0.4005782902240753, "grad_norm": 25.183116912841797, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.2504292130470276, "logits/rejected": -0.2585201859474182, "logps/chosen": -146.2684326171875, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -297.04425048828125, "loss": 1.0712, "margin_dpo/margin_mean": 110.6509017944336, "margin_dpo/margin_std": 138.2140655517578, "step": 188 }, { "KL/chosen_KL_mean": -115.58564758300781, "KL/mean": -160.01602172851562, "KL/rejected_KL_mean": -204.44639587402344, "KL/std": 127.6397933959961, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.004090734291821718, "fcm_dpo/delta": 0.055509161204099655, "fcm_dpo/margin": 88.86076354980469, "fcm_dpo/q_t": 0.42382127046585083, "grad_norm": 27.4331111907959, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.271611750125885, "logits/rejected": -0.2570100426673889, "logps/chosen": -167.98460388183594, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -276.6137390136719, "loss": 1.1472, "margin_dpo/margin_mean": 88.86076354980469, "margin_dpo/margin_std": 158.424560546875, "step": 189 }, { "KL/chosen_KL_mean": -124.001708984375, "KL/mean": -178.27090454101562, "KL/rejected_KL_mean": -232.54010009765625, "KL/std": 137.54371643066406, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.004083447623997927, "fcm_dpo/delta": -0.09416892379522324, "fcm_dpo/margin": 108.53840637207031, "fcm_dpo/q_t": 0.3950834274291992, "grad_norm": 19.966215133666992, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.3038903474807739, "logits/rejected": -0.29708611965179443, "logps/chosen": -188.68478393554688, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -335.09063720703125, "loss": 1.1112, "margin_dpo/margin_mean": 108.53840637207031, "margin_dpo/margin_std": 159.21173095703125, "step": 190 }, { "KL/chosen_KL_mean": -116.2418212890625, "KL/mean": -192.55517578125, "KL/rejected_KL_mean": -268.8685302734375, "KL/std": 155.87002563476562, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.0028747329488396645, "fcm_dpo/delta": -0.08564233034849167, "fcm_dpo/margin": 152.626708984375, "fcm_dpo/q_t": 0.3907443881034851, "grad_norm": 17.792726516723633, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.2973872721195221, "logits/rejected": -0.2730400264263153, "logps/chosen": -184.90069580078125, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -379.0082092285156, "loss": 1.0538, "margin_dpo/margin_mean": 152.626708984375, "margin_dpo/margin_std": 175.80136108398438, "step": 191 }, { "KL/chosen_KL_mean": -149.68504333496094, "KL/mean": -200.70745849609375, "KL/rejected_KL_mean": -251.7298583984375, "KL/std": 139.31866455078125, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.003366319462656975, "fcm_dpo/delta": 0.079339399933815, "fcm_dpo/margin": 102.04479217529297, "fcm_dpo/q_t": 0.42711859941482544, "grad_norm": 20.006580352783203, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.2600586414337158, "logits/rejected": -0.26135045289993286, "logps/chosen": -219.41195678710938, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -355.05120849609375, "loss": 1.1503, "margin_dpo/margin_mean": 102.0447998046875, "margin_dpo/margin_std": 174.04171752929688, "step": 192 }, { "KL/chosen_KL_mean": -149.03636169433594, "KL/mean": -177.56857299804688, "KL/rejected_KL_mean": -206.10079956054688, "KL/std": 129.29698181152344, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.003643119242042303, "fcm_dpo/delta": 0.06691880524158478, "fcm_dpo/margin": 57.06443786621094, "fcm_dpo/q_t": 0.45318782329559326, "grad_norm": 28.55337142944336, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.27016884088516235, "logits/rejected": -0.2688116431236267, "logps/chosen": -209.22686767578125, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -282.50836181640625, "loss": 1.2736, "margin_dpo/margin_mean": 57.06443786621094, "margin_dpo/margin_std": 165.7989501953125, "step": 193 }, { "KL/chosen_KL_mean": -90.87747955322266, "KL/mean": -138.6411590576172, "KL/rejected_KL_mean": -186.40484619140625, "KL/std": 105.31715393066406, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.0039482079446315765, "fcm_dpo/delta": 0.034811198711395264, "fcm_dpo/margin": 95.52735900878906, "fcm_dpo/q_t": 0.4142157733440399, "grad_norm": 17.623666763305664, "learning_rate": 4.510405240853854e-07, "logits/chosen": -0.1908482015132904, "logits/rejected": -0.17822426557540894, "logps/chosen": -128.7178497314453, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -247.08963012695312, "loss": 1.0982, "margin_dpo/margin_mean": 95.52735900878906, "margin_dpo/margin_std": 116.02978515625, "step": 194 }, { "KL/chosen_KL_mean": -144.2040557861328, "KL/mean": -196.9357147216797, "KL/rejected_KL_mean": -249.66738891601562, "KL/std": 133.17889404296875, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0039472421631217, "fcm_dpo/delta": -0.028900716453790665, "fcm_dpo/margin": 105.46333312988281, "fcm_dpo/q_t": 0.40308016538619995, "grad_norm": 22.52235984802246, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.23734885454177856, "logits/rejected": -0.25579947233200073, "logps/chosen": -199.0956268310547, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -346.4383544921875, "loss": 1.0821, "margin_dpo/margin_mean": 105.46334075927734, "margin_dpo/margin_std": 139.16427612304688, "step": 195 }, { "KL/chosen_KL_mean": -111.86429595947266, "KL/mean": -169.64901733398438, "KL/rejected_KL_mean": -227.4337158203125, "KL/std": 129.91050720214844, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.003521410282701254, "fcm_dpo/delta": -0.028155531734228134, "fcm_dpo/margin": 115.56942749023438, "fcm_dpo/q_t": 0.40525445342063904, "grad_norm": 16.956315994262695, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.24712811410427094, "logits/rejected": -0.2348405420780182, "logps/chosen": -165.1095428466797, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -303.4866638183594, "loss": 1.0964, "margin_dpo/margin_mean": 115.56942749023438, "margin_dpo/margin_std": 156.6468505859375, "step": 196 }, { "KL/chosen_KL_mean": -114.23474884033203, "KL/mean": -157.83114624023438, "KL/rejected_KL_mean": -201.42755126953125, "KL/std": 111.18183135986328, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.004171357490122318, "fcm_dpo/delta": 0.04900194704532623, "fcm_dpo/margin": 87.19279479980469, "fcm_dpo/q_t": 0.4201440215110779, "grad_norm": 31.859424591064453, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.24368175864219666, "logits/rejected": -0.24027302861213684, "logps/chosen": -174.65509033203125, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -278.636474609375, "loss": 1.1232, "margin_dpo/margin_mean": 87.19279479980469, "margin_dpo/margin_std": 128.5411376953125, "step": 197 }, { "KL/chosen_KL_mean": -124.44345092773438, "KL/mean": -183.01571655273438, "KL/rejected_KL_mean": -241.5879669189453, "KL/std": 141.48675537109375, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.003705207258462906, "fcm_dpo/delta": -0.06200866401195526, "fcm_dpo/margin": 117.14451599121094, "fcm_dpo/q_t": 0.39838171005249023, "grad_norm": 22.00115966796875, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.29106560349464417, "logits/rejected": -0.2917174696922302, "logps/chosen": -179.47964477539062, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -338.83123779296875, "loss": 1.0749, "margin_dpo/margin_mean": 117.14451599121094, "margin_dpo/margin_std": 159.01614379882812, "step": 198 }, { "KL/chosen_KL_mean": -120.00575256347656, "KL/mean": -176.61056518554688, "KL/rejected_KL_mean": -233.21536254882812, "KL/std": 125.2437515258789, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.00345216179266572, "fcm_dpo/delta": 0.006332229822874069, "fcm_dpo/margin": 113.2095947265625, "fcm_dpo/q_t": 0.4100239872932434, "grad_norm": 18.751543045043945, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.2945285141468048, "logits/rejected": -0.2768348157405853, "logps/chosen": -176.8345947265625, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -317.86358642578125, "loss": 1.106, "margin_dpo/margin_mean": 113.2095947265625, "margin_dpo/margin_std": 155.6266632080078, "step": 199 }, { "KL/chosen_KL_mean": -117.3885498046875, "KL/mean": -176.23748779296875, "KL/rejected_KL_mean": -235.08642578125, "KL/std": 138.85617065429688, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.0034625413827598095, "fcm_dpo/delta": -0.016614485532045364, "fcm_dpo/margin": 117.6978988647461, "fcm_dpo/q_t": 0.4070635437965393, "grad_norm": 20.583110809326172, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.2415538728237152, "logits/rejected": -0.22881929576396942, "logps/chosen": -170.4556121826172, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -315.69488525390625, "loss": 1.1079, "margin_dpo/margin_mean": 117.6978988647461, "margin_dpo/margin_std": 178.77822875976562, "step": 200 }, { "KL/chosen_KL_mean": -128.0514373779297, "KL/mean": -184.71365356445312, "KL/rejected_KL_mean": -241.3758544921875, "KL/std": 148.18923950195312, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.00359878852032125, "fcm_dpo/delta": -0.015944721177220345, "fcm_dpo/margin": 113.32442474365234, "fcm_dpo/q_t": 0.40622514486312866, "grad_norm": 24.77713966369629, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.3444761037826538, "logits/rejected": -0.3457464575767517, "logps/chosen": -203.4536590576172, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -356.18408203125, "loss": 1.1082, "margin_dpo/margin_mean": 113.32442474365234, "margin_dpo/margin_std": 168.1522979736328, "step": 201 }, { "KL/chosen_KL_mean": -129.18331909179688, "KL/mean": -168.78958129882812, "KL/rejected_KL_mean": -208.39584350585938, "KL/std": 127.8756103515625, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.004097277298569679, "fcm_dpo/delta": 0.10765650868415833, "fcm_dpo/margin": 79.21253204345703, "fcm_dpo/q_t": 0.4345342516899109, "grad_norm": 22.265583038330078, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.27213054895401, "logits/rejected": -0.28951022028923035, "logps/chosen": -179.28463745117188, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -295.380859375, "loss": 1.1947, "margin_dpo/margin_mean": 79.21253967285156, "margin_dpo/margin_std": 168.17898559570312, "step": 202 }, { "KL/chosen_KL_mean": -128.6010284423828, "KL/mean": -178.1983642578125, "KL/rejected_KL_mean": -227.79568481445312, "KL/std": 133.08248901367188, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.004127629101276398, "fcm_dpo/delta": -0.01639389619231224, "fcm_dpo/margin": 99.19467163085938, "fcm_dpo/q_t": 0.40751299262046814, "grad_norm": 20.381248474121094, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.303178995847702, "logits/rejected": -0.29665249586105347, "logps/chosen": -189.21072387695312, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -313.691650390625, "loss": 1.1046, "margin_dpo/margin_mean": 99.19467163085938, "margin_dpo/margin_std": 150.01708984375, "step": 203 }, { "KL/chosen_KL_mean": -141.3732452392578, "KL/mean": -185.82644653320312, "KL/rejected_KL_mean": -230.27967834472656, "KL/std": 142.70265197753906, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.0043134503066539764, "fcm_dpo/delta": 0.026382368057966232, "fcm_dpo/margin": 88.90643310546875, "fcm_dpo/q_t": 0.4196096360683441, "grad_norm": 31.27920150756836, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.38292059302330017, "logits/rejected": -0.34853625297546387, "logps/chosen": -221.5382080078125, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -317.9755859375, "loss": 1.1628, "margin_dpo/margin_mean": 88.90642547607422, "margin_dpo/margin_std": 170.709228515625, "step": 204 }, { "KL/chosen_KL_mean": -131.44627380371094, "KL/mean": -190.60824584960938, "KL/rejected_KL_mean": -249.77024841308594, "KL/std": 140.37823486328125, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.0035104253329336643, "fcm_dpo/delta": -0.07186352461576462, "fcm_dpo/margin": 118.3239517211914, "fcm_dpo/q_t": 0.3940643072128296, "grad_norm": 19.9980411529541, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.290398508310318, "logits/rejected": -0.2605670094490051, "logps/chosen": -190.8310089111328, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -334.8952941894531, "loss": 1.0949, "margin_dpo/margin_mean": 118.3239517211914, "margin_dpo/margin_std": 153.96115112304688, "step": 205 }, { "KL/chosen_KL_mean": -112.48219299316406, "KL/mean": -175.43797302246094, "KL/rejected_KL_mean": -238.3937530517578, "KL/std": 123.48064422607422, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.0033509135246276855, "fcm_dpo/delta": -0.047678425908088684, "fcm_dpo/margin": 125.91154479980469, "fcm_dpo/q_t": 0.39560678601264954, "grad_norm": 30.06896209716797, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.26723650097846985, "logits/rejected": -0.2706051170825958, "logps/chosen": -159.44668579101562, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -337.34722900390625, "loss": 1.0582, "margin_dpo/margin_mean": 125.91154479980469, "margin_dpo/margin_std": 135.0882568359375, "step": 206 }, { "KL/chosen_KL_mean": -106.53071594238281, "KL/mean": -176.65072631835938, "KL/rejected_KL_mean": -246.77072143554688, "KL/std": 144.35787963867188, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.0031110041309148073, "fcm_dpo/delta": -0.06504091620445251, "fcm_dpo/margin": 140.24000549316406, "fcm_dpo/q_t": 0.3961068391799927, "grad_norm": 30.773921966552734, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.35365355014801025, "logits/rejected": -0.3321405053138733, "logps/chosen": -162.58697509765625, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -331.218505859375, "loss": 1.0617, "margin_dpo/margin_mean": 140.24000549316406, "margin_dpo/margin_std": 172.24667358398438, "step": 207 }, { "KL/chosen_KL_mean": -163.8634490966797, "KL/mean": -219.51443481445312, "KL/rejected_KL_mean": -275.1654052734375, "KL/std": 141.97909545898438, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.0032390430569648743, "fcm_dpo/delta": 0.06209877133369446, "fcm_dpo/margin": 111.30195617675781, "fcm_dpo/q_t": 0.42106446623802185, "grad_norm": 23.454050064086914, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.34735146164894104, "logits/rejected": -0.33190101385116577, "logps/chosen": -230.93106079101562, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -369.4523010253906, "loss": 1.1334, "margin_dpo/margin_mean": 111.30195617675781, "margin_dpo/margin_std": 172.6982879638672, "step": 208 }, { "KL/chosen_KL_mean": -133.59063720703125, "KL/mean": -184.37820434570312, "KL/rejected_KL_mean": -235.165771484375, "KL/std": 125.87327575683594, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.0036668735556304455, "fcm_dpo/delta": 0.042608536779880524, "fcm_dpo/margin": 101.57514190673828, "fcm_dpo/q_t": 0.41783520579338074, "grad_norm": 21.281137466430664, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.257140576839447, "logits/rejected": -0.24332758784294128, "logps/chosen": -189.77232360839844, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -316.1072998046875, "loss": 1.127, "margin_dpo/margin_mean": 101.57514953613281, "margin_dpo/margin_std": 156.05691528320312, "step": 209 }, { "KL/chosen_KL_mean": -117.53192138671875, "KL/mean": -174.57296752929688, "KL/rejected_KL_mean": -231.614013671875, "KL/std": 124.3192367553711, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.0036693743895739317, "fcm_dpo/delta": -0.03410163149237633, "fcm_dpo/margin": 114.08209228515625, "fcm_dpo/q_t": 0.4016813635826111, "grad_norm": 30.14597511291504, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.3081471621990204, "logits/rejected": -0.30119192600250244, "logps/chosen": -163.90374755859375, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -308.2956237792969, "loss": 1.0831, "margin_dpo/margin_mean": 114.08209228515625, "margin_dpo/margin_std": 149.70401000976562, "step": 210 }, { "KL/chosen_KL_mean": -176.67413330078125, "KL/mean": -227.01296997070312, "KL/rejected_KL_mean": -277.351806640625, "KL/std": 148.22705078125, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.0037167323753237724, "fcm_dpo/delta": 0.03998677805066109, "fcm_dpo/margin": 100.67768859863281, "fcm_dpo/q_t": 0.42228320240974426, "grad_norm": 28.76727294921875, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.31403297185897827, "logits/rejected": -0.27679428458213806, "logps/chosen": -255.60647583007812, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -364.17279052734375, "loss": 1.1458, "margin_dpo/margin_mean": 100.67768859863281, "margin_dpo/margin_std": 173.43338012695312, "step": 211 }, { "KL/chosen_KL_mean": -147.537109375, "KL/mean": -220.67324829101562, "KL/rejected_KL_mean": -293.80938720703125, "KL/std": 159.14773559570312, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.0029512690380215645, "fcm_dpo/delta": -0.09743355959653854, "fcm_dpo/margin": 146.27227783203125, "fcm_dpo/q_t": 0.3903607130050659, "grad_norm": 18.54225730895996, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.26962530612945557, "logits/rejected": -0.26270633935928345, "logps/chosen": -205.73411560058594, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -396.86724853515625, "loss": 1.0686, "margin_dpo/margin_mean": 146.27227783203125, "margin_dpo/margin_std": 174.09445190429688, "step": 212 }, { "KL/chosen_KL_mean": -143.3365478515625, "KL/mean": -215.00723266601562, "KL/rejected_KL_mean": -286.67791748046875, "KL/std": 147.20899963378906, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.002868292387574911, "fcm_dpo/delta": -0.02426821179687977, "fcm_dpo/margin": 143.34136962890625, "fcm_dpo/q_t": 0.4022514820098877, "grad_norm": 18.237730026245117, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.2986302077770233, "logits/rejected": -0.27630025148391724, "logps/chosen": -210.84925842285156, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -380.5926513671875, "loss": 1.0765, "margin_dpo/margin_mean": 143.3413848876953, "margin_dpo/margin_std": 173.92288208007812, "step": 213 }, { "KL/chosen_KL_mean": -125.64663696289062, "KL/mean": -189.3560028076172, "KL/rejected_KL_mean": -253.06533813476562, "KL/std": 145.16001892089844, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.0030471929349005222, "fcm_dpo/delta": 0.018730763345956802, "fcm_dpo/margin": 127.41871643066406, "fcm_dpo/q_t": 0.4126247763633728, "grad_norm": 18.90703010559082, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.21006491780281067, "logits/rejected": -0.2164653241634369, "logps/chosen": -167.25152587890625, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -330.582763671875, "loss": 1.0976, "margin_dpo/margin_mean": 127.41871643066406, "margin_dpo/margin_std": 170.29165649414062, "step": 214 }, { "KL/chosen_KL_mean": -150.6676025390625, "KL/mean": -216.49374389648438, "KL/rejected_KL_mean": -282.31988525390625, "KL/std": 148.04367065429688, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.0030098496936261654, "fcm_dpo/delta": 0.005100354552268982, "fcm_dpo/margin": 131.6522979736328, "fcm_dpo/q_t": 0.4081616997718811, "grad_norm": 22.060007095336914, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.24693366885185242, "logits/rejected": -0.23978078365325928, "logps/chosen": -203.94686889648438, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -372.2845458984375, "loss": 1.0802, "margin_dpo/margin_mean": 131.6522979736328, "margin_dpo/margin_std": 154.73556518554688, "step": 215 }, { "KL/chosen_KL_mean": -155.56214904785156, "KL/mean": -220.8539581298828, "KL/rejected_KL_mean": -286.145751953125, "KL/std": 156.0887451171875, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.0031287854071706533, "fcm_dpo/delta": -0.01758376508951187, "fcm_dpo/margin": 130.5836181640625, "fcm_dpo/q_t": 0.40592771768569946, "grad_norm": 22.239622116088867, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.2485334426164627, "logits/rejected": -0.23272496461868286, "logps/chosen": -204.449951171875, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -363.3446960449219, "loss": 1.1032, "margin_dpo/margin_mean": 130.5836181640625, "margin_dpo/margin_std": 189.82901000976562, "step": 216 }, { "KL/chosen_KL_mean": -158.0336456298828, "KL/mean": -235.93881225585938, "KL/rejected_KL_mean": -313.843994140625, "KL/std": 162.12493896484375, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.0027228975668549538, "fcm_dpo/delta": -0.045385316014289856, "fcm_dpo/margin": 155.81036376953125, "fcm_dpo/q_t": 0.39862823486328125, "grad_norm": 20.71597671508789, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.25405189394950867, "logits/rejected": -0.2514030337333679, "logps/chosen": -207.8789520263672, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -413.92230224609375, "loss": 1.0651, "margin_dpo/margin_mean": 155.81036376953125, "margin_dpo/margin_std": 185.34475708007812, "step": 217 }, { "KL/chosen_KL_mean": -167.77676391601562, "KL/mean": -227.49313354492188, "KL/rejected_KL_mean": -287.2095031738281, "KL/std": 159.71775817871094, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.0029854662716388702, "fcm_dpo/delta": 0.0675802081823349, "fcm_dpo/margin": 119.43275451660156, "fcm_dpo/q_t": 0.42418262362480164, "grad_norm": 21.031944274902344, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.2911514639854431, "logits/rejected": -0.28862863779067993, "logps/chosen": -226.3534393310547, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -375.055908203125, "loss": 1.1395, "margin_dpo/margin_mean": 119.43275451660156, "margin_dpo/margin_std": 189.45452880859375, "step": 218 }, { "KL/chosen_KL_mean": -179.43991088867188, "KL/mean": -230.53575134277344, "KL/rejected_KL_mean": -281.631591796875, "KL/std": 166.12942504882812, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.003073344938457012, "fcm_dpo/delta": -0.012668075039982796, "fcm_dpo/margin": 102.19166564941406, "fcm_dpo/q_t": 0.42826542258262634, "grad_norm": 32.95735549926758, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.2939431965351105, "logits/rejected": -0.29159048199653625, "logps/chosen": -240.52377319335938, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -367.4620361328125, "loss": 1.1924, "margin_dpo/margin_mean": 102.19166564941406, "margin_dpo/margin_std": 205.6397247314453, "step": 219 }, { "KL/chosen_KL_mean": -201.14923095703125, "KL/mean": -243.14886474609375, "KL/rejected_KL_mean": -285.1484680175781, "KL/std": 140.5084228515625, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.003665705444291234, "fcm_dpo/delta": 0.13320602476596832, "fcm_dpo/margin": 83.9992446899414, "fcm_dpo/q_t": 0.4367624521255493, "grad_norm": 45.616695404052734, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.3121609687805176, "logits/rejected": -0.2908840775489807, "logps/chosen": -271.1805419921875, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -372.833984375, "loss": 1.176, "margin_dpo/margin_mean": 83.9992446899414, "margin_dpo/margin_std": 149.43328857421875, "step": 220 }, { "KL/chosen_KL_mean": -161.646728515625, "KL/mean": -256.31317138671875, "KL/rejected_KL_mean": -350.9796142578125, "KL/std": 178.03521728515625, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.0027057200204581022, "fcm_dpo/delta": -0.25114211440086365, "fcm_dpo/margin": 189.3328857421875, "fcm_dpo/q_t": 0.35794326663017273, "grad_norm": 40.98723602294922, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.3036612868309021, "logits/rejected": -0.3129799962043762, "logps/chosen": -213.80145263671875, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -457.44732666015625, "loss": 1.0037, "margin_dpo/margin_mean": 189.3328857421875, "margin_dpo/margin_std": 184.73748779296875, "step": 221 }, { "KL/chosen_KL_mean": -166.11956787109375, "KL/mean": -236.33322143554688, "KL/rejected_KL_mean": -306.546875, "KL/std": 162.46240234375, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.00264683086425066, "fcm_dpo/delta": 0.03757670894265175, "fcm_dpo/margin": 140.42733764648438, "fcm_dpo/q_t": 0.4158090353012085, "grad_norm": 24.47178840637207, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.3462597131729126, "logits/rejected": -0.3537750244140625, "logps/chosen": -227.09066772460938, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -406.54803466796875, "loss": 1.1057, "margin_dpo/margin_mean": 140.42733764648438, "margin_dpo/margin_std": 183.39102172851562, "step": 222 }, { "KL/chosen_KL_mean": -183.58261108398438, "KL/mean": -243.48226928710938, "KL/rejected_KL_mean": -303.38189697265625, "KL/std": 159.4056396484375, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.0028988937847316265, "fcm_dpo/delta": 0.0806143581867218, "fcm_dpo/margin": 119.79930114746094, "fcm_dpo/q_t": 0.4264669120311737, "grad_norm": 21.68163299560547, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.2645799219608307, "logits/rejected": -0.2657017111778259, "logps/chosen": -236.22317504882812, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -386.2069396972656, "loss": 1.1509, "margin_dpo/margin_mean": 119.79930114746094, "margin_dpo/margin_std": 204.67282104492188, "step": 223 }, { "KL/chosen_KL_mean": -170.26473999023438, "KL/mean": -239.272216796875, "KL/rejected_KL_mean": -308.27972412109375, "KL/std": 185.91464233398438, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.002813777420669794, "fcm_dpo/delta": 0.0016366764903068542, "fcm_dpo/margin": 138.0149383544922, "fcm_dpo/q_t": 0.41247671842575073, "grad_norm": 31.99540901184082, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.27732497453689575, "logits/rejected": -0.26288285851478577, "logps/chosen": -218.86013793945312, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -385.39617919921875, "loss": 1.1282, "margin_dpo/margin_mean": 138.0149383544922, "margin_dpo/margin_std": 214.30078125, "step": 224 }, { "KL/chosen_KL_mean": -199.21405029296875, "KL/mean": -281.2215576171875, "KL/rejected_KL_mean": -363.2290954589844, "KL/std": 178.91641235351562, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.0026704357005655766, "fcm_dpo/delta": -0.07286377251148224, "fcm_dpo/margin": 164.01502990722656, "fcm_dpo/q_t": 0.3952982723712921, "grad_norm": 19.550594329833984, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.24977388978004456, "logits/rejected": -0.24200385808944702, "logps/chosen": -257.21453857421875, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -463.13201904296875, "loss": 1.0792, "margin_dpo/margin_mean": 164.01502990722656, "margin_dpo/margin_std": 227.45672607421875, "step": 225 }, { "KL/chosen_KL_mean": -171.4937286376953, "KL/mean": -237.76144409179688, "KL/rejected_KL_mean": -304.0291748046875, "KL/std": 178.0089111328125, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.0027313155587762594, "fcm_dpo/delta": 0.056763019412755966, "fcm_dpo/margin": 132.5354461669922, "fcm_dpo/q_t": 0.42256051301956177, "grad_norm": 24.24056053161621, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.32699283957481384, "logits/rejected": -0.29330500960350037, "logps/chosen": -230.39251708984375, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -382.7169189453125, "loss": 1.139, "margin_dpo/margin_mean": 132.53546142578125, "margin_dpo/margin_std": 209.61676025390625, "step": 226 }, { "KL/chosen_KL_mean": -197.4924774169922, "KL/mean": -280.9031677246094, "KL/rejected_KL_mean": -364.3138427734375, "KL/std": 201.27569580078125, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.002555706538259983, "fcm_dpo/delta": -0.05992633104324341, "fcm_dpo/margin": 166.82138061523438, "fcm_dpo/q_t": 0.3967394530773163, "grad_norm": 19.361919403076172, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.2924739718437195, "logits/rejected": -0.2799687385559082, "logps/chosen": -256.56463623046875, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -463.7261962890625, "loss": 1.072, "margin_dpo/margin_mean": 166.82138061523438, "margin_dpo/margin_std": 210.5462646484375, "step": 227 }, { "KL/chosen_KL_mean": -204.37904357910156, "KL/mean": -260.72320556640625, "KL/rejected_KL_mean": -317.0673828125, "KL/std": 166.0177001953125, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.0030394818168133497, "fcm_dpo/delta": 0.08101288229227066, "fcm_dpo/margin": 112.68833923339844, "fcm_dpo/q_t": 0.427123486995697, "grad_norm": 26.379526138305664, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.28412914276123047, "logits/rejected": -0.26933881640434265, "logps/chosen": -270.27032470703125, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -408.1161193847656, "loss": 1.1678, "margin_dpo/margin_mean": 112.68833923339844, "margin_dpo/margin_std": 209.85467529296875, "step": 228 }, { "KL/chosen_KL_mean": -214.5729522705078, "KL/mean": -271.60009765625, "KL/rejected_KL_mean": -328.62725830078125, "KL/std": 190.02706909179688, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.0032839591149240732, "fcm_dpo/delta": 0.04005454480648041, "fcm_dpo/margin": 114.0543212890625, "fcm_dpo/q_t": 0.4181785583496094, "grad_norm": 38.73786544799805, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.3614084720611572, "logits/rejected": -0.3334274888038635, "logps/chosen": -285.2793273925781, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -413.1546630859375, "loss": 1.1432, "margin_dpo/margin_mean": 114.0543212890625, "margin_dpo/margin_std": 198.13516235351562, "step": 229 }, { "KL/chosen_KL_mean": -155.9505157470703, "KL/mean": -249.08929443359375, "KL/rejected_KL_mean": -342.22808837890625, "KL/std": 169.89035034179688, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.0025410507805645466, "fcm_dpo/delta": -0.15831325948238373, "fcm_dpo/margin": 186.27757263183594, "fcm_dpo/q_t": 0.3732534348964691, "grad_norm": 28.544626235961914, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.25178319215774536, "logits/rejected": -0.25868701934814453, "logps/chosen": -195.23251342773438, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -427.8499755859375, "loss": 1.0197, "margin_dpo/margin_mean": 186.27755737304688, "margin_dpo/margin_std": 184.69554138183594, "step": 230 }, { "KL/chosen_KL_mean": -179.1609344482422, "KL/mean": -240.4979705810547, "KL/rejected_KL_mean": -301.8349914550781, "KL/std": 151.0618896484375, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.0027899318374693394, "fcm_dpo/delta": 0.0851934552192688, "fcm_dpo/margin": 122.67406463623047, "fcm_dpo/q_t": 0.427597314119339, "grad_norm": 24.108116149902344, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.3525630831718445, "logits/rejected": -0.32520967721939087, "logps/chosen": -242.4373779296875, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -375.9588928222656, "loss": 1.1322, "margin_dpo/margin_mean": 122.67406463623047, "margin_dpo/margin_std": 179.02493286132812, "step": 231 }, { "KL/chosen_KL_mean": -216.9395294189453, "KL/mean": -265.2451171875, "KL/rejected_KL_mean": -313.55072021484375, "KL/std": 175.61700439453125, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.0033727658446878195, "fcm_dpo/delta": 0.10968147218227386, "fcm_dpo/margin": 96.61114501953125, "fcm_dpo/q_t": 0.435283899307251, "grad_norm": 27.411245346069336, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.3760000467300415, "logits/rejected": -0.35748612880706787, "logps/chosen": -287.68829345703125, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -397.52777099609375, "loss": 1.1718, "margin_dpo/margin_mean": 96.61115264892578, "margin_dpo/margin_std": 181.7001495361328, "step": 232 }, { "KL/chosen_KL_mean": -181.86721801757812, "KL/mean": -262.99560546875, "KL/rejected_KL_mean": -344.12396240234375, "KL/std": 183.73455810546875, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.0029199281707406044, "fcm_dpo/delta": -0.14302438497543335, "fcm_dpo/margin": 162.25674438476562, "fcm_dpo/q_t": 0.38662809133529663, "grad_norm": 21.30043601989746, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.3732132017612457, "logits/rejected": -0.37482750415802, "logps/chosen": -236.75015258789062, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -451.6040344238281, "loss": 1.0754, "margin_dpo/margin_mean": 162.25674438476562, "margin_dpo/margin_std": 236.0518035888672, "step": 233 }, { "KL/chosen_KL_mean": -163.74453735351562, "KL/mean": -245.33642578125, "KL/rejected_KL_mean": -326.9283447265625, "KL/std": 157.0659942626953, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.0024456209503114223, "fcm_dpo/delta": -0.015596166253089905, "fcm_dpo/margin": 163.18380737304688, "fcm_dpo/q_t": 0.403796911239624, "grad_norm": 22.082944869995117, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.36095279455184937, "logits/rejected": -0.3873627483844757, "logps/chosen": -207.83905029296875, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -426.93499755859375, "loss": 1.0777, "margin_dpo/margin_mean": 163.18380737304688, "margin_dpo/margin_std": 176.453857421875, "step": 234 }, { "KL/chosen_KL_mean": -201.49075317382812, "KL/mean": -251.1799774169922, "KL/rejected_KL_mean": -300.86920166015625, "KL/std": 150.57394409179688, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.002739574294537306, "fcm_dpo/delta": 0.046936918050050735, "fcm_dpo/margin": 99.37845611572266, "fcm_dpo/q_t": 0.43904921412467957, "grad_norm": 21.031633377075195, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.38550442457199097, "logits/rejected": -0.3654111921787262, "logps/chosen": -263.7286682128906, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -391.2642822265625, "loss": 1.1915, "margin_dpo/margin_mean": 99.37844848632812, "margin_dpo/margin_std": 185.09054565429688, "step": 235 }, { "KL/chosen_KL_mean": -145.79144287109375, "KL/mean": -228.65101623535156, "KL/rejected_KL_mean": -311.5105895996094, "KL/std": 160.1380157470703, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.0026162322610616684, "fcm_dpo/delta": -0.06015552580356598, "fcm_dpo/margin": 165.71914672851562, "fcm_dpo/q_t": 0.3923841118812561, "grad_norm": 37.16407775878906, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.3568821847438812, "logits/rejected": -0.3578595221042633, "logps/chosen": -195.1328125, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -415.022216796875, "loss": 1.0388, "margin_dpo/margin_mean": 165.71914672851562, "margin_dpo/margin_std": 155.69921875, "step": 236 }, { "KL/chosen_KL_mean": -203.95053100585938, "KL/mean": -268.9967041015625, "KL/rejected_KL_mean": -334.04290771484375, "KL/std": 166.29592895507812, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.0028002336621284485, "fcm_dpo/delta": 0.05379777029156685, "fcm_dpo/margin": 130.09239196777344, "fcm_dpo/q_t": 0.4198486804962158, "grad_norm": 27.22985076904297, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.32136499881744385, "logits/rejected": -0.3153060972690582, "logps/chosen": -258.11865234375, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -428.82330322265625, "loss": 1.1202, "margin_dpo/margin_mean": 130.09239196777344, "margin_dpo/margin_std": 190.1111602783203, "step": 237 }, { "KL/chosen_KL_mean": -188.8069610595703, "KL/mean": -254.78656005859375, "KL/rejected_KL_mean": -320.7662048339844, "KL/std": 168.50390625, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.0028396020643413067, "fcm_dpo/delta": 0.035806380212306976, "fcm_dpo/margin": 131.95924377441406, "fcm_dpo/q_t": 0.418282687664032, "grad_norm": 23.320514678955078, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.31921494007110596, "logits/rejected": -0.33235523104667664, "logps/chosen": -242.78009033203125, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -410.18414306640625, "loss": 1.116, "margin_dpo/margin_mean": 131.95924377441406, "margin_dpo/margin_std": 188.6099395751953, "step": 238 }, { "KL/chosen_KL_mean": -209.04046630859375, "KL/mean": -283.45135498046875, "KL/rejected_KL_mean": -357.8622741699219, "KL/std": 165.2271728515625, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.0027087791822850704, "fcm_dpo/delta": -0.025651507079601288, "fcm_dpo/margin": 148.8218231201172, "fcm_dpo/q_t": 0.4025055170059204, "grad_norm": 26.025190353393555, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.3110630214214325, "logits/rejected": -0.3010329604148865, "logps/chosen": -267.1383056640625, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -451.4552307128906, "loss": 1.0886, "margin_dpo/margin_mean": 148.8218231201172, "margin_dpo/margin_std": 182.97344970703125, "step": 239 }, { "KL/chosen_KL_mean": -216.27822875976562, "KL/mean": -275.22637939453125, "KL/rejected_KL_mean": -334.17449951171875, "KL/std": 174.17791748046875, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.003052940359339118, "fcm_dpo/delta": 0.06172945722937584, "fcm_dpo/margin": 117.89629364013672, "fcm_dpo/q_t": 0.42239585518836975, "grad_norm": 30.3389949798584, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.34589529037475586, "logits/rejected": -0.32597455382347107, "logps/chosen": -276.8927001953125, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -408.29302978515625, "loss": 1.1595, "margin_dpo/margin_mean": 117.89629364013672, "margin_dpo/margin_std": 205.7227020263672, "step": 240 }, { "KL/chosen_KL_mean": -189.63717651367188, "KL/mean": -283.8388366699219, "KL/rejected_KL_mean": -378.04052734375, "KL/std": 193.572021484375, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.002562090754508972, "fcm_dpo/delta": -0.16558168828487396, "fcm_dpo/margin": 188.4033203125, "fcm_dpo/q_t": 0.3756968379020691, "grad_norm": 19.404237747192383, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.3840920329093933, "logits/rejected": -0.36620739102363586, "logps/chosen": -255.72824096679688, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -466.1014099121094, "loss": 1.033, "margin_dpo/margin_mean": 188.4033203125, "margin_dpo/margin_std": 218.57650756835938, "step": 241 }, { "KL/chosen_KL_mean": -209.334716796875, "KL/mean": -275.63055419921875, "KL/rejected_KL_mean": -341.9263610839844, "KL/std": 168.1392364501953, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.0026945569552481174, "fcm_dpo/delta": 0.052527863532304764, "fcm_dpo/margin": 132.59165954589844, "fcm_dpo/q_t": 0.4214203357696533, "grad_norm": 27.632558822631836, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.3013432025909424, "logits/rejected": -0.27941906452178955, "logps/chosen": -277.1986389160156, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -425.2867126464844, "loss": 1.1318, "margin_dpo/margin_mean": 132.59165954589844, "margin_dpo/margin_std": 201.74139404296875, "step": 242 }, { "KL/chosen_KL_mean": -208.54391479492188, "KL/mean": -289.72686767578125, "KL/rejected_KL_mean": -370.9097900390625, "KL/std": 181.60488891601562, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.0025174664333462715, "fcm_dpo/delta": -0.015545245260000229, "fcm_dpo/margin": 162.36587524414062, "fcm_dpo/q_t": 0.4042058289051056, "grad_norm": 18.131187438964844, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.2907285690307617, "logits/rejected": -0.26282238960266113, "logps/chosen": -271.628173828125, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -447.24542236328125, "loss": 1.0785, "margin_dpo/margin_mean": 162.36585998535156, "margin_dpo/margin_std": 201.73919677734375, "step": 243 }, { "KL/chosen_KL_mean": -176.4449462890625, "KL/mean": -262.936767578125, "KL/rejected_KL_mean": -349.4285888671875, "KL/std": 177.73191833496094, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.002386314095929265, "fcm_dpo/delta": -0.024043917655944824, "fcm_dpo/margin": 172.983642578125, "fcm_dpo/q_t": 0.4017675518989563, "grad_norm": 22.16912841796875, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.37594205141067505, "logits/rejected": -0.3709765076637268, "logps/chosen": -237.58563232421875, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -444.3205261230469, "loss": 1.061, "margin_dpo/margin_mean": 172.983642578125, "margin_dpo/margin_std": 182.35865783691406, "step": 244 }, { "KL/chosen_KL_mean": -210.67608642578125, "KL/mean": -275.7692565917969, "KL/rejected_KL_mean": -340.8624267578125, "KL/std": 175.44842529296875, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.002276795916259289, "fcm_dpo/delta": -0.028318271040916443, "fcm_dpo/margin": 130.18634033203125, "fcm_dpo/q_t": 0.42754846811294556, "grad_norm": 21.60332489013672, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.38238322734832764, "logits/rejected": -0.35831815004348755, "logps/chosen": -277.9383544921875, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -428.50250244140625, "loss": 1.1617, "margin_dpo/margin_mean": 130.1863250732422, "margin_dpo/margin_std": 192.90383911132812, "step": 245 }, { "KL/chosen_KL_mean": -203.3482666015625, "KL/mean": -271.3511657714844, "KL/rejected_KL_mean": -339.3540344238281, "KL/std": 193.9945068359375, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.002540751826018095, "fcm_dpo/delta": 0.08369514346122742, "fcm_dpo/margin": 136.00575256347656, "fcm_dpo/q_t": 0.42736515402793884, "grad_norm": 29.36928939819336, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.43162864446640015, "logits/rejected": -0.39635908603668213, "logps/chosen": -270.04522705078125, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -423.70037841796875, "loss": 1.139, "margin_dpo/margin_mean": 136.00576782226562, "margin_dpo/margin_std": 211.0897216796875, "step": 246 }, { "KL/chosen_KL_mean": -238.23548889160156, "KL/mean": -336.6441345214844, "KL/rejected_KL_mean": -435.05279541015625, "KL/std": 201.9400634765625, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.002273306716233492, "fcm_dpo/delta": -0.09434907883405685, "fcm_dpo/margin": 196.8173065185547, "fcm_dpo/q_t": 0.3884933590888977, "grad_norm": 24.273239135742188, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.35952645540237427, "logits/rejected": -0.35816654562950134, "logps/chosen": -294.8408203125, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -541.3460693359375, "loss": 1.0487, "margin_dpo/margin_mean": 196.81732177734375, "margin_dpo/margin_std": 221.4100799560547, "step": 247 }, { "KL/chosen_KL_mean": -203.11044311523438, "KL/mean": -287.45843505859375, "KL/rejected_KL_mean": -371.806396484375, "KL/std": 163.28530883789062, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.002244535367935896, "fcm_dpo/delta": 0.031689293682575226, "fcm_dpo/margin": 168.69598388671875, "fcm_dpo/q_t": 0.4118279814720154, "grad_norm": 20.322803497314453, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.37604576349258423, "logits/rejected": -0.39418306946754456, "logps/chosen": -247.15365600585938, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -463.66326904296875, "loss": 1.0792, "margin_dpo/margin_mean": 168.69598388671875, "margin_dpo/margin_std": 161.9207763671875, "step": 248 }, { "KL/chosen_KL_mean": -261.7310791015625, "KL/mean": -307.686279296875, "KL/rejected_KL_mean": -353.6415100097656, "KL/std": 179.0096435546875, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.002784975804388523, "fcm_dpo/delta": 0.0796995759010315, "fcm_dpo/margin": 91.9104232788086, "fcm_dpo/q_t": 0.4464285671710968, "grad_norm": 43.7970085144043, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.40871942043304443, "logits/rejected": -0.38268932700157166, "logps/chosen": -324.1734313964844, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -434.10955810546875, "loss": 1.2311, "margin_dpo/margin_mean": 91.9104232788086, "margin_dpo/margin_std": 214.4239959716797, "step": 249 }, { "KL/chosen_KL_mean": -232.26882934570312, "KL/mean": -313.94488525390625, "KL/rejected_KL_mean": -395.6209716796875, "KL/std": 176.32708740234375, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.002672810573130846, "fcm_dpo/delta": -0.06773751974105835, "fcm_dpo/margin": 163.3521270751953, "fcm_dpo/q_t": 0.3925377130508423, "grad_norm": 46.0150032043457, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.4073798656463623, "logits/rejected": -0.3506365418434143, "logps/chosen": -297.905517578125, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -469.4928283691406, "loss": 1.059, "margin_dpo/margin_mean": 163.3521270751953, "margin_dpo/margin_std": 180.181884765625, "step": 250 }, { "KL/chosen_KL_mean": -246.01177978515625, "KL/mean": -302.828857421875, "KL/rejected_KL_mean": -359.64593505859375, "KL/std": 187.79159545898438, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.0028411387465894222, "fcm_dpo/delta": 0.11464589834213257, "fcm_dpo/margin": 113.63416290283203, "fcm_dpo/q_t": 0.43381422758102417, "grad_norm": 41.136512756347656, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.37190985679626465, "logits/rejected": -0.3395519554615021, "logps/chosen": -303.1944885253906, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -437.30938720703125, "loss": 1.1687, "margin_dpo/margin_mean": 113.63416290283203, "margin_dpo/margin_std": 201.274169921875, "step": 251 }, { "KL/chosen_KL_mean": -241.49786376953125, "KL/mean": -317.1058654785156, "KL/rejected_KL_mean": -392.7138671875, "KL/std": 162.52325439453125, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.0027777920477092266, "fcm_dpo/delta": -0.0407252199947834, "fcm_dpo/margin": 151.21600341796875, "fcm_dpo/q_t": 0.3957163393497467, "grad_norm": 25.602100372314453, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.42717283964157104, "logits/rejected": -0.38557177782058716, "logps/chosen": -313.1835021972656, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -477.47186279296875, "loss": 1.0596, "margin_dpo/margin_mean": 151.21600341796875, "margin_dpo/margin_std": 162.9154815673828, "step": 252 }, { "KL/chosen_KL_mean": -202.2740478515625, "KL/mean": -278.83538818359375, "KL/rejected_KL_mean": -355.396728515625, "KL/std": 169.63818359375, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.0027958948630839586, "fcm_dpo/delta": -0.05647484213113785, "fcm_dpo/margin": 153.12266540527344, "fcm_dpo/q_t": 0.3956605792045593, "grad_norm": 26.383848190307617, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.42702776193618774, "logits/rejected": -0.4189242720603943, "logps/chosen": -271.4079895019531, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -454.0992736816406, "loss": 1.0808, "margin_dpo/margin_mean": 153.1226806640625, "margin_dpo/margin_std": 188.74114990234375, "step": 253 }, { "KL/chosen_KL_mean": -190.6053924560547, "KL/mean": -255.74807739257812, "KL/rejected_KL_mean": -320.8907470703125, "KL/std": 174.94302368164062, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.002762971445918083, "fcm_dpo/delta": 0.06207642704248428, "fcm_dpo/margin": 130.28536987304688, "fcm_dpo/q_t": 0.4237990379333496, "grad_norm": 26.932371139526367, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.40052559971809387, "logits/rejected": -0.38795092701911926, "logps/chosen": -244.76039123535156, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -411.1983947753906, "loss": 1.1403, "margin_dpo/margin_mean": 130.28536987304688, "margin_dpo/margin_std": 217.71649169921875, "step": 254 }, { "KL/chosen_KL_mean": -196.3863983154297, "KL/mean": -262.91351318359375, "KL/rejected_KL_mean": -329.44061279296875, "KL/std": 153.0009002685547, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.002854404039680958, "fcm_dpo/delta": 0.02931365743279457, "fcm_dpo/margin": 133.05422973632812, "fcm_dpo/q_t": 0.41438183188438416, "grad_norm": 25.421916961669922, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.35080671310424805, "logits/rejected": -0.3392289876937866, "logps/chosen": -253.528076171875, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -419.64923095703125, "loss": 1.1183, "margin_dpo/margin_mean": 133.05422973632812, "margin_dpo/margin_std": 194.57473754882812, "step": 255 }, { "KL/chosen_KL_mean": -169.05984497070312, "KL/mean": -243.46347045898438, "KL/rejected_KL_mean": -317.8670959472656, "KL/std": 165.87603759765625, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.002850246848538518, "fcm_dpo/delta": -0.04200413078069687, "fcm_dpo/margin": 148.8072509765625, "fcm_dpo/q_t": 0.39909690618515015, "grad_norm": 28.732080459594727, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.2951539158821106, "logits/rejected": -0.3040773868560791, "logps/chosen": -224.22332763671875, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -410.42999267578125, "loss": 1.075, "margin_dpo/margin_mean": 148.8072509765625, "margin_dpo/margin_std": 190.7834014892578, "step": 256 }, { "KL/chosen_KL_mean": -165.35116577148438, "KL/mean": -242.6610107421875, "KL/rejected_KL_mean": -319.9708557128906, "KL/std": 172.49417114257812, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.002569081261754036, "fcm_dpo/delta": -0.008627064526081085, "fcm_dpo/margin": 154.6196746826172, "fcm_dpo/q_t": 0.40691322088241577, "grad_norm": 21.039451599121094, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.33848869800567627, "logits/rejected": -0.3249668478965759, "logps/chosen": -214.77487182617188, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -399.5087890625, "loss": 1.0937, "margin_dpo/margin_mean": 154.6196746826172, "margin_dpo/margin_std": 196.26806640625, "step": 257 }, { "KL/chosen_KL_mean": -220.045654296875, "KL/mean": -296.9225158691406, "KL/rejected_KL_mean": -373.79937744140625, "KL/std": 184.12759399414062, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.0026925005950033665, "fcm_dpo/delta": -0.024290982633829117, "fcm_dpo/margin": 153.7537078857422, "fcm_dpo/q_t": 0.4024388790130615, "grad_norm": 26.345495223999023, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.3109281659126282, "logits/rejected": -0.3115309476852417, "logps/chosen": -279.4297790527344, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -469.78948974609375, "loss": 1.0982, "margin_dpo/margin_mean": 153.7537078857422, "margin_dpo/margin_std": 222.58351135253906, "step": 258 }, { "KL/chosen_KL_mean": -209.99713134765625, "KL/mean": -273.5628356933594, "KL/rejected_KL_mean": -337.1285400390625, "KL/std": 172.61416625976562, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.0028006834909319878, "fcm_dpo/delta": 0.06836118549108505, "fcm_dpo/margin": 127.13138580322266, "fcm_dpo/q_t": 0.42346620559692383, "grad_norm": 21.578977584838867, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.3506283462047577, "logits/rejected": -0.3485710322856903, "logps/chosen": -262.82550048828125, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -426.3201904296875, "loss": 1.137, "margin_dpo/margin_mean": 127.13138580322266, "margin_dpo/margin_std": 197.51239013671875, "step": 259 }, { "KL/chosen_KL_mean": -210.90850830078125, "KL/mean": -298.40771484375, "KL/rejected_KL_mean": -385.90692138671875, "KL/std": 181.40826416015625, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.002633176278322935, "fcm_dpo/delta": -0.11144484579563141, "fcm_dpo/margin": 174.99844360351562, "fcm_dpo/q_t": 0.3861405551433563, "grad_norm": 28.88026237487793, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.31534501910209656, "logits/rejected": -0.33186644315719604, "logps/chosen": -258.326171875, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -480.9967041015625, "loss": 1.0402, "margin_dpo/margin_mean": 174.99844360351562, "margin_dpo/margin_std": 197.2706298828125, "step": 260 }, { "KL/chosen_KL_mean": -220.949462890625, "KL/mean": -301.2371826171875, "KL/rejected_KL_mean": -381.52496337890625, "KL/std": 196.11915588378906, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0024081161245703697, "fcm_dpo/delta": 0.020973602309823036, "fcm_dpo/margin": 160.57550048828125, "fcm_dpo/q_t": 0.41286349296569824, "grad_norm": 18.689228057861328, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.2601136565208435, "logits/rejected": -0.2584311366081238, "logps/chosen": -273.9808349609375, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -470.0399169921875, "loss": 1.1028, "margin_dpo/margin_mean": 160.57550048828125, "margin_dpo/margin_std": 221.1414337158203, "step": 261 }, { "KL/chosen_KL_mean": -260.35552978515625, "KL/mean": -325.7707214355469, "KL/rejected_KL_mean": -391.1859130859375, "KL/std": 179.41098022460938, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.002791064791381359, "fcm_dpo/delta": 0.04618440195918083, "fcm_dpo/margin": 130.8303680419922, "fcm_dpo/q_t": 0.4176996350288391, "grad_norm": 33.41985321044922, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.29340440034866333, "logits/rejected": -0.26309913396835327, "logps/chosen": -319.9756774902344, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -477.60443115234375, "loss": 1.1146, "margin_dpo/margin_mean": 130.83035278320312, "margin_dpo/margin_std": 180.4952392578125, "step": 262 }, { "KL/chosen_KL_mean": -233.8137969970703, "KL/mean": -327.53448486328125, "KL/rejected_KL_mean": -421.2552185058594, "KL/std": 211.76744079589844, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.0023032082244753838, "fcm_dpo/delta": -0.07411213964223862, "fcm_dpo/margin": 187.44140625, "fcm_dpo/q_t": 0.39327767491340637, "grad_norm": 18.634008407592773, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.3248862028121948, "logits/rejected": -0.30047309398651123, "logps/chosen": -293.2347412109375, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -518.1124267578125, "loss": 1.0674, "margin_dpo/margin_mean": 187.44142150878906, "margin_dpo/margin_std": 231.066162109375, "step": 263 }, { "KL/chosen_KL_mean": -241.53146362304688, "KL/mean": -323.41162109375, "KL/rejected_KL_mean": -405.291748046875, "KL/std": 194.24461364746094, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.002298943232744932, "fcm_dpo/delta": 0.028040196746587753, "fcm_dpo/margin": 163.76028442382812, "fcm_dpo/q_t": 0.4143482446670532, "grad_norm": 20.764347076416016, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.3569568693637848, "logits/rejected": -0.34849122166633606, "logps/chosen": -304.2535400390625, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -499.14794921875, "loss": 1.1174, "margin_dpo/margin_mean": 163.76028442382812, "margin_dpo/margin_std": 224.1405487060547, "step": 264 }, { "KL/chosen_KL_mean": -264.88433837890625, "KL/mean": -340.1336669921875, "KL/rejected_KL_mean": -415.3829650878906, "KL/std": 215.2719268798828, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0025752345100045204, "fcm_dpo/delta": 0.020102664828300476, "fcm_dpo/margin": 150.49859619140625, "fcm_dpo/q_t": 0.4153306484222412, "grad_norm": 23.874309539794922, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.31058114767074585, "logits/rejected": -0.2928779423236847, "logps/chosen": -326.8558349609375, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -503.403564453125, "loss": 1.1349, "margin_dpo/margin_mean": 150.49859619140625, "margin_dpo/margin_std": 257.16748046875, "step": 265 }, { "KL/chosen_KL_mean": -264.1705322265625, "KL/mean": -323.71405029296875, "KL/rejected_KL_mean": -383.25750732421875, "KL/std": 179.75607299804688, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0029307270888239145, "fcm_dpo/delta": 0.07655003666877747, "fcm_dpo/margin": 119.08696746826172, "fcm_dpo/q_t": 0.42644163966178894, "grad_norm": 46.22469711303711, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.3636462092399597, "logits/rejected": -0.3230510354042053, "logps/chosen": -331.27020263671875, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -451.228759765625, "loss": 1.1556, "margin_dpo/margin_mean": 119.08696746826172, "margin_dpo/margin_std": 211.2481689453125, "step": 266 }, { "KL/chosen_KL_mean": -236.5076904296875, "KL/mean": -311.24676513671875, "KL/rejected_KL_mean": -385.98590087890625, "KL/std": 193.96978759765625, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.00275492575019598, "fcm_dpo/delta": -0.029174409806728363, "fcm_dpo/margin": 149.47821044921875, "fcm_dpo/q_t": 0.4047339856624603, "grad_norm": 41.7883415222168, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.3469699025154114, "logits/rejected": -0.3102799355983734, "logps/chosen": -305.47845458984375, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -476.1543273925781, "loss": 1.1054, "margin_dpo/margin_mean": 149.47821044921875, "margin_dpo/margin_std": 224.31707763671875, "step": 267 }, { "KL/chosen_KL_mean": -240.66119384765625, "KL/mean": -317.0205078125, "KL/rejected_KL_mean": -393.37982177734375, "KL/std": 185.6231689453125, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.002668085042387247, "fcm_dpo/delta": -0.01804983615875244, "fcm_dpo/margin": 152.71859741210938, "fcm_dpo/q_t": 0.40581750869750977, "grad_norm": 28.408159255981445, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.35881197452545166, "logits/rejected": -0.36506927013397217, "logps/chosen": -296.5615234375, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -495.0274353027344, "loss": 1.1018, "margin_dpo/margin_mean": 152.71859741210938, "margin_dpo/margin_std": 222.39694213867188, "step": 268 }, { "KL/chosen_KL_mean": -257.70648193359375, "KL/mean": -346.0221862792969, "KL/rejected_KL_mean": -434.337890625, "KL/std": 184.19381713867188, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.00247764540836215, "fcm_dpo/delta": -0.06811105459928513, "fcm_dpo/margin": 176.63140869140625, "fcm_dpo/q_t": 0.3938090205192566, "grad_norm": 22.41498374938965, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.38506025075912476, "logits/rejected": -0.3642171025276184, "logps/chosen": -327.74603271484375, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -541.687255859375, "loss": 1.0757, "margin_dpo/margin_mean": 176.6313934326172, "margin_dpo/margin_std": 235.69393920898438, "step": 269 }, { "KL/chosen_KL_mean": -222.79367065429688, "KL/mean": -289.4638671875, "KL/rejected_KL_mean": -356.134033203125, "KL/std": 165.93325805664062, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0027032734360545874, "fcm_dpo/delta": 0.05463853478431702, "fcm_dpo/margin": 133.3403778076172, "fcm_dpo/q_t": 0.4207903742790222, "grad_norm": 25.0926570892334, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.43856489658355713, "logits/rejected": -0.4321529269218445, "logps/chosen": -292.3271484375, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -466.06268310546875, "loss": 1.1388, "margin_dpo/margin_mean": 133.3403778076172, "margin_dpo/margin_std": 216.5740509033203, "step": 270 }, { "KL/chosen_KL_mean": -203.2021942138672, "KL/mean": -287.8795166015625, "KL/rejected_KL_mean": -372.5568542480469, "KL/std": 168.32081604003906, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0024533187970519066, "fcm_dpo/delta": -0.03273054212331772, "fcm_dpo/margin": 169.3546905517578, "fcm_dpo/q_t": 0.4006693959236145, "grad_norm": 23.280786514282227, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.3510928452014923, "logits/rejected": -0.33294203877449036, "logps/chosen": -259.9667663574219, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -465.0706787109375, "loss": 1.0663, "margin_dpo/margin_mean": 169.3546905517578, "margin_dpo/margin_std": 192.665283203125, "step": 271 }, { "KL/chosen_KL_mean": -191.9091339111328, "KL/mean": -292.7960205078125, "KL/rejected_KL_mean": -393.68292236328125, "KL/std": 186.9519805908203, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.002250020857900381, "fcm_dpo/delta": -0.09791300445795059, "fcm_dpo/margin": 201.77377319335938, "fcm_dpo/q_t": 0.38448095321655273, "grad_norm": 28.866960525512695, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.29558974504470825, "logits/rejected": -0.3049769401550293, "logps/chosen": -241.4062957763672, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -499.2257080078125, "loss": 1.0321, "margin_dpo/margin_mean": 201.7737579345703, "margin_dpo/margin_std": 194.00624084472656, "step": 272 }, { "KL/chosen_KL_mean": -226.32107543945312, "KL/mean": -325.10498046875, "KL/rejected_KL_mean": -423.8888244628906, "KL/std": 189.16091918945312, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.0020193979144096375, "fcm_dpo/delta": 0.0005125030875205994, "fcm_dpo/margin": 197.56777954101562, "fcm_dpo/q_t": 0.4059392511844635, "grad_norm": 17.918615341186523, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.31307101249694824, "logits/rejected": -0.29330283403396606, "logps/chosen": -289.2964782714844, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -516.3873901367188, "loss": 1.0715, "margin_dpo/margin_mean": 197.56777954101562, "margin_dpo/margin_std": 214.41448974609375, "step": 273 }, { "KL/chosen_KL_mean": -267.08245849609375, "KL/mean": -342.52392578125, "KL/rejected_KL_mean": -417.9654541015625, "KL/std": 174.84378051757812, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.002351115457713604, "fcm_dpo/delta": 0.06252269446849823, "fcm_dpo/margin": 150.8830108642578, "fcm_dpo/q_t": 0.420854389667511, "grad_norm": 22.64203453063965, "learning_rate": 3.75e-07, "logits/chosen": -0.23737865686416626, "logits/rejected": -0.22025717794895172, "logps/chosen": -322.7501525878906, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -495.29852294921875, "loss": 1.1162, "margin_dpo/margin_mean": 150.88299560546875, "margin_dpo/margin_std": 200.55410766601562, "step": 274 }, { "KL/chosen_KL_mean": -211.66796875, "KL/mean": -292.65130615234375, "KL/rejected_KL_mean": -373.6346435546875, "KL/std": 179.12078857421875, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.002439269330352545, "fcm_dpo/delta": 0.004620321094989777, "fcm_dpo/margin": 161.9666748046875, "fcm_dpo/q_t": 0.40825700759887695, "grad_norm": 23.12116241455078, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.3093240261077881, "logits/rejected": -0.316268652677536, "logps/chosen": -260.2626953125, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -466.9383544921875, "loss": 1.0894, "margin_dpo/margin_mean": 161.9666748046875, "margin_dpo/margin_std": 203.50302124023438, "step": 275 }, { "KL/chosen_KL_mean": -241.7791290283203, "KL/mean": -322.0965576171875, "KL/rejected_KL_mean": -402.4139404296875, "KL/std": 190.34759521484375, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.0024003759026527405, "fcm_dpo/delta": 0.02319333702325821, "fcm_dpo/margin": 160.63482666015625, "fcm_dpo/q_t": 0.4141028821468353, "grad_norm": 29.356189727783203, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.23649513721466064, "logits/rejected": -0.20997360348701477, "logps/chosen": -298.3565368652344, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -472.77960205078125, "loss": 1.1107, "margin_dpo/margin_mean": 160.63482666015625, "margin_dpo/margin_std": 233.87583923339844, "step": 276 }, { "KL/chosen_KL_mean": -263.56640625, "KL/mean": -349.49713134765625, "KL/rejected_KL_mean": -435.42779541015625, "KL/std": 192.81695556640625, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.002432500245049596, "fcm_dpo/delta": -0.032088592648506165, "fcm_dpo/margin": 171.86138916015625, "fcm_dpo/q_t": 0.4016081690788269, "grad_norm": 26.00898551940918, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.26711416244506836, "logits/rejected": -0.24979907274246216, "logps/chosen": -319.8379821777344, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -528.30908203125, "loss": 1.0858, "margin_dpo/margin_mean": 171.86138916015625, "margin_dpo/margin_std": 228.77777099609375, "step": 277 }, { "KL/chosen_KL_mean": -235.44161987304688, "KL/mean": -336.4327392578125, "KL/rejected_KL_mean": -437.42388916015625, "KL/std": 201.9891357421875, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.002094062976539135, "fcm_dpo/delta": -0.045741938054561615, "fcm_dpo/margin": 201.98226928710938, "fcm_dpo/q_t": 0.3976041078567505, "grad_norm": 22.05190658569336, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.24980834126472473, "logits/rejected": -0.25416260957717896, "logps/chosen": -288.383544921875, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -528.677490234375, "loss": 1.0605, "margin_dpo/margin_mean": 201.9822998046875, "margin_dpo/margin_std": 226.97244262695312, "step": 278 }, { "KL/chosen_KL_mean": -271.568115234375, "KL/mean": -371.5149841308594, "KL/rejected_KL_mean": -471.46185302734375, "KL/std": 216.6588897705078, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.0019927441608160734, "fcm_dpo/delta": -0.005218621343374252, "fcm_dpo/margin": 199.89370727539062, "fcm_dpo/q_t": 0.40749794244766235, "grad_norm": 26.85765838623047, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.19662827253341675, "logits/rejected": -0.19574469327926636, "logps/chosen": -320.20947265625, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -559.3132934570312, "loss": 1.0963, "margin_dpo/margin_mean": 199.89370727539062, "margin_dpo/margin_std": 265.7958984375, "step": 279 }, { "KL/chosen_KL_mean": -264.79876708984375, "KL/mean": -365.7987060546875, "KL/rejected_KL_mean": -466.79864501953125, "KL/std": 195.9139862060547, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.0020362103823572397, "fcm_dpo/delta": -0.01925138756632805, "fcm_dpo/margin": 201.9998779296875, "fcm_dpo/q_t": 0.4014127254486084, "grad_norm": 20.498315811157227, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.24768587946891785, "logits/rejected": -0.24966652691364288, "logps/chosen": -323.59588623046875, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -565.41748046875, "loss": 1.0628, "margin_dpo/margin_mean": 201.99989318847656, "margin_dpo/margin_std": 216.71530151367188, "step": 280 }, { "KL/chosen_KL_mean": -247.58489990234375, "KL/mean": -335.40484619140625, "KL/rejected_KL_mean": -423.2247619628906, "KL/std": 196.88302612304688, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0021747334394603968, "fcm_dpo/delta": 0.022978195920586586, "fcm_dpo/margin": 175.6398162841797, "fcm_dpo/q_t": 0.4103030562400818, "grad_norm": 21.528810501098633, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.3240095376968384, "logits/rejected": -0.3004445433616638, "logps/chosen": -303.07342529296875, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -504.1073303222656, "loss": 1.0948, "margin_dpo/margin_mean": 175.6398162841797, "margin_dpo/margin_std": 213.6923828125, "step": 281 }, { "KL/chosen_KL_mean": -268.12432861328125, "KL/mean": -337.4071044921875, "KL/rejected_KL_mean": -406.68988037109375, "KL/std": 196.039794921875, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0021595642901957035, "fcm_dpo/delta": 0.02335960976779461, "fcm_dpo/margin": 138.56552124023438, "fcm_dpo/q_t": 0.4331875145435333, "grad_norm": 21.117961883544922, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.3865717053413391, "logits/rejected": -0.3700195550918579, "logps/chosen": -341.1944885253906, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -502.0408935546875, "loss": 1.1627, "margin_dpo/margin_mean": 138.56552124023438, "margin_dpo/margin_std": 225.9492645263672, "step": 282 }, { "KL/chosen_KL_mean": -283.1618347167969, "KL/mean": -388.3852233886719, "KL/rejected_KL_mean": -493.608642578125, "KL/std": 241.67697143554688, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.0020841769874095917, "fcm_dpo/delta": -0.06936129927635193, "fcm_dpo/margin": 210.44679260253906, "fcm_dpo/q_t": 0.3913659453392029, "grad_norm": 29.183977127075195, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.33291661739349365, "logits/rejected": -0.3372425436973572, "logps/chosen": -345.0602722167969, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -590.59521484375, "loss": 1.057, "margin_dpo/margin_mean": 210.44679260253906, "margin_dpo/margin_std": 231.2135009765625, "step": 283 }, { "KL/chosen_KL_mean": -270.32830810546875, "KL/mean": -369.215087890625, "KL/rejected_KL_mean": -468.1018371582031, "KL/std": 219.54708862304688, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.0019610195886343718, "fcm_dpo/delta": 0.01984664611518383, "fcm_dpo/margin": 197.77349853515625, "fcm_dpo/q_t": 0.4114644527435303, "grad_norm": 25.379192352294922, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.37090370059013367, "logits/rejected": -0.3533616065979004, "logps/chosen": -328.76385498046875, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -561.5711059570312, "loss": 1.0843, "margin_dpo/margin_mean": 197.77349853515625, "margin_dpo/margin_std": 232.99339294433594, "step": 284 }, { "KL/chosen_KL_mean": -313.6859130859375, "KL/mean": -400.13983154296875, "KL/rejected_KL_mean": -486.59375, "KL/std": 192.8905029296875, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.002196455840021372, "fcm_dpo/delta": 0.021015789359807968, "fcm_dpo/margin": 172.90782165527344, "fcm_dpo/q_t": 0.41028088331222534, "grad_norm": 23.027700424194336, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.36354196071624756, "logits/rejected": -0.3486790955066681, "logps/chosen": -379.9181213378906, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -585.7205810546875, "loss": 1.1035, "margin_dpo/margin_mean": 172.9078369140625, "margin_dpo/margin_std": 218.47084045410156, "step": 285 }, { "KL/chosen_KL_mean": -327.0200500488281, "KL/mean": -422.5919189453125, "KL/rejected_KL_mean": -518.163818359375, "KL/std": 230.7489776611328, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.002110408851876855, "fcm_dpo/delta": -0.006586743518710136, "fcm_dpo/margin": 191.14376831054688, "fcm_dpo/q_t": 0.4062705934047699, "grad_norm": 23.25826644897461, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.35514867305755615, "logits/rejected": -0.3393225073814392, "logps/chosen": -399.9710693359375, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -606.7522583007812, "loss": 1.0915, "margin_dpo/margin_mean": 191.14376831054688, "margin_dpo/margin_std": 254.77488708496094, "step": 286 }, { "KL/chosen_KL_mean": -300.4835510253906, "KL/mean": -379.34942626953125, "KL/rejected_KL_mean": -458.21527099609375, "KL/std": 201.38296508789062, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.0022800066508352757, "fcm_dpo/delta": 0.06143353134393692, "fcm_dpo/margin": 157.731689453125, "fcm_dpo/q_t": 0.42126035690307617, "grad_norm": 27.160978317260742, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.30109989643096924, "logits/rejected": -0.27528104186058044, "logps/chosen": -362.02471923828125, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -535.9113159179688, "loss": 1.1154, "margin_dpo/margin_mean": 157.731689453125, "margin_dpo/margin_std": 216.85557556152344, "step": 287 }, { "KL/chosen_KL_mean": -276.4866943359375, "KL/mean": -374.0993347167969, "KL/rejected_KL_mean": -471.71197509765625, "KL/std": 193.3370361328125, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.002058174926787615, "fcm_dpo/delta": -0.026749789714813232, "fcm_dpo/margin": 195.2252655029297, "fcm_dpo/q_t": 0.40044307708740234, "grad_norm": 21.408998489379883, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.34782230854034424, "logits/rejected": -0.3332597613334656, "logps/chosen": -333.1479187011719, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -559.0476684570312, "loss": 1.0755, "margin_dpo/margin_mean": 195.2252655029297, "margin_dpo/margin_std": 204.141357421875, "step": 288 }, { "KL/chosen_KL_mean": -259.8858337402344, "KL/mean": -367.64276123046875, "KL/rejected_KL_mean": -475.3996887207031, "KL/std": 213.38723754882812, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.0020438190549612045, "fcm_dpo/delta": -0.07187651097774506, "fcm_dpo/margin": 215.51380920410156, "fcm_dpo/q_t": 0.3916912078857422, "grad_norm": 24.203519821166992, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.3048250675201416, "logits/rejected": -0.3042747378349304, "logps/chosen": -305.1162414550781, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -563.0423583984375, "loss": 1.0505, "margin_dpo/margin_mean": 215.51380920410156, "margin_dpo/margin_std": 233.81417846679688, "step": 289 }, { "KL/chosen_KL_mean": -269.5323791503906, "KL/mean": -379.81329345703125, "KL/rejected_KL_mean": -490.09423828125, "KL/std": 221.88650512695312, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.0018460073042660952, "fcm_dpo/delta": -0.013119934126734734, "fcm_dpo/margin": 220.56182861328125, "fcm_dpo/q_t": 0.4045451879501343, "grad_norm": 20.667232513427734, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.2957327961921692, "logits/rejected": -0.31704530119895935, "logps/chosen": -325.0038757324219, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -606.8027954101562, "loss": 1.0762, "margin_dpo/margin_mean": 220.56182861328125, "margin_dpo/margin_std": 266.3393859863281, "step": 290 }, { "KL/chosen_KL_mean": -240.43844604492188, "KL/mean": -341.17254638671875, "KL/rejected_KL_mean": -441.9066467285156, "KL/std": 196.33567810058594, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.001962371403351426, "fcm_dpo/delta": 0.0012248605489730835, "fcm_dpo/margin": 201.46820068359375, "fcm_dpo/q_t": 0.40641385316848755, "grad_norm": 25.23223114013672, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.3080219626426697, "logits/rejected": -0.29609572887420654, "logps/chosen": -303.69879150390625, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -531.2037353515625, "loss": 1.0851, "margin_dpo/margin_mean": 201.46820068359375, "margin_dpo/margin_std": 237.0089874267578, "step": 291 }, { "KL/chosen_KL_mean": -267.9251708984375, "KL/mean": -372.0924987792969, "KL/rejected_KL_mean": -476.25982666015625, "KL/std": 229.24436950683594, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0019257422536611557, "fcm_dpo/delta": -0.005529198795557022, "fcm_dpo/margin": 208.33460998535156, "fcm_dpo/q_t": 0.40545937418937683, "grad_norm": 21.263906478881836, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.3315708637237549, "logits/rejected": -0.3213496208190918, "logps/chosen": -321.8437194824219, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -566.22119140625, "loss": 1.0907, "margin_dpo/margin_mean": 208.33460998535156, "margin_dpo/margin_std": 271.5143737792969, "step": 292 }, { "KL/chosen_KL_mean": -278.5457763671875, "KL/mean": -362.8836975097656, "KL/rejected_KL_mean": -447.22161865234375, "KL/std": 229.81143188476562, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.002093075541779399, "fcm_dpo/delta": 0.06820766627788544, "fcm_dpo/margin": 168.67581176757812, "fcm_dpo/q_t": 0.42393064498901367, "grad_norm": 32.55642318725586, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.3256397843360901, "logits/rejected": -0.3210792541503906, "logps/chosen": -338.92181396484375, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -525.0740356445312, "loss": 1.1334, "margin_dpo/margin_mean": 168.67581176757812, "margin_dpo/margin_std": 262.6167907714844, "step": 293 }, { "KL/chosen_KL_mean": -259.8635559082031, "KL/mean": -352.86517333984375, "KL/rejected_KL_mean": -445.8667907714844, "KL/std": 204.66827392578125, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.002107615116983652, "fcm_dpo/delta": 0.012143617495894432, "fcm_dpo/margin": 186.0032196044922, "fcm_dpo/q_t": 0.4118291139602661, "grad_norm": 22.22089195251465, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.3044741451740265, "logits/rejected": -0.311285138130188, "logps/chosen": -307.95111083984375, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -527.7637939453125, "loss": 1.1023, "margin_dpo/margin_mean": 186.0032196044922, "margin_dpo/margin_std": 260.2086181640625, "step": 294 }, { "KL/chosen_KL_mean": -320.80810546875, "KL/mean": -423.1761474609375, "KL/rejected_KL_mean": -525.5442504882812, "KL/std": 238.36337280273438, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.002073909156024456, "fcm_dpo/delta": -0.04264184087514877, "fcm_dpo/margin": 204.73614501953125, "fcm_dpo/q_t": 0.4011147618293762, "grad_norm": 27.336824417114258, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.3063458800315857, "logits/rejected": -0.31039565801620483, "logps/chosen": -370.7327880859375, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -613.0005493164062, "loss": 1.0927, "margin_dpo/margin_mean": 204.73614501953125, "margin_dpo/margin_std": 289.61798095703125, "step": 295 }, { "KL/chosen_KL_mean": -394.3350830078125, "KL/mean": -464.76324462890625, "KL/rejected_KL_mean": -535.19140625, "KL/std": 229.95870971679688, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.0020087361335754395, "fcm_dpo/delta": 0.00424933061003685, "fcm_dpo/margin": 140.85630798339844, "fcm_dpo/q_t": 0.4354059398174286, "grad_norm": 33.60810852050781, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.3529552221298218, "logits/rejected": -0.3394961953163147, "logps/chosen": -459.8262939453125, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -628.2804565429688, "loss": 1.2109, "margin_dpo/margin_mean": 140.85629272460938, "margin_dpo/margin_std": 299.87567138671875, "step": 296 }, { "KL/chosen_KL_mean": -344.30731201171875, "KL/mean": -423.664794921875, "KL/rejected_KL_mean": -503.0222473144531, "KL/std": 215.01690673828125, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.002223505638539791, "fcm_dpo/delta": 0.07282247394323349, "fcm_dpo/margin": 158.7149200439453, "fcm_dpo/q_t": 0.4236125946044922, "grad_norm": 42.23075485229492, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.3576158583164215, "logits/rejected": -0.36280083656311035, "logps/chosen": -400.7842712402344, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -598.1607666015625, "loss": 1.1244, "margin_dpo/margin_mean": 158.71493530273438, "margin_dpo/margin_std": 226.28025817871094, "step": 297 }, { "KL/chosen_KL_mean": -358.29290771484375, "KL/mean": -466.8505859375, "KL/rejected_KL_mean": -575.408203125, "KL/std": 281.4233093261719, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0020931584294885397, "fcm_dpo/delta": -0.0989355742931366, "fcm_dpo/margin": 217.11532592773438, "fcm_dpo/q_t": 0.39393287897109985, "grad_norm": 29.66607093811035, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.3804464340209961, "logits/rejected": -0.39784708619117737, "logps/chosen": -425.61810302734375, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -692.0704345703125, "loss": 1.0936, "margin_dpo/margin_mean": 217.1153106689453, "margin_dpo/margin_std": 334.7890930175781, "step": 298 }, { "KL/chosen_KL_mean": -290.83984375, "KL/mean": -384.0028076171875, "KL/rejected_KL_mean": -477.165771484375, "KL/std": 211.8370361328125, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.0020763694774359465, "fcm_dpo/delta": 0.006563708186149597, "fcm_dpo/margin": 186.32595825195312, "fcm_dpo/q_t": 0.41148841381073, "grad_norm": 29.92995262145996, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.35604220628738403, "logits/rejected": -0.3601064085960388, "logps/chosen": -339.80194091796875, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -561.4940185546875, "loss": 1.1168, "margin_dpo/margin_mean": 186.32595825195312, "margin_dpo/margin_std": 269.77178955078125, "step": 299 }, { "KL/chosen_KL_mean": -361.88922119140625, "KL/mean": -479.45281982421875, "KL/rejected_KL_mean": -597.016357421875, "KL/std": 256.1278381347656, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0018352132756263018, "fcm_dpo/delta": -0.055538419634103775, "fcm_dpo/margin": 235.1271514892578, "fcm_dpo/q_t": 0.3963469862937927, "grad_norm": 22.620500564575195, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.35153138637542725, "logits/rejected": -0.3557186424732208, "logps/chosen": -420.96295166015625, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -692.9827880859375, "loss": 1.0815, "margin_dpo/margin_mean": 235.12716674804688, "margin_dpo/margin_std": 311.29791259765625, "step": 300 }, { "KL/chosen_KL_mean": -296.9462890625, "KL/mean": -410.1385498046875, "KL/rejected_KL_mean": -523.3307495117188, "KL/std": 233.12229919433594, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.0017267554067075253, "fcm_dpo/delta": 0.014057676307857037, "fcm_dpo/margin": 226.38442993164062, "fcm_dpo/q_t": 0.4108361601829529, "grad_norm": 18.437192916870117, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.3524608612060547, "logits/rejected": -0.3491095006465912, "logps/chosen": -354.1956787109375, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -615.684326171875, "loss": 1.0883, "margin_dpo/margin_mean": 226.38442993164062, "margin_dpo/margin_std": 280.14007568359375, "step": 301 }, { "KL/chosen_KL_mean": -269.1903076171875, "KL/mean": -361.92718505859375, "KL/rejected_KL_mean": -454.6640625, "KL/std": 212.72869873046875, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.001909639802761376, "fcm_dpo/delta": 0.07072426378726959, "fcm_dpo/margin": 185.4737548828125, "fcm_dpo/q_t": 0.42330536246299744, "grad_norm": 20.071674346923828, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.34278440475463867, "logits/rejected": -0.34728020429611206, "logps/chosen": -320.3883056640625, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -551.8904418945312, "loss": 1.1251, "margin_dpo/margin_mean": 185.4737548828125, "margin_dpo/margin_std": 256.4941101074219, "step": 302 }, { "KL/chosen_KL_mean": -279.39471435546875, "KL/mean": -365.64605712890625, "KL/rejected_KL_mean": -451.89739990234375, "KL/std": 218.69338989257812, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.0021402090787887573, "fcm_dpo/delta": 0.04843229800462723, "fcm_dpo/margin": 172.50267028808594, "fcm_dpo/q_t": 0.41948962211608887, "grad_norm": 23.985990524291992, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.38367849588394165, "logits/rejected": -0.3712496757507324, "logps/chosen": -346.1086730957031, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -538.8428344726562, "loss": 1.1289, "margin_dpo/margin_mean": 172.50267028808594, "margin_dpo/margin_std": 265.6961669921875, "step": 303 }, { "KL/chosen_KL_mean": -255.20101928710938, "KL/mean": -347.8723449707031, "KL/rejected_KL_mean": -440.54364013671875, "KL/std": 201.79244995117188, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.0022568381391465664, "fcm_dpo/delta": -0.03559745475649834, "fcm_dpo/margin": 185.34263610839844, "fcm_dpo/q_t": 0.3972512185573578, "grad_norm": 40.94980239868164, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.40435200929641724, "logits/rejected": -0.39599835872650146, "logps/chosen": -327.1517333984375, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -531.0156860351562, "loss": 1.0579, "margin_dpo/margin_mean": 185.34262084960938, "margin_dpo/margin_std": 180.45718383789062, "step": 304 }, { "KL/chosen_KL_mean": -266.6454772949219, "KL/mean": -351.7230224609375, "KL/rejected_KL_mean": -436.8005676269531, "KL/std": 221.42990112304688, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.0021582874469459057, "fcm_dpo/delta": 0.04888404905796051, "fcm_dpo/margin": 170.1551055908203, "fcm_dpo/q_t": 0.41974300146102905, "grad_norm": 22.135257720947266, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.4137483537197113, "logits/rejected": -0.39837193489074707, "logps/chosen": -333.4407043457031, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -529.55517578125, "loss": 1.1212, "margin_dpo/margin_mean": 170.1551055908203, "margin_dpo/margin_std": 243.96319580078125, "step": 305 }, { "KL/chosen_KL_mean": -261.8374938964844, "KL/mean": -343.0050354003906, "KL/rejected_KL_mean": -424.1725769042969, "KL/std": 194.5423583984375, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.002403826918452978, "fcm_dpo/delta": 0.015124435536563396, "fcm_dpo/margin": 162.33505249023438, "fcm_dpo/q_t": 0.4113726019859314, "grad_norm": 23.578460693359375, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.3788735866546631, "logits/rejected": -0.35274261236190796, "logps/chosen": -331.5213928222656, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -509.3317565917969, "loss": 1.0925, "margin_dpo/margin_mean": 162.33505249023438, "margin_dpo/margin_std": 210.62339782714844, "step": 306 }, { "KL/chosen_KL_mean": -237.50689697265625, "KL/mean": -322.65093994140625, "KL/rejected_KL_mean": -407.7949523925781, "KL/std": 183.3559112548828, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0023515899665653706, "fcm_dpo/delta": -0.0012738704681396484, "fcm_dpo/margin": 170.28805541992188, "fcm_dpo/q_t": 0.40794315934181213, "grad_norm": 24.69138526916504, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.37078455090522766, "logits/rejected": -0.35139191150665283, "logps/chosen": -307.67230224609375, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -494.76727294921875, "loss": 1.0939, "margin_dpo/margin_mean": 170.28805541992188, "margin_dpo/margin_std": 231.32247924804688, "step": 307 }, { "KL/chosen_KL_mean": -242.2062225341797, "KL/mean": -339.5050964355469, "KL/rejected_KL_mean": -436.803955078125, "KL/std": 205.6837158203125, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.002198460977524519, "fcm_dpo/delta": -0.049677349627017975, "fcm_dpo/margin": 194.59776306152344, "fcm_dpo/q_t": 0.3950149416923523, "grad_norm": 29.672807693481445, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.3967479467391968, "logits/rejected": -0.3871362507343292, "logps/chosen": -297.45123291015625, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -516.1762084960938, "loss": 1.05, "margin_dpo/margin_mean": 194.59774780273438, "margin_dpo/margin_std": 198.84432983398438, "step": 308 }, { "KL/chosen_KL_mean": -249.229248046875, "KL/mean": -345.9730224609375, "KL/rejected_KL_mean": -442.71673583984375, "KL/std": 216.7500457763672, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.00216893432661891, "fcm_dpo/delta": -0.040406569838523865, "fcm_dpo/margin": 193.48745727539062, "fcm_dpo/q_t": 0.3975731432437897, "grad_norm": 26.32208251953125, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.32539457082748413, "logits/rejected": -0.3188524842262268, "logps/chosen": -298.1883544921875, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -525.0574951171875, "loss": 1.0796, "margin_dpo/margin_mean": 193.4874725341797, "margin_dpo/margin_std": 231.7923583984375, "step": 309 }, { "KL/chosen_KL_mean": -287.5223388671875, "KL/mean": -375.1282958984375, "KL/rejected_KL_mean": -462.7342224121094, "KL/std": 191.27525329589844, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.002167648635804653, "fcm_dpo/delta": 0.02439631149172783, "fcm_dpo/margin": 175.21188354492188, "fcm_dpo/q_t": 0.41132253408432007, "grad_norm": 37.333168029785156, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.37113621830940247, "logits/rejected": -0.35927319526672363, "logps/chosen": -350.2641296386719, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -542.66455078125, "loss": 1.0851, "margin_dpo/margin_mean": 175.21188354492188, "margin_dpo/margin_std": 188.38232421875, "step": 310 }, { "KL/chosen_KL_mean": -305.24285888671875, "KL/mean": -408.00628662109375, "KL/rejected_KL_mean": -510.7697448730469, "KL/std": 250.8982696533203, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.002006505150347948, "fcm_dpo/delta": -0.021621335297822952, "fcm_dpo/margin": 205.52688598632812, "fcm_dpo/q_t": 0.40415090322494507, "grad_norm": 23.370407104492188, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.3467109501361847, "logits/rejected": -0.32372719049453735, "logps/chosen": -358.2708435058594, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -588.2079467773438, "loss": 1.0815, "margin_dpo/margin_mean": 205.52688598632812, "margin_dpo/margin_std": 265.2549743652344, "step": 311 }, { "KL/chosen_KL_mean": -289.9778747558594, "KL/mean": -387.76654052734375, "KL/rejected_KL_mean": -485.55523681640625, "KL/std": 229.4159393310547, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.0020645607728511095, "fcm_dpo/delta": -0.011410795152187347, "fcm_dpo/margin": 195.57733154296875, "fcm_dpo/q_t": 0.4080260396003723, "grad_norm": 22.71463394165039, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.2678723633289337, "logits/rejected": -0.2562296390533447, "logps/chosen": -339.3700866699219, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -561.3480224609375, "loss": 1.0977, "margin_dpo/margin_mean": 195.57733154296875, "margin_dpo/margin_std": 275.55340576171875, "step": 312 }, { "KL/chosen_KL_mean": -276.6165771484375, "KL/mean": -375.31329345703125, "KL/rejected_KL_mean": -474.01007080078125, "KL/std": 237.2071533203125, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0020586608443409204, "fcm_dpo/delta": -0.02704712748527527, "fcm_dpo/margin": 197.3934326171875, "fcm_dpo/q_t": 0.40283501148223877, "grad_norm": 26.37751579284668, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.3565303385257721, "logits/rejected": -0.3560790419578552, "logps/chosen": -326.76934814453125, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -560.416259765625, "loss": 1.1012, "margin_dpo/margin_mean": 197.3934326171875, "margin_dpo/margin_std": 258.5960693359375, "step": 313 }, { "KL/chosen_KL_mean": -297.798828125, "KL/mean": -391.349853515625, "KL/rejected_KL_mean": -484.90087890625, "KL/std": 228.58383178710938, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0019475116860121489, "fcm_dpo/delta": 0.055439822375774384, "fcm_dpo/margin": 187.10211181640625, "fcm_dpo/q_t": 0.4213699698448181, "grad_norm": 22.879201889038086, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.39260512590408325, "logits/rejected": -0.39466455578804016, "logps/chosen": -355.036376953125, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -582.4974365234375, "loss": 1.1347, "margin_dpo/margin_mean": 187.10211181640625, "margin_dpo/margin_std": 293.64080810546875, "step": 314 }, { "KL/chosen_KL_mean": -271.39031982421875, "KL/mean": -353.8436279296875, "KL/rejected_KL_mean": -436.296875, "KL/std": 202.65625, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0022210520692169666, "fcm_dpo/delta": 0.05339854210615158, "fcm_dpo/margin": 164.90655517578125, "fcm_dpo/q_t": 0.41980287432670593, "grad_norm": 24.21739959716797, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.3449453115463257, "logits/rejected": -0.3495122194290161, "logps/chosen": -320.45989990234375, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -521.977783203125, "loss": 1.1145, "margin_dpo/margin_mean": 164.90655517578125, "margin_dpo/margin_std": 229.23095703125, "step": 315 }, { "KL/chosen_KL_mean": -270.35205078125, "KL/mean": -393.6520690917969, "KL/rejected_KL_mean": -516.9520263671875, "KL/std": 250.11061096191406, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0017885593697428703, "fcm_dpo/delta": -0.11213979125022888, "fcm_dpo/margin": 246.60000610351562, "fcm_dpo/q_t": 0.3872528076171875, "grad_norm": 22.613483428955078, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.40570446848869324, "logits/rejected": -0.40869593620300293, "logps/chosen": -324.6128234863281, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -618.2335205078125, "loss": 1.0531, "margin_dpo/margin_mean": 246.60000610351562, "margin_dpo/margin_std": 279.0342102050781, "step": 316 }, { "KL/chosen_KL_mean": -269.0587158203125, "KL/mean": -400.5162658691406, "KL/rejected_KL_mean": -531.9737548828125, "KL/std": 222.888427734375, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0017005549743771553, "fcm_dpo/delta": -0.08523563295602798, "fcm_dpo/margin": 262.91510009765625, "fcm_dpo/q_t": 0.3873101472854614, "grad_norm": 26.969892501831055, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.3938244581222534, "logits/rejected": -0.3835000693798065, "logps/chosen": -325.1529235839844, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -632.6728515625, "loss": 1.0363, "margin_dpo/margin_mean": 262.91510009765625, "margin_dpo/margin_std": 250.01504516601562, "step": 317 }, { "KL/chosen_KL_mean": -297.5737609863281, "KL/mean": -394.74908447265625, "KL/rejected_KL_mean": -491.9244384765625, "KL/std": 229.82669067382812, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.0017898095538839698, "fcm_dpo/delta": 0.07254049926996231, "fcm_dpo/margin": 194.35072326660156, "fcm_dpo/q_t": 0.42293781042099, "grad_norm": 21.67091178894043, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.45766058564186096, "logits/rejected": -0.4315134584903717, "logps/chosen": -362.2194519042969, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -574.688720703125, "loss": 1.1147, "margin_dpo/margin_mean": 194.35073852539062, "margin_dpo/margin_std": 247.84156799316406, "step": 318 }, { "KL/chosen_KL_mean": -268.0478820800781, "KL/mean": -386.1382751464844, "KL/rejected_KL_mean": -504.2286682128906, "KL/std": 255.11056518554688, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0017438679933547974, "fcm_dpo/delta": -0.02019650489091873, "fcm_dpo/margin": 236.18075561523438, "fcm_dpo/q_t": 0.40276533365249634, "grad_norm": 24.052961349487305, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.40444713830947876, "logits/rejected": -0.4185525178909302, "logps/chosen": -317.431640625, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -618.1351318359375, "loss": 1.0723, "margin_dpo/margin_mean": 236.18075561523438, "margin_dpo/margin_std": 282.4808349609375, "step": 319 }, { "KL/chosen_KL_mean": -281.9687194824219, "KL/mean": -408.4984130859375, "KL/rejected_KL_mean": -535.0281372070312, "KL/std": 261.2070007324219, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0016149728326126933, "fcm_dpo/delta": -0.019406653940677643, "fcm_dpo/margin": 253.0594482421875, "fcm_dpo/q_t": 0.40273725986480713, "grad_norm": 27.48386001586914, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.39280545711517334, "logits/rejected": -0.397488534450531, "logps/chosen": -341.4736022949219, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -632.6953125, "loss": 1.0693, "margin_dpo/margin_mean": 253.05943298339844, "margin_dpo/margin_std": 285.04510498046875, "step": 320 }, { "KL/chosen_KL_mean": -347.18927001953125, "KL/mean": -454.4679260253906, "KL/rejected_KL_mean": -561.74658203125, "KL/std": 268.3890686035156, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.0017120807897299528, "fcm_dpo/delta": 0.0495997853577137, "fcm_dpo/margin": 214.55735778808594, "fcm_dpo/q_t": 0.41874903440475464, "grad_norm": 21.842844009399414, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.3386973738670349, "logits/rejected": -0.3240874409675598, "logps/chosen": -408.7379150390625, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -653.3876953125, "loss": 1.1147, "margin_dpo/margin_mean": 214.55735778808594, "margin_dpo/margin_std": 290.58978271484375, "step": 321 }, { "KL/chosen_KL_mean": -306.6402893066406, "KL/mean": -424.40594482421875, "KL/rejected_KL_mean": -542.171630859375, "KL/std": 241.84188842773438, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0017245409544557333, "fcm_dpo/delta": -0.015119031071662903, "fcm_dpo/margin": 235.53131103515625, "fcm_dpo/q_t": 0.4033818244934082, "grad_norm": 24.67373275756836, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.3635835349559784, "logits/rejected": -0.36071500182151794, "logps/chosen": -363.9303283691406, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -637.9215698242188, "loss": 1.0722, "margin_dpo/margin_mean": 235.5313262939453, "margin_dpo/margin_std": 266.08941650390625, "step": 322 }, { "KL/chosen_KL_mean": -327.5718994140625, "KL/mean": -426.56634521484375, "KL/rejected_KL_mean": -525.560791015625, "KL/std": 240.03286743164062, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.0018759286031126976, "fcm_dpo/delta": 0.04566890001296997, "fcm_dpo/margin": 197.98890686035156, "fcm_dpo/q_t": 0.41689807176589966, "grad_norm": 30.36820411682129, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.391653448343277, "logits/rejected": -0.3899046778678894, "logps/chosen": -378.8058776855469, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -600.6227416992188, "loss": 1.1077, "margin_dpo/margin_mean": 197.98890686035156, "margin_dpo/margin_std": 260.3305969238281, "step": 323 }, { "KL/chosen_KL_mean": -376.3759765625, "KL/mean": -470.93865966796875, "KL/rejected_KL_mean": -565.5013427734375, "KL/std": 256.7306823730469, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.002103945007547736, "fcm_dpo/delta": -0.02135651558637619, "fcm_dpo/margin": 189.12530517578125, "fcm_dpo/q_t": 0.41041427850723267, "grad_norm": 50.850955963134766, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.3911237120628357, "logits/rejected": -0.37464720010757446, "logps/chosen": -441.51116943359375, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -651.9788818359375, "loss": 1.1359, "margin_dpo/margin_mean": 189.12530517578125, "margin_dpo/margin_std": 302.71319580078125, "step": 324 }, { "KL/chosen_KL_mean": -296.9546203613281, "KL/mean": -379.6038513183594, "KL/rejected_KL_mean": -462.25311279296875, "KL/std": 226.74127197265625, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.002113199792802334, "fcm_dpo/delta": 0.0753774344921112, "fcm_dpo/margin": 165.2984619140625, "fcm_dpo/q_t": 0.4240267872810364, "grad_norm": 27.387161254882812, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.4187164306640625, "logits/rejected": -0.3953896760940552, "logps/chosen": -353.17022705078125, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -532.3390502929688, "loss": 1.1291, "margin_dpo/margin_mean": 165.2984619140625, "margin_dpo/margin_std": 237.98086547851562, "step": 325 }, { "KL/chosen_KL_mean": -324.411865234375, "KL/mean": -408.02496337890625, "KL/rejected_KL_mean": -491.6380920410156, "KL/std": 210.09243774414062, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0022600889205932617, "fcm_dpo/delta": 0.03556614741683006, "fcm_dpo/margin": 167.2262420654297, "fcm_dpo/q_t": 0.414311021566391, "grad_norm": 37.292301177978516, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.44619232416152954, "logits/rejected": -0.42077717185020447, "logps/chosen": -397.1368408203125, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -571.48486328125, "loss": 1.0978, "margin_dpo/margin_mean": 167.2262420654297, "margin_dpo/margin_std": 205.65249633789062, "step": 326 }, { "KL/chosen_KL_mean": -280.4002685546875, "KL/mean": -404.8081970214844, "KL/rejected_KL_mean": -529.2161254882812, "KL/std": 223.6777801513672, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0018240961944684386, "fcm_dpo/delta": -0.12385020405054092, "fcm_dpo/margin": 248.81582641601562, "fcm_dpo/q_t": 0.3799220621585846, "grad_norm": 32.990352630615234, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.41425788402557373, "logits/rejected": -0.4029478132724762, "logps/chosen": -349.53472900390625, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -641.14990234375, "loss": 1.0343, "margin_dpo/margin_mean": 248.81582641601562, "margin_dpo/margin_std": 254.188720703125, "step": 327 }, { "KL/chosen_KL_mean": -289.2528381347656, "KL/mean": -397.5467224121094, "KL/rejected_KL_mean": -505.84063720703125, "KL/std": 236.40066528320312, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0018365057185292244, "fcm_dpo/delta": 0.003600106341764331, "fcm_dpo/margin": 216.58779907226562, "fcm_dpo/q_t": 0.4082072973251343, "grad_norm": 25.665807723999023, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.35888296365737915, "logits/rejected": -0.36398810148239136, "logps/chosen": -348.9400329589844, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -596.6956176757812, "loss": 1.0842, "margin_dpo/margin_mean": 216.58779907226562, "margin_dpo/margin_std": 268.95428466796875, "step": 328 }, { "KL/chosen_KL_mean": -328.11090087890625, "KL/mean": -442.0875244140625, "KL/rejected_KL_mean": -556.0640869140625, "KL/std": 269.01434326171875, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0017261970788240433, "fcm_dpo/delta": 0.0008079372346401215, "fcm_dpo/margin": 227.95321655273438, "fcm_dpo/q_t": 0.40766558051109314, "grad_norm": 24.41579818725586, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.3722694516181946, "logits/rejected": -0.3635803461074829, "logps/chosen": -393.35711669921875, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -656.7618408203125, "loss": 1.1045, "margin_dpo/margin_mean": 227.95321655273438, "margin_dpo/margin_std": 301.1634521484375, "step": 329 }, { "KL/chosen_KL_mean": -275.129150390625, "KL/mean": -394.5272216796875, "KL/rejected_KL_mean": -513.92529296875, "KL/std": 246.2072296142578, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0017482815310359001, "fcm_dpo/delta": -0.03028486669063568, "fcm_dpo/margin": 238.79615783691406, "fcm_dpo/q_t": 0.400304913520813, "grad_norm": 20.50057029724121, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.3955565094947815, "logits/rejected": -0.39654165506362915, "logps/chosen": -322.12750244140625, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -600.8021240234375, "loss": 1.0642, "margin_dpo/margin_mean": 238.79615783691406, "margin_dpo/margin_std": 271.3484191894531, "step": 330 }, { "KL/chosen_KL_mean": -305.6733093261719, "KL/mean": -425.43841552734375, "KL/rejected_KL_mean": -545.2034912109375, "KL/std": 206.7650146484375, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.0017030672170221806, "fcm_dpo/delta": -0.013409378007054329, "fcm_dpo/margin": 239.53012084960938, "fcm_dpo/q_t": 0.40146273374557495, "grad_norm": 22.680070877075195, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.37283700704574585, "logits/rejected": -0.3688333034515381, "logps/chosen": -356.19757080078125, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -634.2188720703125, "loss": 1.0483, "margin_dpo/margin_mean": 239.5301513671875, "margin_dpo/margin_std": 205.19898986816406, "step": 331 }, { "KL/chosen_KL_mean": -309.19793701171875, "KL/mean": -390.8100891113281, "KL/rejected_KL_mean": -472.4222412109375, "KL/std": 211.47320556640625, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.0020041917450726032, "fcm_dpo/delta": 0.10211435705423355, "fcm_dpo/margin": 163.22427368164062, "fcm_dpo/q_t": 0.43064454197883606, "grad_norm": 23.14905548095703, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.360334187746048, "logits/rejected": -0.33985477685928345, "logps/chosen": -358.37823486328125, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -548.9073486328125, "loss": 1.139, "margin_dpo/margin_mean": 163.22427368164062, "margin_dpo/margin_std": 238.90945434570312, "step": 332 }, { "KL/chosen_KL_mean": -336.2814636230469, "KL/mean": -449.5226135253906, "KL/rejected_KL_mean": -562.7637939453125, "KL/std": 258.14825439453125, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.001787829678505659, "fcm_dpo/delta": -0.0357210636138916, "fcm_dpo/margin": 226.48231506347656, "fcm_dpo/q_t": 0.4021400809288025, "grad_norm": 37.613563537597656, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.34511464834213257, "logits/rejected": -0.3372688889503479, "logps/chosen": -400.0372314453125, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -657.807861328125, "loss": 1.1071, "margin_dpo/margin_mean": 226.48233032226562, "margin_dpo/margin_std": 314.77203369140625, "step": 333 }, { "KL/chosen_KL_mean": -324.66705322265625, "KL/mean": -422.1019287109375, "KL/rejected_KL_mean": -519.5368041992188, "KL/std": 276.22686767578125, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0019669593311846256, "fcm_dpo/delta": 0.0270000621676445, "fcm_dpo/margin": 194.86973571777344, "fcm_dpo/q_t": 0.41571369767189026, "grad_norm": 21.602262496948242, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.38413316011428833, "logits/rejected": -0.3805462718009949, "logps/chosen": -391.6468200683594, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -614.853759765625, "loss": 1.1271, "margin_dpo/margin_mean": 194.86972045898438, "margin_dpo/margin_std": 309.7537536621094, "step": 334 }, { "KL/chosen_KL_mean": -354.06585693359375, "KL/mean": -434.64276123046875, "KL/rejected_KL_mean": -515.2196655273438, "KL/std": 258.8491516113281, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.002282127272337675, "fcm_dpo/delta": 0.03639799728989601, "fcm_dpo/margin": 161.15380859375, "fcm_dpo/q_t": 0.41994214057922363, "grad_norm": 31.488386154174805, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.41236335039138794, "logits/rejected": -0.39202946424484253, "logps/chosen": -416.60833740234375, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -602.83740234375, "loss": 1.1504, "margin_dpo/margin_mean": 161.15379333496094, "margin_dpo/margin_std": 279.69989013671875, "step": 335 }, { "KL/chosen_KL_mean": -347.204345703125, "KL/mean": -475.4479064941406, "KL/rejected_KL_mean": -603.69140625, "KL/std": 295.57818603515625, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0018394170328974724, "fcm_dpo/delta": -0.1347188502550125, "fcm_dpo/margin": 256.4870910644531, "fcm_dpo/q_t": 0.3836958408355713, "grad_norm": 27.45176887512207, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.38979193568229675, "logits/rejected": -0.3902202248573303, "logps/chosen": -401.7355041503906, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -708.095703125, "loss": 1.0657, "margin_dpo/margin_mean": 256.4870910644531, "margin_dpo/margin_std": 350.75457763671875, "step": 336 }, { "KL/chosen_KL_mean": -316.01263427734375, "KL/mean": -441.3327941894531, "KL/rejected_KL_mean": -566.6529541015625, "KL/std": 243.086181640625, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.001608079532161355, "fcm_dpo/delta": -0.006862752139568329, "fcm_dpo/margin": 250.64028930664062, "fcm_dpo/q_t": 0.4053046405315399, "grad_norm": 20.08615493774414, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.41743162274360657, "logits/rejected": -0.40591827034950256, "logps/chosen": -381.1413269042969, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -668.3799438476562, "loss": 1.0743, "margin_dpo/margin_mean": 250.6403045654297, "margin_dpo/margin_std": 288.3606872558594, "step": 337 }, { "KL/chosen_KL_mean": -287.96209716796875, "KL/mean": -410.0386962890625, "KL/rejected_KL_mean": -532.1152954101562, "KL/std": 230.76153564453125, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.0016326969489455223, "fcm_dpo/delta": 0.002140873111784458, "fcm_dpo/margin": 244.1531524658203, "fcm_dpo/q_t": 0.4057929515838623, "grad_norm": 22.323871612548828, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.42750677466392517, "logits/rejected": -0.4222105145454407, "logps/chosen": -346.3847961425781, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -621.183837890625, "loss": 1.0645, "margin_dpo/margin_mean": 244.15316772460938, "margin_dpo/margin_std": 244.08584594726562, "step": 338 }, { "KL/chosen_KL_mean": -303.424560546875, "KL/mean": -419.2534484863281, "KL/rejected_KL_mean": -535.0823974609375, "KL/std": 239.62149047851562, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.0017491846811026335, "fcm_dpo/delta": -0.029546871781349182, "fcm_dpo/margin": 231.65774536132812, "fcm_dpo/q_t": 0.40035557746887207, "grad_norm": 24.648235321044922, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.43223291635513306, "logits/rejected": -0.42046141624450684, "logps/chosen": -363.4198913574219, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -638.9932861328125, "loss": 1.0808, "margin_dpo/margin_mean": 231.65774536132812, "margin_dpo/margin_std": 242.7178497314453, "step": 339 }, { "KL/chosen_KL_mean": -324.7750244140625, "KL/mean": -426.1435546875, "KL/rejected_KL_mean": -527.5120239257812, "KL/std": 234.62620544433594, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0017105863662436604, "fcm_dpo/delta": 0.08201248943805695, "fcm_dpo/margin": 202.73699951171875, "fcm_dpo/q_t": 0.425212025642395, "grad_norm": 34.08943176269531, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.3556247353553772, "logits/rejected": -0.33346790075302124, "logps/chosen": -377.60528564453125, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -600.6192626953125, "loss": 1.1282, "margin_dpo/margin_mean": 202.73699951171875, "margin_dpo/margin_std": 283.2412109375, "step": 340 }, { "KL/chosen_KL_mean": -331.2342224121094, "KL/mean": -450.0869140625, "KL/rejected_KL_mean": -568.9395751953125, "KL/std": 259.69732666015625, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0017094245413318276, "fcm_dpo/delta": -0.016936201602220535, "fcm_dpo/margin": 237.70541381835938, "fcm_dpo/q_t": 0.4038848578929901, "grad_norm": 25.37321662902832, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.37913063168525696, "logits/rejected": -0.37905052304267883, "logps/chosen": -379.134033203125, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -670.74951171875, "loss": 1.0753, "margin_dpo/margin_mean": 237.70538330078125, "margin_dpo/margin_std": 275.2982177734375, "step": 341 }, { "KL/chosen_KL_mean": -320.9766540527344, "KL/mean": -426.2503967285156, "KL/rejected_KL_mean": -531.524169921875, "KL/std": 245.80764770507812, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0018736084457486868, "fcm_dpo/delta": 0.0011573433876037598, "fcm_dpo/margin": 210.54747009277344, "fcm_dpo/q_t": 0.4087076187133789, "grad_norm": 27.59446144104004, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.45016488432884216, "logits/rejected": -0.42709416151046753, "logps/chosen": -392.9732971191406, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -624.11376953125, "loss": 1.1059, "margin_dpo/margin_mean": 210.5474853515625, "margin_dpo/margin_std": 294.087158203125, "step": 342 }, { "KL/chosen_KL_mean": -307.4436950683594, "KL/mean": -433.9495849609375, "KL/rejected_KL_mean": -560.4554443359375, "KL/std": 248.22842407226562, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.001662734430283308, "fcm_dpo/delta": -0.036752138286828995, "fcm_dpo/margin": 253.01181030273438, "fcm_dpo/q_t": 0.3963671922683716, "grad_norm": 21.48690414428711, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.4163956344127655, "logits/rejected": -0.4220476746559143, "logps/chosen": -361.84930419921875, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -671.4969482421875, "loss": 1.042, "margin_dpo/margin_mean": 253.01181030273438, "margin_dpo/margin_std": 219.65103149414062, "step": 343 }, { "KL/chosen_KL_mean": -313.14569091796875, "KL/mean": -424.7760925292969, "KL/rejected_KL_mean": -536.406494140625, "KL/std": 274.6375427246094, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.0015421407297253609, "fcm_dpo/delta": -0.06822478026151657, "fcm_dpo/margin": 223.26084899902344, "fcm_dpo/q_t": 0.41599759459495544, "grad_norm": 17.046581268310547, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.3974248170852661, "logits/rejected": -0.39894139766693115, "logps/chosen": -367.1103515625, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -627.0298461914062, "loss": 1.1263, "margin_dpo/margin_mean": 223.26084899902344, "margin_dpo/margin_std": 279.2683410644531, "step": 344 }, { "KL/chosen_KL_mean": -370.6162414550781, "KL/mean": -496.4261474609375, "KL/rejected_KL_mean": -622.236083984375, "KL/std": 260.423095703125, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.001530057517811656, "fcm_dpo/delta": 0.020667294040322304, "fcm_dpo/margin": 251.619873046875, "fcm_dpo/q_t": 0.4105876684188843, "grad_norm": 20.95164680480957, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.42145538330078125, "logits/rejected": -0.41829943656921387, "logps/chosen": -432.30194091796875, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -721.7265014648438, "loss": 1.084, "margin_dpo/margin_mean": 251.619873046875, "margin_dpo/margin_std": 285.166259765625, "step": 345 }, { "KL/chosen_KL_mean": -365.87664794921875, "KL/mean": -480.89178466796875, "KL/rejected_KL_mean": -595.906982421875, "KL/std": 254.52523803710938, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.001609054277651012, "fcm_dpo/delta": 0.04702121764421463, "fcm_dpo/margin": 230.03033447265625, "fcm_dpo/q_t": 0.415949285030365, "grad_norm": 20.622108459472656, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.3826184570789337, "logits/rejected": -0.37241339683532715, "logps/chosen": -421.1329040527344, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -673.322265625, "loss": 1.0974, "margin_dpo/margin_mean": 230.03033447265625, "margin_dpo/margin_std": 276.1398620605469, "step": 346 }, { "KL/chosen_KL_mean": -361.59759521484375, "KL/mean": -479.4732666015625, "KL/rejected_KL_mean": -597.348876953125, "KL/std": 272.4322509765625, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.0016747593181207776, "fcm_dpo/delta": 0.00839798804372549, "fcm_dpo/margin": 235.7513427734375, "fcm_dpo/q_t": 0.4093061089515686, "grad_norm": 21.934925079345703, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.4126300811767578, "logits/rejected": -0.40708625316619873, "logps/chosen": -419.163818359375, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -689.7039794921875, "loss": 1.0905, "margin_dpo/margin_mean": 235.7513427734375, "margin_dpo/margin_std": 303.5050964355469, "step": 347 }, { "KL/chosen_KL_mean": -326.0738220214844, "KL/mean": -424.66363525390625, "KL/rejected_KL_mean": -523.25341796875, "KL/std": 231.39590454101562, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.001778720528818667, "fcm_dpo/delta": 0.07379812002182007, "fcm_dpo/margin": 197.17959594726562, "fcm_dpo/q_t": 0.42386046051979065, "grad_norm": 24.249563217163086, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.44509440660476685, "logits/rejected": -0.44437336921691895, "logps/chosen": -382.39154052734375, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -612.391845703125, "loss": 1.1286, "margin_dpo/margin_mean": 197.1796112060547, "margin_dpo/margin_std": 269.8843994140625, "step": 348 }, { "KL/chosen_KL_mean": -308.29888916015625, "KL/mean": -417.86456298828125, "KL/rejected_KL_mean": -527.4302978515625, "KL/std": 253.47750854492188, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.0019209685269743204, "fcm_dpo/delta": -0.03923729062080383, "fcm_dpo/margin": 219.1314239501953, "fcm_dpo/q_t": 0.40259507298469543, "grad_norm": 23.687149047851562, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.45386502146720886, "logits/rejected": -0.45663371682167053, "logps/chosen": -366.32440185546875, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -624.9354248046875, "loss": 1.0951, "margin_dpo/margin_mean": 219.13143920898438, "margin_dpo/margin_std": 312.9615478515625, "step": 349 }, { "KL/chosen_KL_mean": -332.3165283203125, "KL/mean": -443.63629150390625, "KL/rejected_KL_mean": -554.9560546875, "KL/std": 248.83816528320312, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.0017454567132517695, "fcm_dpo/delta": 0.013433754444122314, "fcm_dpo/margin": 222.63954162597656, "fcm_dpo/q_t": 0.4100583791732788, "grad_norm": 26.530868530273438, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.4750177562236786, "logits/rejected": -0.4820059835910797, "logps/chosen": -396.64703369140625, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -644.8277587890625, "loss": 1.1064, "margin_dpo/margin_mean": 222.63955688476562, "margin_dpo/margin_std": 307.46893310546875, "step": 350 }, { "KL/chosen_KL_mean": -296.4862060546875, "KL/mean": -429.1023254394531, "KL/rejected_KL_mean": -561.718505859375, "KL/std": 283.80633544921875, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0016240214463323355, "fcm_dpo/delta": -0.061422545462846756, "fcm_dpo/margin": 265.2322998046875, "fcm_dpo/q_t": 0.3965432643890381, "grad_norm": 31.153547286987305, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.4499565362930298, "logits/rejected": -0.4501733183860779, "logps/chosen": -357.1583251953125, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -663.283935546875, "loss": 1.0671, "margin_dpo/margin_mean": 265.2322998046875, "margin_dpo/margin_std": 327.75909423828125, "step": 351 }, { "KL/chosen_KL_mean": -349.35406494140625, "KL/mean": -436.3336181640625, "KL/rejected_KL_mean": -523.313232421875, "KL/std": 263.14996337890625, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.0018782506231218576, "fcm_dpo/delta": 0.10981118679046631, "fcm_dpo/margin": 173.9591064453125, "fcm_dpo/q_t": 0.4318041503429413, "grad_norm": 46.439903259277344, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.45800480246543884, "logits/rejected": -0.4245471954345703, "logps/chosen": -420.2975158691406, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -599.9552001953125, "loss": 1.155, "margin_dpo/margin_mean": 173.9591064453125, "margin_dpo/margin_std": 280.7679748535156, "step": 352 }, { "KL/chosen_KL_mean": -317.61041259765625, "KL/mean": -430.0752258300781, "KL/rejected_KL_mean": -542.5400390625, "KL/std": 259.1496276855469, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.0018013006774708629, "fcm_dpo/delta": -0.02872556447982788, "fcm_dpo/margin": 224.9296112060547, "fcm_dpo/q_t": 0.40363016724586487, "grad_norm": 23.421403884887695, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.4782019555568695, "logits/rejected": -0.4654806852340698, "logps/chosen": -376.0057373046875, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -622.8755493164062, "loss": 1.0947, "margin_dpo/margin_mean": 224.92959594726562, "margin_dpo/margin_std": 295.62744140625, "step": 353 }, { "KL/chosen_KL_mean": -272.10107421875, "KL/mean": -383.7615966796875, "KL/rejected_KL_mean": -495.422119140625, "KL/std": 226.24905395507812, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.001837678486481309, "fcm_dpo/delta": -0.01862015575170517, "fcm_dpo/margin": 223.32101440429688, "fcm_dpo/q_t": 0.4018193483352661, "grad_norm": 24.753582000732422, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.45796817541122437, "logits/rejected": -0.4484265446662903, "logps/chosen": -331.9040832519531, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -584.1796264648438, "loss": 1.0623, "margin_dpo/margin_mean": 223.32101440429688, "margin_dpo/margin_std": 235.79531860351562, "step": 354 }, { "KL/chosen_KL_mean": -267.42645263671875, "KL/mean": -380.90118408203125, "KL/rejected_KL_mean": -494.37591552734375, "KL/std": 236.37852478027344, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.0017689072992652655, "fcm_dpo/delta": -0.005881348624825478, "fcm_dpo/margin": 226.94949340820312, "fcm_dpo/q_t": 0.40436455607414246, "grad_norm": 38.343528747558594, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.48708575963974, "logits/rejected": -0.4752500653266907, "logps/chosen": -321.554931640625, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -576.781982421875, "loss": 1.0671, "margin_dpo/margin_mean": 226.9495086669922, "margin_dpo/margin_std": 235.01458740234375, "step": 355 }, { "KL/chosen_KL_mean": -329.96331787109375, "KL/mean": -393.64111328125, "KL/rejected_KL_mean": -457.3189697265625, "KL/std": 250.74232482910156, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.00188022933434695, "fcm_dpo/delta": 0.029842089861631393, "fcm_dpo/margin": 127.35562896728516, "fcm_dpo/q_t": 0.4473668932914734, "grad_norm": 28.56047248840332, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.47035759687423706, "logits/rejected": -0.44759708642959595, "logps/chosen": -394.6371154785156, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -533.21826171875, "loss": 1.2407, "margin_dpo/margin_mean": 127.35563659667969, "margin_dpo/margin_std": 307.847412109375, "step": 356 }, { "KL/chosen_KL_mean": -290.78863525390625, "KL/mean": -393.8023681640625, "KL/rejected_KL_mean": -496.8161315917969, "KL/std": 240.8219757080078, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.0018875239184126258, "fcm_dpo/delta": 0.013352084904909134, "fcm_dpo/margin": 206.02752685546875, "fcm_dpo/q_t": 0.4096463918685913, "grad_norm": 27.79951286315918, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.45739173889160156, "logits/rejected": -0.44537484645843506, "logps/chosen": -343.514404296875, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -583.6572875976562, "loss": 1.0937, "margin_dpo/margin_mean": 206.02752685546875, "margin_dpo/margin_std": 253.822509765625, "step": 357 }, { "KL/chosen_KL_mean": -269.3954162597656, "KL/mean": -368.242919921875, "KL/rejected_KL_mean": -467.09039306640625, "KL/std": 235.274658203125, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0020065298303961754, "fcm_dpo/delta": 0.005489196628332138, "fcm_dpo/margin": 197.69497680664062, "fcm_dpo/q_t": 0.410032719373703, "grad_norm": 28.569393157958984, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.48102307319641113, "logits/rejected": -0.463295042514801, "logps/chosen": -332.6008605957031, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -555.4637451171875, "loss": 1.0932, "margin_dpo/margin_mean": 197.69496154785156, "margin_dpo/margin_std": 264.3070983886719, "step": 358 }, { "KL/chosen_KL_mean": -304.4889221191406, "KL/mean": -419.318359375, "KL/rejected_KL_mean": -534.1477661132812, "KL/std": 241.59100341796875, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0018390321638435125, "fcm_dpo/delta": -0.04261445254087448, "fcm_dpo/margin": 229.6588134765625, "fcm_dpo/q_t": 0.3999358117580414, "grad_norm": 27.864505767822266, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.4233545958995819, "logits/rejected": -0.40909135341644287, "logps/chosen": -360.859130859375, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -616.321533203125, "loss": 1.0733, "margin_dpo/margin_mean": 229.6588134765625, "margin_dpo/margin_std": 284.9479064941406, "step": 359 }, { "KL/chosen_KL_mean": -301.6378479003906, "KL/mean": -391.5552978515625, "KL/rejected_KL_mean": -481.4727478027344, "KL/std": 206.68411254882812, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.0020297402516007423, "fcm_dpo/delta": 0.05278204008936882, "fcm_dpo/margin": 179.8349151611328, "fcm_dpo/q_t": 0.4177628755569458, "grad_norm": 40.045162200927734, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.43576139211654663, "logits/rejected": -0.422180712223053, "logps/chosen": -353.0982360839844, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -551.3116455078125, "loss": 1.0958, "margin_dpo/margin_mean": 179.83489990234375, "margin_dpo/margin_std": 205.51583862304688, "step": 360 }, { "KL/chosen_KL_mean": -332.83001708984375, "KL/mean": -420.3536376953125, "KL/rejected_KL_mean": -507.8772277832031, "KL/std": 242.87753295898438, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.0021930173970758915, "fcm_dpo/delta": 0.021988222375512123, "fcm_dpo/margin": 175.04721069335938, "fcm_dpo/q_t": 0.4129222631454468, "grad_norm": 37.02199172973633, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.4316215217113495, "logits/rejected": -0.427333801984787, "logps/chosen": -386.69952392578125, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -598.646484375, "loss": 1.115, "margin_dpo/margin_mean": 175.04721069335938, "margin_dpo/margin_std": 257.5013427734375, "step": 361 }, { "KL/chosen_KL_mean": -281.8548583984375, "KL/mean": -425.60888671875, "KL/rejected_KL_mean": -569.3629150390625, "KL/std": 254.64712524414062, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0015978740993887186, "fcm_dpo/delta": -0.14796875417232513, "fcm_dpo/margin": 287.50799560546875, "fcm_dpo/q_t": 0.37716490030288696, "grad_norm": 23.246320724487305, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.4437373876571655, "logits/rejected": -0.45072150230407715, "logps/chosen": -340.49395751953125, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -674.94482421875, "loss": 1.0236, "margin_dpo/margin_mean": 287.50799560546875, "margin_dpo/margin_std": 274.0611572265625, "step": 362 }, { "KL/chosen_KL_mean": -271.1270751953125, "KL/mean": -413.6645202636719, "KL/rejected_KL_mean": -556.2019653320312, "KL/std": 249.9840087890625, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0015145809156820178, "fcm_dpo/delta": -0.056021787226200104, "fcm_dpo/margin": 285.07489013671875, "fcm_dpo/q_t": 0.39367860555648804, "grad_norm": 20.8101749420166, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.43102937936782837, "logits/rejected": -0.4243617057800293, "logps/chosen": -315.6854553222656, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -630.8969116210938, "loss": 1.0437, "margin_dpo/margin_mean": 285.07489013671875, "margin_dpo/margin_std": 276.6128234863281, "step": 363 }, { "KL/chosen_KL_mean": -300.5224914550781, "KL/mean": -406.80938720703125, "KL/rejected_KL_mean": -513.0963134765625, "KL/std": 243.1720733642578, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0016487597022205591, "fcm_dpo/delta": 0.06513446569442749, "fcm_dpo/margin": 212.57382202148438, "fcm_dpo/q_t": 0.42112141847610474, "grad_norm": 21.724754333496094, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.44869840145111084, "logits/rejected": -0.4586051106452942, "logps/chosen": -349.4171142578125, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -604.4920654296875, "loss": 1.1204, "margin_dpo/margin_mean": 212.57383728027344, "margin_dpo/margin_std": 284.5193176269531, "step": 364 }, { "KL/chosen_KL_mean": -304.2714538574219, "KL/mean": -416.9530029296875, "KL/rejected_KL_mean": -529.634521484375, "KL/std": 258.83819580078125, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0016951878787949681, "fcm_dpo/delta": 0.02898254618048668, "fcm_dpo/margin": 225.36306762695312, "fcm_dpo/q_t": 0.41397157311439514, "grad_norm": 20.870635986328125, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.43810129165649414, "logits/rejected": -0.44645947217941284, "logps/chosen": -355.7641906738281, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -622.336181640625, "loss": 1.0909, "margin_dpo/margin_mean": 225.36306762695312, "margin_dpo/margin_std": 272.38427734375, "step": 365 }, { "KL/chosen_KL_mean": -287.7502746582031, "KL/mean": -404.9134826660156, "KL/rejected_KL_mean": -522.0767211914062, "KL/std": 256.9874267578125, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.0016629542224109173, "fcm_dpo/delta": 0.010643713176250458, "fcm_dpo/margin": 234.326416015625, "fcm_dpo/q_t": 0.4102753698825836, "grad_norm": 19.262928009033203, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.4263162612915039, "logits/rejected": -0.44171929359436035, "logps/chosen": -332.4708557128906, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -605.3870849609375, "loss": 1.0992, "margin_dpo/margin_mean": 234.326416015625, "margin_dpo/margin_std": 304.7642517089844, "step": 366 }, { "KL/chosen_KL_mean": -303.0289611816406, "KL/mean": -399.514892578125, "KL/rejected_KL_mean": -496.0008544921875, "KL/std": 225.67916870117188, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0019213332561776042, "fcm_dpo/delta": 0.04300880804657936, "fcm_dpo/margin": 192.971923828125, "fcm_dpo/q_t": 0.4157758355140686, "grad_norm": 21.952829360961914, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.48216262459754944, "logits/rejected": -0.4676123857498169, "logps/chosen": -361.43438720703125, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -572.752197265625, "loss": 1.1045, "margin_dpo/margin_mean": 192.971923828125, "margin_dpo/margin_std": 250.1029052734375, "step": 367 }, { "KL/chosen_KL_mean": -282.49639892578125, "KL/mean": -436.16912841796875, "KL/rejected_KL_mean": -589.8418579101562, "KL/std": 253.77154541015625, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0015261652879416943, "fcm_dpo/delta": -0.14009416103363037, "fcm_dpo/margin": 307.345458984375, "fcm_dpo/q_t": 0.37267598509788513, "grad_norm": 34.17594528198242, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.4614467918872833, "logits/rejected": -0.4720000922679901, "logps/chosen": -326.94891357421875, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -688.3970947265625, "loss": 0.9994, "margin_dpo/margin_mean": 307.345458984375, "margin_dpo/margin_std": 224.4804229736328, "step": 368 }, { "KL/chosen_KL_mean": -367.19921875, "KL/mean": -449.5543212890625, "KL/rejected_KL_mean": -531.909423828125, "KL/std": 254.9052276611328, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0015358589589595795, "fcm_dpo/delta": 0.05729483440518379, "fcm_dpo/margin": 164.710205078125, "fcm_dpo/q_t": 0.4428786039352417, "grad_norm": 24.15337371826172, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.4904573857784271, "logits/rejected": -0.4741830825805664, "logps/chosen": -438.58074951171875, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -623.2052001953125, "loss": 1.1956, "margin_dpo/margin_mean": 164.710205078125, "margin_dpo/margin_std": 289.94000244140625, "step": 369 }, { "KL/chosen_KL_mean": -390.330810546875, "KL/mean": -462.7243347167969, "KL/rejected_KL_mean": -535.1178588867188, "KL/std": 265.82781982421875, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0017272934783250093, "fcm_dpo/delta": 0.06001188978552818, "fcm_dpo/margin": 144.7870635986328, "fcm_dpo/q_t": 0.44495099782943726, "grad_norm": 28.623899459838867, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.4887716770172119, "logits/rejected": -0.4825999140739441, "logps/chosen": -461.93829345703125, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -632.377685546875, "loss": 1.2167, "margin_dpo/margin_mean": 144.7870635986328, "margin_dpo/margin_std": 307.05999755859375, "step": 370 }, { "KL/chosen_KL_mean": -373.00421142578125, "KL/mean": -484.24078369140625, "KL/rejected_KL_mean": -595.477294921875, "KL/std": 274.64886474609375, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0018450857605785131, "fcm_dpo/delta": -0.020257379859685898, "fcm_dpo/margin": 222.4730987548828, "fcm_dpo/q_t": 0.4056010842323303, "grad_norm": 29.032577514648438, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.5206031799316406, "logits/rejected": -0.5084042549133301, "logps/chosen": -442.418701171875, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -694.6495361328125, "loss": 1.1008, "margin_dpo/margin_mean": 222.4730987548828, "margin_dpo/margin_std": 321.52008056640625, "step": 371 }, { "KL/chosen_KL_mean": -360.9215393066406, "KL/mean": -495.5597229003906, "KL/rejected_KL_mean": -630.1979370117188, "KL/std": 315.78912353515625, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.001595328445546329, "fcm_dpo/delta": -0.05453532934188843, "fcm_dpo/margin": 269.2763671875, "fcm_dpo/q_t": 0.39929690957069397, "grad_norm": 26.51140594482422, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.4976493716239929, "logits/rejected": -0.4843401610851288, "logps/chosen": -422.7395324707031, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -708.7374267578125, "loss": 1.0767, "margin_dpo/margin_mean": 269.2763671875, "margin_dpo/margin_std": 363.7861328125, "step": 372 }, { "KL/chosen_KL_mean": -385.86328125, "KL/mean": -511.8228759765625, "KL/rejected_KL_mean": -637.7823486328125, "KL/std": 299.7609558105469, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.001618098933249712, "fcm_dpo/delta": -0.01854773610830307, "fcm_dpo/margin": 251.91912841796875, "fcm_dpo/q_t": 0.40265533328056335, "grad_norm": 23.972576141357422, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.46941909193992615, "logits/rejected": -0.4492412209510803, "logps/chosen": -450.0804138183594, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -723.7420043945312, "loss": 1.0837, "margin_dpo/margin_mean": 251.91912841796875, "margin_dpo/margin_std": 299.7061767578125, "step": 373 }, { "KL/chosen_KL_mean": -349.892333984375, "KL/mean": -465.20263671875, "KL/rejected_KL_mean": -580.512939453125, "KL/std": 325.4590148925781, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.0015723302494734526, "fcm_dpo/delta": 0.05567358061671257, "fcm_dpo/margin": 230.62063598632812, "fcm_dpo/q_t": 0.42263877391815186, "grad_norm": 22.6434383392334, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.5414470434188843, "logits/rejected": -0.524927020072937, "logps/chosen": -420.54254150390625, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -674.1531372070312, "loss": 1.1406, "margin_dpo/margin_mean": 230.62063598632812, "margin_dpo/margin_std": 373.28961181640625, "step": 374 }, { "KL/chosen_KL_mean": -364.77288818359375, "KL/mean": -473.304931640625, "KL/rejected_KL_mean": -581.8369750976562, "KL/std": 262.2808837890625, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.0017019468359649181, "fcm_dpo/delta": 0.04178474843502045, "fcm_dpo/margin": 217.06411743164062, "fcm_dpo/q_t": 0.41703087091445923, "grad_norm": 31.42391586303711, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.4924180507659912, "logits/rejected": -0.4902787208557129, "logps/chosen": -424.85308837890625, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -670.7752685546875, "loss": 1.1287, "margin_dpo/margin_mean": 217.06411743164062, "margin_dpo/margin_std": 315.9866943359375, "step": 375 }, { "KL/chosen_KL_mean": -347.1155090332031, "KL/mean": -484.00567626953125, "KL/rejected_KL_mean": -620.8958740234375, "KL/std": 285.42181396484375, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0016071631107479334, "fcm_dpo/delta": -0.07716603577136993, "fcm_dpo/margin": 273.7803955078125, "fcm_dpo/q_t": 0.39170658588409424, "grad_norm": 24.55660629272461, "learning_rate": 2.5e-07, "logits/chosen": -0.49682873487472534, "logits/rejected": -0.48542171716690063, "logps/chosen": -409.77581787109375, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -726.4224853515625, "loss": 1.0607, "margin_dpo/margin_mean": 273.7803955078125, "margin_dpo/margin_std": 332.501708984375, "step": 376 }, { "KL/chosen_KL_mean": -349.8211975097656, "KL/mean": -483.8100280761719, "KL/rejected_KL_mean": -617.7988891601562, "KL/std": 303.7757263183594, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.0015564009081572294, "fcm_dpo/delta": -0.030124623328447342, "fcm_dpo/margin": 267.9776916503906, "fcm_dpo/q_t": 0.401019811630249, "grad_norm": 21.409746170043945, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.4336928129196167, "logits/rejected": -0.432683527469635, "logps/chosen": -404.2999267578125, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -716.502197265625, "loss": 1.072, "margin_dpo/margin_mean": 267.9776611328125, "margin_dpo/margin_std": 321.41864013671875, "step": 377 }, { "KL/chosen_KL_mean": -324.5780029296875, "KL/mean": -463.517333984375, "KL/rejected_KL_mean": -602.4566650390625, "KL/std": 269.98370361328125, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.0014617801643908024, "fcm_dpo/delta": -0.010532225482165813, "fcm_dpo/margin": 277.878662109375, "fcm_dpo/q_t": 0.40280961990356445, "grad_norm": 24.20557403564453, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.43811696767807007, "logits/rejected": -0.4507744312286377, "logps/chosen": -369.5985107421875, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -690.503662109375, "loss": 1.0649, "margin_dpo/margin_mean": 277.878662109375, "margin_dpo/margin_std": 294.4283447265625, "step": 378 }, { "KL/chosen_KL_mean": -359.47418212890625, "KL/mean": -495.9452209472656, "KL/rejected_KL_mean": -632.416259765625, "KL/std": 277.9902038574219, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0014259706949815154, "fcm_dpo/delta": 0.0134861059486866, "fcm_dpo/margin": 272.94207763671875, "fcm_dpo/q_t": 0.41009724140167236, "grad_norm": 26.755817413330078, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.38554373383522034, "logits/rejected": -0.4059056043624878, "logps/chosen": -412.65625, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -746.7164306640625, "loss": 1.0908, "margin_dpo/margin_mean": 272.9421081542969, "margin_dpo/margin_std": 334.01593017578125, "step": 379 }, { "KL/chosen_KL_mean": -353.51654052734375, "KL/mean": -504.0564270019531, "KL/rejected_KL_mean": -654.5963134765625, "KL/std": 314.81463623046875, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.0014171417569741607, "fcm_dpo/delta": -0.046321943402290344, "fcm_dpo/margin": 301.0798034667969, "fcm_dpo/q_t": 0.39565783739089966, "grad_norm": 22.568410873413086, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.4838710427284241, "logits/rejected": -0.5105802416801453, "logps/chosen": -404.86956787109375, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -758.7880249023438, "loss": 1.0598, "margin_dpo/margin_mean": 301.079833984375, "margin_dpo/margin_std": 341.14447021484375, "step": 380 }, { "KL/chosen_KL_mean": -365.193359375, "KL/mean": -466.3520812988281, "KL/rejected_KL_mean": -567.5108642578125, "KL/std": 262.38165283203125, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.001597442664206028, "fcm_dpo/delta": 0.11401475220918655, "fcm_dpo/margin": 202.31747436523438, "fcm_dpo/q_t": 0.43302297592163086, "grad_norm": 25.53712272644043, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.4542039632797241, "logits/rejected": -0.4321713149547577, "logps/chosen": -422.99639892578125, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -646.730224609375, "loss": 1.1718, "margin_dpo/margin_mean": 202.31747436523438, "margin_dpo/margin_std": 359.52716064453125, "step": 381 }, { "KL/chosen_KL_mean": -346.081298828125, "KL/mean": -476.01800537109375, "KL/rejected_KL_mean": -605.9547119140625, "KL/std": 252.94512939453125, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0016131680458784103, "fcm_dpo/delta": -0.03418818861246109, "fcm_dpo/margin": 259.8733825683594, "fcm_dpo/q_t": 0.3991495966911316, "grad_norm": 20.419843673706055, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.5054559707641602, "logits/rejected": -0.5107045769691467, "logps/chosen": -412.10162353515625, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -716.6648559570312, "loss": 1.0605, "margin_dpo/margin_mean": 259.8733825683594, "margin_dpo/margin_std": 285.6113586425781, "step": 382 }, { "KL/chosen_KL_mean": -361.154541015625, "KL/mean": -490.5584716796875, "KL/rejected_KL_mean": -619.96240234375, "KL/std": 283.15301513671875, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0015578651800751686, "fcm_dpo/delta": -0.00645710714161396, "fcm_dpo/margin": 258.8077392578125, "fcm_dpo/q_t": 0.4064059555530548, "grad_norm": 27.733203887939453, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.4914437532424927, "logits/rejected": -0.5008552670478821, "logps/chosen": -411.5460510253906, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -713.67822265625, "loss": 1.1016, "margin_dpo/margin_mean": 258.8077392578125, "margin_dpo/margin_std": 370.54718017578125, "step": 383 }, { "KL/chosen_KL_mean": -375.8470458984375, "KL/mean": -478.16192626953125, "KL/rejected_KL_mean": -580.4768676757812, "KL/std": 256.4949645996094, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.0017840572400018573, "fcm_dpo/delta": 0.04306403547525406, "fcm_dpo/margin": 204.6298065185547, "fcm_dpo/q_t": 0.41565489768981934, "grad_norm": 26.314607620239258, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.501458466053009, "logits/rejected": -0.4933622181415558, "logps/chosen": -427.89312744140625, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -666.23779296875, "loss": 1.1048, "margin_dpo/margin_mean": 204.62982177734375, "margin_dpo/margin_std": 249.5997314453125, "step": 384 }, { "KL/chosen_KL_mean": -345.30267333984375, "KL/mean": -467.48583984375, "KL/rejected_KL_mean": -589.6690063476562, "KL/std": 224.0086669921875, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0016768588684499264, "fcm_dpo/delta": -0.016689352691173553, "fcm_dpo/margin": 244.36630249023438, "fcm_dpo/q_t": 0.40174418687820435, "grad_norm": 26.779409408569336, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.5289189219474792, "logits/rejected": -0.500619649887085, "logps/chosen": -410.8548583984375, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -667.4969482421875, "loss": 1.0615, "margin_dpo/margin_mean": 244.36631774902344, "margin_dpo/margin_std": 253.60830688476562, "step": 385 }, { "KL/chosen_KL_mean": -361.995361328125, "KL/mean": -493.2794494628906, "KL/rejected_KL_mean": -624.5635375976562, "KL/std": 274.184326171875, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0015988796949386597, "fcm_dpo/delta": -0.034051790833473206, "fcm_dpo/margin": 262.5681457519531, "fcm_dpo/q_t": 0.400490939617157, "grad_norm": 25.8364200592041, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.5255472660064697, "logits/rejected": -0.5233632922172546, "logps/chosen": -420.21722412109375, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -716.8909912109375, "loss": 1.0737, "margin_dpo/margin_mean": 262.56817626953125, "margin_dpo/margin_std": 327.55181884765625, "step": 386 }, { "KL/chosen_KL_mean": -404.6907958984375, "KL/mean": -515.4647827148438, "KL/rejected_KL_mean": -626.23876953125, "KL/std": 265.8984375, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0016458687605336308, "fcm_dpo/delta": 0.05550061911344528, "fcm_dpo/margin": 221.54798889160156, "fcm_dpo/q_t": 0.4187946617603302, "grad_norm": 29.908674240112305, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.5720622539520264, "logits/rejected": -0.5503807067871094, "logps/chosen": -471.1102600097656, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -718.407958984375, "loss": 1.1074, "margin_dpo/margin_mean": 221.54798889160156, "margin_dpo/margin_std": 279.3563232421875, "step": 387 }, { "KL/chosen_KL_mean": -380.31915283203125, "KL/mean": -527.940185546875, "KL/rejected_KL_mean": -675.5611572265625, "KL/std": 304.70587158203125, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.001483785454183817, "fcm_dpo/delta": -0.07916902750730515, "fcm_dpo/margin": 295.241943359375, "fcm_dpo/q_t": 0.39270198345184326, "grad_norm": 27.048213958740234, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.4747141897678375, "logits/rejected": -0.4871397018432617, "logps/chosen": -430.4486083984375, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -779.9942016601562, "loss": 1.0573, "margin_dpo/margin_mean": 295.2419738769531, "margin_dpo/margin_std": 350.07147216796875, "step": 388 }, { "KL/chosen_KL_mean": -417.1252136230469, "KL/mean": -543.790771484375, "KL/rejected_KL_mean": -670.4564208984375, "KL/std": 307.47637939453125, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.001513138646259904, "fcm_dpo/delta": 0.026595965027809143, "fcm_dpo/margin": 253.3311767578125, "fcm_dpo/q_t": 0.41341066360473633, "grad_norm": 33.48180389404297, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.5469924211502075, "logits/rejected": -0.5381365418434143, "logps/chosen": -475.03179931640625, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -748.3709716796875, "loss": 1.1045, "margin_dpo/margin_mean": 253.3311767578125, "margin_dpo/margin_std": 345.8287353515625, "step": 389 }, { "KL/chosen_KL_mean": -417.957763671875, "KL/mean": -541.5369262695312, "KL/rejected_KL_mean": -665.1160888671875, "KL/std": 302.0540771484375, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.0015666976105421782, "fcm_dpo/delta": 0.019385023042559624, "fcm_dpo/margin": 247.1583251953125, "fcm_dpo/q_t": 0.4156228303909302, "grad_norm": 23.625947952270508, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.46019166707992554, "logits/rejected": -0.45252659916877747, "logps/chosen": -467.1836853027344, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -750.644287109375, "loss": 1.1168, "margin_dpo/margin_mean": 247.1583251953125, "margin_dpo/margin_std": 383.4603271484375, "step": 390 }, { "KL/chosen_KL_mean": -418.42572021484375, "KL/mean": -490.8452453613281, "KL/rejected_KL_mean": -563.2647705078125, "KL/std": 288.76385498046875, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0020397088956087828, "fcm_dpo/delta": 0.14491389691829681, "fcm_dpo/margin": 144.83901977539062, "fcm_dpo/q_t": 0.4414842128753662, "grad_norm": 52.0184440612793, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.472456693649292, "logits/rejected": -0.4619515538215637, "logps/chosen": -482.75537109375, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -650.0029296875, "loss": 1.2088, "margin_dpo/margin_mean": 144.83901977539062, "margin_dpo/margin_std": 298.0270690917969, "step": 391 }, { "KL/chosen_KL_mean": -343.19525146484375, "KL/mean": -493.396484375, "KL/rejected_KL_mean": -643.5977783203125, "KL/std": 300.6968994140625, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0016628424637019634, "fcm_dpo/delta": -0.20023079216480255, "fcm_dpo/margin": 300.4024658203125, "fcm_dpo/q_t": 0.36630189418792725, "grad_norm": 26.650352478027344, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.4742690920829773, "logits/rejected": -0.46840304136276245, "logps/chosen": -396.69921875, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -745.943603515625, "loss": 1.0092, "margin_dpo/margin_mean": 300.4024658203125, "margin_dpo/margin_std": 297.166748046875, "step": 392 }, { "KL/chosen_KL_mean": -344.81060791015625, "KL/mean": -455.6504821777344, "KL/rejected_KL_mean": -566.4903564453125, "KL/std": 291.33221435546875, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0016052678693085909, "fcm_dpo/delta": 0.06851550936698914, "fcm_dpo/margin": 221.67974853515625, "fcm_dpo/q_t": 0.4255250096321106, "grad_norm": 21.591970443725586, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.4540834426879883, "logits/rejected": -0.451620489358902, "logps/chosen": -391.2845458984375, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -638.459228515625, "loss": 1.1418, "margin_dpo/margin_mean": 221.67974853515625, "margin_dpo/margin_std": 372.8785400390625, "step": 393 }, { "KL/chosen_KL_mean": -376.7562561035156, "KL/mean": -496.7586669921875, "KL/rejected_KL_mean": -616.7610473632812, "KL/std": 303.6330261230469, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.001655565807595849, "fcm_dpo/delta": 0.003836357966065407, "fcm_dpo/margin": 240.00477600097656, "fcm_dpo/q_t": 0.4095136225223541, "grad_norm": 21.343175888061523, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.46841922402381897, "logits/rejected": -0.4693748354911804, "logps/chosen": -429.6678161621094, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -707.5836791992188, "loss": 1.0918, "margin_dpo/margin_mean": 240.00477600097656, "margin_dpo/margin_std": 317.42437744140625, "step": 394 }, { "KL/chosen_KL_mean": -362.38995361328125, "KL/mean": -493.2774658203125, "KL/rejected_KL_mean": -624.1649780273438, "KL/std": 293.3901672363281, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.0015726612182334065, "fcm_dpo/delta": -0.024214228615164757, "fcm_dpo/margin": 261.7750244140625, "fcm_dpo/q_t": 0.4036139249801636, "grad_norm": 21.289175033569336, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.5199534893035889, "logits/rejected": -0.5158529281616211, "logps/chosen": -424.9360656738281, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -707.9476318359375, "loss": 1.0849, "margin_dpo/margin_mean": 261.7750244140625, "margin_dpo/margin_std": 343.44354248046875, "step": 395 }, { "KL/chosen_KL_mean": -373.7277526855469, "KL/mean": -489.446533203125, "KL/rejected_KL_mean": -605.1654052734375, "KL/std": 288.4603576660156, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.0016512691508978605, "fcm_dpo/delta": 0.028826599940657616, "fcm_dpo/margin": 231.4375457763672, "fcm_dpo/q_t": 0.41250765323638916, "grad_norm": 22.668109893798828, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.4854172468185425, "logits/rejected": -0.4645005464553833, "logps/chosen": -442.72369384765625, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -693.81201171875, "loss": 1.0875, "margin_dpo/margin_mean": 231.43756103515625, "margin_dpo/margin_std": 270.1766662597656, "step": 396 }, { "KL/chosen_KL_mean": -345.4547119140625, "KL/mean": -491.2574462890625, "KL/rejected_KL_mean": -637.0601806640625, "KL/std": 283.0622253417969, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.0014739397447556257, "fcm_dpo/delta": -0.06422150880098343, "fcm_dpo/margin": 291.6054382324219, "fcm_dpo/q_t": 0.39274823665618896, "grad_norm": 29.06831169128418, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.46980020403862, "logits/rejected": -0.47478750348091125, "logps/chosen": -406.73187255859375, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -740.17626953125, "loss": 1.0495, "margin_dpo/margin_mean": 291.6054382324219, "margin_dpo/margin_std": 302.38970947265625, "step": 397 }, { "KL/chosen_KL_mean": -366.6300354003906, "KL/mean": -499.76983642578125, "KL/rejected_KL_mean": -632.90966796875, "KL/std": 293.9355773925781, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.0014921356923878193, "fcm_dpo/delta": 0.004290161654353142, "fcm_dpo/margin": 266.2795715332031, "fcm_dpo/q_t": 0.40941908955574036, "grad_norm": 20.657440185546875, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.5278111696243286, "logits/rejected": -0.5256218314170837, "logps/chosen": -434.7815856933594, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -741.4332275390625, "loss": 1.0907, "margin_dpo/margin_mean": 266.2795715332031, "margin_dpo/margin_std": 350.179443359375, "step": 398 }, { "KL/chosen_KL_mean": -315.19482421875, "KL/mean": -432.66326904296875, "KL/rejected_KL_mean": -550.1317138671875, "KL/std": 257.30950927734375, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0015734569169580936, "fcm_dpo/delta": 0.047637663781642914, "fcm_dpo/margin": 234.9368133544922, "fcm_dpo/q_t": 0.41743379831314087, "grad_norm": 21.86760711669922, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.4569128751754761, "logits/rejected": -0.41413915157318115, "logps/chosen": -376.0846252441406, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -628.0972900390625, "loss": 1.1148, "margin_dpo/margin_mean": 234.93682861328125, "margin_dpo/margin_std": 323.26910400390625, "step": 399 }, { "KL/chosen_KL_mean": -298.4742431640625, "KL/mean": -467.8236389160156, "KL/rejected_KL_mean": -637.1730346679688, "KL/std": 272.68536376953125, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.00134001555852592, "fcm_dpo/delta": -0.11473602056503296, "fcm_dpo/margin": 338.6988220214844, "fcm_dpo/q_t": 0.37980222702026367, "grad_norm": 19.321056365966797, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.45275765657424927, "logits/rejected": -0.4457949995994568, "logps/chosen": -362.1178283691406, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -742.4257202148438, "loss": 1.0206, "margin_dpo/margin_mean": 338.6988220214844, "margin_dpo/margin_std": 297.51031494140625, "step": 400 }, { "KL/chosen_KL_mean": -347.76898193359375, "KL/mean": -442.2925109863281, "KL/rejected_KL_mean": -536.8160400390625, "KL/std": 280.5494384765625, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.00160554563626647, "fcm_dpo/delta": 0.13878443837165833, "fcm_dpo/margin": 189.04698181152344, "fcm_dpo/q_t": 0.43937820196151733, "grad_norm": 21.756427764892578, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.4988451600074768, "logits/rejected": -0.492392897605896, "logps/chosen": -404.9320373535156, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -620.6085205078125, "loss": 1.1925, "margin_dpo/margin_mean": 189.0469970703125, "margin_dpo/margin_std": 374.1167297363281, "step": 401 }, { "KL/chosen_KL_mean": -258.14959716796875, "KL/mean": -438.02703857421875, "KL/rejected_KL_mean": -617.9044799804688, "KL/std": 310.0885009765625, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.0013267784379422665, "fcm_dpo/delta": -0.1641661822795868, "fcm_dpo/margin": 359.75494384765625, "fcm_dpo/q_t": 0.37116238474845886, "grad_norm": 20.190622329711914, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.4920314848423004, "logits/rejected": -0.480332612991333, "logps/chosen": -308.88995361328125, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -698.9505615234375, "loss": 1.0061, "margin_dpo/margin_mean": 359.75494384765625, "margin_dpo/margin_std": 319.1515808105469, "step": 402 }, { "KL/chosen_KL_mean": -292.50390625, "KL/mean": -428.873291015625, "KL/rejected_KL_mean": -565.24267578125, "KL/std": 291.69244384765625, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0013493448495864868, "fcm_dpo/delta": 0.050415799021720886, "fcm_dpo/margin": 272.73876953125, "fcm_dpo/q_t": 0.4170788526535034, "grad_norm": 22.705007553100586, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.4615529775619507, "logits/rejected": -0.4557945728302002, "logps/chosen": -339.6512145996094, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -642.50927734375, "loss": 1.0941, "margin_dpo/margin_mean": 272.73876953125, "margin_dpo/margin_std": 306.66265869140625, "step": 403 }, { "KL/chosen_KL_mean": -336.2072448730469, "KL/mean": -466.8104248046875, "KL/rejected_KL_mean": -597.41357421875, "KL/std": 278.45245361328125, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.001444081193767488, "fcm_dpo/delta": 0.03668762370944023, "fcm_dpo/margin": 261.20635986328125, "fcm_dpo/q_t": 0.41537854075431824, "grad_norm": 29.176677703857422, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.45944684743881226, "logits/rejected": -0.4487238824367523, "logps/chosen": -384.08251953125, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -674.568603515625, "loss": 1.1104, "margin_dpo/margin_mean": 261.20635986328125, "margin_dpo/margin_std": 361.2408447265625, "step": 404 }, { "KL/chosen_KL_mean": -388.7740478515625, "KL/mean": -496.8353271484375, "KL/rejected_KL_mean": -604.8966064453125, "KL/std": 312.28668212890625, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0014538425020873547, "fcm_dpo/delta": -0.026516491547226906, "fcm_dpo/margin": 216.12255859375, "fcm_dpo/q_t": 0.42757102847099304, "grad_norm": 29.22574234008789, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.5201287865638733, "logits/rejected": -0.5093380212783813, "logps/chosen": -453.9369812011719, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -692.0834350585938, "loss": 1.1806, "margin_dpo/margin_mean": 216.12255859375, "margin_dpo/margin_std": 396.87908935546875, "step": 405 }, { "KL/chosen_KL_mean": -355.637451171875, "KL/mean": -501.5604248046875, "KL/rejected_KL_mean": -647.4833984375, "KL/std": 317.0197448730469, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0014266908401623368, "fcm_dpo/delta": -0.033352650701999664, "fcm_dpo/margin": 291.8459167480469, "fcm_dpo/q_t": 0.4017193913459778, "grad_norm": 21.75078773498535, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.5137743949890137, "logits/rejected": -0.5227109789848328, "logps/chosen": -405.3782958984375, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -739.56201171875, "loss": 1.0792, "margin_dpo/margin_mean": 291.8459167480469, "margin_dpo/margin_std": 360.86712646484375, "step": 406 }, { "KL/chosen_KL_mean": -375.3631896972656, "KL/mean": -470.50189208984375, "KL/rejected_KL_mean": -565.6405639648438, "KL/std": 258.78857421875, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0013535934267565608, "fcm_dpo/delta": 0.02188667096197605, "fcm_dpo/margin": 190.27740478515625, "fcm_dpo/q_t": 0.43996331095695496, "grad_norm": 23.94710350036621, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.5585223436355591, "logits/rejected": -0.5322338938713074, "logps/chosen": -431.69390869140625, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -643.1527099609375, "loss": 1.1953, "margin_dpo/margin_mean": 190.2774200439453, "margin_dpo/margin_std": 333.7882385253906, "step": 407 }, { "KL/chosen_KL_mean": -389.05035400390625, "KL/mean": -490.5750427246094, "KL/rejected_KL_mean": -592.0997314453125, "KL/std": 244.0631866455078, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0016291262581944466, "fcm_dpo/delta": 0.09969305992126465, "fcm_dpo/margin": 203.04937744140625, "fcm_dpo/q_t": 0.4286070466041565, "grad_norm": 28.127901077270508, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.5659944415092468, "logits/rejected": -0.5408339500427246, "logps/chosen": -458.83966064453125, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -682.1966552734375, "loss": 1.1308, "margin_dpo/margin_mean": 203.04937744140625, "margin_dpo/margin_std": 274.0846862792969, "step": 408 }, { "KL/chosen_KL_mean": -381.6026306152344, "KL/mean": -485.71112060546875, "KL/rejected_KL_mean": -589.819580078125, "KL/std": 286.2164306640625, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0018248842097818851, "fcm_dpo/delta": 0.02699420042335987, "fcm_dpo/margin": 208.21694946289062, "fcm_dpo/q_t": 0.4149779677391052, "grad_norm": 42.70933532714844, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.4948524832725525, "logits/rejected": -0.47649848461151123, "logps/chosen": -448.9200744628906, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -674.723876953125, "loss": 1.1244, "margin_dpo/margin_mean": 208.21694946289062, "margin_dpo/margin_std": 319.69189453125, "step": 409 }, { "KL/chosen_KL_mean": -351.9015808105469, "KL/mean": -478.4359130859375, "KL/rejected_KL_mean": -604.9702758789062, "KL/std": 285.3361511230469, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0016538414638489485, "fcm_dpo/delta": -0.03480095416307449, "fcm_dpo/margin": 253.06863403320312, "fcm_dpo/q_t": 0.3992757201194763, "grad_norm": 35.037715911865234, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.5824193954467773, "logits/rejected": -0.5856485366821289, "logps/chosen": -403.366943359375, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -688.169189453125, "loss": 1.079, "margin_dpo/margin_mean": 253.06863403320312, "margin_dpo/margin_std": 324.7110595703125, "step": 410 }, { "KL/chosen_KL_mean": -374.0163269042969, "KL/mean": -491.8541259765625, "KL/rejected_KL_mean": -609.69189453125, "KL/std": 309.1568603515625, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0014088283060118556, "fcm_dpo/delta": -0.08271211385726929, "fcm_dpo/margin": 235.67556762695312, "fcm_dpo/q_t": 0.4166325628757477, "grad_norm": 32.5038948059082, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.5615779757499695, "logits/rejected": -0.566271185874939, "logps/chosen": -426.3236083984375, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -690.3868408203125, "loss": 1.1475, "margin_dpo/margin_mean": 235.67556762695312, "margin_dpo/margin_std": 364.743408203125, "step": 411 }, { "KL/chosen_KL_mean": -376.010009765625, "KL/mean": -504.2699279785156, "KL/rejected_KL_mean": -632.5298461914062, "KL/std": 297.99005126953125, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0014877468347549438, "fcm_dpo/delta": 0.028690597042441368, "fcm_dpo/margin": 256.51983642578125, "fcm_dpo/q_t": 0.4138166904449463, "grad_norm": 27.262081146240234, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.6011987924575806, "logits/rejected": -0.6305391788482666, "logps/chosen": -429.1541442871094, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -732.5906372070312, "loss": 1.1073, "margin_dpo/margin_mean": 256.51983642578125, "margin_dpo/margin_std": 358.5880126953125, "step": 412 }, { "KL/chosen_KL_mean": -413.91949462890625, "KL/mean": -544.1796264648438, "KL/rejected_KL_mean": -674.4398193359375, "KL/std": 308.14947509765625, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0015282519161701202, "fcm_dpo/delta": 0.0018488653004169464, "fcm_dpo/margin": 260.52032470703125, "fcm_dpo/q_t": 0.4066236615180969, "grad_norm": 31.31689453125, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.5308142304420471, "logits/rejected": -0.5381065607070923, "logps/chosen": -475.5014343261719, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -773.9132080078125, "loss": 1.0953, "margin_dpo/margin_mean": 260.52032470703125, "margin_dpo/margin_std": 349.33416748046875, "step": 413 }, { "KL/chosen_KL_mean": -376.8102722167969, "KL/mean": -513.2676391601562, "KL/rejected_KL_mean": -649.7249755859375, "KL/std": 290.8931884765625, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0015321837272495031, "fcm_dpo/delta": -0.043847713619470596, "fcm_dpo/margin": 272.91473388671875, "fcm_dpo/q_t": 0.3983882963657379, "grad_norm": 32.68032455444336, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.5271749496459961, "logits/rejected": -0.5175067186355591, "logps/chosen": -423.4417724609375, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -737.3715209960938, "loss": 1.0877, "margin_dpo/margin_mean": 272.91473388671875, "margin_dpo/margin_std": 341.33642578125, "step": 414 }, { "KL/chosen_KL_mean": -399.9847412109375, "KL/mean": -505.3280029296875, "KL/rejected_KL_mean": -610.6712646484375, "KL/std": 276.31341552734375, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0015684034442529082, "fcm_dpo/delta": 0.10484690964221954, "fcm_dpo/margin": 210.68653869628906, "fcm_dpo/q_t": 0.42983919382095337, "grad_norm": 25.915630340576172, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.5725701451301575, "logits/rejected": -0.567946195602417, "logps/chosen": -478.60302734375, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -711.1488037109375, "loss": 1.144, "margin_dpo/margin_mean": 210.68653869628906, "margin_dpo/margin_std": 311.3158264160156, "step": 415 }, { "KL/chosen_KL_mean": -365.5072021484375, "KL/mean": -527.9451904296875, "KL/rejected_KL_mean": -690.3831787109375, "KL/std": 313.2249755859375, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.0014323981013149023, "fcm_dpo/delta": -0.1226118803024292, "fcm_dpo/margin": 324.8759460449219, "fcm_dpo/q_t": 0.37992632389068604, "grad_norm": 27.974157333374023, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.5598719120025635, "logits/rejected": -0.564556896686554, "logps/chosen": -423.7863464355469, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -780.951904296875, "loss": 1.024, "margin_dpo/margin_mean": 324.8759460449219, "margin_dpo/margin_std": 312.55938720703125, "step": 416 }, { "KL/chosen_KL_mean": -353.57537841796875, "KL/mean": -489.65478515625, "KL/rejected_KL_mean": -625.7341918945312, "KL/std": 285.030029296875, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.0013993415050208569, "fcm_dpo/delta": 0.030016159638762474, "fcm_dpo/margin": 272.1588134765625, "fcm_dpo/q_t": 0.4119567573070526, "grad_norm": 29.711999893188477, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.5493444204330444, "logits/rejected": -0.5241958498954773, "logps/chosen": -403.7740783691406, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -693.8860473632812, "loss": 1.0801, "margin_dpo/margin_mean": 272.1588439941406, "margin_dpo/margin_std": 288.48846435546875, "step": 417 }, { "KL/chosen_KL_mean": -389.98394775390625, "KL/mean": -531.188232421875, "KL/rejected_KL_mean": -672.392578125, "KL/std": 321.02679443359375, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.0014645163901150227, "fcm_dpo/delta": -0.04145345836877823, "fcm_dpo/margin": 282.40869140625, "fcm_dpo/q_t": 0.4018082022666931, "grad_norm": 22.506240844726562, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.6127409338951111, "logits/rejected": -0.5916212201118469, "logps/chosen": -471.96240234375, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -777.0841064453125, "loss": 1.1022, "margin_dpo/margin_mean": 282.40869140625, "margin_dpo/margin_std": 376.09161376953125, "step": 418 }, { "KL/chosen_KL_mean": -354.58636474609375, "KL/mean": -506.98297119140625, "KL/rejected_KL_mean": -659.379638671875, "KL/std": 276.9751281738281, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0013055275194346905, "fcm_dpo/delta": 0.0034003122709691525, "fcm_dpo/margin": 304.79327392578125, "fcm_dpo/q_t": 0.4058445692062378, "grad_norm": 25.18451499938965, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.5361112356185913, "logits/rejected": -0.5325924158096313, "logps/chosen": -407.5350341796875, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -750.9627685546875, "loss": 1.0692, "margin_dpo/margin_mean": 304.79327392578125, "margin_dpo/margin_std": 320.16162109375, "step": 419 }, { "KL/chosen_KL_mean": -448.81329345703125, "KL/mean": -544.2679443359375, "KL/rejected_KL_mean": -639.7225341796875, "KL/std": 302.1771240234375, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.001609490136615932, "fcm_dpo/delta": 0.12760579586029053, "fcm_dpo/margin": 190.9093475341797, "fcm_dpo/q_t": 0.43786126375198364, "grad_norm": 57.40446090698242, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.5600706338882446, "logits/rejected": -0.5160965919494629, "logps/chosen": -526.5831909179688, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -709.04248046875, "loss": 1.2016, "margin_dpo/margin_mean": 190.9093475341797, "margin_dpo/margin_std": 403.58026123046875, "step": 420 }, { "KL/chosen_KL_mean": -383.8134765625, "KL/mean": -487.9678955078125, "KL/rejected_KL_mean": -592.122314453125, "KL/std": 293.6025390625, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0018344377167522907, "fcm_dpo/delta": 0.018156960606575012, "fcm_dpo/margin": 208.3088836669922, "fcm_dpo/q_t": 0.4109729826450348, "grad_norm": 32.440040588378906, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.6202581524848938, "logits/rejected": -0.6075379848480225, "logps/chosen": -437.579345703125, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -681.40380859375, "loss": 1.1322, "margin_dpo/margin_mean": 208.3088836669922, "margin_dpo/margin_std": 319.2204895019531, "step": 421 }, { "KL/chosen_KL_mean": -423.02252197265625, "KL/mean": -545.48046875, "KL/rejected_KL_mean": -667.9382934570312, "KL/std": 296.6886901855469, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0016887241508811712, "fcm_dpo/delta": -0.023193474858999252, "fcm_dpo/margin": 244.91578674316406, "fcm_dpo/q_t": 0.40264299511909485, "grad_norm": 44.55949783325195, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.5801171064376831, "logits/rejected": -0.5803545713424683, "logps/chosen": -491.65631103515625, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -755.8018188476562, "loss": 1.0928, "margin_dpo/margin_mean": 244.91578674316406, "margin_dpo/margin_std": 343.1561584472656, "step": 422 }, { "KL/chosen_KL_mean": -408.7502136230469, "KL/mean": -549.06884765625, "KL/rejected_KL_mean": -689.3875122070312, "KL/std": 292.96270751953125, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0015233319718390703, "fcm_dpo/delta": -0.05018993839621544, "fcm_dpo/margin": 280.6373291015625, "fcm_dpo/q_t": 0.3961368799209595, "grad_norm": 26.388099670410156, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.5856159329414368, "logits/rejected": -0.5923899412155151, "logps/chosen": -463.7405090332031, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -775.694091796875, "loss": 1.0592, "margin_dpo/margin_mean": 280.6373596191406, "margin_dpo/margin_std": 320.4229431152344, "step": 423 }, { "KL/chosen_KL_mean": -368.3785400390625, "KL/mean": -496.5576477050781, "KL/rejected_KL_mean": -624.7366943359375, "KL/std": 285.81842041015625, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0015009802300482988, "fcm_dpo/delta": 0.022251751273870468, "fcm_dpo/margin": 256.358154296875, "fcm_dpo/q_t": 0.4135265052318573, "grad_norm": 33.62800598144531, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.6009132266044617, "logits/rejected": -0.5754563808441162, "logps/chosen": -424.39044189453125, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -691.2156982421875, "loss": 1.106, "margin_dpo/margin_mean": 256.358154296875, "margin_dpo/margin_std": 352.6119384765625, "step": 424 }, { "KL/chosen_KL_mean": -417.15570068359375, "KL/mean": -515.7978515625, "KL/rejected_KL_mean": -614.43994140625, "KL/std": 278.95025634765625, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.0017970511689782143, "fcm_dpo/delta": 0.062237467616796494, "fcm_dpo/margin": 197.28427124023438, "fcm_dpo/q_t": 0.4205625057220459, "grad_norm": 29.63746452331543, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.5688070058822632, "logits/rejected": -0.5690561532974243, "logps/chosen": -464.024658203125, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -710.3654174804688, "loss": 1.1556, "margin_dpo/margin_mean": 197.28427124023438, "margin_dpo/margin_std": 347.9555358886719, "step": 425 }, { "KL/chosen_KL_mean": -396.1029052734375, "KL/mean": -517.9900512695312, "KL/rejected_KL_mean": -639.877197265625, "KL/std": 274.98834228515625, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0017517898231744766, "fcm_dpo/delta": -0.0511149987578392, "fcm_dpo/margin": 243.7742919921875, "fcm_dpo/q_t": 0.3971126079559326, "grad_norm": 32.52616882324219, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.5753895044326782, "logits/rejected": -0.5384079813957214, "logps/chosen": -472.6864318847656, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -721.143798828125, "loss": 1.0773, "margin_dpo/margin_mean": 243.7742919921875, "margin_dpo/margin_std": 307.3427734375, "step": 426 }, { "KL/chosen_KL_mean": -360.6373596191406, "KL/mean": -457.64605712890625, "KL/rejected_KL_mean": -554.6547241210938, "KL/std": 240.3179473876953, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.001793712843209505, "fcm_dpo/delta": 0.08003441989421844, "fcm_dpo/margin": 194.01736450195312, "fcm_dpo/q_t": 0.4255453646183014, "grad_norm": 28.326711654663086, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.6111181974411011, "logits/rejected": -0.5922002792358398, "logps/chosen": -425.4912414550781, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -633.220703125, "loss": 1.1332, "margin_dpo/margin_mean": 194.01736450195312, "margin_dpo/margin_std": 291.666015625, "step": 427 }, { "KL/chosen_KL_mean": -443.464111328125, "KL/mean": -577.4393310546875, "KL/rejected_KL_mean": -711.41455078125, "KL/std": 321.87640380859375, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.001614520326256752, "fcm_dpo/delta": -0.07145649194717407, "fcm_dpo/margin": 267.9504089355469, "fcm_dpo/q_t": 0.394869327545166, "grad_norm": 29.2352294921875, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.5986416339874268, "logits/rejected": -0.6018632650375366, "logps/chosen": -506.10076904296875, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -814.6963500976562, "loss": 1.0951, "margin_dpo/margin_mean": 267.9504089355469, "margin_dpo/margin_std": 396.5560302734375, "step": 428 }, { "KL/chosen_KL_mean": -453.14031982421875, "KL/mean": -558.6182861328125, "KL/rejected_KL_mean": -664.09619140625, "KL/std": 281.262939453125, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0015741197858005762, "fcm_dpo/delta": -0.040317293256521225, "fcm_dpo/margin": 210.95590209960938, "fcm_dpo/q_t": 0.4206579625606537, "grad_norm": 29.22441864013672, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.6086622476577759, "logits/rejected": -0.581855297088623, "logps/chosen": -534.3743286132812, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -755.89111328125, "loss": 1.1602, "margin_dpo/margin_mean": 210.95590209960938, "margin_dpo/margin_std": 341.42901611328125, "step": 429 }, { "KL/chosen_KL_mean": -381.8338623046875, "KL/mean": -485.0985107421875, "KL/rejected_KL_mean": -588.3631591796875, "KL/std": 265.72589111328125, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.001708697760477662, "fcm_dpo/delta": 0.06908033788204193, "fcm_dpo/margin": 206.52932739257812, "fcm_dpo/q_t": 0.4227146804332733, "grad_norm": 40.607749938964844, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.5892158150672913, "logits/rejected": -0.5879628658294678, "logps/chosen": -442.754150390625, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -692.7860107421875, "loss": 1.1248, "margin_dpo/margin_mean": 206.52932739257812, "margin_dpo/margin_std": 297.7901611328125, "step": 430 }, { "KL/chosen_KL_mean": -354.79736328125, "KL/mean": -504.74810791015625, "KL/rejected_KL_mean": -654.6988525390625, "KL/std": 294.78875732421875, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.0014870925806462765, "fcm_dpo/delta": -0.0890204980969429, "fcm_dpo/margin": 299.9014892578125, "fcm_dpo/q_t": 0.3885713517665863, "grad_norm": 27.04310417175293, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.6147454977035522, "logits/rejected": -0.6116945743560791, "logps/chosen": -412.1461181640625, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -747.5390625, "loss": 1.0492, "margin_dpo/margin_mean": 299.9014892578125, "margin_dpo/margin_std": 333.67730712890625, "step": 431 }, { "KL/chosen_KL_mean": -351.54315185546875, "KL/mean": -501.9591979980469, "KL/rejected_KL_mean": -652.3753051757812, "KL/std": 292.12811279296875, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0014298271853476763, "fcm_dpo/delta": -0.05924157053232193, "fcm_dpo/margin": 300.8321533203125, "fcm_dpo/q_t": 0.39376306533813477, "grad_norm": 31.670360565185547, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.6010767221450806, "logits/rejected": -0.5897285342216492, "logps/chosen": -422.617919921875, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -750.954833984375, "loss": 1.0523, "margin_dpo/margin_mean": 300.8321533203125, "margin_dpo/margin_std": 296.0356140136719, "step": 432 }, { "KL/chosen_KL_mean": -445.81146240234375, "KL/mean": -543.8163452148438, "KL/rejected_KL_mean": -641.8211669921875, "KL/std": 302.93316650390625, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0015859071863815188, "fcm_dpo/delta": 0.12226778268814087, "fcm_dpo/margin": 196.00973510742188, "fcm_dpo/q_t": 0.43242147564888, "grad_norm": 38.35377502441406, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.6371247172355652, "logits/rejected": -0.6237683892250061, "logps/chosen": -504.08465576171875, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -737.7720947265625, "loss": 1.1783, "margin_dpo/margin_mean": 196.00973510742188, "margin_dpo/margin_std": 365.50811767578125, "step": 433 }, { "KL/chosen_KL_mean": -385.5121765136719, "KL/mean": -504.4186096191406, "KL/rejected_KL_mean": -623.3250122070312, "KL/std": 285.87884521484375, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0016222589183598757, "fcm_dpo/delta": 0.019110068678855896, "fcm_dpo/margin": 237.81283569335938, "fcm_dpo/q_t": 0.4163554310798645, "grad_norm": 31.304502487182617, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.6609657406806946, "logits/rejected": -0.6543369889259338, "logps/chosen": -447.48590087890625, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -701.8236083984375, "loss": 1.113, "margin_dpo/margin_mean": 237.8128204345703, "margin_dpo/margin_std": 357.0226745605469, "step": 434 }, { "KL/chosen_KL_mean": -366.02606201171875, "KL/mean": -493.39617919921875, "KL/rejected_KL_mean": -620.766357421875, "KL/std": 283.81414794921875, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.001610426465049386, "fcm_dpo/delta": -0.020799141377210617, "fcm_dpo/margin": 254.740234375, "fcm_dpo/q_t": 0.4033718705177307, "grad_norm": 30.6375789642334, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.6943074464797974, "logits/rejected": -0.6805083751678467, "logps/chosen": -417.5281066894531, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -708.333251953125, "loss": 1.0828, "margin_dpo/margin_mean": 254.74024963378906, "margin_dpo/margin_std": 321.0318603515625, "step": 435 }, { "KL/chosen_KL_mean": -373.72186279296875, "KL/mean": -489.437744140625, "KL/rejected_KL_mean": -605.153564453125, "KL/std": 257.1832580566406, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0016507278196513653, "fcm_dpo/delta": 0.02712010033428669, "fcm_dpo/margin": 231.43165588378906, "fcm_dpo/q_t": 0.41315099596977234, "grad_norm": 30.587175369262695, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.6558730602264404, "logits/rejected": -0.6176996827125549, "logps/chosen": -445.1256103515625, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -687.88134765625, "loss": 1.103, "margin_dpo/margin_mean": 231.43167114257812, "margin_dpo/margin_std": 304.65887451171875, "step": 436 }, { "KL/chosen_KL_mean": -369.8938293457031, "KL/mean": -485.63037109375, "KL/rejected_KL_mean": -601.366943359375, "KL/std": 242.96083068847656, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0017139371484518051, "fcm_dpo/delta": 0.004689780995249748, "fcm_dpo/margin": 231.47308349609375, "fcm_dpo/q_t": 0.40768975019454956, "grad_norm": 25.386877059936523, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.6826403141021729, "logits/rejected": -0.6562011241912842, "logps/chosen": -434.6380920410156, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -683.4104614257812, "loss": 1.0842, "margin_dpo/margin_mean": 231.47308349609375, "margin_dpo/margin_std": 282.5240478515625, "step": 437 }, { "KL/chosen_KL_mean": -385.8797302246094, "KL/mean": -514.9544677734375, "KL/rejected_KL_mean": -644.0291748046875, "KL/std": 284.6157531738281, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0015868828631937504, "fcm_dpo/delta": -0.027686219662427902, "fcm_dpo/margin": 258.14947509765625, "fcm_dpo/q_t": 0.4011117219924927, "grad_norm": 36.56245803833008, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.6592687368392944, "logits/rejected": -0.6328068971633911, "logps/chosen": -444.8984069824219, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -727.1060180664062, "loss": 1.0781, "margin_dpo/margin_mean": 258.14947509765625, "margin_dpo/margin_std": 307.8912658691406, "step": 438 }, { "KL/chosen_KL_mean": -397.440185546875, "KL/mean": -502.10455322265625, "KL/rejected_KL_mean": -606.7689208984375, "KL/std": 281.3536682128906, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.0014228572836145759, "fcm_dpo/delta": -0.07547041773796082, "fcm_dpo/margin": 209.32879638671875, "fcm_dpo/q_t": 0.4233711063861847, "grad_norm": 26.591917037963867, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.6513150930404663, "logits/rejected": -0.6519348621368408, "logps/chosen": -451.2242431640625, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -690.75439453125, "loss": 1.1686, "margin_dpo/margin_mean": 209.32876586914062, "margin_dpo/margin_std": 326.3582763671875, "step": 439 }, { "KL/chosen_KL_mean": -424.1907043457031, "KL/mean": -549.408203125, "KL/rejected_KL_mean": -674.625732421875, "KL/std": 355.38433837890625, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0015238459454849362, "fcm_dpo/delta": 0.02612234093248844, "fcm_dpo/margin": 250.43495178222656, "fcm_dpo/q_t": 0.41634005308151245, "grad_norm": 36.65753936767578, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.6994473934173584, "logits/rejected": -0.6900968551635742, "logps/chosen": -502.7574462890625, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -771.1234741210938, "loss": 1.1072, "margin_dpo/margin_mean": 250.43496704101562, "margin_dpo/margin_std": 356.3500671386719, "step": 440 }, { "KL/chosen_KL_mean": -473.6324462890625, "KL/mean": -599.330322265625, "KL/rejected_KL_mean": -725.0281982421875, "KL/std": 360.2265625, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.0015725798439234495, "fcm_dpo/delta": 0.00260746106505394, "fcm_dpo/margin": 251.39569091796875, "fcm_dpo/q_t": 0.41514575481414795, "grad_norm": 36.14642333984375, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.6896128058433533, "logits/rejected": -0.6732909679412842, "logps/chosen": -534.4569091796875, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -821.4990234375, "loss": 1.1421, "margin_dpo/margin_mean": 251.39569091796875, "margin_dpo/margin_std": 446.50982666015625, "step": 441 }, { "KL/chosen_KL_mean": -402.5406494140625, "KL/mean": -554.17041015625, "KL/rejected_KL_mean": -705.8001098632812, "KL/std": 349.5821533203125, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0014577994588762522, "fcm_dpo/delta": -0.08352486789226532, "fcm_dpo/margin": 303.25946044921875, "fcm_dpo/q_t": 0.3944917619228363, "grad_norm": 34.06388473510742, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.6561492085456848, "logits/rejected": -0.6455131769180298, "logps/chosen": -449.5518798828125, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -782.33935546875, "loss": 1.0824, "margin_dpo/margin_mean": 303.2594299316406, "margin_dpo/margin_std": 401.3167419433594, "step": 442 }, { "KL/chosen_KL_mean": -470.502197265625, "KL/mean": -563.5130615234375, "KL/rejected_KL_mean": -656.52392578125, "KL/std": 353.9302673339844, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0013765229377895594, "fcm_dpo/delta": 0.0382419228553772, "fcm_dpo/margin": 186.021728515625, "fcm_dpo/q_t": 0.4441724121570587, "grad_norm": 40.15092086791992, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.6763448119163513, "logits/rejected": -0.6484371423721313, "logps/chosen": -541.7752075195312, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -743.203857421875, "loss": 1.2325, "margin_dpo/margin_mean": 186.021728515625, "margin_dpo/margin_std": 431.4246826171875, "step": 443 }, { "KL/chosen_KL_mean": -461.93658447265625, "KL/mean": -613.6505126953125, "KL/rejected_KL_mean": -765.364501953125, "KL/std": 370.37298583984375, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0013226409209892154, "fcm_dpo/delta": -0.005307670682668686, "fcm_dpo/margin": 303.42791748046875, "fcm_dpo/q_t": 0.4082014858722687, "grad_norm": 33.01715087890625, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.6438364386558533, "logits/rejected": -0.6463167667388916, "logps/chosen": -519.1502685546875, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -862.619384765625, "loss": 1.0982, "margin_dpo/margin_mean": 303.42791748046875, "margin_dpo/margin_std": 423.27032470703125, "step": 444 }, { "KL/chosen_KL_mean": -402.2928771972656, "KL/mean": -540.4663696289062, "KL/rejected_KL_mean": -678.639892578125, "KL/std": 285.4811706542969, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0014118524268269539, "fcm_dpo/delta": 0.015325665473937988, "fcm_dpo/margin": 276.34698486328125, "fcm_dpo/q_t": 0.4115728735923767, "grad_norm": 34.8456916809082, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.6687377691268921, "logits/rejected": -0.6554244756698608, "logps/chosen": -469.5926818847656, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -771.322509765625, "loss": 1.0992, "margin_dpo/margin_mean": 276.34698486328125, "margin_dpo/margin_std": 367.69573974609375, "step": 445 }, { "KL/chosen_KL_mean": -360.5770568847656, "KL/mean": -518.53955078125, "KL/rejected_KL_mean": -676.5020751953125, "KL/std": 312.20379638671875, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.00137224025093019, "fcm_dpo/delta": -0.06505385041236877, "fcm_dpo/margin": 315.92498779296875, "fcm_dpo/q_t": 0.39190033078193665, "grad_norm": 26.528141021728516, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.6326192617416382, "logits/rejected": -0.6437954902648926, "logps/chosen": -419.675537109375, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -777.7662353515625, "loss": 1.0557, "margin_dpo/margin_mean": 315.92498779296875, "margin_dpo/margin_std": 325.07269287109375, "step": 446 }, { "KL/chosen_KL_mean": -383.163330078125, "KL/mean": -541.930908203125, "KL/rejected_KL_mean": -700.698486328125, "KL/std": 360.7058410644531, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0012335461797192693, "fcm_dpo/delta": 0.013036707416176796, "fcm_dpo/margin": 317.53509521484375, "fcm_dpo/q_t": 0.41153937578201294, "grad_norm": 25.36223602294922, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.6875368356704712, "logits/rejected": -0.7032911777496338, "logps/chosen": -439.2386779785156, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -799.3931884765625, "loss": 1.0899, "margin_dpo/margin_mean": 317.5351257324219, "margin_dpo/margin_std": 410.7313232421875, "step": 447 }, { "KL/chosen_KL_mean": -426.30487060546875, "KL/mean": -575.4131469726562, "KL/rejected_KL_mean": -724.521484375, "KL/std": 317.2506103515625, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0013242221903055906, "fcm_dpo/delta": 0.004273921251296997, "fcm_dpo/margin": 298.21661376953125, "fcm_dpo/q_t": 0.4076346755027771, "grad_norm": 29.84330940246582, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.5890108942985535, "logits/rejected": -0.5925085544586182, "logps/chosen": -486.3087158203125, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -826.7861328125, "loss": 1.0881, "margin_dpo/margin_mean": 298.21661376953125, "margin_dpo/margin_std": 366.0950927734375, "step": 448 }, { "KL/chosen_KL_mean": -432.274658203125, "KL/mean": -588.0142822265625, "KL/rejected_KL_mean": -743.7538452148438, "KL/std": 391.2838134765625, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.0012739100493490696, "fcm_dpo/delta": 0.005134426523000002, "fcm_dpo/margin": 311.4792175292969, "fcm_dpo/q_t": 0.4119264483451843, "grad_norm": 24.36716079711914, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.6395070552825928, "logits/rejected": -0.6301548480987549, "logps/chosen": -499.80126953125, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -832.3507690429688, "loss": 1.1068, "margin_dpo/margin_mean": 311.4792175292969, "margin_dpo/margin_std": 466.9681396484375, "step": 449 }, { "KL/chosen_KL_mean": -350.74542236328125, "KL/mean": -518.7066040039062, "KL/rejected_KL_mean": -686.6678466796875, "KL/std": 333.5655822753906, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0012484046164900064, "fcm_dpo/delta": -0.03335545212030411, "fcm_dpo/margin": 335.9223937988281, "fcm_dpo/q_t": 0.3976227641105652, "grad_norm": 36.70199203491211, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.6501277089118958, "logits/rejected": -0.6594793796539307, "logps/chosen": -407.853515625, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -789.4227294921875, "loss": 1.052, "margin_dpo/margin_mean": 335.92236328125, "margin_dpo/margin_std": 331.643310546875, "step": 450 }, { "KL/chosen_KL_mean": -478.4472351074219, "KL/mean": -596.5457763671875, "KL/rejected_KL_mean": -714.6444091796875, "KL/std": 388.7025146484375, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0014245854690670967, "fcm_dpo/delta": 0.08672139793634415, "fcm_dpo/margin": 236.19717407226562, "fcm_dpo/q_t": 0.42556342482566833, "grad_norm": 40.344722747802734, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -0.6830481290817261, "logits/rejected": -0.6519405245780945, "logps/chosen": -536.9160766601562, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -787.5737915039062, "loss": 1.1783, "margin_dpo/margin_mean": 236.19717407226562, "margin_dpo/margin_std": 461.38665771484375, "step": 451 }, { "KL/chosen_KL_mean": -339.47760009765625, "KL/mean": -487.0719299316406, "KL/rejected_KL_mean": -634.666259765625, "KL/std": 301.2289733886719, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.0013748779892921448, "fcm_dpo/delta": -0.012026989832520485, "fcm_dpo/margin": 295.1885986328125, "fcm_dpo/q_t": 0.40248599648475647, "grad_norm": 21.888635635375977, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.6212879419326782, "logits/rejected": -0.638474702835083, "logps/chosen": -389.8681640625, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -732.4376831054688, "loss": 1.0635, "margin_dpo/margin_mean": 295.1885986328125, "margin_dpo/margin_std": 304.22344970703125, "step": 452 }, { "KL/chosen_KL_mean": -394.113525390625, "KL/mean": -535.056884765625, "KL/rejected_KL_mean": -676.0001831054688, "KL/std": 311.166259765625, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0013874982250854373, "fcm_dpo/delta": 0.012876830995082855, "fcm_dpo/margin": 281.88671875, "fcm_dpo/q_t": 0.4120361804962158, "grad_norm": 28.710693359375, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -0.6545387506484985, "logits/rejected": -0.635535478591919, "logps/chosen": -451.828369140625, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -758.2076416015625, "loss": 1.0973, "margin_dpo/margin_mean": 281.8866882324219, "margin_dpo/margin_std": 380.42938232421875, "step": 453 }, { "KL/chosen_KL_mean": -493.164306640625, "KL/mean": -647.89892578125, "KL/rejected_KL_mean": -802.6336669921875, "KL/std": 370.6231689453125, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0013456812594085932, "fcm_dpo/delta": -0.030497819185256958, "fcm_dpo/margin": 309.46929931640625, "fcm_dpo/q_t": 0.4019313454627991, "grad_norm": 25.663516998291016, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -0.6898310780525208, "logits/rejected": -0.676418662071228, "logps/chosen": -554.1099853515625, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -887.5844116210938, "loss": 1.0874, "margin_dpo/margin_mean": 309.46929931640625, "margin_dpo/margin_std": 427.5249938964844, "step": 454 }, { "KL/chosen_KL_mean": -418.80328369140625, "KL/mean": -605.3193359375, "KL/rejected_KL_mean": -791.8353881835938, "KL/std": 368.81707763671875, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0012299984227865934, "fcm_dpo/delta": -0.10926243662834167, "fcm_dpo/margin": 373.0321044921875, "fcm_dpo/q_t": 0.38569873571395874, "grad_norm": 30.85955810546875, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.6721550226211548, "logits/rejected": -0.6930463314056396, "logps/chosen": -463.69000244140625, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -907.1368408203125, "loss": 1.0505, "margin_dpo/margin_mean": 373.0321044921875, "margin_dpo/margin_std": 423.5682678222656, "step": 455 }, { "KL/chosen_KL_mean": -449.27508544921875, "KL/mean": -635.3360595703125, "KL/rejected_KL_mean": -821.3970947265625, "KL/std": 394.14239501953125, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0010888128308579326, "fcm_dpo/delta": -0.00909638125449419, "fcm_dpo/margin": 372.12200927734375, "fcm_dpo/q_t": 0.40475431084632874, "grad_norm": 28.842674255371094, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.6700366139411926, "logits/rejected": -0.6916704773902893, "logps/chosen": -506.3118591308594, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -926.6149291992188, "loss": 1.063, "margin_dpo/margin_mean": 372.12200927734375, "margin_dpo/margin_std": 389.7081298828125, "step": 456 }, { "KL/chosen_KL_mean": -442.96075439453125, "KL/mean": -624.3992919921875, "KL/rejected_KL_mean": -805.837646484375, "KL/std": 381.4212646484375, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.0010792326647788286, "fcm_dpo/delta": 0.01370345987379551, "fcm_dpo/margin": 362.876953125, "fcm_dpo/q_t": 0.40837815403938293, "grad_norm": 28.204694747924805, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.6881248950958252, "logits/rejected": -0.68438720703125, "logps/chosen": -497.20330810546875, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -890.947265625, "loss": 1.0737, "margin_dpo/margin_mean": 362.876953125, "margin_dpo/margin_std": 379.9857177734375, "step": 457 }, { "KL/chosen_KL_mean": -499.1065368652344, "KL/mean": -653.4949951171875, "KL/rejected_KL_mean": -807.8834228515625, "KL/std": 364.198974609375, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0011557680554687977, "fcm_dpo/delta": 0.06521759927272797, "fcm_dpo/margin": 308.7769775390625, "fcm_dpo/q_t": 0.4231434762477875, "grad_norm": 32.81315231323242, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.7310912609100342, "logits/rejected": -0.7260030508041382, "logps/chosen": -554.515380859375, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -905.5667114257812, "loss": 1.123, "margin_dpo/margin_mean": 308.7769775390625, "margin_dpo/margin_std": 427.2161560058594, "step": 458 }, { "KL/chosen_KL_mean": -529.4247436523438, "KL/mean": -706.0894165039062, "KL/rejected_KL_mean": -882.7540283203125, "KL/std": 432.647216796875, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0011564318556338549, "fcm_dpo/delta": -0.021825630217790604, "fcm_dpo/margin": 353.329345703125, "fcm_dpo/q_t": 0.40472641587257385, "grad_norm": 32.47217559814453, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.7430839538574219, "logits/rejected": -0.7598717212677002, "logps/chosen": -575.9822387695312, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -968.922607421875, "loss": 1.0937, "margin_dpo/margin_mean": 353.329345703125, "margin_dpo/margin_std": 485.6087341308594, "step": 459 }, { "KL/chosen_KL_mean": -571.9495239257812, "KL/mean": -779.2578125, "KL/rejected_KL_mean": -986.5660400390625, "KL/std": 431.83953857421875, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0010669762268662453, "fcm_dpo/delta": -0.07710142433643341, "fcm_dpo/margin": 414.6165466308594, "fcm_dpo/q_t": 0.3918076157569885, "grad_norm": 30.98316764831543, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.830254316329956, "logits/rejected": -0.8624805212020874, "logps/chosen": -623.58447265625, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -1090.6854248046875, "loss": 1.0514, "margin_dpo/margin_mean": 414.6165466308594, "margin_dpo/margin_std": 476.18707275390625, "step": 460 }, { "KL/chosen_KL_mean": -605.7537841796875, "KL/mean": -766.1744384765625, "KL/rejected_KL_mean": -926.59521484375, "KL/std": 430.42913818359375, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0011395696783438325, "fcm_dpo/delta": 0.04857172816991806, "fcm_dpo/margin": 320.8414611816406, "fcm_dpo/q_t": 0.41997230052948, "grad_norm": 24.931995391845703, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -0.8122773170471191, "logits/rejected": -0.8271867036819458, "logps/chosen": -660.9356689453125, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -1013.0721435546875, "loss": 1.1224, "margin_dpo/margin_mean": 320.8414306640625, "margin_dpo/margin_std": 479.56878662109375, "step": 461 }, { "KL/chosen_KL_mean": -618.9212036132812, "KL/mean": -744.9371337890625, "KL/rejected_KL_mean": -870.9530639648438, "KL/std": 416.94464111328125, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.001313498243689537, "fcm_dpo/delta": 0.10240471363067627, "fcm_dpo/margin": 252.0318603515625, "fcm_dpo/q_t": 0.43281617760658264, "grad_norm": 40.46854019165039, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -0.8632235527038574, "logits/rejected": -0.8471617698669434, "logps/chosen": -688.8492431640625, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -949.794189453125, "loss": 1.1744, "margin_dpo/margin_mean": 252.03182983398438, "margin_dpo/margin_std": 479.5788269042969, "step": 462 }, { "KL/chosen_KL_mean": -603.7901611328125, "KL/mean": -779.239990234375, "KL/rejected_KL_mean": -954.6898803710938, "KL/std": 431.516357421875, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.0012737201759591699, "fcm_dpo/delta": -0.08446250110864639, "fcm_dpo/margin": 350.899658203125, "fcm_dpo/q_t": 0.39394649863243103, "grad_norm": 37.86167526245117, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.8599103689193726, "logits/rejected": -0.8692607879638672, "logps/chosen": -659.0645751953125, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -1043.71484375, "loss": 1.0823, "margin_dpo/margin_mean": 350.899658203125, "margin_dpo/margin_std": 486.9836120605469, "step": 463 }, { "KL/chosen_KL_mean": -577.685302734375, "KL/mean": -836.02734375, "KL/rejected_KL_mean": -1094.369384765625, "KL/std": 506.09954833984375, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0008683481719344854, "fcm_dpo/delta": -0.13791523873806, "fcm_dpo/margin": 516.6839599609375, "fcm_dpo/q_t": 0.37923869490623474, "grad_norm": 35.81996536254883, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.8401867151260376, "logits/rejected": -0.9024742841720581, "logps/chosen": -628.59765625, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -1196.858642578125, "loss": 1.0364, "margin_dpo/margin_mean": 516.6839599609375, "margin_dpo/margin_std": 519.8881225585938, "step": 464 }, { "KL/chosen_KL_mean": -585.9309692382812, "KL/mean": -794.7791748046875, "KL/rejected_KL_mean": -1003.6273193359375, "KL/std": 513.0881958007812, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.0009388852631673217, "fcm_dpo/delta": 0.007542621344327927, "fcm_dpo/margin": 417.69635009765625, "fcm_dpo/q_t": 0.4073883295059204, "grad_norm": 24.084501266479492, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.7998400926589966, "logits/rejected": -0.8214449882507324, "logps/chosen": -646.0478515625, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -1117.5733642578125, "loss": 1.0968, "margin_dpo/margin_mean": 417.69635009765625, "margin_dpo/margin_std": 542.79931640625, "step": 465 }, { "KL/chosen_KL_mean": -639.7386474609375, "KL/mean": -817.8701171875, "KL/rejected_KL_mean": -996.0015258789062, "KL/std": 474.357666015625, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0009801898850128055, "fcm_dpo/delta": 0.0780898779630661, "fcm_dpo/margin": 356.26287841796875, "fcm_dpo/q_t": 0.42381781339645386, "grad_norm": 24.35711669921875, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -0.8397573232650757, "logits/rejected": -0.840815007686615, "logps/chosen": -692.6595458984375, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -1086.3170166015625, "loss": 1.1348, "margin_dpo/margin_mean": 356.26287841796875, "margin_dpo/margin_std": 522.5521240234375, "step": 466 }, { "KL/chosen_KL_mean": -801.0821533203125, "KL/mean": -986.1063232421875, "KL/rejected_KL_mean": -1171.1304931640625, "KL/std": 607.9132080078125, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0009247926063835621, "fcm_dpo/delta": -0.06586673855781555, "fcm_dpo/margin": 370.04833984375, "fcm_dpo/q_t": 0.41659867763519287, "grad_norm": 38.04802703857422, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -0.9427855610847473, "logits/rejected": -0.934941291809082, "logps/chosen": -879.7979736328125, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -1273.99072265625, "loss": 1.179, "margin_dpo/margin_mean": 370.04833984375, "margin_dpo/margin_std": 719.1068115234375, "step": 467 }, { "KL/chosen_KL_mean": -630.8428955078125, "KL/mean": -867.4342041015625, "KL/rejected_KL_mean": -1104.025634765625, "KL/std": 526.3124389648438, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0008766874088905752, "fcm_dpo/delta": -0.026581648737192154, "fcm_dpo/margin": 473.18267822265625, "fcm_dpo/q_t": 0.4034311771392822, "grad_norm": 31.51830291748047, "learning_rate": 1.362737437810114e-07, "logits/chosen": -0.8827992677688599, "logits/rejected": -0.8895440101623535, "logps/chosen": -700.7782592773438, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -1205.054443359375, "loss": 1.0778, "margin_dpo/margin_mean": 473.18267822265625, "margin_dpo/margin_std": 609.1591796875, "step": 468 }, { "KL/chosen_KL_mean": -650.935302734375, "KL/mean": -885.42578125, "KL/rejected_KL_mean": -1119.916259765625, "KL/std": 477.8541259765625, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0008414213662035763, "fcm_dpo/delta": 0.0032104477286338806, "fcm_dpo/margin": 468.98089599609375, "fcm_dpo/q_t": 0.4071598947048187, "grad_norm": 36.97335433959961, "learning_rate": 1.351323902551631e-07, "logits/chosen": -0.899463415145874, "logits/rejected": -0.9126312732696533, "logps/chosen": -719.0599975585938, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -1224.70263671875, "loss": 1.0813, "margin_dpo/margin_mean": 468.98089599609375, "margin_dpo/margin_std": 533.3753662109375, "step": 469 }, { "KL/chosen_KL_mean": -553.0526123046875, "KL/mean": -755.4017333984375, "KL/rejected_KL_mean": -957.7508544921875, "KL/std": 496.32080078125, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0009463757742196321, "fcm_dpo/delta": 0.022916335612535477, "fcm_dpo/margin": 404.69830322265625, "fcm_dpo/q_t": 0.41216540336608887, "grad_norm": 23.679851531982422, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.8496056199073792, "logits/rejected": -0.8591802716255188, "logps/chosen": -596.844482421875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -1040.4537353515625, "loss": 1.1079, "margin_dpo/margin_mean": 404.69830322265625, "margin_dpo/margin_std": 562.834228515625, "step": 470 }, { "KL/chosen_KL_mean": -711.413330078125, "KL/mean": -888.9149169921875, "KL/rejected_KL_mean": -1066.4163818359375, "KL/std": 547.1355590820312, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.0008889293530955911, "fcm_dpo/delta": -0.01932104118168354, "fcm_dpo/margin": 355.0030517578125, "fcm_dpo/q_t": 0.4268658757209778, "grad_norm": 23.575862884521484, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -0.9526507258415222, "logits/rejected": -0.9646092057228088, "logps/chosen": -774.7528686523438, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -1150.02685546875, "loss": 1.1604, "margin_dpo/margin_mean": 355.0030517578125, "margin_dpo/margin_std": 585.3763427734375, "step": 471 }, { "KL/chosen_KL_mean": -698.0546875, "KL/mean": -902.4146728515625, "KL/rejected_KL_mean": -1106.774658203125, "KL/std": 597.7765502929688, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.0009217117330990732, "fcm_dpo/delta": 0.0372597835958004, "fcm_dpo/margin": 408.719970703125, "fcm_dpo/q_t": 0.41907206177711487, "grad_norm": 28.87525749206543, "learning_rate": 1.317266107909975e-07, "logits/chosen": -0.9155426025390625, "logits/rejected": -0.8952895402908325, "logps/chosen": -781.7208251953125, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -1223.98388671875, "loss": 1.1323, "margin_dpo/margin_mean": 408.719970703125, "margin_dpo/margin_std": 667.8204345703125, "step": 472 }, { "KL/chosen_KL_mean": -824.3423461914062, "KL/mean": -920.7506103515625, "KL/rejected_KL_mean": -1017.1588134765625, "KL/std": 608.287353515625, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0009577026357874274, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 192.81646728515625, "fcm_dpo/q_t": 0.4590563476085663, "grad_norm": 100.81672668457031, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -0.9225292801856995, "logits/rejected": -0.8966630697250366, "logps/chosen": -887.83935546875, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -1098.305419921875, "loss": 1.3432, "margin_dpo/margin_mean": 192.81646728515625, "margin_dpo/margin_std": 805.4339599609375, "step": 473 }, { "KL/chosen_KL_mean": -674.1361083984375, "KL/mean": -858.5537109375, "KL/rejected_KL_mean": -1042.971435546875, "KL/std": 571.548583984375, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0008901376277208328, "fcm_dpo/delta": -0.0760503038764, "fcm_dpo/margin": 368.83526611328125, "fcm_dpo/q_t": 0.42114460468292236, "grad_norm": 29.184354782104492, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -0.8849027156829834, "logits/rejected": -0.8879544734954834, "logps/chosen": -726.748046875, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -1133.0517578125, "loss": 1.1794, "margin_dpo/margin_mean": 368.83526611328125, "margin_dpo/margin_std": 658.4710693359375, "step": 474 }, { "KL/chosen_KL_mean": -489.204345703125, "KL/mean": -716.9005737304688, "KL/rejected_KL_mean": -944.5968627929688, "KL/std": 447.86822509765625, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0008446001447737217, "fcm_dpo/delta": 0.025007635354995728, "fcm_dpo/margin": 455.3924560546875, "fcm_dpo/q_t": 0.4114686846733093, "grad_norm": 35.35837173461914, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -0.9520416855812073, "logits/rejected": -0.979211688041687, "logps/chosen": -531.6995849609375, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -1034.6597900390625, "loss": 1.0813, "margin_dpo/margin_mean": 455.3924560546875, "margin_dpo/margin_std": 503.96844482421875, "step": 475 }, { "KL/chosen_KL_mean": -619.6912231445312, "KL/mean": -803.4817504882812, "KL/rejected_KL_mean": -987.272216796875, "KL/std": 514.7440185546875, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0009646163089200854, "fcm_dpo/delta": 0.06908048689365387, "fcm_dpo/margin": 367.5810546875, "fcm_dpo/q_t": 0.42334458231925964, "grad_norm": 42.48811721801758, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.8881396055221558, "logits/rejected": -0.904177188873291, "logps/chosen": -662.640625, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -1060.982421875, "loss": 1.1232, "margin_dpo/margin_mean": 367.5810546875, "margin_dpo/margin_std": 530.9556884765625, "step": 476 }, { "KL/chosen_KL_mean": -674.9053955078125, "KL/mean": -861.569580078125, "KL/rejected_KL_mean": -1048.2337646484375, "KL/std": 548.8595581054688, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0010453263530507684, "fcm_dpo/delta": 0.012534024193882942, "fcm_dpo/margin": 373.328369140625, "fcm_dpo/q_t": 0.4125494360923767, "grad_norm": 29.967121124267578, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -0.974365234375, "logits/rejected": -0.9425362348556519, "logps/chosen": -745.677978515625, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -1124.37109375, "loss": 1.145, "margin_dpo/margin_mean": 373.3283386230469, "margin_dpo/margin_std": 659.33203125, "step": 477 }, { "KL/chosen_KL_mean": -521.8460693359375, "KL/mean": -730.65234375, "KL/rejected_KL_mean": -939.4588012695312, "KL/std": 461.107421875, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.001001200987957418, "fcm_dpo/delta": -0.0321248322725296, "fcm_dpo/margin": 417.6127014160156, "fcm_dpo/q_t": 0.4019705057144165, "grad_norm": 34.509376525878906, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.8223329782485962, "logits/rejected": -0.8430248498916626, "logps/chosen": -563.2865600585938, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -1024.82080078125, "loss": 1.081, "margin_dpo/margin_mean": 417.61273193359375, "margin_dpo/margin_std": 542.504150390625, "step": 478 }, { "KL/chosen_KL_mean": -683.9410400390625, "KL/mean": -882.384033203125, "KL/rejected_KL_mean": -1080.8271484375, "KL/std": 582.9780883789062, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0010136861819773912, "fcm_dpo/delta": -0.020786762237548828, "fcm_dpo/margin": 396.88604736328125, "fcm_dpo/q_t": 0.4105514883995056, "grad_norm": 26.009479522705078, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.8741980791091919, "logits/rejected": -0.9063311815261841, "logps/chosen": -737.8489990234375, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -1175.943359375, "loss": 1.14, "margin_dpo/margin_mean": 396.88604736328125, "margin_dpo/margin_std": 691.9199829101562, "step": 479 }, { "KL/chosen_KL_mean": -823.0316772460938, "KL/mean": -973.1109619140625, "KL/rejected_KL_mean": -1123.190185546875, "KL/std": 528.6361083984375, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0009059295989573002, "fcm_dpo/delta": -0.010956360027194023, "fcm_dpo/margin": 300.15850830078125, "fcm_dpo/q_t": 0.43630528450012207, "grad_norm": 46.897727966308594, "learning_rate": 1.227838333989088e-07, "logits/chosen": -0.9384490251541138, "logits/rejected": -0.9253517389297485, "logps/chosen": -881.71435546875, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -1206.1226806640625, "loss": 1.2068, "margin_dpo/margin_mean": 300.15850830078125, "margin_dpo/margin_std": 578.85498046875, "step": 480 }, { "KL/chosen_KL_mean": -666.0030517578125, "KL/mean": -902.3043212890625, "KL/rejected_KL_mean": -1138.605712890625, "KL/std": 549.5169677734375, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.0008731147972866893, "fcm_dpo/delta": -0.021470826119184494, "fcm_dpo/margin": 472.6026611328125, "fcm_dpo/q_t": 0.4040641784667969, "grad_norm": 27.580970764160156, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -0.9624149799346924, "logits/rejected": -0.9881491661071777, "logps/chosen": -720.96728515625, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -1231.026123046875, "loss": 1.0827, "margin_dpo/margin_mean": 472.6026611328125, "margin_dpo/margin_std": 616.1715087890625, "step": 481 }, { "KL/chosen_KL_mean": -744.04248046875, "KL/mean": -855.4127197265625, "KL/rejected_KL_mean": -966.782958984375, "KL/std": 563.21630859375, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.0009502337779849768, "fcm_dpo/delta": 0.09700674563646317, "fcm_dpo/margin": 222.74044799804688, "fcm_dpo/q_t": 0.4541202783584595, "grad_norm": 35.13661575317383, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.8878397941589355, "logits/rejected": -0.8769594430923462, "logps/chosen": -811.595947265625, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -1054.37255859375, "loss": 1.2788, "margin_dpo/margin_mean": 222.740478515625, "margin_dpo/margin_std": 668.005859375, "step": 482 }, { "KL/chosen_KL_mean": -656.9723510742188, "KL/mean": -896.6019897460938, "KL/rejected_KL_mean": -1136.2315673828125, "KL/std": 552.5379028320312, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0009095891728065908, "fcm_dpo/delta": -0.07079771906137466, "fcm_dpo/margin": 479.25921630859375, "fcm_dpo/q_t": 0.3960753083229065, "grad_norm": 31.120384216308594, "learning_rate": 1.194847979251979e-07, "logits/chosen": -0.8999602198600769, "logits/rejected": -0.9098832607269287, "logps/chosen": -720.3021850585938, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -1232.0185546875, "loss": 1.0695, "margin_dpo/margin_mean": 479.2591857910156, "margin_dpo/margin_std": 622.5853271484375, "step": 483 }, { "KL/chosen_KL_mean": -550.8277587890625, "KL/mean": -768.7025146484375, "KL/rejected_KL_mean": -986.5772705078125, "KL/std": 529.3361206054688, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0009340384276583791, "fcm_dpo/delta": -0.01913183555006981, "fcm_dpo/margin": 435.74957275390625, "fcm_dpo/q_t": 0.40439409017562866, "grad_norm": 38.8234748840332, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -0.9227362871170044, "logits/rejected": -0.9442682266235352, "logps/chosen": -609.9659423828125, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -1070.94873046875, "loss": 1.0868, "margin_dpo/margin_mean": 435.74951171875, "margin_dpo/margin_std": 546.264892578125, "step": 484 }, { "KL/chosen_KL_mean": -558.8453369140625, "KL/mean": -764.8858642578125, "KL/rejected_KL_mean": -970.9264526367188, "KL/std": 525.61865234375, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0009324835264123976, "fcm_dpo/delta": 0.020539814606308937, "fcm_dpo/margin": 412.0811767578125, "fcm_dpo/q_t": 0.41461336612701416, "grad_norm": 38.73042678833008, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.8426438570022583, "logits/rejected": -0.8627835512161255, "logps/chosen": -617.6948852539062, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -1074.29052734375, "loss": 1.1164, "margin_dpo/margin_mean": 412.0811462402344, "margin_dpo/margin_std": 618.0966186523438, "step": 485 }, { "KL/chosen_KL_mean": -644.1904296875, "KL/mean": -887.1826782226562, "KL/rejected_KL_mean": -1130.1748046875, "KL/std": 590.3195190429688, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.0008638256113044918, "fcm_dpo/delta": -0.034225355833768845, "fcm_dpo/margin": 485.98443603515625, "fcm_dpo/q_t": 0.4027414321899414, "grad_norm": 39.733394622802734, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -0.9089898467063904, "logits/rejected": -0.9190336465835571, "logps/chosen": -699.4500732421875, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -1222.314208984375, "loss": 1.0991, "margin_dpo/margin_mean": 485.9844665527344, "margin_dpo/margin_std": 727.2791748046875, "step": 486 }, { "KL/chosen_KL_mean": -683.022216796875, "KL/mean": -864.6463012695312, "KL/rejected_KL_mean": -1046.2703857421875, "KL/std": 544.7926025390625, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0009704558178782463, "fcm_dpo/delta": 0.06513335555791855, "fcm_dpo/margin": 363.2481689453125, "fcm_dpo/q_t": 0.4213024377822876, "grad_norm": 40.45846939086914, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -0.9383945465087891, "logits/rejected": -0.9428784847259521, "logps/chosen": -736.0855712890625, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -1138.689208984375, "loss": 1.1385, "margin_dpo/margin_mean": 363.2481689453125, "margin_dpo/margin_std": 577.7797241210938, "step": 487 }, { "KL/chosen_KL_mean": -545.5859985351562, "KL/mean": -734.630126953125, "KL/rejected_KL_mean": -923.67431640625, "KL/std": 473.2789306640625, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0010280082933604717, "fcm_dpo/delta": 0.012521795928478241, "fcm_dpo/margin": 378.0882873535156, "fcm_dpo/q_t": 0.41051536798477173, "grad_norm": 22.53078269958496, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -0.8381102681159973, "logits/rejected": -0.8391183614730835, "logps/chosen": -597.814208984375, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -1007.680908203125, "loss": 1.0927, "margin_dpo/margin_mean": 378.0882873535156, "margin_dpo/margin_std": 473.35400390625, "step": 488 }, { "KL/chosen_KL_mean": -499.30194091796875, "KL/mean": -672.4368286132812, "KL/rejected_KL_mean": -845.5716552734375, "KL/std": 457.94464111328125, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.00108251697383821, "fcm_dpo/delta": 0.03405838832259178, "fcm_dpo/margin": 346.2696838378906, "fcm_dpo/q_t": 0.41736501455307007, "grad_norm": 24.369661331176758, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -0.8864847421646118, "logits/rejected": -0.8804982900619507, "logps/chosen": -555.2916259765625, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -924.9697265625, "loss": 1.1145, "margin_dpo/margin_mean": 346.2696838378906, "margin_dpo/margin_std": 504.04510498046875, "step": 489 }, { "KL/chosen_KL_mean": -606.05126953125, "KL/mean": -801.578125, "KL/rejected_KL_mean": -997.1050415039062, "KL/std": 577.8994140625, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0010517244227230549, "fcm_dpo/delta": -0.019802920520305634, "fcm_dpo/margin": 391.0537414550781, "fcm_dpo/q_t": 0.4080085754394531, "grad_norm": 39.334407806396484, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -0.8914596438407898, "logits/rejected": -0.9301384687423706, "logps/chosen": -658.4176025390625, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -1107.51416015625, "loss": 1.1285, "margin_dpo/margin_mean": 391.05377197265625, "margin_dpo/margin_std": 660.88671875, "step": 490 }, { "KL/chosen_KL_mean": -556.3938598632812, "KL/mean": -685.6237182617188, "KL/rejected_KL_mean": -814.8536376953125, "KL/std": 486.525634765625, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0012323971604928374, "fcm_dpo/delta": 0.1136833056807518, "fcm_dpo/margin": 258.459716796875, "fcm_dpo/q_t": 0.4345433712005615, "grad_norm": 34.78307342529297, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -0.8827264904975891, "logits/rejected": -0.8719267845153809, "logps/chosen": -616.5101318359375, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -888.1264038085938, "loss": 1.175, "margin_dpo/margin_mean": 258.4597473144531, "margin_dpo/margin_std": 484.7692565917969, "step": 491 }, { "KL/chosen_KL_mean": -558.9266357421875, "KL/mean": -683.21923828125, "KL/rejected_KL_mean": -807.511962890625, "KL/std": 487.9819030761719, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0014084703288972378, "fcm_dpo/delta": 0.07727247476577759, "fcm_dpo/margin": 248.58529663085938, "fcm_dpo/q_t": 0.43000033497810364, "grad_norm": 36.50166320800781, "learning_rate": 1.097764975115576e-07, "logits/chosen": -0.8998019695281982, "logits/rejected": -0.8745296001434326, "logps/chosen": -612.9207763671875, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -880.1715087890625, "loss": 1.209, "margin_dpo/margin_mean": 248.58529663085938, "margin_dpo/margin_std": 559.0795288085938, "step": 492 }, { "KL/chosen_KL_mean": -606.264892578125, "KL/mean": -745.5481567382812, "KL/rejected_KL_mean": -884.8314208984375, "KL/std": 519.5693359375, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0013839509338140488, "fcm_dpo/delta": 0.0048267170786857605, "fcm_dpo/margin": 278.56646728515625, "fcm_dpo/q_t": 0.412563681602478, "grad_norm": 41.9803466796875, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -0.9656150341033936, "logits/rejected": -0.9288890361785889, "logps/chosen": -681.7621459960938, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -972.1544189453125, "loss": 1.1628, "margin_dpo/margin_mean": 278.56646728515625, "margin_dpo/margin_std": 508.8519287109375, "step": 493 }, { "KL/chosen_KL_mean": -456.39111328125, "KL/mean": -672.3492431640625, "KL/rejected_KL_mean": -888.307373046875, "KL/std": 460.58941650390625, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.001108669675886631, "fcm_dpo/delta": -0.17697980999946594, "fcm_dpo/margin": 431.91619873046875, "fcm_dpo/q_t": 0.37332338094711304, "grad_norm": 37.33299255371094, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -0.8224232196807861, "logits/rejected": -0.8543698191642761, "logps/chosen": -497.7503662109375, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -974.398681640625, "loss": 1.0276, "margin_dpo/margin_mean": 431.91619873046875, "margin_dpo/margin_std": 471.99713134765625, "step": 494 }, { "KL/chosen_KL_mean": -516.5604248046875, "KL/mean": -703.7841796875, "KL/rejected_KL_mean": -891.0079956054688, "KL/std": 482.6785888671875, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0010678664548322558, "fcm_dpo/delta": -4.4368207454681396e-05, "fcm_dpo/margin": 374.44757080078125, "fcm_dpo/q_t": 0.4114726483821869, "grad_norm": 35.54365539550781, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -0.9496263265609741, "logits/rejected": -0.9522305727005005, "logps/chosen": -580.095458984375, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -982.432373046875, "loss": 1.1106, "margin_dpo/margin_mean": 374.44757080078125, "margin_dpo/margin_std": 582.944580078125, "step": 495 }, { "KL/chosen_KL_mean": -618.7561645507812, "KL/mean": -733.871337890625, "KL/rejected_KL_mean": -848.986572265625, "KL/std": 380.02642822265625, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0013148835860192776, "fcm_dpo/delta": 0.13359305262565613, "fcm_dpo/margin": 230.23040771484375, "fcm_dpo/q_t": 0.43739110231399536, "grad_norm": 59.39929962158203, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -0.9213845729827881, "logits/rejected": -0.8853435516357422, "logps/chosen": -691.3480834960938, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -933.31591796875, "loss": 1.1831, "margin_dpo/margin_mean": 230.23040771484375, "margin_dpo/margin_std": 436.0897216796875, "step": 496 }, { "KL/chosen_KL_mean": -592.6030883789062, "KL/mean": -699.4375, "KL/rejected_KL_mean": -806.27197265625, "KL/std": 449.3829345703125, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0013049728004261851, "fcm_dpo/delta": -0.028186559677124023, "fcm_dpo/margin": 213.6688232421875, "fcm_dpo/q_t": 0.4349159598350525, "grad_norm": 47.662078857421875, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -0.8769658803939819, "logits/rejected": -0.8549270629882812, "logps/chosen": -651.1970825195312, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -882.560302734375, "loss": 1.2114, "margin_dpo/margin_mean": 213.6688232421875, "margin_dpo/margin_std": 445.9394836425781, "step": 497 }, { "KL/chosen_KL_mean": -555.0482788085938, "KL/mean": -724.8245849609375, "KL/rejected_KL_mean": -894.6010131835938, "KL/std": 492.70159912109375, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0012368912575766444, "fcm_dpo/delta": -0.03477949649095535, "fcm_dpo/margin": 339.552734375, "fcm_dpo/q_t": 0.4025610685348511, "grad_norm": 32.19281768798828, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -0.93885338306427, "logits/rejected": -0.9247368574142456, "logps/chosen": -626.25390625, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -978.55908203125, "loss": 1.1152, "margin_dpo/margin_mean": 339.552734375, "margin_dpo/margin_std": 550.4183349609375, "step": 498 }, { "KL/chosen_KL_mean": -468.7248840332031, "KL/mean": -666.8063354492188, "KL/rejected_KL_mean": -864.8878173828125, "KL/std": 467.00946044921875, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0010947990231215954, "fcm_dpo/delta": -0.062151163816452026, "fcm_dpo/margin": 396.1629638671875, "fcm_dpo/q_t": 0.39568614959716797, "grad_norm": 36.04325485229492, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -0.8788598775863647, "logits/rejected": -0.9086207151412964, "logps/chosen": -519.9801025390625, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -965.966552734375, "loss": 1.0746, "margin_dpo/margin_mean": 396.1629638671875, "margin_dpo/margin_std": 527.5292358398438, "step": 499 }, { "KL/chosen_KL_mean": -586.3583374023438, "KL/mean": -761.30419921875, "KL/rejected_KL_mean": -936.25, "KL/std": 430.88531494140625, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0010993874166160822, "fcm_dpo/delta": 0.024825483560562134, "fcm_dpo/margin": 349.8916931152344, "fcm_dpo/q_t": 0.4142201840877533, "grad_norm": 32.015804290771484, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -0.9118782877922058, "logits/rejected": -0.9368890523910522, "logps/chosen": -643.3857421875, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -1030.1842041015625, "loss": 1.1177, "margin_dpo/margin_mean": 349.8916931152344, "margin_dpo/margin_std": 529.33837890625, "step": 500 }, { "KL/chosen_KL_mean": -496.50091552734375, "KL/mean": -679.2710571289062, "KL/rejected_KL_mean": -862.041259765625, "KL/std": 453.33331298828125, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.001129016513004899, "fcm_dpo/delta": -0.023394692689180374, "fcm_dpo/margin": 365.5402526855469, "fcm_dpo/q_t": 0.4061323404312134, "grad_norm": 33.857051849365234, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -0.8639429211616516, "logits/rejected": -0.8466420769691467, "logps/chosen": -550.8604736328125, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -942.1979370117188, "loss": 1.1061, "margin_dpo/margin_mean": 365.540283203125, "margin_dpo/margin_std": 553.092041015625, "step": 501 }, { "KL/chosen_KL_mean": -463.086181640625, "KL/mean": -645.1473388671875, "KL/rejected_KL_mean": -827.20849609375, "KL/std": 410.1119689941406, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0010994459735229611, "fcm_dpo/delta": -0.0017848927527666092, "fcm_dpo/margin": 364.122314453125, "fcm_dpo/q_t": 0.40732836723327637, "grad_norm": 22.971820831298828, "learning_rate": 9.934134090518592e-08, "logits/chosen": -0.8409848809242249, "logits/rejected": -0.8154308795928955, "logps/chosen": -530.6866455078125, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -910.1572265625, "loss": 1.0803, "margin_dpo/margin_mean": 364.1223449707031, "margin_dpo/margin_std": 437.77301025390625, "step": 502 }, { "KL/chosen_KL_mean": -455.322998046875, "KL/mean": -625.4855346679688, "KL/rejected_KL_mean": -795.6480712890625, "KL/std": 388.8305358886719, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.00111018109600991, "fcm_dpo/delta": 0.03548618406057358, "fcm_dpo/margin": 340.3250427246094, "fcm_dpo/q_t": 0.4157389998435974, "grad_norm": 23.465837478637695, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.8313039541244507, "logits/rejected": -0.8144758939743042, "logps/chosen": -510.40142822265625, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -878.1535034179688, "loss": 1.0957, "margin_dpo/margin_mean": 340.3250427246094, "margin_dpo/margin_std": 424.44854736328125, "step": 503 }, { "KL/chosen_KL_mean": -486.9976806640625, "KL/mean": -691.9973754882812, "KL/rejected_KL_mean": -896.9971313476562, "KL/std": 449.76153564453125, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.001057411776855588, "fcm_dpo/delta": -0.059654317796230316, "fcm_dpo/margin": 409.99945068359375, "fcm_dpo/q_t": 0.39566490054130554, "grad_norm": 23.978740692138672, "learning_rate": 9.730107739932805e-08, "logits/chosen": -0.8678739070892334, "logits/rejected": -0.8862636089324951, "logps/chosen": -546.96337890625, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -1000.75927734375, "loss": 1.0674, "margin_dpo/margin_mean": 409.9994812011719, "margin_dpo/margin_std": 505.0823974609375, "step": 504 }, { "KL/chosen_KL_mean": -583.2789306640625, "KL/mean": -693.8592529296875, "KL/rejected_KL_mean": -804.4395751953125, "KL/std": 454.05352783203125, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0013018188765272498, "fcm_dpo/delta": 0.1528121829032898, "fcm_dpo/margin": 221.16064453125, "fcm_dpo/q_t": 0.44206857681274414, "grad_norm": 44.39717102050781, "learning_rate": 9.628696786995188e-08, "logits/chosen": -0.9133349657058716, "logits/rejected": -0.8864049911499023, "logps/chosen": -659.433837890625, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -893.02490234375, "loss": 1.1987, "margin_dpo/margin_mean": 221.16065979003906, "margin_dpo/margin_std": 430.33538818359375, "step": 505 }, { "KL/chosen_KL_mean": -461.5542907714844, "KL/mean": -646.5904541015625, "KL/rejected_KL_mean": -831.626708984375, "KL/std": 457.093505859375, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.0011723710922524333, "fcm_dpo/delta": -0.07656194269657135, "fcm_dpo/margin": 370.07244873046875, "fcm_dpo/q_t": 0.3941624164581299, "grad_norm": 31.93734359741211, "learning_rate": 9.527690882192635e-08, "logits/chosen": -0.8899002075195312, "logits/rejected": -0.8990935683250427, "logps/chosen": -510.5147705078125, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -910.041748046875, "loss": 1.083, "margin_dpo/margin_mean": 370.07244873046875, "margin_dpo/margin_std": 513.3609619140625, "step": 506 }, { "KL/chosen_KL_mean": -530.3336791992188, "KL/mean": -686.3297729492188, "KL/rejected_KL_mean": -842.3258666992188, "KL/std": 511.7806701660156, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0012264869874343276, "fcm_dpo/delta": 0.028089439496397972, "fcm_dpo/margin": 311.9921569824219, "fcm_dpo/q_t": 0.41973698139190674, "grad_norm": 32.90348434448242, "learning_rate": 9.427092687124691e-08, "logits/chosen": -0.8918719291687012, "logits/rejected": -0.889752984046936, "logps/chosen": -597.1351928710938, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -937.69873046875, "loss": 1.1451, "margin_dpo/margin_mean": 311.9921569824219, "margin_dpo/margin_std": 565.5931396484375, "step": 507 }, { "KL/chosen_KL_mean": -583.1611328125, "KL/mean": -721.2760009765625, "KL/rejected_KL_mean": -859.390869140625, "KL/std": 509.71832275390625, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0013669917825609446, "fcm_dpo/delta": 0.031157607212662697, "fcm_dpo/margin": 276.22967529296875, "fcm_dpo/q_t": 0.4213915169239044, "grad_norm": 53.349178314208984, "learning_rate": 9.326904852647344e-08, "logits/chosen": -0.8866854906082153, "logits/rejected": -0.883845329284668, "logps/chosen": -654.464599609375, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -955.0184326171875, "loss": 1.1942, "margin_dpo/margin_mean": 276.2297058105469, "margin_dpo/margin_std": 601.1476440429688, "step": 508 }, { "KL/chosen_KL_mean": -447.341552734375, "KL/mean": -596.3013916015625, "KL/rejected_KL_mean": -745.26123046875, "KL/std": 366.6440734863281, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0013803851325064898, "fcm_dpo/delta": -0.028025750070810318, "fcm_dpo/margin": 297.9196472167969, "fcm_dpo/q_t": 0.4056088328361511, "grad_norm": 36.49995803833008, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.8121344447135925, "logits/rejected": -0.8039395809173584, "logps/chosen": -511.1605224609375, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -828.5177001953125, "loss": 1.1197, "margin_dpo/margin_mean": 297.9196472167969, "margin_dpo/margin_std": 458.0248107910156, "step": 509 }, { "KL/chosen_KL_mean": -541.6021728515625, "KL/mean": -733.678466796875, "KL/rejected_KL_mean": -925.7547607421875, "KL/std": 435.32470703125, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0011522183194756508, "fcm_dpo/delta": -0.07610686868429184, "fcm_dpo/margin": 384.152587890625, "fcm_dpo/q_t": 0.3902145326137543, "grad_norm": 38.18740463256836, "learning_rate": 9.127770814751932e-08, "logits/chosen": -0.7922756671905518, "logits/rejected": -0.8119780421257019, "logps/chosen": -593.4805908203125, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -1028.5198974609375, "loss": 1.0452, "margin_dpo/margin_mean": 384.152587890625, "margin_dpo/margin_std": 409.4080810546875, "step": 510 }, { "KL/chosen_KL_mean": -491.181884765625, "KL/mean": -649.146484375, "KL/rejected_KL_mean": -807.1109619140625, "KL/std": 461.1341247558594, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.0011797586921602488, "fcm_dpo/delta": 0.0413035973906517, "fcm_dpo/margin": 315.92913818359375, "fcm_dpo/q_t": 0.41845548152923584, "grad_norm": 29.213090896606445, "learning_rate": 9.028829858700973e-08, "logits/chosen": -0.8354673385620117, "logits/rejected": -0.8356435298919678, "logps/chosen": -551.4199829101562, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -899.9677734375, "loss": 1.1461, "margin_dpo/margin_mean": 315.92913818359375, "margin_dpo/margin_std": 560.2075805664062, "step": 511 }, { "KL/chosen_KL_mean": -411.56103515625, "KL/mean": -608.6541137695312, "KL/rejected_KL_mean": -805.7471923828125, "KL/std": 410.18994140625, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0010748871136456728, "fcm_dpo/delta": -0.04578953981399536, "fcm_dpo/margin": 394.1861572265625, "fcm_dpo/q_t": 0.3963681757450104, "grad_norm": 52.93358612060547, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.8261533975601196, "logits/rejected": -0.8413060307502747, "logps/chosen": -466.466552734375, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -887.623046875, "loss": 1.0543, "margin_dpo/margin_mean": 394.1861572265625, "margin_dpo/margin_std": 417.51800537109375, "step": 512 }, { "KL/chosen_KL_mean": -532.857421875, "KL/mean": -685.5048828125, "KL/rejected_KL_mean": -838.1524658203125, "KL/std": 409.397705078125, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.0009999822359532118, "fcm_dpo/delta": -0.038525860756635666, "fcm_dpo/margin": 305.2950134277344, "fcm_dpo/q_t": 0.42581143975257874, "grad_norm": 34.72964859008789, "learning_rate": 8.832213108254863e-08, "logits/chosen": -0.8888071775436401, "logits/rejected": -0.8702949285507202, "logps/chosen": -597.7738647460938, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -914.2149047851562, "loss": 1.1629, "margin_dpo/margin_mean": 305.2950134277344, "margin_dpo/margin_std": 478.2789306640625, "step": 513 }, { "KL/chosen_KL_mean": -564.9537353515625, "KL/mean": -707.5296630859375, "KL/rejected_KL_mean": -850.1055908203125, "KL/std": 448.4078063964844, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.0011754620354622602, "fcm_dpo/delta": 0.09163933992385864, "fcm_dpo/margin": 285.15191650390625, "fcm_dpo/q_t": 0.4304603934288025, "grad_norm": 32.26676559448242, "learning_rate": 8.734542494893954e-08, "logits/chosen": -0.8373386859893799, "logits/rejected": -0.8227113485336304, "logps/chosen": -639.1832885742188, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -929.0511474609375, "loss": 1.1577, "margin_dpo/margin_mean": 285.15191650390625, "margin_dpo/margin_std": 502.1484375, "step": 514 }, { "KL/chosen_KL_mean": -494.32861328125, "KL/mean": -618.3717041015625, "KL/rejected_KL_mean": -742.414794921875, "KL/std": 406.7989501953125, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0013804570771753788, "fcm_dpo/delta": 0.08468376100063324, "fcm_dpo/margin": 248.086181640625, "fcm_dpo/q_t": 0.4263339042663574, "grad_norm": 40.3935546875, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.8123036623001099, "logits/rejected": -0.8226903676986694, "logps/chosen": -544.730224609375, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -829.5125122070312, "loss": 1.1724, "margin_dpo/margin_mean": 248.086181640625, "margin_dpo/margin_std": 483.9718322753906, "step": 515 }, { "KL/chosen_KL_mean": -526.0533447265625, "KL/mean": -701.6831665039062, "KL/rejected_KL_mean": -877.3131103515625, "KL/std": 438.8985900878906, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0012882768642157316, "fcm_dpo/delta": -0.0951680988073349, "fcm_dpo/margin": 351.25970458984375, "fcm_dpo/q_t": 0.3881225287914276, "grad_norm": 36.28194046020508, "learning_rate": 8.540489660386064e-08, "logits/chosen": -0.9183084964752197, "logits/rejected": -0.9395666122436523, "logps/chosen": -590.702880859375, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -989.0354614257812, "loss": 1.0661, "margin_dpo/margin_mean": 351.25970458984375, "margin_dpo/margin_std": 446.1391906738281, "step": 516 }, { "KL/chosen_KL_mean": -552.42333984375, "KL/mean": -758.86474609375, "KL/rejected_KL_mean": -965.3060302734375, "KL/std": 489.52093505859375, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0010481756180524826, "fcm_dpo/delta": -0.0622154101729393, "fcm_dpo/margin": 412.88262939453125, "fcm_dpo/q_t": 0.39706072211265564, "grad_norm": 25.22186279296875, "learning_rate": 8.444112552711752e-08, "logits/chosen": -0.8430509567260742, "logits/rejected": -0.837881326675415, "logps/chosen": -613.3369140625, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -1054.38916015625, "loss": 1.0697, "margin_dpo/margin_mean": 412.88262939453125, "margin_dpo/margin_std": 533.8587646484375, "step": 517 }, { "KL/chosen_KL_mean": -519.568603515625, "KL/mean": -694.2239990234375, "KL/rejected_KL_mean": -868.8794555664062, "KL/std": 415.3473815917969, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0010671718046069145, "fcm_dpo/delta": 0.0419648140668869, "fcm_dpo/margin": 349.3109130859375, "fcm_dpo/q_t": 0.4160323143005371, "grad_norm": 33.90839385986328, "learning_rate": 8.348171708068747e-08, "logits/chosen": -0.875517725944519, "logits/rejected": -0.8914189338684082, "logps/chosen": -577.0244140625, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -954.192138671875, "loss": 1.112, "margin_dpo/margin_mean": 349.3109130859375, "margin_dpo/margin_std": 470.837158203125, "step": 518 }, { "KL/chosen_KL_mean": -517.7393798828125, "KL/mean": -643.4759521484375, "KL/rejected_KL_mean": -769.21240234375, "KL/std": 388.196044921875, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0011561332503333688, "fcm_dpo/delta": 0.02302495203912258, "fcm_dpo/margin": 251.4730224609375, "fcm_dpo/q_t": 0.4338497221469879, "grad_norm": 42.36438751220703, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.889582097530365, "logits/rejected": -0.8709315061569214, "logps/chosen": -591.802734375, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -873.6566162109375, "loss": 1.1946, "margin_dpo/margin_mean": 251.4730224609375, "margin_dpo/margin_std": 490.158447265625, "step": 519 }, { "KL/chosen_KL_mean": -572.123291015625, "KL/mean": -733.942138671875, "KL/rejected_KL_mean": -895.7611083984375, "KL/std": 462.9925537109375, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.001188506605103612, "fcm_dpo/delta": 0.023475412279367447, "fcm_dpo/margin": 323.6377868652344, "fcm_dpo/q_t": 0.41749107837677, "grad_norm": 31.222883224487305, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.8482314348220825, "logits/rejected": -0.8507933616638184, "logps/chosen": -642.423095703125, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -995.742431640625, "loss": 1.1305, "margin_dpo/margin_mean": 323.6378173828125, "margin_dpo/margin_std": 529.4666748046875, "step": 520 }, { "KL/chosen_KL_mean": -498.2612609863281, "KL/mean": -677.0108032226562, "KL/rejected_KL_mean": -855.76025390625, "KL/std": 452.33221435546875, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.001218126737512648, "fcm_dpo/delta": -0.07213665544986725, "fcm_dpo/margin": 357.4990234375, "fcm_dpo/q_t": 0.39487892389297485, "grad_norm": 34.73341369628906, "learning_rate": 8.062991975753378e-08, "logits/chosen": -0.8869306445121765, "logits/rejected": -0.8860381841659546, "logps/chosen": -556.4041748046875, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -939.0408935546875, "loss": 1.0806, "margin_dpo/margin_mean": 357.4990234375, "margin_dpo/margin_std": 450.37408447265625, "step": 521 }, { "KL/chosen_KL_mean": -573.8514404296875, "KL/mean": -739.8829345703125, "KL/rejected_KL_mean": -905.91455078125, "KL/std": 490.18310546875, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.0011361883953213692, "fcm_dpo/delta": 0.036084454506635666, "fcm_dpo/margin": 332.06304931640625, "fcm_dpo/q_t": 0.41713184118270874, "grad_norm": 28.38233757019043, "learning_rate": 7.968821348583643e-08, "logits/chosen": -0.9024146199226379, "logits/rejected": -0.9035881757736206, "logps/chosen": -620.3991088867188, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -971.928466796875, "loss": 1.128, "margin_dpo/margin_mean": 332.0630798339844, "margin_dpo/margin_std": 528.5509033203125, "step": 522 }, { "KL/chosen_KL_mean": -621.0657348632812, "KL/mean": -792.4852905273438, "KL/rejected_KL_mean": -963.9049072265625, "KL/std": 544.6927490234375, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0011524436995387077, "fcm_dpo/delta": 0.007822713814675808, "fcm_dpo/margin": 342.8391418457031, "fcm_dpo/q_t": 0.4123551845550537, "grad_norm": 35.35528564453125, "learning_rate": 7.875099508810484e-08, "logits/chosen": -0.9692133665084839, "logits/rejected": -0.9667251110076904, "logps/chosen": -682.8353271484375, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -1047.666259765625, "loss": 1.1316, "margin_dpo/margin_mean": 342.8391418457031, "margin_dpo/margin_std": 576.16796875, "step": 523 }, { "KL/chosen_KL_mean": -588.4208984375, "KL/mean": -767.6956787109375, "KL/rejected_KL_mean": -946.9705810546875, "KL/std": 500.2874755859375, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.0011021224781870842, "fcm_dpo/delta": -0.0017808079719543457, "fcm_dpo/margin": 358.5496826171875, "fcm_dpo/q_t": 0.40673327445983887, "grad_norm": 34.97015380859375, "learning_rate": 7.781828926091535e-08, "logits/chosen": -0.9925479888916016, "logits/rejected": -0.9842585325241089, "logps/chosen": -666.492919921875, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -1028.2725830078125, "loss": 1.1098, "margin_dpo/margin_mean": 358.5496520996094, "margin_dpo/margin_std": 513.2574462890625, "step": 524 }, { "KL/chosen_KL_mean": -575.912109375, "KL/mean": -803.885986328125, "KL/rejected_KL_mean": -1031.85986328125, "KL/std": 526.4970092773438, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0009674688335508108, "fcm_dpo/delta": -0.09075939655303955, "fcm_dpo/margin": 455.9477233886719, "fcm_dpo/q_t": 0.39121919870376587, "grad_norm": 35.0758056640625, "learning_rate": 7.689012058193384e-08, "logits/chosen": -0.89338219165802, "logits/rejected": -0.9303810596466064, "logps/chosen": -626.739990234375, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -1131.912841796875, "loss": 1.06, "margin_dpo/margin_mean": 455.94775390625, "margin_dpo/margin_std": 559.2699584960938, "step": 525 }, { "KL/chosen_KL_mean": -609.7196044921875, "KL/mean": -831.7645874023438, "KL/rejected_KL_mean": -1053.8095703125, "KL/std": 516.262939453125, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0009489007061347365, "fcm_dpo/delta": -0.03767908737063408, "fcm_dpo/margin": 444.0899658203125, "fcm_dpo/q_t": 0.3990199565887451, "grad_norm": 31.720500946044922, "learning_rate": 7.596651350926836e-08, "logits/chosen": -0.921222984790802, "logits/rejected": -0.9137049317359924, "logps/chosen": -672.8868408203125, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -1140.118896484375, "loss": 1.0826, "margin_dpo/margin_mean": 444.0899658203125, "margin_dpo/margin_std": 581.716064453125, "step": 526 }, { "KL/chosen_KL_mean": -647.0347290039062, "KL/mean": -790.537841796875, "KL/rejected_KL_mean": -934.0408935546875, "KL/std": 527.8538208007812, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0010856003500521183, "fcm_dpo/delta": 0.1282484382390976, "fcm_dpo/margin": 287.0062255859375, "fcm_dpo/q_t": 0.4351726174354553, "grad_norm": 38.458126068115234, "learning_rate": 7.504749238082414e-08, "logits/chosen": -1.1125977039337158, "logits/rejected": -1.0749619007110596, "logps/chosen": -718.1634521484375, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -1012.3834838867188, "loss": 1.1598, "margin_dpo/margin_mean": 287.0062255859375, "margin_dpo/margin_std": 455.161376953125, "step": 527 }, { "KL/chosen_KL_mean": -650.195068359375, "KL/mean": -832.568359375, "KL/rejected_KL_mean": -1014.9417724609375, "KL/std": 509.81414794921875, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0011271900730207562, "fcm_dpo/delta": -0.019259048625826836, "fcm_dpo/margin": 364.7466735839844, "fcm_dpo/q_t": 0.4092450439929962, "grad_norm": 36.67366409301758, "learning_rate": 7.413308141366254e-08, "logits/chosen": -0.981177568435669, "logits/rejected": -0.9641728401184082, "logps/chosen": -718.2845458984375, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -1108.851806640625, "loss": 1.1235, "margin_dpo/margin_mean": 364.7467041015625, "margin_dpo/margin_std": 606.7515258789062, "step": 528 }, { "KL/chosen_KL_mean": -796.822509765625, "KL/mean": -921.7469482421875, "KL/rejected_KL_mean": -1046.67138671875, "KL/std": 494.61181640625, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.0011494512436911464, "fcm_dpo/delta": 0.01170584000647068, "fcm_dpo/margin": 249.8488006591797, "fcm_dpo/q_t": 0.4349808990955353, "grad_norm": 53.19778823852539, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.048140525817871, "logits/rejected": -1.0602138042449951, "logps/chosen": -852.3974609375, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -1135.88037109375, "loss": 1.2429, "margin_dpo/margin_mean": 249.84881591796875, "margin_dpo/margin_std": 625.3258056640625, "step": 529 }, { "KL/chosen_KL_mean": -660.2740478515625, "KL/mean": -862.5133666992188, "KL/rejected_KL_mean": -1064.7528076171875, "KL/std": 578.7081298828125, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0010660856496542692, "fcm_dpo/delta": -0.05491678789258003, "fcm_dpo/margin": 404.4787902832031, "fcm_dpo/q_t": 0.4025976061820984, "grad_norm": 38.45912551879883, "learning_rate": 7.231818622338822e-08, "logits/chosen": -0.9426896572113037, "logits/rejected": -0.9390045404434204, "logps/chosen": -707.8754272460938, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -1152.037353515625, "loss": 1.136, "margin_dpo/margin_mean": 404.4787902832031, "margin_dpo/margin_std": 729.6232299804688, "step": 530 }, { "KL/chosen_KL_mean": -751.2568359375, "KL/mean": -909.684814453125, "KL/rejected_KL_mean": -1068.11279296875, "KL/std": 594.5966186523438, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.001148285111412406, "fcm_dpo/delta": 0.05123718082904816, "fcm_dpo/margin": 316.85595703125, "fcm_dpo/q_t": 0.42305827140808105, "grad_norm": 43.48565673828125, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.0459530353546143, "logits/rejected": -1.0223256349563599, "logps/chosen": -806.5028686523438, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -1138.71875, "loss": 1.1531, "margin_dpo/margin_mean": 316.85595703125, "margin_dpo/margin_std": 567.612060546875, "step": 531 }, { "KL/chosen_KL_mean": -734.84375, "KL/mean": -924.059326171875, "KL/rejected_KL_mean": -1113.27490234375, "KL/std": 581.4156494140625, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0010310335783287883, "fcm_dpo/delta": -0.008013017475605011, "fcm_dpo/margin": 378.4312438964844, "fcm_dpo/q_t": 0.41106435656547546, "grad_norm": 68.37248229980469, "learning_rate": 7.052201923388953e-08, "logits/chosen": -0.9860376119613647, "logits/rejected": -0.961615264415741, "logps/chosen": -805.1297607421875, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1199.8662109375, "loss": 1.1619, "margin_dpo/margin_mean": 378.43121337890625, "margin_dpo/margin_std": 688.8643798828125, "step": 532 }, { "KL/chosen_KL_mean": -660.6729125976562, "KL/mean": -794.4534301757812, "KL/rejected_KL_mean": -928.23388671875, "KL/std": 490.07489013671875, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0011233469704166055, "fcm_dpo/delta": 0.00041316391434520483, "fcm_dpo/margin": 267.5610046386719, "fcm_dpo/q_t": 0.4326951205730438, "grad_norm": 41.96712875366211, "learning_rate": 6.963101805503646e-08, "logits/chosen": -1.028921127319336, "logits/rejected": -1.0061641931533813, "logps/chosen": -725.5280151367188, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -1004.822021484375, "loss": 1.2077, "margin_dpo/margin_mean": 267.5610046386719, "margin_dpo/margin_std": 582.5079345703125, "step": 533 }, { "KL/chosen_KL_mean": -676.6849365234375, "KL/mean": -862.72900390625, "KL/rejected_KL_mean": -1048.7730712890625, "KL/std": 513.5587158203125, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.0010465008672326803, "fcm_dpo/delta": 0.0019133538007736206, "fcm_dpo/margin": 372.08819580078125, "fcm_dpo/q_t": 0.4103556275367737, "grad_norm": 26.502838134765625, "learning_rate": 6.874476976660184e-08, "logits/chosen": -0.9815844297409058, "logits/rejected": -0.9804989099502563, "logps/chosen": -736.8043212890625, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -1127.3165283203125, "loss": 1.1194, "margin_dpo/margin_mean": 372.08819580078125, "margin_dpo/margin_std": 547.061279296875, "step": 534 }, { "KL/chosen_KL_mean": -587.0618286132812, "KL/mean": -801.0487060546875, "KL/rejected_KL_mean": -1015.0355224609375, "KL/std": 525.9488525390625, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.001057377434335649, "fcm_dpo/delta": -0.10026499629020691, "fcm_dpo/margin": 427.9737854003906, "fcm_dpo/q_t": 0.390084445476532, "grad_norm": 25.19883155822754, "learning_rate": 6.786329772205246e-08, "logits/chosen": -0.9359257817268372, "logits/rejected": -0.9411388039588928, "logps/chosen": -641.39208984375, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -1111.34326171875, "loss": 1.0808, "margin_dpo/margin_mean": 427.9737854003906, "margin_dpo/margin_std": 552.81982421875, "step": 535 }, { "KL/chosen_KL_mean": -524.899658203125, "KL/mean": -780.73291015625, "KL/rejected_KL_mean": -1036.5660400390625, "KL/std": 589.3612060546875, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0008447887958027422, "fcm_dpo/delta": -0.05769486725330353, "fcm_dpo/margin": 511.6664733886719, "fcm_dpo/q_t": 0.3999031186103821, "grad_norm": 24.389259338378906, "learning_rate": 6.698662514899638e-08, "logits/chosen": -0.9003403186798096, "logits/rejected": -0.9327056407928467, "logps/chosen": -571.980224609375, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -1125.6639404296875, "loss": 1.0767, "margin_dpo/margin_mean": 511.66644287109375, "margin_dpo/margin_std": 714.7777709960938, "step": 536 }, { "KL/chosen_KL_mean": -562.1661376953125, "KL/mean": -730.231201171875, "KL/rejected_KL_mean": -898.2962646484375, "KL/std": 485.2312316894531, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0008401878294534981, "fcm_dpo/delta": 0.021016618236899376, "fcm_dpo/margin": 336.1300354003906, "fcm_dpo/q_t": 0.4347127377986908, "grad_norm": 33.1090202331543, "learning_rate": 6.611477514857114e-08, "logits/chosen": -0.9427402019500732, "logits/rejected": -0.9209805727005005, "logps/chosen": -619.9136962890625, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -968.734619140625, "loss": 1.1829, "margin_dpo/margin_mean": 336.1300354003906, "margin_dpo/margin_std": 590.2587890625, "step": 537 }, { "KL/chosen_KL_mean": -686.517822265625, "KL/mean": -878.772705078125, "KL/rejected_KL_mean": -1071.027587890625, "KL/std": 506.6947937011719, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0009338191011920571, "fcm_dpo/delta": 0.064121313393116, "fcm_dpo/margin": 384.5097351074219, "fcm_dpo/q_t": 0.42180943489074707, "grad_norm": 26.116352081298828, "learning_rate": 6.524777069483525e-08, "logits/chosen": -0.9635825753211975, "logits/rejected": -0.9489821195602417, "logps/chosen": -752.9337768554688, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -1155.255615234375, "loss": 1.1175, "margin_dpo/margin_mean": 384.5097351074219, "margin_dpo/margin_std": 529.1771240234375, "step": 538 }, { "KL/chosen_KL_mean": -581.152099609375, "KL/mean": -763.506103515625, "KL/rejected_KL_mean": -945.8601684570312, "KL/std": 429.1192321777344, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.0010440616169944406, "fcm_dpo/delta": 0.02792687714099884, "fcm_dpo/margin": 364.7080383300781, "fcm_dpo/q_t": 0.4141731858253479, "grad_norm": 33.319786071777344, "learning_rate": 6.438563463416221e-08, "logits/chosen": -0.9643697738647461, "logits/rejected": -0.9533451795578003, "logps/chosen": -639.6449584960938, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -1037.714111328125, "loss": 1.101, "margin_dpo/margin_mean": 364.70806884765625, "margin_dpo/margin_std": 481.9061279296875, "step": 539 }, { "KL/chosen_KL_mean": -593.5488891601562, "KL/mean": -833.126708984375, "KL/rejected_KL_mean": -1072.70458984375, "KL/std": 541.5235595703125, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0009419465786777437, "fcm_dpo/delta": -0.09322336316108704, "fcm_dpo/margin": 479.15557861328125, "fcm_dpo/q_t": 0.3929232060909271, "grad_norm": 30.28093147277832, "learning_rate": 6.352838968463919e-08, "logits/chosen": -0.9041993618011475, "logits/rejected": -0.9246673583984375, "logps/chosen": -657.0314331054688, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -1189.134521484375, "loss": 1.0719, "margin_dpo/margin_mean": 479.15557861328125, "margin_dpo/margin_std": 642.5653686523438, "step": 540 }, { "KL/chosen_KL_mean": -716.245849609375, "KL/mean": -844.7294921875, "KL/rejected_KL_mean": -973.2132568359375, "KL/std": 497.0907287597656, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.0008998019620776176, "fcm_dpo/delta": 0.02505185268819332, "fcm_dpo/margin": 256.9674072265625, "fcm_dpo/q_t": 0.44805556535720825, "grad_norm": 39.39170455932617, "learning_rate": 6.267605843546767e-08, "logits/chosen": -1.0489933490753174, "logits/rejected": -1.043041706085205, "logps/chosen": -794.5261840820312, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -1076.48681640625, "loss": 1.241, "margin_dpo/margin_mean": 256.9674377441406, "margin_dpo/margin_std": 595.6408081054688, "step": 541 }, { "KL/chosen_KL_mean": -606.02099609375, "KL/mean": -850.662841796875, "KL/rejected_KL_mean": -1095.3046875, "KL/std": 556.0362548828125, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.000812588375993073, "fcm_dpo/delta": -0.015589557588100433, "fcm_dpo/margin": 489.2837829589844, "fcm_dpo/q_t": 0.40638577938079834, "grad_norm": 43.08638000488281, "learning_rate": 6.182866334636888e-08, "logits/chosen": -1.0107741355895996, "logits/rejected": -1.045440435409546, "logps/chosen": -663.5059814453125, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -1191.77978515625, "loss": 1.1037, "margin_dpo/margin_mean": 489.2837829589844, "margin_dpo/margin_std": 664.2471313476562, "step": 542 }, { "KL/chosen_KL_mean": -686.9749145507812, "KL/mean": -857.873046875, "KL/rejected_KL_mean": -1028.7711181640625, "KL/std": 652.3204956054688, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0009889693465083838, "fcm_dpo/delta": 0.09416899085044861, "fcm_dpo/margin": 341.7962646484375, "fcm_dpo/q_t": 0.43743741512298584, "grad_norm": 41.49184799194336, "learning_rate": 6.098622674699147e-08, "logits/chosen": -0.9908128976821899, "logits/rejected": -1.023362398147583, "logps/chosen": -747.5924072265625, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -1134.3701171875, "loss": 1.2031, "margin_dpo/margin_mean": 341.7962646484375, "margin_dpo/margin_std": 781.2564086914062, "step": 543 }, { "KL/chosen_KL_mean": -705.4442749023438, "KL/mean": -904.148193359375, "KL/rejected_KL_mean": -1102.85205078125, "KL/std": 532.5914306640625, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0010228848550468683, "fcm_dpo/delta": -0.011566650122404099, "fcm_dpo/margin": 397.4078369140625, "fcm_dpo/q_t": 0.40789809823036194, "grad_norm": 28.750972747802734, "learning_rate": 6.01487708363232e-08, "logits/chosen": -0.9685473442077637, "logits/rejected": -0.9889190196990967, "logps/chosen": -765.0865478515625, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -1203.8067626953125, "loss": 1.1104, "margin_dpo/margin_mean": 397.4078369140625, "margin_dpo/margin_std": 614.56640625, "step": 544 }, { "KL/chosen_KL_mean": -649.4158935546875, "KL/mean": -885.8656616210938, "KL/rejected_KL_mean": -1122.3154296875, "KL/std": 538.6456298828125, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0009213591692969203, "fcm_dpo/delta": -0.06585489213466644, "fcm_dpo/margin": 472.8994140625, "fcm_dpo/q_t": 0.3961293399333954, "grad_norm": 26.994535446166992, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.8998408913612366, "logits/rejected": -0.9357277154922485, "logps/chosen": -717.0645141601562, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -1218.223388671875, "loss": 1.0743, "margin_dpo/margin_mean": 472.8994140625, "margin_dpo/margin_std": 626.8919677734375, "step": 545 }, { "KL/chosen_KL_mean": -610.9400024414062, "KL/mean": -767.9046630859375, "KL/rejected_KL_mean": -924.8692626953125, "KL/std": 476.8546142578125, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.0010576581116765738, "fcm_dpo/delta": 0.0986643135547638, "fcm_dpo/margin": 313.9293212890625, "fcm_dpo/q_t": 0.4284512400627136, "grad_norm": 36.017555236816406, "learning_rate": 5.848888922025552e-08, "logits/chosen": -0.9738471508026123, "logits/rejected": -0.9590544104576111, "logps/chosen": -661.6842041015625, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -1006.7355346679688, "loss": 1.1519, "margin_dpo/margin_mean": 313.9293518066406, "margin_dpo/margin_std": 505.626220703125, "step": 546 }, { "KL/chosen_KL_mean": -610.9646606445312, "KL/mean": -821.750732421875, "KL/rejected_KL_mean": -1032.536865234375, "KL/std": 533.4896240234375, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.001024933997541666, "fcm_dpo/delta": -0.056229621171951294, "fcm_dpo/margin": 421.57220458984375, "fcm_dpo/q_t": 0.3993460237979889, "grad_norm": 39.7552490234375, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -0.9101555943489075, "logits/rejected": -0.9197220802307129, "logps/chosen": -684.65234375, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -1123.29833984375, "loss": 1.0807, "margin_dpo/margin_mean": 421.57220458984375, "margin_dpo/margin_std": 582.0226440429688, "step": 547 }, { "KL/chosen_KL_mean": -647.4964599609375, "KL/mean": -836.765869140625, "KL/rejected_KL_mean": -1026.035400390625, "KL/std": 553.211181640625, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0010135022457689047, "fcm_dpo/delta": 0.026592375710606575, "fcm_dpo/margin": 378.53887939453125, "fcm_dpo/q_t": 0.4186980724334717, "grad_norm": 30.77254295349121, "learning_rate": 5.684919345471029e-08, "logits/chosen": -0.9787480235099792, "logits/rejected": -0.9837968349456787, "logps/chosen": -712.7427978515625, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -1120.1534423828125, "loss": 1.124, "margin_dpo/margin_mean": 378.53887939453125, "margin_dpo/margin_std": 625.75048828125, "step": 548 }, { "KL/chosen_KL_mean": -664.22216796875, "KL/mean": -802.9415283203125, "KL/rejected_KL_mean": -941.660888671875, "KL/std": 461.15252685546875, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0010441699996590614, "fcm_dpo/delta": 0.008397620171308517, "fcm_dpo/margin": 277.43865966796875, "fcm_dpo/q_t": 0.4350124001502991, "grad_norm": 47.32408142089844, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.0104793310165405, "logits/rejected": -1.0005792379379272, "logps/chosen": -713.4345703125, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -1015.5711669921875, "loss": 1.1953, "margin_dpo/margin_mean": 277.43865966796875, "margin_dpo/margin_std": 547.7349243164062, "step": 549 }, { "KL/chosen_KL_mean": -645.0916748046875, "KL/mean": -818.9498291015625, "KL/rejected_KL_mean": -992.80810546875, "KL/std": 506.2775573730469, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0010925406822934747, "fcm_dpo/delta": 0.03250662982463837, "fcm_dpo/margin": 347.7164001464844, "fcm_dpo/q_t": 0.41644904017448425, "grad_norm": 42.81041717529297, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -0.9073216915130615, "logits/rejected": -0.9299839735031128, "logps/chosen": -701.8985595703125, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -1087.933837890625, "loss": 1.124, "margin_dpo/margin_mean": 347.7164306640625, "margin_dpo/margin_std": 549.0792846679688, "step": 550 }, { "KL/chosen_KL_mean": -558.291259765625, "KL/mean": -833.9419555664062, "KL/rejected_KL_mean": -1109.5927734375, "KL/std": 536.7257080078125, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.0008524754084646702, "fcm_dpo/delta": -0.15517154335975647, "fcm_dpo/margin": 551.3014526367188, "fcm_dpo/q_t": 0.37387317419052124, "grad_norm": 49.040679931640625, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -0.8993926048278809, "logits/rejected": -0.954781174659729, "logps/chosen": -617.3975830078125, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -1221.2655029296875, "loss": 1.0186, "margin_dpo/margin_mean": 551.3014526367188, "margin_dpo/margin_std": 538.538330078125, "step": 551 }, { "KL/chosen_KL_mean": -533.4443359375, "KL/mean": -838.3875732421875, "KL/rejected_KL_mean": -1143.330810546875, "KL/std": 620.9227294921875, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0006882591405883431, "fcm_dpo/delta": -0.06372275948524475, "fcm_dpo/margin": 609.8865356445312, "fcm_dpo/q_t": 0.3962107002735138, "grad_norm": 35.4857063293457, "learning_rate": 5.363104864490034e-08, "logits/chosen": -0.9574573040008545, "logits/rejected": -0.9972236156463623, "logps/chosen": -595.7989501953125, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -1247.892822265625, "loss": 1.0688, "margin_dpo/margin_mean": 609.8865356445312, "margin_dpo/margin_std": 702.5086669921875, "step": 552 }, { "KL/chosen_KL_mean": -621.122802734375, "KL/mean": -784.8905029296875, "KL/rejected_KL_mean": -948.6583251953125, "KL/std": 512.807861328125, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0009008180350065231, "fcm_dpo/delta": 0.1481509506702423, "fcm_dpo/margin": 327.53546142578125, "fcm_dpo/q_t": 0.44127804040908813, "grad_norm": 25.92827033996582, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -0.9558432698249817, "logits/rejected": -0.9528769254684448, "logps/chosen": -689.381591796875, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -1046.7554931640625, "loss": 1.1731, "margin_dpo/margin_mean": 327.53546142578125, "margin_dpo/margin_std": 559.0849609375, "step": 553 }, { "KL/chosen_KL_mean": -633.1609497070312, "KL/mean": -847.7535400390625, "KL/rejected_KL_mean": -1062.3460693359375, "KL/std": 536.1700439453125, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0009824573062360287, "fcm_dpo/delta": -0.04598114266991615, "fcm_dpo/margin": 429.18511962890625, "fcm_dpo/q_t": 0.4033704102039337, "grad_norm": 57.81589126586914, "learning_rate": 5.205293880283551e-08, "logits/chosen": -0.9455916285514832, "logits/rejected": -0.9259899258613586, "logps/chosen": -701.108642578125, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1152.1287841796875, "loss": 1.1217, "margin_dpo/margin_mean": 429.18511962890625, "margin_dpo/margin_std": 682.432861328125, "step": 554 }, { "KL/chosen_KL_mean": -634.130615234375, "KL/mean": -891.5674438476562, "KL/rejected_KL_mean": -1149.0042724609375, "KL/std": 582.853515625, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.000833613914437592, "fcm_dpo/delta": -0.051001857966184616, "fcm_dpo/margin": 514.8737182617188, "fcm_dpo/q_t": 0.40110278129577637, "grad_norm": 41.96858215332031, "learning_rate": 5.127169765359515e-08, "logits/chosen": -0.9838435649871826, "logits/rejected": -1.0397891998291016, "logps/chosen": -687.4610595703125, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -1257.483642578125, "loss": 1.093, "margin_dpo/margin_mean": 514.8736572265625, "margin_dpo/margin_std": 755.9553833007812, "step": 555 }, { "KL/chosen_KL_mean": -619.962646484375, "KL/mean": -767.575439453125, "KL/rejected_KL_mean": -915.1883544921875, "KL/std": 455.46258544921875, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.0010035810992121696, "fcm_dpo/delta": 0.14432799816131592, "fcm_dpo/margin": 295.22576904296875, "fcm_dpo/q_t": 0.4395366907119751, "grad_norm": 32.15121841430664, "learning_rate": 5.049569317994012e-08, "logits/chosen": -0.9909765720367432, "logits/rejected": -0.9854000210762024, "logps/chosen": -678.6070556640625, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -1016.5287475585938, "loss": 1.163, "margin_dpo/margin_mean": 295.22576904296875, "margin_dpo/margin_std": 462.01824951171875, "step": 556 }, { "KL/chosen_KL_mean": -678.8939208984375, "KL/mean": -913.603759765625, "KL/rejected_KL_mean": -1148.3134765625, "KL/std": 606.7928466796875, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.0009204911766573787, "fcm_dpo/delta": -0.06962708383798599, "fcm_dpo/margin": 469.4195556640625, "fcm_dpo/q_t": 0.39773887395858765, "grad_norm": 51.71062088012695, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -0.9994686841964722, "logits/rejected": -1.0355925559997559, "logps/chosen": -746.734619140625, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -1258.253173828125, "loss": 1.0935, "margin_dpo/margin_mean": 469.4195556640625, "margin_dpo/margin_std": 688.9013671875, "step": 557 }, { "KL/chosen_KL_mean": -629.8007202148438, "KL/mean": -937.6903076171875, "KL/rejected_KL_mean": -1245.579833984375, "KL/std": 600.0916748046875, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0007529731374233961, "fcm_dpo/delta": -0.12873995304107666, "fcm_dpo/margin": 615.7791748046875, "fcm_dpo/q_t": 0.37874358892440796, "grad_norm": 26.18462562561035, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -1.0540165901184082, "logits/rejected": -1.0735430717468262, "logps/chosen": -692.1689453125, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -1347.740966796875, "loss": 1.037, "margin_dpo/margin_mean": 615.7791748046875, "margin_dpo/margin_std": 673.7421264648438, "step": 558 }, { "KL/chosen_KL_mean": -701.1896362304688, "KL/mean": -954.133544921875, "KL/rejected_KL_mean": -1207.0775146484375, "KL/std": 592.470458984375, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0007604295969940722, "fcm_dpo/delta": 0.023233801126480103, "fcm_dpo/margin": 505.8878479003906, "fcm_dpo/q_t": 0.41252970695495605, "grad_norm": 24.069955825805664, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -1.1368141174316406, "logits/rejected": -1.1402065753936768, "logps/chosen": -761.9420166015625, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -1300.519775390625, "loss": 1.0951, "margin_dpo/margin_mean": 505.8878479003906, "margin_dpo/margin_std": 648.712158203125, "step": 559 }, { "KL/chosen_KL_mean": -657.3939208984375, "KL/mean": -828.5643310546875, "KL/rejected_KL_mean": -999.7348022460938, "KL/std": 511.27996826171875, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0008098391117528081, "fcm_dpo/delta": 0.034772761166095734, "fcm_dpo/margin": 342.3409118652344, "fcm_dpo/q_t": 0.437608540058136, "grad_norm": 26.729141235351562, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.9040641784667969, "logits/rejected": -0.8838648796081543, "logps/chosen": -715.4976806640625, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -1079.72607421875, "loss": 1.1829, "margin_dpo/margin_mean": 342.3409423828125, "margin_dpo/margin_std": 584.12548828125, "step": 560 }, { "KL/chosen_KL_mean": -759.172119140625, "KL/mean": -920.302490234375, "KL/rejected_KL_mean": -1081.432861328125, "KL/std": 530.906494140625, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0009913041722029448, "fcm_dpo/delta": 0.1052679792046547, "fcm_dpo/margin": 322.26068115234375, "fcm_dpo/q_t": 0.43175017833709717, "grad_norm": 45.66500473022461, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.0488073825836182, "logits/rejected": -1.0699024200439453, "logps/chosen": -810.0850219726562, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -1180.50146484375, "loss": 1.1919, "margin_dpo/margin_mean": 322.2607116699219, "margin_dpo/margin_std": 664.1737060546875, "step": 561 }, { "KL/chosen_KL_mean": -713.2972412109375, "KL/mean": -922.2352905273438, "KL/rejected_KL_mean": -1131.1732177734375, "KL/std": 585.4476928710938, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.000937954755499959, "fcm_dpo/delta": 0.0012168213725090027, "fcm_dpo/margin": 417.87603759765625, "fcm_dpo/q_t": 0.4111784100532532, "grad_norm": 32.701107025146484, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -1.0261307954788208, "logits/rejected": -1.029954433441162, "logps/chosen": -772.7616577148438, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1227.7158203125, "loss": 1.1174, "margin_dpo/margin_mean": 417.87603759765625, "margin_dpo/margin_std": 627.3534545898438, "step": 562 }, { "KL/chosen_KL_mean": -804.9948120117188, "KL/mean": -967.0625, "KL/rejected_KL_mean": -1129.1302490234375, "KL/std": 637.6392211914062, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0009516784339211881, "fcm_dpo/delta": -0.052980802953243256, "fcm_dpo/margin": 324.13543701171875, "fcm_dpo/q_t": 0.42142850160598755, "grad_norm": 52.949893951416016, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.0193402767181396, "logits/rejected": -1.0229434967041016, "logps/chosen": -865.60302734375, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -1223.697998046875, "loss": 1.2444, "margin_dpo/margin_mean": 324.13543701171875, "margin_dpo/margin_std": 768.7203369140625, "step": 563 }, { "KL/chosen_KL_mean": -731.5855712890625, "KL/mean": -927.5992431640625, "KL/rejected_KL_mean": -1123.6129150390625, "KL/std": 567.4203491210938, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.000967438449151814, "fcm_dpo/delta": 0.030362479388713837, "fcm_dpo/margin": 392.0273132324219, "fcm_dpo/q_t": 0.41628676652908325, "grad_norm": 39.21650695800781, "learning_rate": 4.447860229910544e-08, "logits/chosen": -1.102457880973816, "logits/rejected": -1.0930469036102295, "logps/chosen": -805.8539428710938, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -1216.85107421875, "loss": 1.1161, "margin_dpo/margin_mean": 392.0273132324219, "margin_dpo/margin_std": 574.1221923828125, "step": 564 }, { "KL/chosen_KL_mean": -767.8232421875, "KL/mean": -991.4110107421875, "KL/rejected_KL_mean": -1214.998779296875, "KL/std": 661.7870483398438, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.0009264518739655614, "fcm_dpo/delta": -0.024389147758483887, "fcm_dpo/margin": 447.1756286621094, "fcm_dpo/q_t": 0.4096825420856476, "grad_norm": 40.5957145690918, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.025184154510498, "logits/rejected": -1.0267276763916016, "logps/chosen": -836.8431396484375, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -1300.77783203125, "loss": 1.1389, "margin_dpo/margin_mean": 447.1756286621094, "margin_dpo/margin_std": 811.365234375, "step": 565 }, { "KL/chosen_KL_mean": -710.8924560546875, "KL/mean": -951.005615234375, "KL/rejected_KL_mean": -1191.1187744140625, "KL/std": 655.048828125, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0008984344312921166, "fcm_dpo/delta": -0.06253774464130402, "fcm_dpo/margin": 480.226318359375, "fcm_dpo/q_t": 0.3989356458187103, "grad_norm": 32.377769470214844, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -0.997527003288269, "logits/rejected": -1.0165774822235107, "logps/chosen": -777.4378051757812, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -1294.988037109375, "loss": 1.1279, "margin_dpo/margin_mean": 480.226318359375, "margin_dpo/margin_std": 777.289794921875, "step": 566 }, { "KL/chosen_KL_mean": -701.48583984375, "KL/mean": -886.89306640625, "KL/rejected_KL_mean": -1072.3001708984375, "KL/std": 463.2765197753906, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.000920231977943331, "fcm_dpo/delta": 0.0880487933754921, "fcm_dpo/margin": 370.814453125, "fcm_dpo/q_t": 0.42590731382369995, "grad_norm": 27.92223358154297, "learning_rate": 4.231101308059165e-08, "logits/chosen": -1.1106657981872559, "logits/rejected": -1.1169605255126953, "logps/chosen": -754.3441162109375, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -1157.671142578125, "loss": 1.1274, "margin_dpo/margin_mean": 370.814453125, "margin_dpo/margin_std": 517.0704345703125, "step": 567 }, { "KL/chosen_KL_mean": -680.7457885742188, "KL/mean": -934.899658203125, "KL/rejected_KL_mean": -1189.053466796875, "KL/std": 556.766845703125, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0008575194515287876, "fcm_dpo/delta": -0.06648175418376923, "fcm_dpo/margin": 508.3077087402344, "fcm_dpo/q_t": 0.3927525281906128, "grad_norm": 30.725919723510742, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -0.9947052001953125, "logits/rejected": -1.032778024673462, "logps/chosen": -725.9381713867188, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1278.14599609375, "loss": 1.0558, "margin_dpo/margin_mean": 508.3077392578125, "margin_dpo/margin_std": 588.199951171875, "step": 568 }, { "KL/chosen_KL_mean": -766.2860717773438, "KL/mean": -974.7943115234375, "KL/rejected_KL_mean": -1183.3026123046875, "KL/std": 686.7083740234375, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.0009196175960823894, "fcm_dpo/delta": 0.012853320688009262, "fcm_dpo/margin": 417.0164794921875, "fcm_dpo/q_t": 0.41270917654037476, "grad_norm": 42.41626739501953, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.0407395362854004, "logits/rejected": -1.0457051992416382, "logps/chosen": -830.0066528320312, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1262.40576171875, "loss": 1.1548, "margin_dpo/margin_mean": 417.0164794921875, "margin_dpo/margin_std": 728.3641357421875, "step": 569 }, { "KL/chosen_KL_mean": -706.3309326171875, "KL/mean": -890.91357421875, "KL/rejected_KL_mean": -1075.49609375, "KL/std": 532.2640380859375, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0009554984280839562, "fcm_dpo/delta": 0.07299959659576416, "fcm_dpo/margin": 369.1651306152344, "fcm_dpo/q_t": 0.424812376499176, "grad_norm": 25.38414764404297, "learning_rate": 4.019267817841834e-08, "logits/chosen": -1.12546968460083, "logits/rejected": -1.1144479513168335, "logps/chosen": -767.945556640625, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -1157.637939453125, "loss": 1.1375, "margin_dpo/margin_mean": 369.1651306152344, "margin_dpo/margin_std": 582.8961181640625, "step": 570 }, { "KL/chosen_KL_mean": -729.112548828125, "KL/mean": -953.21875, "KL/rejected_KL_mean": -1177.324951171875, "KL/std": 557.555908203125, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0009227694245055318, "fcm_dpo/delta": -0.02847745269536972, "fcm_dpo/margin": 448.21246337890625, "fcm_dpo/q_t": 0.4045218527317047, "grad_norm": 43.35622024536133, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -0.9972771406173706, "logits/rejected": -1.0120426416397095, "logps/chosen": -782.1666259765625, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1268.661865234375, "loss": 1.1091, "margin_dpo/margin_mean": 448.21246337890625, "margin_dpo/margin_std": 690.9599609375, "step": 571 }, { "KL/chosen_KL_mean": -768.8928833007812, "KL/mean": -1001.8563232421875, "KL/rejected_KL_mean": -1234.8197021484375, "KL/std": 665.1800537109375, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.000880242558196187, "fcm_dpo/delta": -0.021999845281243324, "fcm_dpo/margin": 465.9267883300781, "fcm_dpo/q_t": 0.40960174798965454, "grad_norm": 27.348243713378906, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.0507442951202393, "logits/rejected": -1.080924153327942, "logps/chosen": -817.3521728515625, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1318.376708984375, "loss": 1.13, "margin_dpo/margin_mean": 465.92681884765625, "margin_dpo/margin_std": 805.666015625, "step": 572 }, { "KL/chosen_KL_mean": -752.9931030273438, "KL/mean": -959.6439208984375, "KL/rejected_KL_mean": -1166.294677734375, "KL/std": 583.8150634765625, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.000941689417231828, "fcm_dpo/delta": 0.016215384006500244, "fcm_dpo/margin": 413.30157470703125, "fcm_dpo/q_t": 0.4137089252471924, "grad_norm": 29.199546813964844, "learning_rate": 3.812409996461275e-08, "logits/chosen": -1.084615707397461, "logits/rejected": -1.095890760421753, "logps/chosen": -804.61572265625, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1251.61962890625, "loss": 1.1048, "margin_dpo/margin_mean": 413.30157470703125, "margin_dpo/margin_std": 595.2252197265625, "step": 573 }, { "KL/chosen_KL_mean": -693.208984375, "KL/mean": -919.9398193359375, "KL/rejected_KL_mean": -1146.670654296875, "KL/std": 540.618896484375, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0009166327654384077, "fcm_dpo/delta": -0.027354443445801735, "fcm_dpo/margin": 453.4616394042969, "fcm_dpo/q_t": 0.40286535024642944, "grad_norm": 39.16676712036133, "learning_rate": 3.74457160675965e-08, "logits/chosen": -1.1048978567123413, "logits/rejected": -1.1292717456817627, "logps/chosen": -744.25341796875, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -1239.47705078125, "loss": 1.09, "margin_dpo/margin_mean": 453.46160888671875, "margin_dpo/margin_std": 616.9259643554688, "step": 574 }, { "KL/chosen_KL_mean": -759.01708984375, "KL/mean": -962.711181640625, "KL/rejected_KL_mean": -1166.4052734375, "KL/std": 562.2581787109375, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0007753444951958954, "fcm_dpo/delta": -0.06450725346803665, "fcm_dpo/margin": 407.38818359375, "fcm_dpo/q_t": 0.42131945490837097, "grad_norm": 27.12775993347168, "learning_rate": 3.677293317363864e-08, "logits/chosen": -0.9311203360557556, "logits/rejected": -0.9386662244796753, "logps/chosen": -830.8072509765625, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1261.79150390625, "loss": 1.1753, "margin_dpo/margin_mean": 407.3881530761719, "margin_dpo/margin_std": 729.4576416015625, "step": 575 }, { "KL/chosen_KL_mean": -730.3524169921875, "KL/mean": -889.8451538085938, "KL/rejected_KL_mean": -1049.3380126953125, "KL/std": 508.9241943359375, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.0008465542923659086, "fcm_dpo/delta": 0.043933454900979996, "fcm_dpo/margin": 318.985595703125, "fcm_dpo/q_t": 0.4400612413883209, "grad_norm": 28.7397403717041, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.0479196310043335, "logits/rejected": -1.076101541519165, "logps/chosen": -784.6153564453125, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -1150.09228515625, "loss": 1.1968, "margin_dpo/margin_mean": 318.9855651855469, "margin_dpo/margin_std": 597.7258911132812, "step": 576 }, { "KL/chosen_KL_mean": -645.6124877929688, "KL/mean": -855.6819458007812, "KL/rejected_KL_mean": -1065.75146484375, "KL/std": 571.1446533203125, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0008858378860168159, "fcm_dpo/delta": 0.04430598393082619, "fcm_dpo/margin": 420.138916015625, "fcm_dpo/q_t": 0.4191765785217285, "grad_norm": 24.229158401489258, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.0603196620941162, "logits/rejected": -1.0447897911071777, "logps/chosen": -707.522216796875, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -1149.8221435546875, "loss": 1.1276, "margin_dpo/margin_mean": 420.138916015625, "margin_dpo/margin_std": 631.5208740234375, "step": 577 }, { "KL/chosen_KL_mean": -638.2508544921875, "KL/mean": -869.6015014648438, "KL/rejected_KL_mean": -1100.9521484375, "KL/std": 571.8765258789062, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0008566380711272359, "fcm_dpo/delta": -0.007108829915523529, "fcm_dpo/margin": 462.70135498046875, "fcm_dpo/q_t": 0.40710175037384033, "grad_norm": 31.410873413085938, "learning_rate": 3.478836705390808e-08, "logits/chosen": -0.9340611696243286, "logits/rejected": -0.968732476234436, "logps/chosen": -687.5145263671875, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -1184.388427734375, "loss": 1.0938, "margin_dpo/margin_mean": 462.70135498046875, "margin_dpo/margin_std": 587.1175537109375, "step": 578 }, { "KL/chosen_KL_mean": -741.0704345703125, "KL/mean": -876.5369873046875, "KL/rejected_KL_mean": -1012.0033569335938, "KL/std": 570.01953125, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0010091912699863315, "fcm_dpo/delta": 0.05066795274615288, "fcm_dpo/margin": 270.932861328125, "fcm_dpo/q_t": 0.44031286239624023, "grad_norm": 65.59943389892578, "learning_rate": 3.41381639738331e-08, "logits/chosen": -0.9676034450531006, "logits/rejected": -0.9647072553634644, "logps/chosen": -799.956298828125, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -1106.791015625, "loss": 1.2276, "margin_dpo/margin_mean": 270.93292236328125, "margin_dpo/margin_std": 630.8173217773438, "step": 579 }, { "KL/chosen_KL_mean": -552.2589721679688, "KL/mean": -812.1983642578125, "KL/rejected_KL_mean": -1072.1376953125, "KL/std": 636.0146484375, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0008385708788409829, "fcm_dpo/delta": -0.08361184597015381, "fcm_dpo/margin": 519.8788452148438, "fcm_dpo/q_t": 0.3960821032524109, "grad_norm": 34.06106185913086, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.8970617055892944, "logits/rejected": -0.9277414679527283, "logps/chosen": -600.9658203125, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -1153.896240234375, "loss": 1.0825, "margin_dpo/margin_mean": 519.8788452148438, "margin_dpo/margin_std": 744.65966796875, "step": 580 }, { "KL/chosen_KL_mean": -730.3821411132812, "KL/mean": -920.6229858398438, "KL/rejected_KL_mean": -1110.86376953125, "KL/std": 590.761962890625, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0009422561852261424, "fcm_dpo/delta": 0.06384095549583435, "fcm_dpo/margin": 380.4817199707031, "fcm_dpo/q_t": 0.4236143231391907, "grad_norm": 37.83233642578125, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.045546054840088, "logits/rejected": -1.0597128868103027, "logps/chosen": -792.6044921875, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -1202.599609375, "loss": 1.1464, "margin_dpo/margin_mean": 380.48175048828125, "margin_dpo/margin_std": 654.8671875, "step": 581 }, { "KL/chosen_KL_mean": -663.98291015625, "KL/mean": -884.2652587890625, "KL/rejected_KL_mean": -1104.547607421875, "KL/std": 535.1427001953125, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0009837101679295301, "fcm_dpo/delta": -0.08480895310640335, "fcm_dpo/margin": 440.5647277832031, "fcm_dpo/q_t": 0.3922159671783447, "grad_norm": 33.73926544189453, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.0071823596954346, "logits/rejected": -1.0349555015563965, "logps/chosen": -722.2115478515625, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -1214.6171875, "loss": 1.1097, "margin_dpo/margin_mean": 440.56475830078125, "margin_dpo/margin_std": 578.998046875, "step": 582 }, { "KL/chosen_KL_mean": -755.7252197265625, "KL/mean": -893.9094848632812, "KL/rejected_KL_mean": -1032.0936279296875, "KL/std": 595.7422485351562, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0008164974860846996, "fcm_dpo/delta": -0.0013605619315057993, "fcm_dpo/margin": 276.36834716796875, "fcm_dpo/q_t": 0.4494253695011139, "grad_norm": 38.58605194091797, "learning_rate": 3.159440233840763e-08, "logits/chosen": -0.9783551096916199, "logits/rejected": -0.9746303558349609, "logps/chosen": -812.588134765625, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -1120.49755859375, "loss": 1.2611, "margin_dpo/margin_mean": 276.36834716796875, "margin_dpo/margin_std": 718.41357421875, "step": 583 }, { "KL/chosen_KL_mean": -633.4464721679688, "KL/mean": -902.180908203125, "KL/rejected_KL_mean": -1170.915283203125, "KL/std": 608.067626953125, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.0007747331401333213, "fcm_dpo/delta": -0.028918039053678513, "fcm_dpo/margin": 537.46875, "fcm_dpo/q_t": 0.4025737941265106, "grad_norm": 19.63285255432129, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -0.986220121383667, "logits/rejected": -1.0069942474365234, "logps/chosen": -690.34716796875, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1268.55126953125, "loss": 1.0713, "margin_dpo/margin_mean": 537.46875, "margin_dpo/margin_std": 643.06298828125, "step": 584 }, { "KL/chosen_KL_mean": -779.0941772460938, "KL/mean": -1014.2800903320312, "KL/rejected_KL_mean": -1249.466064453125, "KL/std": 677.1864013671875, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0007966947159729898, "fcm_dpo/delta": 0.03990716487169266, "fcm_dpo/margin": 470.371826171875, "fcm_dpo/q_t": 0.4176916182041168, "grad_norm": 31.820642471313477, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.07651948928833, "logits/rejected": -1.104307770729065, "logps/chosen": -840.068115234375, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -1333.635498046875, "loss": 1.1456, "margin_dpo/margin_mean": 470.371826171875, "margin_dpo/margin_std": 820.4464111328125, "step": 585 }, { "KL/chosen_KL_mean": -788.8521728515625, "KL/mean": -943.93310546875, "KL/rejected_KL_mean": -1099.013916015625, "KL/std": 580.5469970703125, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0010007360251620412, "fcm_dpo/delta": 0.1305122673511505, "fcm_dpo/margin": 310.16180419921875, "fcm_dpo/q_t": 0.43785321712493896, "grad_norm": 55.22218322753906, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.0562876462936401, "logits/rejected": -1.0779566764831543, "logps/chosen": -845.707763671875, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -1190.816650390625, "loss": 1.1985, "margin_dpo/margin_mean": 310.16180419921875, "margin_dpo/margin_std": 638.7469482421875, "step": 586 }, { "KL/chosen_KL_mean": -554.4176635742188, "KL/mean": -765.8219604492188, "KL/rejected_KL_mean": -977.2261962890625, "KL/std": 601.9225463867188, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0010241747368127108, "fcm_dpo/delta": -0.05821375921368599, "fcm_dpo/margin": 422.8085632324219, "fcm_dpo/q_t": 0.39820683002471924, "grad_norm": 31.542490005493164, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -1.0706478357315063, "logits/rejected": -1.0984125137329102, "logps/chosen": -599.1092529296875, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -1059.85009765625, "loss": 1.0901, "margin_dpo/margin_mean": 422.80853271484375, "margin_dpo/margin_std": 611.4573974609375, "step": 587 }, { "KL/chosen_KL_mean": -721.3621826171875, "KL/mean": -904.2818603515625, "KL/rejected_KL_mean": -1087.2015380859375, "KL/std": 524.9571533203125, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0009967966470867395, "fcm_dpo/delta": 0.05296815186738968, "fcm_dpo/margin": 365.8393859863281, "fcm_dpo/q_t": 0.41959983110427856, "grad_norm": 32.92317581176758, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.0092002153396606, "logits/rejected": -1.0338335037231445, "logps/chosen": -771.6571044921875, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -1194.571533203125, "loss": 1.1352, "margin_dpo/margin_mean": 365.83941650390625, "margin_dpo/margin_std": 559.2975463867188, "step": 588 }, { "KL/chosen_KL_mean": -717.5643920898438, "KL/mean": -957.2388916015625, "KL/rejected_KL_mean": -1196.9134521484375, "KL/std": 576.5167846679688, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0009336763177998364, "fcm_dpo/delta": -0.08870557695627213, "fcm_dpo/margin": 479.34906005859375, "fcm_dpo/q_t": 0.39069461822509766, "grad_norm": 31.190114974975586, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -0.9594367146492004, "logits/rejected": -0.968762993812561, "logps/chosen": -777.4942626953125, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -1308.56884765625, "loss": 1.0654, "margin_dpo/margin_mean": 479.34906005859375, "margin_dpo/margin_std": 618.2476806640625, "step": 589 }, { "KL/chosen_KL_mean": -610.7032470703125, "KL/mean": -851.4964599609375, "KL/rejected_KL_mean": -1092.28955078125, "KL/std": 560.0525512695312, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0008607573690824211, "fcm_dpo/delta": -0.025180064141750336, "fcm_dpo/margin": 481.5863037109375, "fcm_dpo/q_t": 0.40354564785957336, "grad_norm": 23.0913028717041, "learning_rate": 2.736501028272095e-08, "logits/chosen": -0.9822874069213867, "logits/rejected": -1.0102424621582031, "logps/chosen": -666.5130615234375, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -1198.3524169921875, "loss": 1.0802, "margin_dpo/margin_mean": 481.5863037109375, "margin_dpo/margin_std": 624.2484741210938, "step": 590 }, { "KL/chosen_KL_mean": -666.4302978515625, "KL/mean": -876.166259765625, "KL/rejected_KL_mean": -1085.902099609375, "KL/std": 542.080322265625, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0008978393743745983, "fcm_dpo/delta": 0.03757140040397644, "fcm_dpo/margin": 419.4718017578125, "fcm_dpo/q_t": 0.41537439823150635, "grad_norm": 31.897592544555664, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.0660505294799805, "logits/rejected": -1.0602378845214844, "logps/chosen": -722.6708984375, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -1169.6884765625, "loss": 1.1202, "margin_dpo/margin_mean": 419.4718017578125, "margin_dpo/margin_std": 628.8763427734375, "step": 591 }, { "KL/chosen_KL_mean": -694.1058349609375, "KL/mean": -893.2886962890625, "KL/rejected_KL_mean": -1092.4716796875, "KL/std": 556.25927734375, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0009542823536321521, "fcm_dpo/delta": 0.03209465742111206, "fcm_dpo/margin": 398.3658447265625, "fcm_dpo/q_t": 0.4169124960899353, "grad_norm": 41.34512710571289, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.0209119319915771, "logits/rejected": -1.0272916555404663, "logps/chosen": -742.0460815429688, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -1168.20458984375, "loss": 1.1721, "margin_dpo/margin_mean": 398.3658447265625, "margin_dpo/margin_std": 758.87109375, "step": 592 }, { "KL/chosen_KL_mean": -679.8154296875, "KL/mean": -824.459228515625, "KL/rejected_KL_mean": -969.10302734375, "KL/std": 587.4564208984375, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.001031426596455276, "fcm_dpo/delta": 0.024800950661301613, "fcm_dpo/margin": 289.28753662109375, "fcm_dpo/q_t": 0.4358917474746704, "grad_norm": 62.813846588134766, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.9074722528457642, "logits/rejected": -0.9004065990447998, "logps/chosen": -728.5061645507812, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -1030.010986328125, "loss": 1.2174, "margin_dpo/margin_mean": 289.28753662109375, "margin_dpo/margin_std": 654.1934204101562, "step": 593 }, { "KL/chosen_KL_mean": -626.3131103515625, "KL/mean": -814.7869873046875, "KL/rejected_KL_mean": -1003.2608642578125, "KL/std": 552.5802001953125, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0010870166588574648, "fcm_dpo/delta": -0.03803686052560806, "fcm_dpo/margin": 376.94769287109375, "fcm_dpo/q_t": 0.4029105305671692, "grad_norm": 54.68209457397461, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.0033159255981445, "logits/rejected": -0.9932465553283691, "logps/chosen": -681.248046875, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -1089.360595703125, "loss": 1.1364, "margin_dpo/margin_mean": 376.94769287109375, "margin_dpo/margin_std": 580.914794921875, "step": 594 }, { "KL/chosen_KL_mean": -590.2716674804688, "KL/mean": -798.0963134765625, "KL/rejected_KL_mean": -1005.9209594726562, "KL/std": 527.621826171875, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0009453489328734577, "fcm_dpo/delta": 0.01016305387020111, "fcm_dpo/margin": 415.6493225097656, "fcm_dpo/q_t": 0.41294392943382263, "grad_norm": 53.49211120605469, "learning_rate": 2.451969280180849e-08, "logits/chosen": -0.970264196395874, "logits/rejected": -0.9880306124687195, "logps/chosen": -639.6920776367188, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -1086.54833984375, "loss": 1.1058, "margin_dpo/margin_mean": 415.6493225097656, "margin_dpo/margin_std": 597.9583129882812, "step": 595 }, { "KL/chosen_KL_mean": -709.765625, "KL/mean": -855.121337890625, "KL/rejected_KL_mean": -1000.47705078125, "KL/std": 552.8822021484375, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.0011489235330373049, "fcm_dpo/delta": 0.09330044686794281, "fcm_dpo/margin": 290.71142578125, "fcm_dpo/q_t": 0.4343324303627014, "grad_norm": 93.66545867919922, "learning_rate": 2.396839494982103e-08, "logits/chosen": -0.9842763543128967, "logits/rejected": -0.9520334005355835, "logps/chosen": -769.5573120117188, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -1080.568115234375, "loss": 1.2066, "margin_dpo/margin_mean": 290.71142578125, "margin_dpo/margin_std": 659.2149658203125, "step": 596 }, { "KL/chosen_KL_mean": -605.57763671875, "KL/mean": -864.177001953125, "KL/rejected_KL_mean": -1122.7763671875, "KL/std": 595.9703369140625, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0008297483436763287, "fcm_dpo/delta": -0.1401767134666443, "fcm_dpo/margin": 517.19873046875, "fcm_dpo/q_t": 0.3865550458431244, "grad_norm": 30.938121795654297, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -0.8776600956916809, "logits/rejected": -0.9235655069351196, "logps/chosen": -662.83837890625, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -1223.469970703125, "loss": 1.0834, "margin_dpo/margin_mean": 517.1986694335938, "margin_dpo/margin_std": 668.316162109375, "step": 597 }, { "KL/chosen_KL_mean": -645.602294921875, "KL/mean": -850.23193359375, "KL/rejected_KL_mean": -1054.861572265625, "KL/std": 531.68505859375, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0008029807358980179, "fcm_dpo/delta": -0.050433676689863205, "fcm_dpo/margin": 409.2593994140625, "fcm_dpo/q_t": 0.41770249605178833, "grad_norm": 26.686721801757812, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -0.9931685328483582, "logits/rejected": -1.0055735111236572, "logps/chosen": -698.1207885742188, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -1144.3055419921875, "loss": 1.1418, "margin_dpo/margin_mean": 409.2593994140625, "margin_dpo/margin_std": 592.2933349609375, "step": 598 }, { "KL/chosen_KL_mean": -626.4500732421875, "KL/mean": -800.4187622070312, "KL/rejected_KL_mean": -974.387451171875, "KL/std": 483.94842529296875, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0007874942384660244, "fcm_dpo/delta": -0.009737352840602398, "fcm_dpo/margin": 347.93743896484375, "fcm_dpo/q_t": 0.4326390027999878, "grad_norm": 21.70595932006836, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -0.8962666988372803, "logits/rejected": -0.9054356813430786, "logps/chosen": -676.2527465820312, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -1057.365966796875, "loss": 1.168, "margin_dpo/margin_mean": 347.93743896484375, "margin_dpo/margin_std": 467.16839599609375, "step": 599 }, { "KL/chosen_KL_mean": -702.6114501953125, "KL/mean": -863.553955078125, "KL/rejected_KL_mean": -1024.496337890625, "KL/std": 520.333251953125, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.0009568602545186877, "fcm_dpo/delta": 0.1319303810596466, "fcm_dpo/margin": 321.88494873046875, "fcm_dpo/q_t": 0.43882617354393005, "grad_norm": 38.444881439208984, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -1.0397906303405762, "logits/rejected": -1.0333709716796875, "logps/chosen": -769.04638671875, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -1109.952880859375, "loss": 1.1911, "margin_dpo/margin_mean": 321.88494873046875, "margin_dpo/margin_std": 643.2849731445312, "step": 600 }, { "KL/chosen_KL_mean": -702.807373046875, "KL/mean": -912.3177490234375, "KL/rejected_KL_mean": -1121.828125, "KL/std": 553.8662109375, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0009801883716136217, "fcm_dpo/delta": -0.01944911666214466, "fcm_dpo/margin": 419.02081298828125, "fcm_dpo/q_t": 0.40293174982070923, "grad_norm": 32.09441375732422, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -0.9758914113044739, "logits/rejected": -0.9933589696884155, "logps/chosen": -761.9410400390625, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -1216.5191650390625, "loss": 1.0856, "margin_dpo/margin_mean": 419.02081298828125, "margin_dpo/margin_std": 550.6053466796875, "step": 601 }, { "KL/chosen_KL_mean": -479.7313232421875, "KL/mean": -729.894287109375, "KL/rejected_KL_mean": -980.057373046875, "KL/std": 495.54962158203125, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0009083700715564191, "fcm_dpo/delta": -0.10118807107210159, "fcm_dpo/margin": 500.3260192871094, "fcm_dpo/q_t": 0.3865256905555725, "grad_norm": 69.2566909790039, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -0.9927771091461182, "logits/rejected": -1.0256961584091187, "logps/chosen": -528.3248291015625, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -1067.725830078125, "loss": 1.0464, "margin_dpo/margin_mean": 500.3260192871094, "margin_dpo/margin_std": 536.3623046875, "step": 602 }, { "KL/chosen_KL_mean": -610.1720581054688, "KL/mean": -848.7678833007812, "KL/rejected_KL_mean": -1087.3636474609375, "KL/std": 582.8988037109375, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0008255116990767419, "fcm_dpo/delta": 0.009567895904183388, "fcm_dpo/margin": 477.191650390625, "fcm_dpo/q_t": 0.41279083490371704, "grad_norm": 32.93044662475586, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -0.9903879165649414, "logits/rejected": -0.9831830263137817, "logps/chosen": -680.586669921875, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -1187.689208984375, "loss": 1.1056, "margin_dpo/margin_mean": 477.191650390625, "margin_dpo/margin_std": 704.5848388671875, "step": 603 }, { "KL/chosen_KL_mean": -604.884521484375, "KL/mean": -850.2882080078125, "KL/rejected_KL_mean": -1095.69189453125, "KL/std": 557.6204833984375, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0008146062027662992, "fcm_dpo/delta": 0.0001351330429315567, "fcm_dpo/margin": 490.8074035644531, "fcm_dpo/q_t": 0.40916040539741516, "grad_norm": 38.27802658081055, "learning_rate": 1.977362051376158e-08, "logits/chosen": -0.9702785015106201, "logits/rejected": -1.004528284072876, "logps/chosen": -651.3425903320312, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -1187.54638671875, "loss": 1.1004, "margin_dpo/margin_mean": 490.8074035644531, "margin_dpo/margin_std": 693.0052490234375, "step": 604 }, { "KL/chosen_KL_mean": -633.1536865234375, "KL/mean": -830.642578125, "KL/rejected_KL_mean": -1028.1314697265625, "KL/std": 524.9966430664062, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0008949829498305917, "fcm_dpo/delta": 0.07238632440567017, "fcm_dpo/margin": 394.977783203125, "fcm_dpo/q_t": 0.42492377758026123, "grad_norm": 38.80448913574219, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -0.9616391658782959, "logits/rejected": -0.9820314645767212, "logps/chosen": -699.4030151367188, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -1130.4365234375, "loss": 1.1385, "margin_dpo/margin_mean": 394.977783203125, "margin_dpo/margin_std": 627.3665771484375, "step": 605 }, { "KL/chosen_KL_mean": -626.211181640625, "KL/mean": -838.0142211914062, "KL/rejected_KL_mean": -1049.8173828125, "KL/std": 531.291748046875, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0009638095507398248, "fcm_dpo/delta": -0.017124518752098083, "fcm_dpo/margin": 423.60614013671875, "fcm_dpo/q_t": 0.4059169888496399, "grad_norm": 26.27326011657715, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -0.9585670232772827, "logits/rejected": -0.9715864658355713, "logps/chosen": -681.0302734375, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -1148.188720703125, "loss": 1.0882, "margin_dpo/margin_mean": 423.60614013671875, "margin_dpo/margin_std": 562.107666015625, "step": 606 }, { "KL/chosen_KL_mean": -664.5488891601562, "KL/mean": -828.581298828125, "KL/rejected_KL_mean": -992.6136474609375, "KL/std": 521.07080078125, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0010647273156791925, "fcm_dpo/delta": 0.06575503945350647, "fcm_dpo/margin": 328.06475830078125, "fcm_dpo/q_t": 0.42434990406036377, "grad_norm": 34.5279655456543, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -0.9723426103591919, "logits/rejected": -0.9582047462463379, "logps/chosen": -722.6329345703125, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -1072.3907470703125, "loss": 1.1543, "margin_dpo/margin_mean": 328.06475830078125, "margin_dpo/margin_std": 572.1151123046875, "step": 607 }, { "KL/chosen_KL_mean": -599.10595703125, "KL/mean": -798.927978515625, "KL/rejected_KL_mean": -998.75, "KL/std": 486.12176513671875, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.001036277157254517, "fcm_dpo/delta": -0.02508939430117607, "fcm_dpo/margin": 399.6440734863281, "fcm_dpo/q_t": 0.4025970697402954, "grad_norm": 39.96210479736328, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.0201570987701416, "logits/rejected": -1.0304514169692993, "logps/chosen": -656.5568237304688, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -1093.5234375, "loss": 1.0788, "margin_dpo/margin_mean": 399.64404296875, "margin_dpo/margin_std": 502.84564208984375, "step": 608 }, { "KL/chosen_KL_mean": -625.5040283203125, "KL/mean": -888.441162109375, "KL/rejected_KL_mean": -1151.378173828125, "KL/std": 666.3944091796875, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0008594167302362621, "fcm_dpo/delta": -0.09702034294605255, "fcm_dpo/margin": 525.8741455078125, "fcm_dpo/q_t": 0.3949633240699768, "grad_norm": 29.249309539794922, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -1.0417685508728027, "logits/rejected": -1.0658833980560303, "logps/chosen": -684.3094482421875, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -1240.1943359375, "loss": 1.0786, "margin_dpo/margin_mean": 525.8742065429688, "margin_dpo/margin_std": 787.166748046875, "step": 609 }, { "KL/chosen_KL_mean": -631.95361328125, "KL/mean": -791.3546142578125, "KL/rejected_KL_mean": -950.7557373046875, "KL/std": 494.2914733886719, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0009960741735994816, "fcm_dpo/delta": 0.11386167258024216, "fcm_dpo/margin": 318.8021240234375, "fcm_dpo/q_t": 0.4325554370880127, "grad_norm": 40.697330474853516, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -0.9712044596672058, "logits/rejected": -0.9452321529388428, "logps/chosen": -697.648681640625, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -1034.1611328125, "loss": 1.1692, "margin_dpo/margin_mean": 318.8021240234375, "margin_dpo/margin_std": 572.7630615234375, "step": 610 }, { "KL/chosen_KL_mean": -635.2578125, "KL/mean": -909.711181640625, "KL/rejected_KL_mean": -1184.1644287109375, "KL/std": 663.8045654296875, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0008432117174379528, "fcm_dpo/delta": -0.12401486933231354, "fcm_dpo/margin": 548.9066162109375, "fcm_dpo/q_t": 0.38857901096343994, "grad_norm": 30.683746337890625, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.0351388454437256, "logits/rejected": -1.0823793411254883, "logps/chosen": -687.8572998046875, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -1270.495361328125, "loss": 1.0702, "margin_dpo/margin_mean": 548.9066162109375, "margin_dpo/margin_std": 768.596923828125, "step": 611 }, { "KL/chosen_KL_mean": -690.9043579101562, "KL/mean": -900.0711669921875, "KL/rejected_KL_mean": -1109.238037109375, "KL/std": 532.022705078125, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0008869701996445656, "fcm_dpo/delta": 0.03892756253480911, "fcm_dpo/margin": 418.3336181640625, "fcm_dpo/q_t": 0.4165932536125183, "grad_norm": 28.735475540161133, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.0041875839233398, "logits/rejected": -1.0075181722640991, "logps/chosen": -750.2280883789062, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1197.55029296875, "loss": 1.1135, "margin_dpo/margin_mean": 418.3336181640625, "margin_dpo/margin_std": 586.0115356445312, "step": 612 }, { "KL/chosen_KL_mean": -596.6041259765625, "KL/mean": -865.2132568359375, "KL/rejected_KL_mean": -1133.8223876953125, "KL/std": 586.6395874023438, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0007895183516666293, "fcm_dpo/delta": -0.046130187809467316, "fcm_dpo/margin": 537.21826171875, "fcm_dpo/q_t": 0.3976578116416931, "grad_norm": 28.2447509765625, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.0270159244537354, "logits/rejected": -1.0836434364318848, "logps/chosen": -656.3341064453125, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -1238.929931640625, "loss": 1.0612, "margin_dpo/margin_mean": 537.21826171875, "margin_dpo/margin_std": 617.8411865234375, "step": 613 }, { "KL/chosen_KL_mean": -683.5602416992188, "KL/mean": -936.4262084960938, "KL/rejected_KL_mean": -1189.292236328125, "KL/std": 602.0802001953125, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0007989507867023349, "fcm_dpo/delta": -0.007680162787437439, "fcm_dpo/margin": 505.73199462890625, "fcm_dpo/q_t": 0.40751612186431885, "grad_norm": 35.26876449584961, "learning_rate": 1.507684480352292e-08, "logits/chosen": -0.9546043276786804, "logits/rejected": -1.0301423072814941, "logps/chosen": -736.4992065429688, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -1293.9715576171875, "loss": 1.101, "margin_dpo/margin_mean": 505.73199462890625, "margin_dpo/margin_std": 729.1088256835938, "step": 614 }, { "KL/chosen_KL_mean": -640.28759765625, "KL/mean": -844.96484375, "KL/rejected_KL_mean": -1049.6419677734375, "KL/std": 611.680419921875, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.000880036037415266, "fcm_dpo/delta": 0.05419084057211876, "fcm_dpo/margin": 409.35443115234375, "fcm_dpo/q_t": 0.4209446310997009, "grad_norm": 33.03697204589844, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -0.9839037656784058, "logits/rejected": -1.0060296058654785, "logps/chosen": -706.1048583984375, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -1144.8194580078125, "loss": 1.1391, "margin_dpo/margin_mean": 409.35443115234375, "margin_dpo/margin_std": 675.0528564453125, "step": 615 }, { "KL/chosen_KL_mean": -780.3349609375, "KL/mean": -949.9927368164062, "KL/rejected_KL_mean": -1119.6505126953125, "KL/std": 530.0982055664062, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0010057191830128431, "fcm_dpo/delta": 0.08487111330032349, "fcm_dpo/margin": 339.3155517578125, "fcm_dpo/q_t": 0.4275793731212616, "grad_norm": 35.978885650634766, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.0499340295791626, "logits/rejected": -1.0310404300689697, "logps/chosen": -845.4678344726562, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -1194.35107421875, "loss": 1.1523, "margin_dpo/margin_mean": 339.3155517578125, "margin_dpo/margin_std": 571.7545166015625, "step": 616 }, { "KL/chosen_KL_mean": -708.6893310546875, "KL/mean": -836.7636108398438, "KL/rejected_KL_mean": -964.837890625, "KL/std": 481.6317443847656, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0012495677219703794, "fcm_dpo/delta": 0.1090506836771965, "fcm_dpo/margin": 256.1485595703125, "fcm_dpo/q_t": 0.4353943467140198, "grad_norm": 90.08831024169922, "learning_rate": 1.378797888467345e-08, "logits/chosen": -0.9055740833282471, "logits/rejected": -0.8715794086456299, "logps/chosen": -771.6948852539062, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -1029.072021484375, "loss": 1.208, "margin_dpo/margin_mean": 256.1485290527344, "margin_dpo/margin_std": 568.8765869140625, "step": 617 }, { "KL/chosen_KL_mean": -740.5645141601562, "KL/mean": -1015.2177124023438, "KL/rejected_KL_mean": -1289.870849609375, "KL/std": 659.4688720703125, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0009442686568945646, "fcm_dpo/delta": -0.24364520609378815, "fcm_dpo/margin": 549.306396484375, "fcm_dpo/q_t": 0.3701603412628174, "grad_norm": 38.955535888671875, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.0169545412063599, "logits/rejected": -1.0575779676437378, "logps/chosen": -807.6658935546875, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -1382.0242919921875, "loss": 1.0903, "margin_dpo/margin_mean": 549.306396484375, "margin_dpo/margin_std": 852.4629516601562, "step": 618 }, { "KL/chosen_KL_mean": -735.8086547851562, "KL/mean": -945.514892578125, "KL/rejected_KL_mean": -1155.22119140625, "KL/std": 635.7939453125, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0008637189166620374, "fcm_dpo/delta": 0.05800282955169678, "fcm_dpo/margin": 419.4125671386719, "fcm_dpo/q_t": 0.4264986217021942, "grad_norm": 39.2636604309082, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.0181564092636108, "logits/rejected": -1.0320208072662354, "logps/chosen": -791.786865234375, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1248.4066162109375, "loss": 1.1748, "margin_dpo/margin_mean": 419.41259765625, "margin_dpo/margin_std": 845.4202880859375, "step": 619 }, { "KL/chosen_KL_mean": -677.5810546875, "KL/mean": -869.0966796875, "KL/rejected_KL_mean": -1060.6123046875, "KL/std": 539.739990234375, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0009379057446494699, "fcm_dpo/delta": 0.06312977522611618, "fcm_dpo/margin": 383.0312194824219, "fcm_dpo/q_t": 0.4227410554885864, "grad_norm": 29.400907516479492, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -1.0447208881378174, "logits/rejected": -1.05232572555542, "logps/chosen": -737.3785400390625, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -1139.02294921875, "loss": 1.1375, "margin_dpo/margin_mean": 383.0312194824219, "margin_dpo/margin_std": 603.0001220703125, "step": 620 }, { "KL/chosen_KL_mean": -654.668701171875, "KL/mean": -947.2188110351562, "KL/rejected_KL_mean": -1239.768798828125, "KL/std": 664.20166015625, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.0008394881151616573, "fcm_dpo/delta": -0.17594929039478302, "fcm_dpo/margin": 585.100341796875, "fcm_dpo/q_t": 0.37430325150489807, "grad_norm": 41.29154968261719, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.0949594974517822, "logits/rejected": -1.1471253633499146, "logps/chosen": -708.6024169921875, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -1328.138427734375, "loss": 1.0593, "margin_dpo/margin_mean": 585.1002807617188, "margin_dpo/margin_std": 730.66455078125, "step": 621 }, { "KL/chosen_KL_mean": -661.12158203125, "KL/mean": -863.3720703125, "KL/rejected_KL_mean": -1065.6224365234375, "KL/std": 502.6478271484375, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0008088588365353644, "fcm_dpo/delta": 0.10930877178907394, "fcm_dpo/margin": 404.5008544921875, "fcm_dpo/q_t": 0.4323442876338959, "grad_norm": 22.646230697631836, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -0.941389799118042, "logits/rejected": -0.9269174337387085, "logps/chosen": -721.407470703125, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -1151.1412353515625, "loss": 1.1511, "margin_dpo/margin_mean": 404.5008544921875, "margin_dpo/margin_std": 631.4569091796875, "step": 622 }, { "KL/chosen_KL_mean": -722.4276733398438, "KL/mean": -880.337158203125, "KL/rejected_KL_mean": -1038.2467041015625, "KL/std": 528.0697631835938, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0010385662317276, "fcm_dpo/delta": 0.10328055918216705, "fcm_dpo/margin": 315.8189697265625, "fcm_dpo/q_t": 0.4342414438724518, "grad_norm": 46.649322509765625, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -1.0891550779342651, "logits/rejected": -1.091712474822998, "logps/chosen": -786.5845947265625, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -1123.3297119140625, "loss": 1.186, "margin_dpo/margin_mean": 315.8189697265625, "margin_dpo/margin_std": 645.6528930664062, "step": 623 }, { "KL/chosen_KL_mean": -704.1187133789062, "KL/mean": -950.7569580078125, "KL/rejected_KL_mean": -1197.3951416015625, "KL/std": 544.427490234375, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0009231247240677476, "fcm_dpo/delta": -0.10394740104675293, "fcm_dpo/margin": 493.2764892578125, "fcm_dpo/q_t": 0.38469332456588745, "grad_norm": 31.07668685913086, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -1.077886700630188, "logits/rejected": -1.079150676727295, "logps/chosen": -776.037353515625, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -1294.5272216796875, "loss": 1.0515, "margin_dpo/margin_mean": 493.2764892578125, "margin_dpo/margin_std": 586.5498046875, "step": 624 }, { "KL/chosen_KL_mean": -672.1764526367188, "KL/mean": -935.4581298828125, "KL/rejected_KL_mean": -1198.7398681640625, "KL/std": 594.755615234375, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0008218766888603568, "fcm_dpo/delta": -0.057466380298137665, "fcm_dpo/margin": 526.5632934570312, "fcm_dpo/q_t": 0.39506271481513977, "grad_norm": 63.15570068359375, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -0.9775293469429016, "logits/rejected": -1.0035572052001953, "logps/chosen": -730.5185546875, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1284.8302001953125, "loss": 1.0508, "margin_dpo/margin_mean": 526.5633544921875, "margin_dpo/margin_std": 566.92041015625, "step": 625 }, { "KL/chosen_KL_mean": -812.67529296875, "KL/mean": -959.8338623046875, "KL/rejected_KL_mean": -1106.9925537109375, "KL/std": 628.1944580078125, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0009906254708766937, "fcm_dpo/delta": 0.14670009911060333, "fcm_dpo/margin": 294.3172607421875, "fcm_dpo/q_t": 0.43976619839668274, "grad_norm": 35.10493469238281, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -0.9490357637405396, "logits/rejected": -0.9404639005661011, "logps/chosen": -887.7879028320312, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1206.1812744140625, "loss": 1.2132, "margin_dpo/margin_mean": 294.3172607421875, "margin_dpo/margin_std": 620.13720703125, "step": 626 }, { "KL/chosen_KL_mean": -542.8727416992188, "KL/mean": -843.132080078125, "KL/rejected_KL_mean": -1143.3914794921875, "KL/std": 703.228271484375, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0008381834486499429, "fcm_dpo/delta": -0.20297735929489136, "fcm_dpo/margin": 600.518798828125, "fcm_dpo/q_t": 0.3782247304916382, "grad_norm": 29.68364715576172, "learning_rate": 9.897955805412e-09, "logits/chosen": -0.9200704097747803, "logits/rejected": -0.9946620464324951, "logps/chosen": -590.6159057617188, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1250.14599609375, "loss": 1.0535, "margin_dpo/margin_mean": 600.518798828125, "margin_dpo/margin_std": 814.1729736328125, "step": 627 }, { "KL/chosen_KL_mean": -741.293701171875, "KL/mean": -966.7863159179688, "KL/rejected_KL_mean": -1192.279052734375, "KL/std": 596.0997924804688, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0007803476182743907, "fcm_dpo/delta": 0.07325306534767151, "fcm_dpo/margin": 450.9853515625, "fcm_dpo/q_t": 0.423714816570282, "grad_norm": 29.094398498535156, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.0479551553726196, "logits/rejected": -1.0646214485168457, "logps/chosen": -801.4765625, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -1293.833740234375, "loss": 1.1308, "margin_dpo/margin_mean": 450.9853515625, "margin_dpo/margin_std": 675.803466796875, "step": 628 }, { "KL/chosen_KL_mean": -719.9290771484375, "KL/mean": -929.900634765625, "KL/rejected_KL_mean": -1139.872314453125, "KL/std": 583.1702270507812, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0008786619873717427, "fcm_dpo/delta": 0.04738582670688629, "fcm_dpo/margin": 419.9432373046875, "fcm_dpo/q_t": 0.4166840612888336, "grad_norm": 32.80850601196289, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.0385822057724, "logits/rejected": -1.0577049255371094, "logps/chosen": -784.142578125, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -1231.52587890625, "loss": 1.112, "margin_dpo/margin_mean": 419.9432373046875, "margin_dpo/margin_std": 578.53662109375, "step": 629 }, { "KL/chosen_KL_mean": -647.0543212890625, "KL/mean": -764.2266845703125, "KL/rejected_KL_mean": -881.39892578125, "KL/std": 556.714599609375, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0009413863299414515, "fcm_dpo/delta": 0.056776098906993866, "fcm_dpo/margin": 234.34454345703125, "fcm_dpo/q_t": 0.4554150104522705, "grad_norm": 49.36244201660156, "learning_rate": 8.85387393063622e-09, "logits/chosen": -0.9865130186080933, "logits/rejected": -0.9569113254547119, "logps/chosen": -706.3453369140625, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -964.9971923828125, "loss": 1.2685, "margin_dpo/margin_mean": 234.34454345703125, "margin_dpo/margin_std": 668.56982421875, "step": 630 }, { "KL/chosen_KL_mean": -776.7286987304688, "KL/mean": -965.1847534179688, "KL/rejected_KL_mean": -1153.6407470703125, "KL/std": 581.8084106445312, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0010326361516490579, "fcm_dpo/delta": 0.016902513802051544, "fcm_dpo/margin": 376.9120788574219, "fcm_dpo/q_t": 0.4136263132095337, "grad_norm": 44.76034164428711, "learning_rate": 8.518543427732949e-09, "logits/chosen": -1.0995469093322754, "logits/rejected": -1.1115614175796509, "logps/chosen": -836.1823120117188, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1234.59228515625, "loss": 1.1518, "margin_dpo/margin_mean": 376.9120788574219, "margin_dpo/margin_std": 690.0167236328125, "step": 631 }, { "KL/chosen_KL_mean": -697.9989013671875, "KL/mean": -890.870361328125, "KL/rejected_KL_mean": -1083.741943359375, "KL/std": 523.5582275390625, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.000998594332486391, "fcm_dpo/delta": 0.01689380779862404, "fcm_dpo/margin": 385.7430114746094, "fcm_dpo/q_t": 0.41343018412590027, "grad_norm": 48.563560485839844, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.047680139541626, "logits/rejected": -1.0479519367218018, "logps/chosen": -759.3504638671875, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1169.902099609375, "loss": 1.1447, "margin_dpo/margin_mean": 385.74298095703125, "margin_dpo/margin_std": 662.6145629882812, "step": 632 }, { "KL/chosen_KL_mean": -745.3228149414062, "KL/mean": -884.4315795898438, "KL/rejected_KL_mean": -1023.540283203125, "KL/std": 512.197509765625, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0012451799120754004, "fcm_dpo/delta": 0.06800185889005661, "fcm_dpo/margin": 278.2175598144531, "fcm_dpo/q_t": 0.4261208176612854, "grad_norm": 68.927978515625, "learning_rate": 7.866980873399015e-09, "logits/chosen": -1.06679105758667, "logits/rejected": -1.0765254497528076, "logps/chosen": -802.6009521484375, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -1115.124267578125, "loss": 1.2008, "margin_dpo/margin_mean": 278.2175598144531, "margin_dpo/margin_std": 601.500732421875, "step": 633 }, { "KL/chosen_KL_mean": -832.7714233398438, "KL/mean": -965.0292358398438, "KL/rejected_KL_mean": -1097.2869873046875, "KL/std": 614.1533203125, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0013293168740347028, "fcm_dpo/delta": 0.07503058761358261, "fcm_dpo/margin": 264.5155334472656, "fcm_dpo/q_t": 0.4308067560195923, "grad_norm": 51.24019241333008, "learning_rate": 7.550765991247654e-09, "logits/chosen": -0.9571331739425659, "logits/rejected": -0.9513822197914124, "logps/chosen": -899.390380859375, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1204.41259765625, "loss": 1.2102, "margin_dpo/margin_mean": 264.5155334472656, "margin_dpo/margin_std": 607.553466796875, "step": 634 }, { "KL/chosen_KL_mean": -726.7911376953125, "KL/mean": -909.9793090820312, "KL/rejected_KL_mean": -1093.16748046875, "KL/std": 651.4637451171875, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0012207648251205683, "fcm_dpo/delta": -0.08999871462583542, "fcm_dpo/margin": 366.37628173828125, "fcm_dpo/q_t": 0.3993282914161682, "grad_norm": 50.3961067199707, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.0328423976898193, "logits/rejected": -1.0168393850326538, "logps/chosen": -800.7467041015625, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -1175.66796875, "loss": 1.1187, "margin_dpo/margin_mean": 366.37628173828125, "margin_dpo/margin_std": 644.3758544921875, "step": 635 }, { "KL/chosen_KL_mean": -682.0125732421875, "KL/mean": -899.14404296875, "KL/rejected_KL_mean": -1116.275634765625, "KL/std": 621.2393798828125, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0010760778095573187, "fcm_dpo/delta": -0.12986694276332855, "fcm_dpo/margin": 434.2630615234375, "fcm_dpo/q_t": 0.38959363102912903, "grad_norm": 29.961071014404297, "learning_rate": 6.937510679537628e-09, "logits/chosen": -0.9651399254798889, "logits/rejected": -0.9689005613327026, "logps/chosen": -741.6414794921875, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1198.25439453125, "loss": 1.0948, "margin_dpo/margin_mean": 434.2630310058594, "margin_dpo/margin_std": 650.568603515625, "step": 636 }, { "KL/chosen_KL_mean": -688.02197265625, "KL/mean": -933.2636108398438, "KL/rejected_KL_mean": -1178.505126953125, "KL/std": 643.35302734375, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0008327056420966983, "fcm_dpo/delta": -0.024866830557584763, "fcm_dpo/margin": 490.4832458496094, "fcm_dpo/q_t": 0.4043170213699341, "grad_norm": 34.2825813293457, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.08000648021698, "logits/rejected": -1.127310872077942, "logps/chosen": -737.6746826171875, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1276.910400390625, "loss": 1.0877, "margin_dpo/margin_mean": 490.48321533203125, "margin_dpo/margin_std": 645.675537109375, "step": 637 }, { "KL/chosen_KL_mean": -676.4507446289062, "KL/mean": -868.2711791992188, "KL/rejected_KL_mean": -1060.0916748046875, "KL/std": 593.851318359375, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0008206696948036551, "fcm_dpo/delta": -0.058288540691137314, "fcm_dpo/margin": 383.64093017578125, "fcm_dpo/q_t": 0.419683039188385, "grad_norm": 30.65635871887207, "learning_rate": 6.349874889624962e-09, "logits/chosen": -0.9378705024719238, "logits/rejected": -0.9226667881011963, "logps/chosen": -734.607421875, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -1139.39306640625, "loss": 1.1863, "margin_dpo/margin_mean": 383.64093017578125, "margin_dpo/margin_std": 691.0709228515625, "step": 638 }, { "KL/chosen_KL_mean": -905.7219848632812, "KL/mean": -986.7338256835938, "KL/rejected_KL_mean": -1067.7457275390625, "KL/std": 534.1470336914062, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0007728881319053471, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 162.023681640625, "fcm_dpo/q_t": 0.47046107053756714, "grad_norm": 76.31160736083984, "learning_rate": 6.065683776815933e-09, "logits/chosen": -0.9101752042770386, "logits/rejected": -0.8493287563323975, "logps/chosen": -978.045166015625, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1142.0206298828125, "loss": 1.3279, "margin_dpo/margin_mean": 162.023681640625, "margin_dpo/margin_std": 657.9573364257812, "step": 639 }, { "KL/chosen_KL_mean": -690.0206298828125, "KL/mean": -973.13671875, "KL/rejected_KL_mean": -1256.2529296875, "KL/std": 648.8939819335938, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.0007562098908238113, "fcm_dpo/delta": -0.05274926871061325, "fcm_dpo/margin": 566.2322387695312, "fcm_dpo/q_t": 0.3993333876132965, "grad_norm": 46.08436965942383, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -0.9405654668807983, "logits/rejected": -0.9669902920722961, "logps/chosen": -746.1549682617188, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -1364.85302734375, "loss": 1.0815, "margin_dpo/margin_mean": 566.2322387695312, "margin_dpo/margin_std": 748.8364868164062, "step": 640 }, { "KL/chosen_KL_mean": -803.405517578125, "KL/mean": -974.4388427734375, "KL/rejected_KL_mean": -1145.47216796875, "KL/std": 547.9729614257812, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0008681797189638019, "fcm_dpo/delta": 0.14330008625984192, "fcm_dpo/margin": 342.0666809082031, "fcm_dpo/q_t": 0.4413103461265564, "grad_norm": 27.740734100341797, "learning_rate": 5.516592558795746e-09, "logits/chosen": -1.0207037925720215, "logits/rejected": -1.02461838722229, "logps/chosen": -868.40234375, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -1232.4644775390625, "loss": 1.2043, "margin_dpo/margin_mean": 342.06671142578125, "margin_dpo/margin_std": 712.2955932617188, "step": 641 }, { "KL/chosen_KL_mean": -779.18896484375, "KL/mean": -998.7464599609375, "KL/rejected_KL_mean": -1218.3040771484375, "KL/std": 729.4473266601562, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0009394697844982147, "fcm_dpo/delta": -0.026245135813951492, "fcm_dpo/margin": 439.1151123046875, "fcm_dpo/q_t": 0.41486814618110657, "grad_norm": 41.30564880371094, "learning_rate": 5.251706922648868e-09, "logits/chosen": -0.9439425468444824, "logits/rejected": -0.9753347635269165, "logps/chosen": -844.878173828125, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -1328.546142578125, "loss": 1.1663, "margin_dpo/margin_mean": 439.1151428222656, "margin_dpo/margin_std": 900.11669921875, "step": 642 }, { "KL/chosen_KL_mean": -674.6640625, "KL/mean": -851.1025390625, "KL/rejected_KL_mean": -1027.5411376953125, "KL/std": 530.1317138671875, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0008457102812826633, "fcm_dpo/delta": -0.019284440204501152, "fcm_dpo/margin": 352.87701416015625, "fcm_dpo/q_t": 0.4285130500793457, "grad_norm": 34.299949645996094, "learning_rate": 4.993270631642038e-09, "logits/chosen": -1.0501961708068848, "logits/rejected": -1.0501909255981445, "logps/chosen": -726.614013671875, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -1115.009521484375, "loss": 1.1622, "margin_dpo/margin_mean": 352.87701416015625, "margin_dpo/margin_std": 535.4759521484375, "step": 643 }, { "KL/chosen_KL_mean": -669.6585693359375, "KL/mean": -843.1557006835938, "KL/rejected_KL_mean": -1016.65283203125, "KL/std": 622.4733276367188, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0009780020918697119, "fcm_dpo/delta": 0.08886226266622543, "fcm_dpo/margin": 346.9942932128906, "fcm_dpo/q_t": 0.43022337555885315, "grad_norm": 48.09732437133789, "learning_rate": 4.741290495811873e-09, "logits/chosen": -0.9770164489746094, "logits/rejected": -0.9840372800827026, "logps/chosen": -728.67626953125, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -1103.78955078125, "loss": 1.1865, "margin_dpo/margin_mean": 346.9942626953125, "margin_dpo/margin_std": 714.44580078125, "step": 644 }, { "KL/chosen_KL_mean": -693.9857788085938, "KL/mean": -786.16162109375, "KL/rejected_KL_mean": -878.3374633789062, "KL/std": 501.67999267578125, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0011059939861297607, "fcm_dpo/delta": 0.04529920592904091, "fcm_dpo/margin": 184.3516387939453, "fcm_dpo/q_t": 0.45926517248153687, "grad_norm": 131.60787963867188, "learning_rate": 4.495773155069299e-09, "logits/chosen": -0.989364743232727, "logits/rejected": -0.97637939453125, "logps/chosen": -749.86181640625, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -976.1182861328125, "loss": 1.3317, "margin_dpo/margin_mean": 184.3516387939453, "margin_dpo/margin_std": 676.8275146484375, "step": 645 }, { "KL/chosen_KL_mean": -658.73046875, "KL/mean": -810.00341796875, "KL/rejected_KL_mean": -961.2762451171875, "KL/std": 463.8369140625, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0011796164326369762, "fcm_dpo/delta": 0.06660034507513046, "fcm_dpo/margin": 302.5458068847656, "fcm_dpo/q_t": 0.42494145035743713, "grad_norm": 58.692771911621094, "learning_rate": 4.256725079024553e-09, "logits/chosen": -0.9866272211074829, "logits/rejected": -0.9638226628303528, "logps/chosen": -720.0062255859375, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -1038.7821044921875, "loss": 1.1639, "margin_dpo/margin_mean": 302.54583740234375, "margin_dpo/margin_std": 556.3814697265625, "step": 646 }, { "KL/chosen_KL_mean": -595.4979248046875, "KL/mean": -783.5799560546875, "KL/rejected_KL_mean": -971.6620483398438, "KL/std": 541.9985961914062, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.001148381270468235, "fcm_dpo/delta": -0.05802955478429794, "fcm_dpo/margin": 376.1640319824219, "fcm_dpo/q_t": 0.3967111110687256, "grad_norm": 51.125606536865234, "learning_rate": 4.024152566816791e-09, "logits/chosen": -0.9212703704833984, "logits/rejected": -0.9506291151046753, "logps/chosen": -650.350341796875, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -1065.181396484375, "loss": 1.0894, "margin_dpo/margin_mean": 376.1640319824219, "margin_dpo/margin_std": 538.5177001953125, "step": 647 }, { "KL/chosen_KL_mean": -622.4876708984375, "KL/mean": -907.7584228515625, "KL/rejected_KL_mean": -1193.0291748046875, "KL/std": 659.01708984375, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.0008362603839486837, "fcm_dpo/delta": -0.17148783802986145, "fcm_dpo/margin": 570.54150390625, "fcm_dpo/q_t": 0.3812508285045624, "grad_norm": 28.91020965576172, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.062262773513794, "logits/rejected": -1.119946002960205, "logps/chosen": -676.6591186523438, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -1291.741943359375, "loss": 1.0583, "margin_dpo/margin_mean": 570.5415649414062, "margin_dpo/margin_std": 795.2802734375, "step": 648 }, { "KL/chosen_KL_mean": -647.7181396484375, "KL/mean": -777.0692138671875, "KL/rejected_KL_mean": -906.4202880859375, "KL/std": 489.11322021484375, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0008656082791276276, "fcm_dpo/delta": 0.07774749398231506, "fcm_dpo/margin": 258.7022399902344, "fcm_dpo/q_t": 0.4531601071357727, "grad_norm": 29.970624923706055, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -1.0473252534866333, "logits/rejected": -1.0361632108688354, "logps/chosen": -710.198486328125, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -986.4974975585938, "loss": 1.2466, "margin_dpo/margin_mean": 258.7022399902344, "margin_dpo/margin_std": 601.4564208984375, "step": 649 }, { "KL/chosen_KL_mean": -711.3486328125, "KL/mean": -921.720703125, "KL/rejected_KL_mean": -1132.0926513671875, "KL/std": 623.22998046875, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0009292041650041938, "fcm_dpo/delta": 0.013731414452195168, "fcm_dpo/margin": 420.7440185546875, "fcm_dpo/q_t": 0.41418665647506714, "grad_norm": 31.185792922973633, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -0.9931234121322632, "logits/rejected": -1.013329029083252, "logps/chosen": -767.4414672851562, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1230.357421875, "loss": 1.1316, "margin_dpo/margin_mean": 420.7440185546875, "margin_dpo/margin_std": 705.7841796875, "step": 650 }, { "KL/chosen_KL_mean": -472.3923034667969, "KL/mean": -752.0784912109375, "KL/rejected_KL_mean": -1031.7647705078125, "KL/std": 590.5972900390625, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0008050062460824847, "fcm_dpo/delta": -0.09981894493103027, "fcm_dpo/margin": 559.3723754882812, "fcm_dpo/q_t": 0.3874790072441101, "grad_norm": 33.94593048095703, "learning_rate": 3.158738163478475e-09, "logits/chosen": -1.0268137454986572, "logits/rejected": -1.0827641487121582, "logps/chosen": -515.8177490234375, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -1131.72265625, "loss": 1.0412, "margin_dpo/margin_mean": 559.3724365234375, "margin_dpo/margin_std": 614.9459228515625, "step": 651 }, { "KL/chosen_KL_mean": -589.6246337890625, "KL/mean": -800.077392578125, "KL/rejected_KL_mean": -1010.5301513671875, "KL/std": 563.1753540039062, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.0008659152081236243, "fcm_dpo/delta": 0.053139444440603256, "fcm_dpo/margin": 420.905517578125, "fcm_dpo/q_t": 0.42015212774276733, "grad_norm": 43.597076416015625, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -1.026230812072754, "logits/rejected": -1.0473260879516602, "logps/chosen": -652.201416015625, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -1122.2979736328125, "loss": 1.1242, "margin_dpo/margin_mean": 420.905517578125, "margin_dpo/margin_std": 636.0498046875, "step": 652 }, { "KL/chosen_KL_mean": -745.919677734375, "KL/mean": -946.8770751953125, "KL/rejected_KL_mean": -1147.834228515625, "KL/std": 630.18896484375, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0009330883622169495, "fcm_dpo/delta": 0.03911864012479782, "fcm_dpo/margin": 401.91461181640625, "fcm_dpo/q_t": 0.4190702438354492, "grad_norm": 33.804931640625, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -1.0814123153686523, "logits/rejected": -1.107730507850647, "logps/chosen": -807.03271484375, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -1251.083984375, "loss": 1.1333, "margin_dpo/margin_mean": 401.9146423339844, "margin_dpo/margin_std": 663.3088989257812, "step": 653 }, { "KL/chosen_KL_mean": -651.044677734375, "KL/mean": -828.8569946289062, "KL/rejected_KL_mean": -1006.66943359375, "KL/std": 504.73419189453125, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.0010234671644866467, "fcm_dpo/delta": 0.05655752867460251, "fcm_dpo/margin": 355.62469482421875, "fcm_dpo/q_t": 0.42311620712280273, "grad_norm": 37.25580978393555, "learning_rate": 2.577954022936174e-09, "logits/chosen": -1.0293757915496826, "logits/rejected": -1.0464175939559937, "logps/chosen": -712.7728271484375, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -1105.4432373046875, "loss": 1.141, "margin_dpo/margin_mean": 355.62469482421875, "margin_dpo/margin_std": 592.0263671875, "step": 654 }, { "KL/chosen_KL_mean": -607.3045654296875, "KL/mean": -795.6822509765625, "KL/rejected_KL_mean": -984.0599365234375, "KL/std": 513.406005859375, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0010560275986790657, "fcm_dpo/delta": 0.003467606846243143, "fcm_dpo/margin": 376.75543212890625, "fcm_dpo/q_t": 0.4126419126987457, "grad_norm": 35.243900299072266, "learning_rate": 2.397392281198729e-09, "logits/chosen": -0.9929590225219727, "logits/rejected": -1.032914638519287, "logps/chosen": -656.88134765625, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -1082.351806640625, "loss": 1.1245, "margin_dpo/margin_mean": 376.75543212890625, "margin_dpo/margin_std": 618.8502197265625, "step": 655 }, { "KL/chosen_KL_mean": -648.4987182617188, "KL/mean": -975.326904296875, "KL/rejected_KL_mean": -1302.1551513671875, "KL/std": 691.7890014648438, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0007785690249875188, "fcm_dpo/delta": -0.22655422985553741, "fcm_dpo/margin": 653.6564331054688, "fcm_dpo/q_t": 0.36488282680511475, "grad_norm": 64.71890258789062, "learning_rate": 2.223355098446622e-09, "logits/chosen": -0.899747371673584, "logits/rejected": -0.968382716178894, "logps/chosen": -701.0481567382812, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -1415.829833984375, "loss": 1.013, "margin_dpo/margin_mean": 653.6564331054688, "margin_dpo/margin_std": 684.8790283203125, "step": 656 }, { "KL/chosen_KL_mean": -607.3179931640625, "KL/mean": -871.7550659179688, "KL/rejected_KL_mean": -1136.192138671875, "KL/std": 643.2208862304688, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0007102171657606959, "fcm_dpo/delta": 0.039193179458379745, "fcm_dpo/margin": 528.8741455078125, "fcm_dpo/q_t": 0.415811151266098, "grad_norm": 30.748796463012695, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.0481717586517334, "logits/rejected": -1.0999813079833984, "logps/chosen": -654.0185546875, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -1234.1070556640625, "loss": 1.0999, "margin_dpo/margin_mean": 528.8741455078125, "margin_dpo/margin_std": 680.64990234375, "step": 657 }, { "KL/chosen_KL_mean": -668.5402221679688, "KL/mean": -876.7470703125, "KL/rejected_KL_mean": -1084.9539794921875, "KL/std": 520.5706176757812, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0008155145333148539, "fcm_dpo/delta": 0.0921507477760315, "fcm_dpo/margin": 416.4137268066406, "fcm_dpo/q_t": 0.4271154999732971, "grad_norm": 30.934783935546875, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -0.9990421533584595, "logits/rejected": -1.0265355110168457, "logps/chosen": -729.4984130859375, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1180.8934326171875, "loss": 1.1332, "margin_dpo/margin_mean": 416.4136962890625, "margin_dpo/margin_std": 572.531494140625, "step": 658 }, { "KL/chosen_KL_mean": -589.103515625, "KL/mean": -787.4786376953125, "KL/rejected_KL_mean": -985.8536376953125, "KL/std": 501.25604248046875, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0009247631533071399, "fcm_dpo/delta": 0.0519876554608345, "fcm_dpo/margin": 396.7500915527344, "fcm_dpo/q_t": 0.42018935084342957, "grad_norm": 30.552549362182617, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -0.9165958762168884, "logits/rejected": -0.885309100151062, "logps/chosen": -665.8465576171875, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -1073.3245849609375, "loss": 1.1172, "margin_dpo/margin_mean": 396.7500915527344, "margin_dpo/margin_std": 562.6476440429688, "step": 659 }, { "KL/chosen_KL_mean": -651.0494384765625, "KL/mean": -908.062744140625, "KL/rejected_KL_mean": -1165.076171875, "KL/std": 607.6834716796875, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0008679507300257683, "fcm_dpo/delta": -0.08382175117731094, "fcm_dpo/margin": 514.0266723632812, "fcm_dpo/q_t": 0.39148008823394775, "grad_norm": 46.10919952392578, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.017472505569458, "logits/rejected": -1.033474326133728, "logps/chosen": -710.0972900390625, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1241.0361328125, "loss": 1.0632, "margin_dpo/margin_mean": 514.0266723632812, "margin_dpo/margin_std": 650.8546142578125, "step": 660 }, { "KL/chosen_KL_mean": -573.46484375, "KL/mean": -808.244140625, "KL/rejected_KL_mean": -1043.0234375, "KL/std": 656.3875732421875, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.0008553254883736372, "fcm_dpo/delta": -0.0076020583510398865, "fcm_dpo/margin": 469.55877685546875, "fcm_dpo/q_t": 0.408011257648468, "grad_norm": 39.97681427001953, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.0107231140136719, "logits/rejected": -1.019051194190979, "logps/chosen": -624.1387939453125, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -1129.0291748046875, "loss": 1.0917, "margin_dpo/margin_mean": 469.5587463378906, "margin_dpo/margin_std": 631.5257568359375, "step": 661 }, { "KL/chosen_KL_mean": -674.645263671875, "KL/mean": -845.860595703125, "KL/rejected_KL_mean": -1017.075927734375, "KL/std": 547.025146484375, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0009593941504135728, "fcm_dpo/delta": 0.10353372991085052, "fcm_dpo/margin": 342.43072509765625, "fcm_dpo/q_t": 0.43190035223960876, "grad_norm": 30.81041717529297, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -0.9726383090019226, "logits/rejected": -0.969027042388916, "logps/chosen": -743.9063110351562, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -1106.1318359375, "loss": 1.1861, "margin_dpo/margin_mean": 342.43072509765625, "margin_dpo/margin_std": 687.1112060546875, "step": 662 }, { "KL/chosen_KL_mean": -622.2049560546875, "KL/mean": -859.8319091796875, "KL/rejected_KL_mean": -1097.458740234375, "KL/std": 645.55224609375, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.000906460452824831, "fcm_dpo/delta": -0.05661872774362564, "fcm_dpo/margin": 475.2537536621094, "fcm_dpo/q_t": 0.40141403675079346, "grad_norm": 25.596982955932617, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -1.029599666595459, "logits/rejected": -1.055091381072998, "logps/chosen": -687.0839233398438, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -1211.3841552734375, "loss": 1.1023, "margin_dpo/margin_mean": 475.25372314453125, "margin_dpo/margin_std": 764.5504760742188, "step": 663 }, { "KL/chosen_KL_mean": -656.9991455078125, "KL/mean": -909.43798828125, "KL/rejected_KL_mean": -1161.876708984375, "KL/std": 617.7022705078125, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0008173753740265965, "fcm_dpo/delta": -0.0294804647564888, "fcm_dpo/margin": 504.87762451171875, "fcm_dpo/q_t": 0.40310847759246826, "grad_norm": 22.38481330871582, "learning_rate": 1.066455926241383e-09, "logits/chosen": -0.987866997718811, "logits/rejected": -1.0178110599517822, "logps/chosen": -717.8876342773438, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -1267.3984375, "loss": 1.0872, "margin_dpo/margin_mean": 504.87762451171875, "margin_dpo/margin_std": 675.4814453125, "step": 664 }, { "KL/chosen_KL_mean": -591.2396240234375, "KL/mean": -797.2940673828125, "KL/rejected_KL_mean": -1003.3485717773438, "KL/std": 507.349609375, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0009045639308169484, "fcm_dpo/delta": 0.04246987774968147, "fcm_dpo/margin": 412.10906982421875, "fcm_dpo/q_t": 0.4166688919067383, "grad_norm": 44.036556243896484, "learning_rate": 9.513254770636137e-10, "logits/chosen": -1.0652823448181152, "logits/rejected": -1.0855448246002197, "logps/chosen": -651.8037109375, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -1088.157470703125, "loss": 1.099, "margin_dpo/margin_mean": 412.10906982421875, "margin_dpo/margin_std": 510.36553955078125, "step": 665 }, { "KL/chosen_KL_mean": -635.3843994140625, "KL/mean": -849.865234375, "KL/rejected_KL_mean": -1064.3460693359375, "KL/std": 543.2855224609375, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0009157652966678143, "fcm_dpo/delta": 0.011229622177779675, "fcm_dpo/margin": 428.96173095703125, "fcm_dpo/q_t": 0.4111603796482086, "grad_norm": 29.72758674621582, "learning_rate": 8.427576920763956e-10, "logits/chosen": -0.9271786212921143, "logits/rejected": -0.9376795291900635, "logps/chosen": -699.8043212890625, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1160.23779296875, "loss": 1.1035, "margin_dpo/margin_mean": 428.96173095703125, "margin_dpo/margin_std": 589.12890625, "step": 666 }, { "KL/chosen_KL_mean": -728.5592041015625, "KL/mean": -976.642822265625, "KL/rejected_KL_mean": -1224.7265625, "KL/std": 594.4561767578125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0008521339623257518, "fcm_dpo/delta": -0.04440337419509888, "fcm_dpo/margin": 496.16729736328125, "fcm_dpo/q_t": 0.39933979511260986, "grad_norm": 36.45888137817383, "learning_rate": 7.407554321417764e-10, "logits/chosen": -0.9821093082427979, "logits/rejected": -0.9815555810928345, "logps/chosen": -797.8362426757812, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -1312.56201171875, "loss": 1.0793, "margin_dpo/margin_mean": 496.1672668457031, "margin_dpo/margin_std": 653.2168579101562, "step": 667 }, { "KL/chosen_KL_mean": -755.283447265625, "KL/mean": -929.7744140625, "KL/rejected_KL_mean": -1104.265380859375, "KL/std": 615.79931640625, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0010012466227635741, "fcm_dpo/delta": 0.061918459832668304, "fcm_dpo/margin": 348.9818420410156, "fcm_dpo/q_t": 0.4277912974357605, "grad_norm": 45.25949478149414, "learning_rate": 6.453213851142225e-10, "logits/chosen": -1.0155105590820312, "logits/rejected": -1.0245213508605957, "logps/chosen": -827.887451171875, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1208.00439453125, "loss": 1.1889, "margin_dpo/margin_mean": 348.9818420410156, "margin_dpo/margin_std": 726.23681640625, "step": 668 }, { "KL/chosen_KL_mean": -597.3612060546875, "KL/mean": -849.0230102539062, "KL/rejected_KL_mean": -1100.6846923828125, "KL/std": 599.0240478515625, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0008747231913730502, "fcm_dpo/delta": -0.07254499942064285, "fcm_dpo/margin": 503.323486328125, "fcm_dpo/q_t": 0.393608957529068, "grad_norm": 22.192842483520508, "learning_rate": 5.564580657695939e-10, "logits/chosen": -0.9759036302566528, "logits/rejected": -0.9822956323623657, "logps/chosen": -643.4776611328125, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1178.609130859375, "loss": 1.0742, "margin_dpo/margin_mean": 503.323486328125, "margin_dpo/margin_std": 677.1585693359375, "step": 669 }, { "KL/chosen_KL_mean": -544.9779052734375, "KL/mean": -801.1505737304688, "KL/rejected_KL_mean": -1057.3232421875, "KL/std": 542.9509887695312, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0008237832807935774, "fcm_dpo/delta": -0.039803534746170044, "fcm_dpo/margin": 512.3453369140625, "fcm_dpo/q_t": 0.3997776508331299, "grad_norm": 40.426666259765625, "learning_rate": 4.741678157389739e-10, "logits/chosen": -0.9458330273628235, "logits/rejected": -0.9657979011535645, "logps/chosen": -607.3236694335938, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -1154.2637939453125, "loss": 1.0816, "margin_dpo/margin_mean": 512.3453369140625, "margin_dpo/margin_std": 667.56640625, "step": 670 }, { "KL/chosen_KL_mean": -693.818603515625, "KL/mean": -904.692626953125, "KL/rejected_KL_mean": -1115.5667724609375, "KL/std": 550.0133056640625, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.0008690550457686186, "fcm_dpo/delta": 0.04318936541676521, "fcm_dpo/margin": 421.74822998046875, "fcm_dpo/q_t": 0.41694390773773193, "grad_norm": 26.835704803466797, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.0066074132919312, "logits/rejected": -1.0331590175628662, "logps/chosen": -741.8187255859375, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1199.3861083984375, "loss": 1.1341, "margin_dpo/margin_mean": 421.74822998046875, "margin_dpo/margin_std": 664.017333984375, "step": 671 }, { "KL/chosen_KL_mean": -766.8887329101562, "KL/mean": -968.2491455078125, "KL/rejected_KL_mean": -1169.6094970703125, "KL/std": 652.8719482421875, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0009030759101733565, "fcm_dpo/delta": 0.057317838072776794, "fcm_dpo/margin": 402.7208251953125, "fcm_dpo/q_t": 0.4237225651741028, "grad_norm": 62.17927932739258, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.085627794265747, "logits/rejected": -1.0907000303268433, "logps/chosen": -825.4720458984375, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1262.749755859375, "loss": 1.1636, "margin_dpo/margin_mean": 402.7208251953125, "margin_dpo/margin_std": 737.9989013671875, "step": 672 }, { "KL/chosen_KL_mean": -674.4365234375, "KL/mean": -871.0324096679688, "KL/rejected_KL_mean": -1067.62841796875, "KL/std": 548.711181640625, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0009767746087163687, "fcm_dpo/delta": 0.02593318186700344, "fcm_dpo/margin": 393.1917724609375, "fcm_dpo/q_t": 0.41595256328582764, "grad_norm": 40.66542053222656, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.0082206726074219, "logits/rejected": -1.0116488933563232, "logps/chosen": -721.15966796875, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1152.924560546875, "loss": 1.119, "margin_dpo/margin_mean": 393.1917724609375, "margin_dpo/margin_std": 624.88330078125, "step": 673 }, { "KL/chosen_KL_mean": -571.8458862304688, "KL/mean": -831.9334716796875, "KL/rejected_KL_mean": -1092.0211181640625, "KL/std": 578.303466796875, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0008713127463124692, "fcm_dpo/delta": -0.09795575588941574, "fcm_dpo/margin": 520.1752319335938, "fcm_dpo/q_t": 0.39291104674339294, "grad_norm": 35.70085144042969, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -0.9462261199951172, "logits/rejected": -0.9662339091300964, "logps/chosen": -617.2913818359375, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -1162.067138671875, "loss": 1.0669, "margin_dpo/margin_mean": 520.1752319335938, "margin_dpo/margin_std": 695.502685546875, "step": 674 }, { "KL/chosen_KL_mean": -642.9573974609375, "KL/mean": -909.2418212890625, "KL/rejected_KL_mean": -1175.526123046875, "KL/std": 614.2586059570312, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0007445673691108823, "fcm_dpo/delta": -0.013315960764884949, "fcm_dpo/margin": 532.56884765625, "fcm_dpo/q_t": 0.4076732397079468, "grad_norm": 23.048574447631836, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.0072154998779297, "logits/rejected": -1.0248122215270996, "logps/chosen": -687.1336669921875, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1249.6181640625, "loss": 1.0911, "margin_dpo/margin_mean": 532.56884765625, "margin_dpo/margin_std": 652.190185546875, "step": 675 }, { "KL/chosen_KL_mean": -695.682373046875, "KL/mean": -932.237060546875, "KL/rejected_KL_mean": -1168.7918701171875, "KL/std": 565.1691284179688, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0008274284191429615, "fcm_dpo/delta": 0.013139697723090649, "fcm_dpo/margin": 473.10943603515625, "fcm_dpo/q_t": 0.409980833530426, "grad_norm": 23.929630279541016, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -0.9380951523780823, "logits/rejected": -0.9486560821533203, "logps/chosen": -767.0808715820312, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1257.150634765625, "loss": 1.086, "margin_dpo/margin_mean": 473.1094665527344, "margin_dpo/margin_std": 570.78515625, "step": 676 }, { "KL/chosen_KL_mean": -701.174560546875, "KL/mean": -933.6365356445312, "KL/rejected_KL_mean": -1166.0985107421875, "KL/std": 588.0880126953125, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.0008385643595829606, "fcm_dpo/delta": 0.01662612520158291, "fcm_dpo/margin": 464.9239501953125, "fcm_dpo/q_t": 0.4153476655483246, "grad_norm": 24.641782760620117, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.0368056297302246, "logits/rejected": -1.0287206172943115, "logps/chosen": -757.7020263671875, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1244.324951171875, "loss": 1.1094, "margin_dpo/margin_mean": 464.9239196777344, "margin_dpo/margin_std": 704.014892578125, "step": 677 }, { "KL/chosen_KL_mean": -589.749267578125, "KL/mean": -871.703369140625, "KL/rejected_KL_mean": -1153.657470703125, "KL/std": 666.5526123046875, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0007895482704043388, "fcm_dpo/delta": -0.08192657679319382, "fcm_dpo/margin": 563.908203125, "fcm_dpo/q_t": 0.39262598752975464, "grad_norm": 28.306751251220703, "learning_rate": 5.270012410216185e-11, "logits/chosen": -0.9746444821357727, "logits/rejected": -1.0132948160171509, "logps/chosen": -635.8837280273438, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1234.2620849609375, "loss": 1.0705, "margin_dpo/margin_mean": 563.9081420898438, "margin_dpo/margin_std": 737.7059326171875, "step": 678 }, { "KL/chosen_KL_mean": -681.0501708984375, "KL/mean": -873.888916015625, "KL/rejected_KL_mean": -1066.727783203125, "KL/std": 521.9443359375, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.0008484461577609181, "fcm_dpo/delta": 0.10628005862236023, "fcm_dpo/margin": 385.67755126953125, "fcm_dpo/q_t": 0.43186846375465393, "grad_norm": 29.822158813476562, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -0.9917019605636597, "logits/rejected": -0.9838023781776428, "logps/chosen": -731.3450927734375, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -1143.3258056640625, "loss": 1.1502, "margin_dpo/margin_mean": 385.67755126953125, "margin_dpo/margin_std": 617.3707275390625, "step": 679 }, { "KL/chosen_KL_mean": -673.279541015625, "KL/mean": -923.5628051757812, "KL/rejected_KL_mean": -1173.8460693359375, "KL/std": 654.016845703125, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0008149506174959242, "fcm_dpo/delta": -0.02367626130580902, "fcm_dpo/margin": 500.56646728515625, "fcm_dpo/q_t": 0.4036046266555786, "grad_norm": 44.08322525024414, "learning_rate": 1.31753782067201e-11, "logits/chosen": -0.9790595173835754, "logits/rejected": -1.0117442607879639, "logps/chosen": -750.1953125, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1286.2308349609375, "loss": 1.1057, "margin_dpo/margin_mean": 500.56646728515625, "margin_dpo/margin_std": 737.4908447265625, "step": 680 }, { "KL/chosen_KL_mean": -678.2127685546875, "KL/mean": -878.902587890625, "KL/rejected_KL_mean": -1079.592529296875, "KL/std": 559.0294799804688, "epoch": 1.0, "fcm_dpo/beta": 0.0008184043690562248, "fcm_dpo/delta": -0.0384669154882431, "fcm_dpo/margin": 401.3796691894531, "fcm_dpo/q_t": 0.4222037196159363, "grad_norm": 26.581893920898438, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -1.0419981479644775, "logits/rejected": -1.0603480339050293, "logps/chosen": -739.1700439453125, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1168.150390625, "loss": 1.1543, "margin_dpo/margin_mean": 401.379638671875, "margin_dpo/margin_std": 625.6992797851562, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.1280472663705745, "train_runtime": 1739.8346, "train_samples_per_second": 25.059, "train_steps_per_second": 0.391 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }