{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.00527191162109375, "KL/mean": 0.016706019639968872, "KL/rejected_KL_mean": 0.028141021728515625, "KL/std": 0.272699236869812, "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.02287006378173828, "fcm_dpo/q_t": 0.5005706548690796, "grad_norm": 83.52085876464844, "learning_rate": 0.0, "logits/chosen": -0.4974287748336792, "logits/rejected": -0.43299180269241333, "logps/chosen": -50.1435661315918, "logps/ref_chosen": -50.14883804321289, "logps/ref_rejected": -74.1280517578125, "logps/rejected": -74.09991455078125, "loss": 1.389, "margin_dpo/margin_mean": -0.02287048101425171, "margin_dpo/margin_std": 0.41920793056488037, "step": 1 }, { "KL/chosen_KL_mean": -0.03498649597167969, "KL/mean": -0.00212840735912323, "KL/rejected_KL_mean": 0.030735015869140625, "KL/std": 0.24797174334526062, "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.06572261452674866, "fcm_dpo/q_t": 0.5016425848007202, "grad_norm": 72.19752502441406, "learning_rate": 7.246376811594203e-09, "logits/chosen": -0.49536412954330444, "logits/rejected": -0.4594460427761078, "logps/chosen": -52.65568923950195, "logps/ref_chosen": -52.620704650878906, "logps/ref_rejected": -75.30413818359375, "logps/rejected": -75.27340698242188, "loss": 1.3932, "margin_dpo/margin_mean": -0.06572240591049194, "margin_dpo/margin_std": 0.35048407316207886, "step": 2 }, { "KL/chosen_KL_mean": -0.017164230346679688, "KL/mean": -0.008938148617744446, "KL/rejected_KL_mean": -0.00070953369140625, "KL/std": 0.25200146436691284, "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.016454964876174927, "fcm_dpo/q_t": 0.500411331653595, "grad_norm": 70.90198516845703, "learning_rate": 1.4492753623188406e-08, "logits/chosen": -0.4816335439682007, "logits/rejected": -0.442240834236145, "logps/chosen": -60.99876022338867, "logps/ref_chosen": -60.981597900390625, "logps/ref_rejected": -68.67259216308594, "logps/rejected": -68.67330932617188, "loss": 1.3883, "margin_dpo/margin_mean": -0.016454339027404785, "margin_dpo/margin_std": 0.3718428313732147, "step": 3 }, { "KL/chosen_KL_mean": 0.0011157989501953125, "KL/mean": 0.030593425035476685, "KL/rejected_KL_mean": 0.060070037841796875, "KL/std": 0.2528386116027832, "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.058952271938323975, "fcm_dpo/q_t": 0.5014750957489014, "grad_norm": 72.30359649658203, "learning_rate": 2.1739130434782606e-08, "logits/chosen": -0.46882253885269165, "logits/rejected": -0.44118714332580566, "logps/chosen": -56.76659393310547, "logps/ref_chosen": -56.7677116394043, "logps/ref_rejected": -86.64710998535156, "logps/rejected": -86.5870361328125, "loss": 1.3926, "margin_dpo/margin_mean": -0.058952778577804565, "margin_dpo/margin_std": 0.3786373734474182, "step": 4 }, { "KL/chosen_KL_mean": 0.04130363464355469, "KL/mean": 0.028823688626289368, "KL/rejected_KL_mean": 0.016345977783203125, "KL/std": 0.2790408730506897, "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.024959057569503784, "fcm_dpo/q_t": 0.4993762671947479, "grad_norm": 89.41995239257812, "learning_rate": 2.898550724637681e-08, "logits/chosen": -0.5145682096481323, "logits/rejected": -0.47082412242889404, "logps/chosen": -53.81806945800781, "logps/ref_chosen": -53.859375, "logps/ref_rejected": -84.14918518066406, "logps/rejected": -84.13284301757812, "loss": 1.3842, "margin_dpo/margin_mean": 0.024959444999694824, "margin_dpo/margin_std": 0.38526445627212524, "step": 5 }, { "KL/chosen_KL_mean": -0.04564094543457031, "KL/mean": -0.020483508706092834, "KL/rejected_KL_mean": 0.00467681884765625, "KL/std": 0.26991650462150574, "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0503142774105072, "fcm_dpo/q_t": 0.50125652551651, "grad_norm": 92.1827621459961, "learning_rate": 3.6231884057971014e-08, "logits/chosen": -0.5298000574111938, "logits/rejected": -0.4896334409713745, "logps/chosen": -63.053123474121094, "logps/ref_chosen": -63.007484436035156, "logps/ref_rejected": -92.64534759521484, "logps/rejected": -92.64067077636719, "loss": 1.3918, "margin_dpo/margin_mean": -0.05031454563140869, "margin_dpo/margin_std": 0.4164780378341675, "step": 6 }, { "KL/chosen_KL_mean": 0.05684852600097656, "KL/mean": 0.05007433891296387, "KL/rejected_KL_mean": 0.043300628662109375, "KL/std": 0.2920554578304291, "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013551175594329834, "fcm_dpo/q_t": 0.4996613562107086, "grad_norm": 82.23982238769531, "learning_rate": 4.347826086956521e-08, "logits/chosen": -0.5054950714111328, "logits/rejected": -0.4716752767562866, "logps/chosen": -57.71797180175781, "logps/ref_chosen": -57.774818420410156, "logps/ref_rejected": -103.92059326171875, "logps/rejected": -103.87728881835938, "loss": 1.3854, "margin_dpo/margin_mean": 0.013551592826843262, "margin_dpo/margin_std": 0.43186938762664795, "step": 7 }, { "KL/chosen_KL_mean": 0.017724990844726562, "KL/mean": 0.02704182267189026, "KL/rejected_KL_mean": 0.0363616943359375, "KL/std": 0.3007723093032837, "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.018639206886291504, "fcm_dpo/q_t": 0.500464916229248, "grad_norm": 78.59706115722656, "learning_rate": 5.0724637681159424e-08, "logits/chosen": -0.5154826641082764, "logits/rejected": -0.4903911352157593, "logps/chosen": -58.69831085205078, "logps/ref_chosen": -58.716033935546875, "logps/ref_rejected": -79.3114242553711, "logps/rejected": -79.27506256103516, "loss": 1.3886, "margin_dpo/margin_mean": -0.01863950490951538, "margin_dpo/margin_std": 0.3895518183708191, "step": 8 }, { "KL/chosen_KL_mean": 0.00308990478515625, "KL/mean": -0.0010503679513931274, "KL/rejected_KL_mean": -0.00518798828125, "KL/std": 0.28215163946151733, "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.008274465799331665, "fcm_dpo/q_t": 0.499793142080307, "grad_norm": 85.33476257324219, "learning_rate": 5.797101449275362e-08, "logits/chosen": -0.5284153819084167, "logits/rejected": -0.4838245213031769, "logps/chosen": -69.86375427246094, "logps/ref_chosen": -69.8668441772461, "logps/ref_rejected": -99.6026611328125, "logps/rejected": -99.60784912109375, "loss": 1.3859, "margin_dpo/margin_mean": 0.008274048566818237, "margin_dpo/margin_std": 0.4120550751686096, "step": 9 }, { "KL/chosen_KL_mean": 0.007932662963867188, "KL/mean": 0.01091562956571579, "KL/rejected_KL_mean": 0.0139007568359375, "KL/std": 0.2518649697303772, "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.00596599280834198, "fcm_dpo/q_t": 0.5001487135887146, "grad_norm": 70.95951080322266, "learning_rate": 6.521739130434782e-08, "logits/chosen": -0.4845648407936096, "logits/rejected": -0.4408423602581024, "logps/chosen": -48.349754333496094, "logps/ref_chosen": -48.35768508911133, "logps/ref_rejected": -80.37206268310547, "logps/rejected": -80.35816192626953, "loss": 1.3872, "margin_dpo/margin_mean": -0.00596579909324646, "margin_dpo/margin_std": 0.3506859540939331, "step": 10 }, { "KL/chosen_KL_mean": 0.06185722351074219, "KL/mean": 0.0011502206325531006, "KL/rejected_KL_mean": -0.0595550537109375, "KL/std": 0.27010950446128845, "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12140899896621704, "fcm_dpo/q_t": 0.4969657063484192, "grad_norm": 67.91629028320312, "learning_rate": 7.246376811594203e-08, "logits/chosen": -0.4723697900772095, "logits/rejected": -0.44793397188186646, "logps/chosen": -52.95499801635742, "logps/ref_chosen": -53.01685333251953, "logps/ref_rejected": -87.78038024902344, "logps/rejected": -87.83993530273438, "loss": 1.3745, "margin_dpo/margin_mean": 0.12140867114067078, "margin_dpo/margin_std": 0.3221217393875122, "step": 11 }, { "KL/chosen_KL_mean": 0.0077342987060546875, "KL/mean": -0.026624195277690887, "KL/rejected_KL_mean": -0.060977935791015625, "KL/std": 0.26408517360687256, "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06871287524700165, "fcm_dpo/q_t": 0.49828261137008667, "grad_norm": 95.73152923583984, "learning_rate": 7.971014492753623e-08, "logits/chosen": -0.5401338338851929, "logits/rejected": -0.5040255784988403, "logps/chosen": -61.79770278930664, "logps/ref_chosen": -61.80543518066406, "logps/ref_rejected": -104.8582763671875, "logps/rejected": -104.91925048828125, "loss": 1.3797, "margin_dpo/margin_mean": 0.06871210038661957, "margin_dpo/margin_std": 0.33830663561820984, "step": 12 }, { "KL/chosen_KL_mean": 0.0069599151611328125, "KL/mean": -0.017511114478111267, "KL/rejected_KL_mean": -0.04198455810546875, "KL/std": 0.3143480122089386, "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.048950523138046265, "fcm_dpo/q_t": 0.4987792670726776, "grad_norm": 79.25921630859375, "learning_rate": 8.695652173913042e-08, "logits/chosen": -0.4748424291610718, "logits/rejected": -0.4463379681110382, "logps/chosen": -64.2533950805664, "logps/ref_chosen": -64.2603530883789, "logps/ref_rejected": -87.20307922363281, "logps/rejected": -87.24507141113281, "loss": 1.382, "margin_dpo/margin_mean": 0.048950642347335815, "margin_dpo/margin_std": 0.44883352518081665, "step": 13 }, { "KL/chosen_KL_mean": -0.021654129028320312, "KL/mean": 0.0024417340755462646, "KL/rejected_KL_mean": 0.0265350341796875, "KL/std": 0.2979321777820587, "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04818415641784668, "fcm_dpo/q_t": 0.5012027025222778, "grad_norm": 85.64781188964844, "learning_rate": 9.420289855072464e-08, "logits/chosen": -0.48348790407180786, "logits/rejected": -0.4443298876285553, "logps/chosen": -58.131866455078125, "logps/ref_chosen": -58.11021041870117, "logps/ref_rejected": -104.04708099365234, "logps/rejected": -104.02054595947266, "loss": 1.3916, "margin_dpo/margin_mean": -0.048183709383010864, "margin_dpo/margin_std": 0.44763094186782837, "step": 14 }, { "KL/chosen_KL_mean": -0.005901336669921875, "KL/mean": -0.012522846460342407, "KL/rejected_KL_mean": -0.019145965576171875, "KL/std": 0.21723134815692902, "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.013249039649963379, "fcm_dpo/q_t": 0.49966850876808167, "grad_norm": 64.4452133178711, "learning_rate": 1.0144927536231885e-07, "logits/chosen": -0.498607337474823, "logits/rejected": -0.48026347160339355, "logps/chosen": -56.972808837890625, "logps/ref_chosen": -56.96691131591797, "logps/ref_rejected": -80.80863952636719, "logps/rejected": -80.82778930664062, "loss": 1.3853, "margin_dpo/margin_mean": 0.013249337673187256, "margin_dpo/margin_std": 0.3437996804714203, "step": 15 }, { "KL/chosen_KL_mean": -0.027292251586914062, "KL/mean": -0.009725570678710938, "KL/rejected_KL_mean": 0.007839202880859375, "KL/std": 0.2627600431442261, "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.035129427909851074, "fcm_dpo/q_t": 0.5008782148361206, "grad_norm": 84.21294403076172, "learning_rate": 1.0869565217391303e-07, "logits/chosen": -0.5192116498947144, "logits/rejected": -0.47780126333236694, "logps/chosen": -61.76718521118164, "logps/ref_chosen": -61.739891052246094, "logps/ref_rejected": -84.36947631835938, "logps/rejected": -84.36163330078125, "loss": 1.3901, "margin_dpo/margin_mean": -0.035130053758621216, "margin_dpo/margin_std": 0.35660019516944885, "step": 16 }, { "KL/chosen_KL_mean": 0.0523681640625, "KL/mean": 0.011688470840454102, "KL/rejected_KL_mean": -0.02899169921875, "KL/std": 0.2670317590236664, "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08135926723480225, "fcm_dpo/q_t": 0.49796566367149353, "grad_norm": 78.71238708496094, "learning_rate": 1.1594202898550725e-07, "logits/chosen": -0.48786813020706177, "logits/rejected": -0.4491962194442749, "logps/chosen": -67.65796661376953, "logps/ref_chosen": -67.71033477783203, "logps/ref_rejected": -85.37865447998047, "logps/rejected": -85.40764617919922, "loss": 1.3785, "margin_dpo/margin_mean": 0.08135849237442017, "margin_dpo/margin_std": 0.38243818283081055, "step": 17 }, { "KL/chosen_KL_mean": 0.039516448974609375, "KL/mean": -0.002365991473197937, "KL/rejected_KL_mean": -0.04425048828125, "KL/std": 0.2152850478887558, "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08376440405845642, "fcm_dpo/q_t": 0.4979061186313629, "grad_norm": 81.61917877197266, "learning_rate": 1.2318840579710146e-07, "logits/chosen": -0.493377149105072, "logits/rejected": -0.43791770935058594, "logps/chosen": -47.699974060058594, "logps/ref_chosen": -47.7394905090332, "logps/ref_rejected": -75.4722900390625, "logps/rejected": -75.51654052734375, "loss": 1.3782, "margin_dpo/margin_mean": 0.08376404643058777, "margin_dpo/margin_std": 0.2884919345378876, "step": 18 }, { "KL/chosen_KL_mean": 0.016864776611328125, "KL/mean": -0.012162566184997559, "KL/rejected_KL_mean": -0.041187286376953125, "KL/std": 0.2685026526451111, "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.058047711849212646, "fcm_dpo/q_t": 0.4985480308532715, "grad_norm": 73.1733627319336, "learning_rate": 1.3043478260869563e-07, "logits/chosen": -0.45881736278533936, "logits/rejected": -0.40635204315185547, "logps/chosen": -70.18849182128906, "logps/ref_chosen": -70.20536041259766, "logps/ref_rejected": -89.7575912475586, "logps/rejected": -89.79878234863281, "loss": 1.3808, "margin_dpo/margin_mean": 0.05804792046546936, "margin_dpo/margin_std": 0.3469369411468506, "step": 19 }, { "KL/chosen_KL_mean": 0.010059356689453125, "KL/mean": -0.0335158109664917, "KL/rejected_KL_mean": -0.07708358764648438, "KL/std": 0.2843707203865051, "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08714669942855835, "fcm_dpo/q_t": 0.4978218078613281, "grad_norm": 74.1033935546875, "learning_rate": 1.3768115942028986e-07, "logits/chosen": -0.5568749904632568, "logits/rejected": -0.5010716915130615, "logps/chosen": -50.793182373046875, "logps/ref_chosen": -50.80324172973633, "logps/ref_rejected": -78.82334899902344, "logps/rejected": -78.90043640136719, "loss": 1.378, "margin_dpo/margin_mean": 0.08714690804481506, "margin_dpo/margin_std": 0.3879868984222412, "step": 20 }, { "KL/chosen_KL_mean": 0.018102645874023438, "KL/mean": -0.054987095296382904, "KL/rejected_KL_mean": -0.12807846069335938, "KL/std": 0.2719336450099945, "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1461833268404007, "fcm_dpo/q_t": 0.49634718894958496, "grad_norm": 77.20344543457031, "learning_rate": 1.4492753623188405e-07, "logits/chosen": -0.5189667344093323, "logits/rejected": -0.49694347381591797, "logps/chosen": -50.04491424560547, "logps/ref_chosen": -50.063018798828125, "logps/ref_rejected": -77.86878967285156, "logps/rejected": -77.99687194824219, "loss": 1.372, "margin_dpo/margin_mean": 0.14618362486362457, "margin_dpo/margin_std": 0.33233124017715454, "step": 21 }, { "KL/chosen_KL_mean": 0.026437759399414062, "KL/mean": -0.06921970844268799, "KL/rejected_KL_mean": -0.16487503051757812, "KL/std": 0.2893703877925873, "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1913151741027832, "fcm_dpo/q_t": 0.49521979689598083, "grad_norm": 84.33903503417969, "learning_rate": 1.5217391304347825e-07, "logits/chosen": -0.4624967873096466, "logits/rejected": -0.41778409481048584, "logps/chosen": -59.03119659423828, "logps/ref_chosen": -59.05763626098633, "logps/ref_rejected": -97.50466918945312, "logps/rejected": -97.66954803466797, "loss": 1.3677, "margin_dpo/margin_mean": 0.1913146674633026, "margin_dpo/margin_std": 0.4002724289894104, "step": 22 }, { "KL/chosen_KL_mean": 0.03466796875, "KL/mean": -0.07599049806594849, "KL/rejected_KL_mean": -0.1866474151611328, "KL/std": 0.3174063563346863, "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.22131741046905518, "fcm_dpo/q_t": 0.49447089433670044, "grad_norm": 80.63067626953125, "learning_rate": 1.5942028985507245e-07, "logits/chosen": -0.5079987645149231, "logits/rejected": -0.48651230335235596, "logps/chosen": -60.04302978515625, "logps/ref_chosen": -60.07769775390625, "logps/ref_rejected": -81.13955688476562, "logps/rejected": -81.32620239257812, "loss": 1.3647, "margin_dpo/margin_mean": 0.221317857503891, "margin_dpo/margin_std": 0.40781861543655396, "step": 23 }, { "KL/chosen_KL_mean": 0.024261474609375, "KL/mean": -0.0957612544298172, "KL/rejected_KL_mean": -0.21578598022460938, "KL/std": 0.27903175354003906, "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24004700779914856, "fcm_dpo/q_t": 0.4940027594566345, "grad_norm": 82.3310546875, "learning_rate": 1.6666666666666665e-07, "logits/chosen": -0.5110853910446167, "logits/rejected": -0.49455416202545166, "logps/chosen": -44.26677322387695, "logps/ref_chosen": -44.29103469848633, "logps/ref_rejected": -99.12521362304688, "logps/rejected": -99.34100341796875, "loss": 1.3628, "margin_dpo/margin_mean": 0.240046888589859, "margin_dpo/margin_std": 0.3710036873817444, "step": 24 }, { "KL/chosen_KL_mean": 0.06036949157714844, "KL/mean": -0.07619968056678772, "KL/rejected_KL_mean": -0.2127685546875, "KL/std": 0.35055142641067505, "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2731361985206604, "fcm_dpo/q_t": 0.49317657947540283, "grad_norm": 73.42306518554688, "learning_rate": 1.7391304347826085e-07, "logits/chosen": -0.5052649974822998, "logits/rejected": -0.4755502939224243, "logps/chosen": -52.4766845703125, "logps/ref_chosen": -52.537052154541016, "logps/ref_rejected": -89.34219360351562, "logps/rejected": -89.55496215820312, "loss": 1.3597, "margin_dpo/margin_mean": 0.2731362283229828, "margin_dpo/margin_std": 0.45307743549346924, "step": 25 }, { "KL/chosen_KL_mean": 0.07877922058105469, "KL/mean": -0.11963352560997009, "KL/rejected_KL_mean": -0.31804656982421875, "KL/std": 0.45149433612823486, "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3968273997306824, "fcm_dpo/q_t": 0.4900946319103241, "grad_norm": 87.18598937988281, "learning_rate": 1.8115942028985507e-07, "logits/chosen": -0.5580284595489502, "logits/rejected": -0.5269525051116943, "logps/chosen": -53.84402847290039, "logps/ref_chosen": -53.92280578613281, "logps/ref_rejected": -103.35971069335938, "logps/rejected": -103.67776489257812, "loss": 1.3479, "margin_dpo/margin_mean": 0.396827757358551, "margin_dpo/margin_std": 0.5908505916595459, "step": 26 }, { "KL/chosen_KL_mean": 0.11952590942382812, "KL/mean": -0.12472324073314667, "KL/rejected_KL_mean": -0.3689727783203125, "KL/std": 0.4575558304786682, "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4884994328022003, "fcm_dpo/q_t": 0.4878042936325073, "grad_norm": 93.66383361816406, "learning_rate": 1.8840579710144927e-07, "logits/chosen": -0.522944450378418, "logits/rejected": -0.48664578795433044, "logps/chosen": -42.77900314331055, "logps/ref_chosen": -42.898529052734375, "logps/ref_rejected": -98.72419738769531, "logps/rejected": -99.09317016601562, "loss": 1.3389, "margin_dpo/margin_mean": 0.4884990155696869, "margin_dpo/margin_std": 0.5788041353225708, "step": 27 }, { "KL/chosen_KL_mean": 0.0053863525390625, "KL/mean": -0.1380460411310196, "KL/rejected_KL_mean": -0.2814826965332031, "KL/std": 0.418185830116272, "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.28686580061912537, "fcm_dpo/q_t": 0.49283337593078613, "grad_norm": 75.52678680419922, "learning_rate": 1.9565217391304347e-07, "logits/chosen": -0.5053206086158752, "logits/rejected": -0.44998180866241455, "logps/chosen": -60.551116943359375, "logps/ref_chosen": -60.55650329589844, "logps/ref_rejected": -91.40111541748047, "logps/rejected": -91.68260192871094, "loss": 1.3587, "margin_dpo/margin_mean": 0.2868659198284149, "margin_dpo/margin_std": 0.5646681785583496, "step": 28 }, { "KL/chosen_KL_mean": 0.10842704772949219, "KL/mean": -0.14363093674182892, "KL/rejected_KL_mean": -0.3956871032714844, "KL/std": 0.4754260182380676, "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5041146278381348, "fcm_dpo/q_t": 0.4874116778373718, "grad_norm": 91.20955657958984, "learning_rate": 2.028985507246377e-07, "logits/chosen": -0.549689531326294, "logits/rejected": -0.503177285194397, "logps/chosen": -57.69935607910156, "logps/ref_chosen": -57.80778503417969, "logps/ref_rejected": -97.39434814453125, "logps/rejected": -97.7900390625, "loss": 1.3374, "margin_dpo/margin_mean": 0.5041155815124512, "margin_dpo/margin_std": 0.5955373644828796, "step": 29 }, { "KL/chosen_KL_mean": 0.14672279357910156, "KL/mean": -0.1726699322462082, "KL/rejected_KL_mean": -0.4920654296875, "KL/std": 0.5893620848655701, "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6387929916381836, "fcm_dpo/q_t": 0.48405760526657104, "grad_norm": 87.42900848388672, "learning_rate": 2.1014492753623187e-07, "logits/chosen": -0.47268378734588623, "logits/rejected": -0.4410501718521118, "logps/chosen": -52.43064880371094, "logps/ref_chosen": -52.577369689941406, "logps/ref_rejected": -98.48920440673828, "logps/rejected": -98.98126983642578, "loss": 1.3245, "margin_dpo/margin_mean": 0.6387931108474731, "margin_dpo/margin_std": 0.6522167921066284, "step": 30 }, { "KL/chosen_KL_mean": 0.12159347534179688, "KL/mean": -0.13642428815364838, "KL/rejected_KL_mean": -0.3944435119628906, "KL/std": 0.5532899498939514, "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5160378217697144, "fcm_dpo/q_t": 0.4871293008327484, "grad_norm": 67.98377990722656, "learning_rate": 2.1739130434782607e-07, "logits/chosen": -0.503361165523529, "logits/rejected": -0.45826011896133423, "logps/chosen": -63.685325622558594, "logps/ref_chosen": -63.806922912597656, "logps/ref_rejected": -72.89400482177734, "logps/rejected": -73.2884521484375, "loss": 1.3366, "margin_dpo/margin_mean": 0.5160377025604248, "margin_dpo/margin_std": 0.6891189813613892, "step": 31 }, { "KL/chosen_KL_mean": 0.1765727996826172, "KL/mean": -0.17987801134586334, "KL/rejected_KL_mean": -0.5363311767578125, "KL/std": 0.6821790933609009, "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7129033803939819, "fcm_dpo/q_t": 0.48223358392715454, "grad_norm": 82.82377624511719, "learning_rate": 2.2463768115942027e-07, "logits/chosen": -0.49510425329208374, "logits/rejected": -0.45262959599494934, "logps/chosen": -62.562950134277344, "logps/ref_chosen": -62.739524841308594, "logps/ref_rejected": -89.3175048828125, "logps/rejected": -89.85383605957031, "loss": 1.3182, "margin_dpo/margin_mean": 0.7129031419754028, "margin_dpo/margin_std": 0.8785291910171509, "step": 32 }, { "KL/chosen_KL_mean": 0.12042808532714844, "KL/mean": -0.18817251920700073, "KL/rejected_KL_mean": -0.49677276611328125, "KL/std": 0.5911757946014404, "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6171959638595581, "fcm_dpo/q_t": 0.4845910668373108, "grad_norm": 72.13778686523438, "learning_rate": 2.318840579710145e-07, "logits/chosen": -0.48218345642089844, "logits/rejected": -0.4558228552341461, "logps/chosen": -53.14054489135742, "logps/ref_chosen": -53.26097106933594, "logps/ref_rejected": -87.8851318359375, "logps/rejected": -88.38191223144531, "loss": 1.3265, "margin_dpo/margin_mean": 0.6171959638595581, "margin_dpo/margin_std": 0.6186330914497375, "step": 33 }, { "KL/chosen_KL_mean": 0.06296157836914062, "KL/mean": -0.34480684995651245, "KL/rejected_KL_mean": -0.7525749206542969, "KL/std": 0.7686243057250977, "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.815531849861145, "fcm_dpo/q_t": 0.4796821177005768, "grad_norm": 77.3115005493164, "learning_rate": 2.391304347826087e-07, "logits/chosen": -0.49842262268066406, "logits/rejected": -0.4809451103210449, "logps/chosen": -50.754364013671875, "logps/ref_chosen": -50.81732940673828, "logps/ref_rejected": -101.92184448242188, "logps/rejected": -102.67442321777344, "loss": 1.3086, "margin_dpo/margin_mean": 0.8155316114425659, "margin_dpo/margin_std": 0.9295395016670227, "step": 34 }, { "KL/chosen_KL_mean": 0.12413406372070312, "KL/mean": -0.4548630118370056, "KL/rejected_KL_mean": -1.0338554382324219, "KL/std": 0.9395226240158081, "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.157991647720337, "fcm_dpo/q_t": 0.4711979627609253, "grad_norm": 83.07164001464844, "learning_rate": 2.463768115942029e-07, "logits/chosen": -0.4888343811035156, "logits/rejected": -0.4522944688796997, "logps/chosen": -50.90035629272461, "logps/ref_chosen": -51.02449035644531, "logps/ref_rejected": -106.82443237304688, "logps/rejected": -107.85828399658203, "loss": 1.2764, "margin_dpo/margin_mean": 1.1579914093017578, "margin_dpo/margin_std": 0.9997678995132446, "step": 35 }, { "KL/chosen_KL_mean": 0.04621315002441406, "KL/mean": -0.5572574138641357, "KL/rejected_KL_mean": -1.1607246398925781, "KL/std": 1.1042115688323975, "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2069365978240967, "fcm_dpo/q_t": 0.4699981212615967, "grad_norm": 72.58307647705078, "learning_rate": 2.536231884057971e-07, "logits/chosen": -0.5532280206680298, "logits/rejected": -0.5167073011398315, "logps/chosen": -51.94527816772461, "logps/ref_chosen": -51.991493225097656, "logps/ref_rejected": -86.0406265258789, "logps/rejected": -87.20135498046875, "loss": 1.2733, "margin_dpo/margin_mean": 1.206936240196228, "margin_dpo/margin_std": 1.2257723808288574, "step": 36 }, { "KL/chosen_KL_mean": 0.00463104248046875, "KL/mean": -0.4934205710887909, "KL/rejected_KL_mean": -0.9914779663085938, "KL/std": 1.0511497259140015, "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.996109127998352, "fcm_dpo/q_t": 0.4753045439720154, "grad_norm": 61.719268798828125, "learning_rate": 2.6086956521739126e-07, "logits/chosen": -0.5050272941589355, "logits/rejected": -0.4607655107975006, "logps/chosen": -62.80247497558594, "logps/ref_chosen": -62.807106018066406, "logps/ref_rejected": -77.89507293701172, "logps/rejected": -78.88655090332031, "loss": 1.2941, "margin_dpo/margin_mean": 0.9961087703704834, "margin_dpo/margin_std": 1.374046802520752, "step": 37 }, { "KL/chosen_KL_mean": 0.1590423583984375, "KL/mean": -0.5295313596725464, "KL/rejected_KL_mean": -1.2181015014648438, "KL/std": 1.374910593032837, "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3771462440490723, "fcm_dpo/q_t": 0.4660264253616333, "grad_norm": 69.59135437011719, "learning_rate": 2.681159420289855e-07, "logits/chosen": -0.5226507186889648, "logits/rejected": -0.4899941682815552, "logps/chosen": -48.231475830078125, "logps/ref_chosen": -48.39051818847656, "logps/ref_rejected": -97.91244506835938, "logps/rejected": -99.13053894042969, "loss": 1.2606, "margin_dpo/margin_mean": 1.3771460056304932, "margin_dpo/margin_std": 1.6853830814361572, "step": 38 }, { "KL/chosen_KL_mean": 0.0872802734375, "KL/mean": -0.7359923124313354, "KL/rejected_KL_mean": -1.55926513671875, "KL/std": 1.2973854541778564, "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.6465413570404053, "fcm_dpo/q_t": 0.4591583013534546, "grad_norm": 74.43284606933594, "learning_rate": 2.753623188405797e-07, "logits/chosen": -0.5463048219680786, "logits/rejected": -0.5056887269020081, "logps/chosen": -50.66319274902344, "logps/ref_chosen": -50.75047302246094, "logps/ref_rejected": -78.56951141357422, "logps/rejected": -80.12877655029297, "loss": 1.2332, "margin_dpo/margin_mean": 1.6465413570404053, "margin_dpo/margin_std": 1.3756870031356812, "step": 39 }, { "KL/chosen_KL_mean": 0.2025127410888672, "KL/mean": -0.5609214305877686, "KL/rejected_KL_mean": -1.3243637084960938, "KL/std": 1.3878769874572754, "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5268778800964355, "fcm_dpo/q_t": 0.4623277187347412, "grad_norm": 60.651695251464844, "learning_rate": 2.8260869565217386e-07, "logits/chosen": -0.5206544399261475, "logits/rejected": -0.4907876253128052, "logps/chosen": -57.782554626464844, "logps/ref_chosen": -57.985069274902344, "logps/ref_rejected": -74.3000717163086, "logps/rejected": -75.62443542480469, "loss": 1.2468, "margin_dpo/margin_mean": 1.5268781185150146, "margin_dpo/margin_std": 1.691199779510498, "step": 40 }, { "KL/chosen_KL_mean": 0.06011772155761719, "KL/mean": -0.9236688613891602, "KL/rejected_KL_mean": -1.9074478149414062, "KL/std": 1.8921372890472412, "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10048507153987885, "fcm_dpo/delta": 0.09654659032821655, "fcm_dpo/margin": 1.967568039894104, "fcm_dpo/q_t": 0.45162275433540344, "grad_norm": 67.96371459960938, "learning_rate": 2.898550724637681e-07, "logits/chosen": -0.5215292572975159, "logits/rejected": -0.48398417234420776, "logps/chosen": -62.63569641113281, "logps/ref_chosen": -62.69581604003906, "logps/ref_rejected": -97.02352905273438, "logps/rejected": -98.93097686767578, "loss": 1.2084, "margin_dpo/margin_mean": 1.9675687551498413, "margin_dpo/margin_std": 1.9688289165496826, "step": 41 }, { "KL/chosen_KL_mean": 0.2504749298095703, "KL/mean": -1.0469075441360474, "KL/rejected_KL_mean": -2.3442840576171875, "KL/std": 2.180742025375366, "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10202906280755997, "fcm_dpo/delta": 0.13704819977283478, "fcm_dpo/margin": 2.5947656631469727, "fcm_dpo/q_t": 0.4359305202960968, "grad_norm": 80.4867935180664, "learning_rate": 2.971014492753623e-07, "logits/chosen": -0.5434771776199341, "logits/rejected": -0.49688485264778137, "logps/chosen": -58.71595001220703, "logps/ref_chosen": -58.966426849365234, "logps/ref_rejected": -109.90837097167969, "logps/rejected": -112.25265502929688, "loss": 1.1538, "margin_dpo/margin_mean": 2.5947659015655518, "margin_dpo/margin_std": 2.4078125953674316, "step": 42 }, { "KL/chosen_KL_mean": 0.5522232055664062, "KL/mean": -0.6794946789741516, "KL/rejected_KL_mean": -1.9112205505371094, "KL/std": 1.821549654006958, "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10344166308641434, "fcm_dpo/delta": 0.14711913466453552, "fcm_dpo/margin": 2.463439464569092, "fcm_dpo/q_t": 0.4379476308822632, "grad_norm": 73.6222915649414, "learning_rate": 3.043478260869565e-07, "logits/chosen": -0.5423204898834229, "logits/rejected": -0.517362117767334, "logps/chosen": -53.60377502441406, "logps/ref_chosen": -54.15599822998047, "logps/ref_rejected": -96.48019409179688, "logps/rejected": -98.39141082763672, "loss": 1.1583, "margin_dpo/margin_mean": 2.463439702987671, "margin_dpo/margin_std": 2.007230758666992, "step": 43 }, { "KL/chosen_KL_mean": 0.2297210693359375, "KL/mean": -1.195556402206421, "KL/rejected_KL_mean": -2.6208343505859375, "KL/std": 2.280576705932617, "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10464578866958618, "fcm_dpo/delta": 0.10327555239200592, "fcm_dpo/margin": 2.8505544662475586, "fcm_dpo/q_t": 0.4275043308734894, "grad_norm": 81.44078063964844, "learning_rate": 3.115942028985507e-07, "logits/chosen": -0.4537878632545471, "logits/rejected": -0.43426036834716797, "logps/chosen": -49.848777770996094, "logps/ref_chosen": -50.07849884033203, "logps/ref_rejected": -108.78376007080078, "logps/rejected": -111.40459442138672, "loss": 1.1233, "margin_dpo/margin_mean": 2.8505539894104004, "margin_dpo/margin_std": 2.22520112991333, "step": 44 }, { "KL/chosen_KL_mean": 0.09650421142578125, "KL/mean": -1.0149602890014648, "KL/rejected_KL_mean": -2.1264266967773438, "KL/std": 2.0060698986053467, "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.1063193678855896, "fcm_dpo/delta": 0.1655273139476776, "fcm_dpo/margin": 2.2229318618774414, "fcm_dpo/q_t": 0.4429905414581299, "grad_norm": 65.44628143310547, "learning_rate": 3.188405797101449e-07, "logits/chosen": -0.46523338556289673, "logits/rejected": -0.4522145688533783, "logps/chosen": -48.318424224853516, "logps/ref_chosen": -48.4149284362793, "logps/ref_rejected": -77.93643188476562, "logps/rejected": -80.06285095214844, "loss": 1.181, "margin_dpo/margin_mean": 2.2229318618774414, "margin_dpo/margin_std": 2.4767987728118896, "step": 45 }, { "KL/chosen_KL_mean": 0.18115234375, "KL/mean": -1.2868940830230713, "KL/rejected_KL_mean": -2.75494384765625, "KL/std": 2.65130352973938, "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10749006271362305, "fcm_dpo/delta": 0.08564689010381699, "fcm_dpo/margin": 2.936098098754883, "fcm_dpo/q_t": 0.4249163269996643, "grad_norm": 71.94483184814453, "learning_rate": 3.260869565217391e-07, "logits/chosen": -0.5257543325424194, "logits/rejected": -0.47459375858306885, "logps/chosen": -55.818275451660156, "logps/ref_chosen": -55.999427795410156, "logps/ref_rejected": -95.652587890625, "logps/rejected": -98.40753173828125, "loss": 1.1239, "margin_dpo/margin_mean": 2.9360976219177246, "margin_dpo/margin_std": 3.2204670906066895, "step": 46 }, { "KL/chosen_KL_mean": 0.41971397399902344, "KL/mean": -1.100320816040039, "KL/rejected_KL_mean": -2.6203575134277344, "KL/std": 2.6521072387695312, "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10835415124893188, "fcm_dpo/delta": 0.07160530984401703, "fcm_dpo/margin": 3.040071487426758, "fcm_dpo/q_t": 0.42072784900665283, "grad_norm": 68.87890625, "learning_rate": 3.333333333333333e-07, "logits/chosen": -0.5598181486129761, "logits/rejected": -0.5062609910964966, "logps/chosen": -57.50636291503906, "logps/ref_chosen": -57.92607879638672, "logps/ref_rejected": -94.67920684814453, "logps/rejected": -97.299560546875, "loss": 1.1053, "margin_dpo/margin_mean": 3.040071487426758, "margin_dpo/margin_std": 2.725712776184082, "step": 47 }, { "KL/chosen_KL_mean": 0.0533294677734375, "KL/mean": -1.5111095905303955, "KL/rejected_KL_mean": -3.075550079345703, "KL/std": 2.7189888954162598, "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10873937606811523, "fcm_dpo/delta": 0.06080545857548714, "fcm_dpo/margin": 3.128880500793457, "fcm_dpo/q_t": 0.41845259070396423, "grad_norm": 75.4119873046875, "learning_rate": 3.4057971014492755e-07, "logits/chosen": -0.5846500396728516, "logits/rejected": -0.5260331630706787, "logps/chosen": -57.134742736816406, "logps/ref_chosen": -57.188072204589844, "logps/ref_rejected": -88.0166015625, "logps/rejected": -91.09214782714844, "loss": 1.1042, "margin_dpo/margin_mean": 3.128880262374878, "margin_dpo/margin_std": 2.981379508972168, "step": 48 }, { "KL/chosen_KL_mean": 0.31061363220214844, "KL/mean": -1.6151647567749023, "KL/rejected_KL_mean": -3.540943145751953, "KL/std": 3.465115785598755, "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10909098386764526, "fcm_dpo/delta": -0.020696185529232025, "fcm_dpo/margin": 3.8515536785125732, "fcm_dpo/q_t": 0.4022524952888489, "grad_norm": 79.5746841430664, "learning_rate": 3.478260869565217e-07, "logits/chosen": -0.5361356139183044, "logits/rejected": -0.4760180115699768, "logps/chosen": -61.374656677246094, "logps/ref_chosen": -61.685272216796875, "logps/ref_rejected": -83.76747131347656, "logps/rejected": -87.30841827392578, "loss": 1.0546, "margin_dpo/margin_mean": 3.8515536785125732, "margin_dpo/margin_std": 3.974454164505005, "step": 49 }, { "KL/chosen_KL_mean": -0.15990066528320312, "KL/mean": -2.221952438354492, "KL/rejected_KL_mean": -4.284000396728516, "KL/std": 3.5255508422851562, "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10840219259262085, "fcm_dpo/delta": -0.048164162784814835, "fcm_dpo/margin": 4.124100208282471, "fcm_dpo/q_t": 0.395396888256073, "grad_norm": 66.33158874511719, "learning_rate": 3.5507246376811595e-07, "logits/chosen": -0.5621310472488403, "logits/rejected": -0.526812732219696, "logps/chosen": -58.88404083251953, "logps/ref_chosen": -58.72413635253906, "logps/ref_rejected": -96.35814666748047, "logps/rejected": -100.64215087890625, "loss": 1.0337, "margin_dpo/margin_mean": 4.1240997314453125, "margin_dpo/margin_std": 4.0859832763671875, "step": 50 }, { "KL/chosen_KL_mean": -0.2936878204345703, "KL/mean": -2.299487590789795, "KL/rejected_KL_mean": -4.305290222167969, "KL/std": 4.18043327331543, "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10777164250612259, "fcm_dpo/delta": -0.03339504823088646, "fcm_dpo/margin": 4.011598587036133, "fcm_dpo/q_t": 0.4028114676475525, "grad_norm": 55.99860382080078, "learning_rate": 3.6231884057971015e-07, "logits/chosen": -0.5201721787452698, "logits/rejected": -0.48702794313430786, "logps/chosen": -61.6673583984375, "logps/ref_chosen": -61.3736686706543, "logps/ref_rejected": -76.00199890136719, "logps/rejected": -80.30729675292969, "loss": 1.0701, "margin_dpo/margin_mean": 4.011598587036133, "margin_dpo/margin_std": 5.134487152099609, "step": 51 }, { "KL/chosen_KL_mean": 0.3607959747314453, "KL/mean": -2.7140278816223145, "KL/rejected_KL_mean": -5.788852691650391, "KL/std": 4.7607879638671875, "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10591427981853485, "fcm_dpo/delta": -0.2597273588180542, "fcm_dpo/margin": 6.149646759033203, "fcm_dpo/q_t": 0.3517502546310425, "grad_norm": 61.00212860107422, "learning_rate": 3.695652173913043e-07, "logits/chosen": -0.5540838241577148, "logits/rejected": -0.4985507130622864, "logps/chosen": -51.9765625, "logps/ref_chosen": -52.33735656738281, "logps/ref_rejected": -79.97391510009766, "logps/rejected": -85.76277160644531, "loss": 0.9009, "margin_dpo/margin_mean": 6.149646759033203, "margin_dpo/margin_std": 5.092857837677002, "step": 52 }, { "KL/chosen_KL_mean": -0.19704818725585938, "KL/mean": -3.3417558670043945, "KL/rejected_KL_mean": -6.486461639404297, "KL/std": 5.458329677581787, "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10362114012241364, "fcm_dpo/delta": -0.2598329186439514, "fcm_dpo/margin": 6.289419174194336, "fcm_dpo/q_t": 0.3608250916004181, "grad_norm": 59.65618133544922, "learning_rate": 3.7681159420289855e-07, "logits/chosen": -0.6251329183578491, "logits/rejected": -0.6039801836013794, "logps/chosen": -53.51169967651367, "logps/ref_chosen": -53.31465148925781, "logps/ref_rejected": -91.78359985351562, "logps/rejected": -98.27005767822266, "loss": 0.9373, "margin_dpo/margin_mean": 6.289420127868652, "margin_dpo/margin_std": 6.279295921325684, "step": 53 }, { "KL/chosen_KL_mean": -0.4445972442626953, "KL/mean": -3.137702703475952, "KL/rejected_KL_mean": -5.830802917480469, "KL/std": 5.121060848236084, "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10144082456827164, "fcm_dpo/delta": -0.15038657188415527, "fcm_dpo/margin": 5.386206150054932, "fcm_dpo/q_t": 0.3753092288970947, "grad_norm": 55.26798629760742, "learning_rate": 3.8405797101449274e-07, "logits/chosen": -0.5941368341445923, "logits/rejected": -0.5407606363296509, "logps/chosen": -51.13325881958008, "logps/ref_chosen": -50.68865966796875, "logps/ref_rejected": -91.71539306640625, "logps/rejected": -97.54619598388672, "loss": 0.9709, "margin_dpo/margin_mean": 5.386206150054932, "margin_dpo/margin_std": 5.081514835357666, "step": 54 }, { "KL/chosen_KL_mean": -0.927947998046875, "KL/mean": -4.146857261657715, "KL/rejected_KL_mean": -7.365764617919922, "KL/std": 6.101950168609619, "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.09932823479175568, "fcm_dpo/delta": -0.24702517688274384, "fcm_dpo/margin": 6.4378156661987305, "fcm_dpo/q_t": 0.36172401905059814, "grad_norm": 53.780818939208984, "learning_rate": 3.9130434782608694e-07, "logits/chosen": -0.6389350891113281, "logits/rejected": -0.5758472084999084, "logps/chosen": -63.543182373046875, "logps/ref_chosen": -62.615234375, "logps/ref_rejected": -88.99349975585938, "logps/rejected": -96.35926818847656, "loss": 0.9584, "margin_dpo/margin_mean": 6.437814712524414, "margin_dpo/margin_std": 7.308642387390137, "step": 55 }, { "KL/chosen_KL_mean": -0.7108230590820312, "KL/mean": -3.80918550491333, "KL/rejected_KL_mean": -6.907550811767578, "KL/std": 6.075893402099609, "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.09705003350973129, "fcm_dpo/delta": -0.20751234889030457, "fcm_dpo/margin": 6.196725845336914, "fcm_dpo/q_t": 0.36958831548690796, "grad_norm": 47.083831787109375, "learning_rate": 3.9855072463768114e-07, "logits/chosen": -0.630185604095459, "logits/rejected": -0.5887142419815063, "logps/chosen": -58.643550872802734, "logps/ref_chosen": -57.9327278137207, "logps/ref_rejected": -94.1744384765625, "logps/rejected": -101.08198547363281, "loss": 0.9834, "margin_dpo/margin_mean": 6.196725845336914, "margin_dpo/margin_std": 7.354310035705566, "step": 56 }, { "KL/chosen_KL_mean": -0.7268028259277344, "KL/mean": -4.224173545837402, "KL/rejected_KL_mean": -7.7215423583984375, "KL/std": 6.010547161102295, "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.09477832913398743, "fcm_dpo/delta": -0.27153968811035156, "fcm_dpo/margin": 6.994732856750488, "fcm_dpo/q_t": 0.351573646068573, "grad_norm": 52.91410827636719, "learning_rate": 4.057971014492754e-07, "logits/chosen": -0.5690896511077881, "logits/rejected": -0.5405641794204712, "logps/chosen": -71.22209167480469, "logps/ref_chosen": -70.49528503417969, "logps/ref_rejected": -95.56546020507812, "logps/rejected": -103.28700256347656, "loss": 0.918, "margin_dpo/margin_mean": 6.994732856750488, "margin_dpo/margin_std": 6.630904674530029, "step": 57 }, { "KL/chosen_KL_mean": -1.010385513305664, "KL/mean": -4.842105865478516, "KL/rejected_KL_mean": -8.673824310302734, "KL/std": 6.785933494567871, "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.09193111956119537, "fcm_dpo/delta": -0.31510716676712036, "fcm_dpo/margin": 7.663437366485596, "fcm_dpo/q_t": 0.349496066570282, "grad_norm": 55.937931060791016, "learning_rate": 4.1304347826086954e-07, "logits/chosen": -0.5902031660079956, "logits/rejected": -0.513656735420227, "logps/chosen": -63.14332580566406, "logps/ref_chosen": -62.13294219970703, "logps/ref_rejected": -84.61729431152344, "logps/rejected": -93.29112243652344, "loss": 0.9191, "margin_dpo/margin_mean": 7.663437843322754, "margin_dpo/margin_std": 8.12877368927002, "step": 58 }, { "KL/chosen_KL_mean": -1.4919471740722656, "KL/mean": -5.709989070892334, "KL/rejected_KL_mean": -9.928028106689453, "KL/std": 7.444372177124023, "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.08817608654499054, "fcm_dpo/delta": -0.35956332087516785, "fcm_dpo/margin": 8.436084747314453, "fcm_dpo/q_t": 0.34159553050994873, "grad_norm": 52.03582763671875, "learning_rate": 4.2028985507246374e-07, "logits/chosen": -0.6292190551757812, "logits/rejected": -0.5891801118850708, "logps/chosen": -53.42447280883789, "logps/ref_chosen": -51.932525634765625, "logps/ref_rejected": -88.88520050048828, "logps/rejected": -98.813232421875, "loss": 0.8944, "margin_dpo/margin_mean": 8.43608570098877, "margin_dpo/margin_std": 8.554616928100586, "step": 59 }, { "KL/chosen_KL_mean": -2.6756553649902344, "KL/mean": -6.041238784790039, "KL/rejected_KL_mean": -9.406822204589844, "KL/std": 6.815237998962402, "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.08637680113315582, "fcm_dpo/delta": -0.18736253678798676, "fcm_dpo/margin": 6.731164932250977, "fcm_dpo/q_t": 0.36844515800476074, "grad_norm": 57.49814987182617, "learning_rate": 4.2753623188405794e-07, "logits/chosen": -0.606688380241394, "logits/rejected": -0.5464133024215698, "logps/chosen": -63.61784362792969, "logps/ref_chosen": -60.94218826293945, "logps/ref_rejected": -85.39340209960938, "logps/rejected": -94.80021667480469, "loss": 0.9875, "margin_dpo/margin_mean": 6.731164932250977, "margin_dpo/margin_std": 7.683201313018799, "step": 60 }, { "KL/chosen_KL_mean": -1.5235176086425781, "KL/mean": -5.651108264923096, "KL/rejected_KL_mean": -9.778701782226562, "KL/std": 9.094491004943848, "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.08403058350086212, "fcm_dpo/delta": -0.3050088882446289, "fcm_dpo/margin": 8.255186080932617, "fcm_dpo/q_t": 0.3618362545967102, "grad_norm": 48.30548858642578, "learning_rate": 4.3478260869565214e-07, "logits/chosen": -0.6002434492111206, "logits/rejected": -0.5660552382469177, "logps/chosen": -62.15704345703125, "logps/ref_chosen": -60.633522033691406, "logps/ref_rejected": -89.85249328613281, "logps/rejected": -99.63119506835938, "loss": 0.9711, "margin_dpo/margin_mean": 8.2551851272583, "margin_dpo/margin_std": 11.305606842041016, "step": 61 }, { "KL/chosen_KL_mean": -1.6561450958251953, "KL/mean": -4.769914150238037, "KL/rejected_KL_mean": -7.8836822509765625, "KL/std": 6.6822967529296875, "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.08298887312412262, "fcm_dpo/delta": -0.11985031515359879, "fcm_dpo/margin": 6.227537155151367, "fcm_dpo/q_t": 0.38729119300842285, "grad_norm": 48.09984588623047, "learning_rate": 4.420289855072464e-07, "logits/chosen": -0.5964776873588562, "logits/rejected": -0.5614856481552124, "logps/chosen": -57.806915283203125, "logps/ref_chosen": -56.15077209472656, "logps/ref_rejected": -75.56619262695312, "logps/rejected": -83.44987487792969, "loss": 1.038, "margin_dpo/margin_mean": 6.227537631988525, "margin_dpo/margin_std": 8.184688568115234, "step": 62 }, { "KL/chosen_KL_mean": -2.681711196899414, "KL/mean": -6.847625732421875, "KL/rejected_KL_mean": -11.013534545898438, "KL/std": 8.392871856689453, "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.08056771755218506, "fcm_dpo/delta": -0.2820628583431244, "fcm_dpo/margin": 8.331822395324707, "fcm_dpo/q_t": 0.35215944051742554, "grad_norm": 49.44816970825195, "learning_rate": 4.4927536231884053e-07, "logits/chosen": -0.6123620271682739, "logits/rejected": -0.5682877898216248, "logps/chosen": -75.8291015625, "logps/ref_chosen": -73.14739227294922, "logps/ref_rejected": -97.61006164550781, "logps/rejected": -108.62359619140625, "loss": 0.9343, "margin_dpo/margin_mean": 8.33182144165039, "margin_dpo/margin_std": 8.752732276916504, "step": 63 }, { "KL/chosen_KL_mean": -1.037282943725586, "KL/mean": -5.9826202392578125, "KL/rejected_KL_mean": -10.927955627441406, "KL/std": 9.150436401367188, "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.07755880057811737, "fcm_dpo/delta": -0.3850880563259125, "fcm_dpo/margin": 9.890669822692871, "fcm_dpo/q_t": 0.3408496081829071, "grad_norm": 44.27583694458008, "learning_rate": 4.5652173913043473e-07, "logits/chosen": -0.605506181716919, "logits/rejected": -0.5743293762207031, "logps/chosen": -55.03588104248047, "logps/ref_chosen": -53.998600006103516, "logps/ref_rejected": -93.53019714355469, "logps/rejected": -104.45816040039062, "loss": 0.9102, "margin_dpo/margin_mean": 9.890670776367188, "margin_dpo/margin_std": 11.207742691040039, "step": 64 }, { "KL/chosen_KL_mean": -3.239490509033203, "KL/mean": -8.281026840209961, "KL/rejected_KL_mean": -13.322566986083984, "KL/std": 10.2528076171875, "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.07488435506820679, "fcm_dpo/delta": -0.37056964635849, "fcm_dpo/margin": 10.08306884765625, "fcm_dpo/q_t": 0.3390854001045227, "grad_norm": 44.772369384765625, "learning_rate": 4.63768115942029e-07, "logits/chosen": -0.6510547995567322, "logits/rejected": -0.6380674839019775, "logps/chosen": -68.07549285888672, "logps/ref_chosen": -64.83599853515625, "logps/ref_rejected": -109.94645690917969, "logps/rejected": -123.26902770996094, "loss": 0.9165, "margin_dpo/margin_mean": 10.083067893981934, "margin_dpo/margin_std": 11.16361141204834, "step": 65 }, { "KL/chosen_KL_mean": -3.0490779876708984, "KL/mean": -7.599049091339111, "KL/rejected_KL_mean": -12.149017333984375, "KL/std": 9.199509620666504, "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.07262101769447327, "fcm_dpo/delta": -0.271577388048172, "fcm_dpo/margin": 9.099935531616211, "fcm_dpo/q_t": 0.36167746782302856, "grad_norm": 41.545040130615234, "learning_rate": 4.7101449275362313e-07, "logits/chosen": -0.655698835849762, "logits/rejected": -0.6236636638641357, "logps/chosen": -54.49260330200195, "logps/ref_chosen": -51.44352722167969, "logps/ref_rejected": -75.63629913330078, "logps/rejected": -87.78531646728516, "loss": 0.9619, "margin_dpo/margin_mean": 9.099935531616211, "margin_dpo/margin_std": 11.122103691101074, "step": 66 }, { "KL/chosen_KL_mean": -2.6720447540283203, "KL/mean": -7.3747406005859375, "KL/rejected_KL_mean": -12.077438354492188, "KL/std": 9.28006362915039, "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.07105744630098343, "fcm_dpo/delta": -0.27725303173065186, "fcm_dpo/margin": 9.405391693115234, "fcm_dpo/q_t": 0.3593307137489319, "grad_norm": 41.94929885864258, "learning_rate": 4.782608695652174e-07, "logits/chosen": -0.6038053035736084, "logits/rejected": -0.5614980459213257, "logps/chosen": -62.012847900390625, "logps/ref_chosen": -59.34080505371094, "logps/ref_rejected": -72.78728485107422, "logps/rejected": -84.8647232055664, "loss": 0.9529, "margin_dpo/margin_mean": 9.405391693115234, "margin_dpo/margin_std": 11.189936637878418, "step": 67 }, { "KL/chosen_KL_mean": -3.055255889892578, "KL/mean": -7.527899742126465, "KL/rejected_KL_mean": -12.000545501708984, "KL/std": 8.336349487304688, "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.06936784088611603, "fcm_dpo/delta": -0.22731636464595795, "fcm_dpo/margin": 8.945289611816406, "fcm_dpo/q_t": 0.36327165365219116, "grad_norm": 40.98445129394531, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.6367079019546509, "logits/rejected": -0.5765199661254883, "logps/chosen": -68.26109313964844, "logps/ref_chosen": -65.2058334350586, "logps/ref_rejected": -77.20724487304688, "logps/rejected": -89.20779418945312, "loss": 0.9496, "margin_dpo/margin_mean": 8.945289611816406, "margin_dpo/margin_std": 9.330879211425781, "step": 68 }, { "KL/chosen_KL_mean": -3.5810699462890625, "KL/mean": -9.05520248413086, "KL/rejected_KL_mean": -14.529338836669922, "KL/std": 9.87652587890625, "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.06693008542060852, "fcm_dpo/delta": -0.346579372882843, "fcm_dpo/margin": 10.948261260986328, "fcm_dpo/q_t": 0.3431288003921509, "grad_norm": 42.85259246826172, "learning_rate": 4.927536231884058e-07, "logits/chosen": -0.6404001712799072, "logits/rejected": -0.6165236830711365, "logps/chosen": -63.40031051635742, "logps/ref_chosen": -59.81924057006836, "logps/ref_rejected": -103.38886260986328, "logps/rejected": -117.91819763183594, "loss": 0.8894, "margin_dpo/margin_mean": 10.948262214660645, "margin_dpo/margin_std": 10.901615142822266, "step": 69 }, { "KL/chosen_KL_mean": -5.2194671630859375, "KL/mean": -11.088415145874023, "KL/rejected_KL_mean": -16.957359313964844, "KL/std": 12.024399757385254, "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.0648043155670166, "fcm_dpo/delta": -0.3743746280670166, "fcm_dpo/margin": 11.73788833618164, "fcm_dpo/q_t": 0.34193557500839233, "grad_norm": 44.00492858886719, "learning_rate": 5e-07, "logits/chosen": -0.6287994384765625, "logits/rejected": -0.5946310758590698, "logps/chosen": -67.15010833740234, "logps/ref_chosen": -61.930641174316406, "logps/ref_rejected": -91.06078338623047, "logps/rejected": -108.01814270019531, "loss": 0.9036, "margin_dpo/margin_mean": 11.737889289855957, "margin_dpo/margin_std": 12.800237655639648, "step": 70 }, { "KL/chosen_KL_mean": -4.846014022827148, "KL/mean": -11.838199615478516, "KL/rejected_KL_mean": -18.830387115478516, "KL/std": 12.62130355834961, "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.061709873378276825, "fcm_dpo/delta": -0.4846467971801758, "fcm_dpo/margin": 13.984375, "fcm_dpo/q_t": 0.32341229915618896, "grad_norm": 39.75981903076172, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.6493800282478333, "logits/rejected": -0.6095075607299805, "logps/chosen": -66.59635162353516, "logps/ref_chosen": -61.750335693359375, "logps/ref_rejected": -97.33662414550781, "logps/rejected": -116.16700744628906, "loss": 0.837, "margin_dpo/margin_mean": 13.984375953674316, "margin_dpo/margin_std": 13.748950004577637, "step": 71 }, { "KL/chosen_KL_mean": -5.580968856811523, "KL/mean": -12.921693801879883, "KL/rejected_KL_mean": -20.262420654296875, "KL/std": 13.505867004394531, "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.05879486724734306, "fcm_dpo/delta": -0.4848293662071228, "fcm_dpo/margin": 14.681446075439453, "fcm_dpo/q_t": 0.3229914903640747, "grad_norm": 41.22248840332031, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.6457772254943848, "logits/rejected": -0.6087461709976196, "logps/chosen": -71.63438415527344, "logps/ref_chosen": -66.05341339111328, "logps/ref_rejected": -95.2869873046875, "logps/rejected": -115.54940795898438, "loss": 0.8629, "margin_dpo/margin_mean": 14.681446075439453, "margin_dpo/margin_std": 15.725846290588379, "step": 72 }, { "KL/chosen_KL_mean": -7.517843246459961, "KL/mean": -14.800748825073242, "KL/rejected_KL_mean": -22.083660125732422, "KL/std": 17.561433792114258, "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.05613835155963898, "fcm_dpo/delta": -0.43695318698883057, "fcm_dpo/margin": 14.565816879272461, "fcm_dpo/q_t": 0.3509829044342041, "grad_norm": 45.127349853515625, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.6935266256332397, "logits/rejected": -0.653118908405304, "logps/chosen": -73.77411651611328, "logps/ref_chosen": -66.25627136230469, "logps/ref_rejected": -90.45613098144531, "logps/rejected": -112.539794921875, "loss": 1.0029, "margin_dpo/margin_mean": 14.565816879272461, "margin_dpo/margin_std": 21.94605255126953, "step": 73 }, { "KL/chosen_KL_mean": -7.943721771240234, "KL/mean": -15.891912460327148, "KL/rejected_KL_mean": -23.840110778808594, "KL/std": 19.308032989501953, "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.05364230275154114, "fcm_dpo/delta": -0.4738168716430664, "fcm_dpo/margin": 15.896381378173828, "fcm_dpo/q_t": 0.3405853509902954, "grad_norm": 44.694664001464844, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.6524510383605957, "logits/rejected": -0.6433727741241455, "logps/chosen": -61.36860656738281, "logps/ref_chosen": -53.42488098144531, "logps/ref_rejected": -95.94693756103516, "logps/rejected": -119.78704833984375, "loss": 0.9488, "margin_dpo/margin_mean": 15.896381378173828, "margin_dpo/margin_std": 23.962190628051758, "step": 74 }, { "KL/chosen_KL_mean": -7.722936630249023, "KL/mean": -18.847137451171875, "KL/rejected_KL_mean": -29.971328735351562, "KL/std": 19.825923919677734, "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.050399381667375565, "fcm_dpo/delta": -0.761603832244873, "fcm_dpo/margin": 22.24839973449707, "fcm_dpo/q_t": 0.2916664481163025, "grad_norm": 36.36934280395508, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.6783813238143921, "logits/rejected": -0.6700829267501831, "logps/chosen": -59.584598541259766, "logps/ref_chosen": -51.861663818359375, "logps/ref_rejected": -111.25398254394531, "logps/rejected": -141.22531127929688, "loss": 0.7699, "margin_dpo/margin_mean": 22.248397827148438, "margin_dpo/margin_std": 22.400943756103516, "step": 75 }, { "KL/chosen_KL_mean": -9.420347213745117, "KL/mean": -16.42698097229004, "KL/rejected_KL_mean": -23.433605194091797, "KL/std": 15.946723937988281, "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.04817916452884674, "fcm_dpo/delta": -0.2846575379371643, "fcm_dpo/margin": 14.013269424438477, "fcm_dpo/q_t": 0.35380053520202637, "grad_norm": 39.79032516479492, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.7044748067855835, "logits/rejected": -0.6629197597503662, "logps/chosen": -62.68638610839844, "logps/ref_chosen": -53.26603698730469, "logps/ref_rejected": -78.21662902832031, "logps/rejected": -101.65023803710938, "loss": 0.9645, "margin_dpo/margin_mean": 14.013269424438477, "margin_dpo/margin_std": 17.342012405395508, "step": 76 }, { "KL/chosen_KL_mean": -8.207674026489258, "KL/mean": -19.680038452148438, "KL/rejected_KL_mean": -31.15240478515625, "KL/std": 21.73928451538086, "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.04527886211872101, "fcm_dpo/delta": -0.677450954914093, "fcm_dpo/margin": 22.944732666015625, "fcm_dpo/q_t": 0.29555851221084595, "grad_norm": 41.42151641845703, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.6860731244087219, "logits/rejected": -0.647629976272583, "logps/chosen": -66.30435180664062, "logps/ref_chosen": -58.0966796875, "logps/ref_rejected": -93.77361297607422, "logps/rejected": -124.92601776123047, "loss": 0.8147, "margin_dpo/margin_mean": 22.944730758666992, "margin_dpo/margin_std": 23.827333450317383, "step": 77 }, { "KL/chosen_KL_mean": -8.151559829711914, "KL/mean": -16.818445205688477, "KL/rejected_KL_mean": -25.485332489013672, "KL/std": 17.356788635253906, "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.043553952127695084, "fcm_dpo/delta": -0.368335485458374, "fcm_dpo/margin": 17.33376693725586, "fcm_dpo/q_t": 0.34413814544677734, "grad_norm": 36.57780838012695, "learning_rate": 4.997892217220159e-07, "logits/chosen": -0.6453582048416138, "logits/rejected": -0.6197341680526733, "logps/chosen": -63.765342712402344, "logps/ref_chosen": -55.61378479003906, "logps/ref_rejected": -84.93436431884766, "logps/rejected": -110.41969299316406, "loss": 0.9188, "margin_dpo/margin_mean": 17.33376693725586, "margin_dpo/margin_std": 19.793365478515625, "step": 78 }, { "KL/chosen_KL_mean": -8.366096496582031, "KL/mean": -17.763093948364258, "KL/rejected_KL_mean": -27.16008758544922, "KL/std": 19.61853790283203, "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.04178548604249954, "fcm_dpo/delta": -0.4011922776699066, "fcm_dpo/margin": 18.793994903564453, "fcm_dpo/q_t": 0.34827283024787903, "grad_norm": 32.12797164916992, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6654260158538818, "logits/rejected": -0.6358423829078674, "logps/chosen": -63.81658172607422, "logps/ref_chosen": -55.45048522949219, "logps/ref_rejected": -87.64756774902344, "logps/rejected": -114.80764770507812, "loss": 0.9432, "margin_dpo/margin_mean": 18.793994903564453, "margin_dpo/margin_std": 24.024066925048828, "step": 79 }, { "KL/chosen_KL_mean": -11.082340240478516, "KL/mean": -19.782291412353516, "KL/rejected_KL_mean": -28.48224639892578, "KL/std": 20.413394927978516, "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.0402105450630188, "fcm_dpo/delta": -0.31318235397338867, "fcm_dpo/margin": 17.399906158447266, "fcm_dpo/q_t": 0.3642631471157074, "grad_norm": 35.137020111083984, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.7078701853752136, "logits/rejected": -0.6634222269058228, "logps/chosen": -69.60163116455078, "logps/ref_chosen": -58.519290924072266, "logps/ref_rejected": -87.54750061035156, "logps/rejected": -116.02974700927734, "loss": 0.9904, "margin_dpo/margin_mean": 17.399904251098633, "margin_dpo/margin_std": 23.675479888916016, "step": 80 }, { "KL/chosen_KL_mean": -9.967578887939453, "KL/mean": -21.815593719482422, "KL/rejected_KL_mean": -33.663612365722656, "KL/std": 24.03949737548828, "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.03838124871253967, "fcm_dpo/delta": -0.5364028811454773, "fcm_dpo/margin": 23.69603729248047, "fcm_dpo/q_t": 0.3254055976867676, "grad_norm": 35.88831329345703, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.7091889381408691, "logits/rejected": -0.6791235208511353, "logps/chosen": -76.41644287109375, "logps/ref_chosen": -66.44886779785156, "logps/ref_rejected": -129.66270446777344, "logps/rejected": -163.32632446289062, "loss": 0.8792, "margin_dpo/margin_mean": 23.69603729248047, "margin_dpo/margin_std": 28.50186538696289, "step": 81 }, { "KL/chosen_KL_mean": -12.100605010986328, "KL/mean": -21.5577392578125, "KL/rejected_KL_mean": -31.014873504638672, "KL/std": 21.757911682128906, "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.03735063970088959, "fcm_dpo/delta": -0.3172946870326996, "fcm_dpo/margin": 18.914270401000977, "fcm_dpo/q_t": 0.3623591363430023, "grad_norm": 41.56498336791992, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.6205165386199951, "logits/rejected": -0.6074250340461731, "logps/chosen": -64.33299255371094, "logps/ref_chosen": -52.232383728027344, "logps/ref_rejected": -90.74325561523438, "logps/rejected": -121.75812530517578, "loss": 1.044, "margin_dpo/margin_mean": 18.914270401000977, "margin_dpo/margin_std": 29.809494018554688, "step": 82 }, { "KL/chosen_KL_mean": -12.361867904663086, "KL/mean": -23.320655822753906, "KL/rejected_KL_mean": -34.27945327758789, "KL/std": 22.589454650878906, "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.035669997334480286, "fcm_dpo/delta": -0.39825981855392456, "fcm_dpo/margin": 21.917579650878906, "fcm_dpo/q_t": 0.3431558609008789, "grad_norm": 36.22183609008789, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.6784165501594543, "logits/rejected": -0.6415207982063293, "logps/chosen": -68.18925476074219, "logps/ref_chosen": -55.82738494873047, "logps/ref_rejected": -103.71589660644531, "logps/rejected": -137.995361328125, "loss": 0.9297, "margin_dpo/margin_mean": 21.917579650878906, "margin_dpo/margin_std": 26.579288482666016, "step": 83 }, { "KL/chosen_KL_mean": -10.984846115112305, "KL/mean": -20.518203735351562, "KL/rejected_KL_mean": -30.051559448242188, "KL/std": 19.22481346130371, "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.034514445811510086, "fcm_dpo/delta": -0.268974244594574, "fcm_dpo/margin": 19.066707611083984, "fcm_dpo/q_t": 0.35393911600112915, "grad_norm": 31.12311553955078, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.6781740188598633, "logits/rejected": -0.6409305334091187, "logps/chosen": -78.1610107421875, "logps/ref_chosen": -67.1761703491211, "logps/ref_rejected": -87.29859924316406, "logps/rejected": -117.35015869140625, "loss": 0.9372, "margin_dpo/margin_mean": 19.066707611083984, "margin_dpo/margin_std": 19.633758544921875, "step": 84 }, { "KL/chosen_KL_mean": -11.446046829223633, "KL/mean": -21.150569915771484, "KL/rejected_KL_mean": -30.855098724365234, "KL/std": 19.64529037475586, "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.03391923010349274, "fcm_dpo/delta": -0.2667713761329651, "fcm_dpo/margin": 19.409046173095703, "fcm_dpo/q_t": 0.36208292841911316, "grad_norm": 31.290390014648438, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.6304664611816406, "logits/rejected": -0.5972647070884705, "logps/chosen": -69.8526611328125, "logps/ref_chosen": -58.4066162109375, "logps/ref_rejected": -78.63880157470703, "logps/rejected": -109.493896484375, "loss": 0.9736, "margin_dpo/margin_mean": 19.409046173095703, "margin_dpo/margin_std": 23.33486557006836, "step": 85 }, { "KL/chosen_KL_mean": -14.669717788696289, "KL/mean": -24.960693359375, "KL/rejected_KL_mean": -35.25166702270508, "KL/std": 25.916988372802734, "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.032901618629693985, "fcm_dpo/delta": -0.2864453196525574, "fcm_dpo/margin": 20.581951141357422, "fcm_dpo/q_t": 0.37629234790802, "grad_norm": 39.49543380737305, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.7171605825424194, "logits/rejected": -0.7046980857849121, "logps/chosen": -70.80718231201172, "logps/ref_chosen": -56.13746643066406, "logps/ref_rejected": -88.12165069580078, "logps/rejected": -123.37332153320312, "loss": 1.0691, "margin_dpo/margin_mean": 20.581951141357422, "margin_dpo/margin_std": 35.439178466796875, "step": 86 }, { "KL/chosen_KL_mean": -14.323598861694336, "KL/mean": -26.19344711303711, "KL/rejected_KL_mean": -38.063297271728516, "KL/std": 25.82670021057129, "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.03160586953163147, "fcm_dpo/delta": -0.36560332775115967, "fcm_dpo/margin": 23.739696502685547, "fcm_dpo/q_t": 0.34734636545181274, "grad_norm": 31.173131942749023, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.7083604335784912, "logits/rejected": -0.6805447340011597, "logps/chosen": -69.9596939086914, "logps/ref_chosen": -55.63609313964844, "logps/ref_rejected": -95.46757507324219, "logps/rejected": -133.5308837890625, "loss": 0.9671, "margin_dpo/margin_mean": 23.739696502685547, "margin_dpo/margin_std": 31.024333953857422, "step": 87 }, { "KL/chosen_KL_mean": -18.130157470703125, "KL/mean": -27.014564514160156, "KL/rejected_KL_mean": -35.89897155761719, "KL/std": 26.533187866210938, "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.030946895480155945, "fcm_dpo/delta": -0.15513765811920166, "fcm_dpo/margin": 17.768814086914062, "fcm_dpo/q_t": 0.3872056007385254, "grad_norm": 33.197364807128906, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.658212423324585, "logits/rejected": -0.6492829322814941, "logps/chosen": -91.80130767822266, "logps/ref_chosen": -73.67115020751953, "logps/ref_rejected": -106.70849609375, "logps/rejected": -142.6074676513672, "loss": 1.0846, "margin_dpo/margin_mean": 17.768814086914062, "margin_dpo/margin_std": 27.587907791137695, "step": 88 }, { "KL/chosen_KL_mean": -10.722139358520508, "KL/mean": -20.540828704833984, "KL/rejected_KL_mean": -30.359527587890625, "KL/std": 22.09980010986328, "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.03054092824459076, "fcm_dpo/delta": -0.2056586593389511, "fcm_dpo/margin": 19.63738250732422, "fcm_dpo/q_t": 0.3710666000843048, "grad_norm": 28.453542709350586, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.7179343104362488, "logits/rejected": -0.6833662986755371, "logps/chosen": -71.34705352783203, "logps/ref_chosen": -60.624916076660156, "logps/ref_rejected": -82.08354949951172, "logps/rejected": -112.44307708740234, "loss": 0.9914, "margin_dpo/margin_mean": 19.63738250732422, "margin_dpo/margin_std": 24.839431762695312, "step": 89 }, { "KL/chosen_KL_mean": -14.105724334716797, "KL/mean": -27.13547706604004, "KL/rejected_KL_mean": -40.16523361206055, "KL/std": 29.942901611328125, "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.029609836637973785, "fcm_dpo/delta": -0.38564369082450867, "fcm_dpo/margin": 26.059507369995117, "fcm_dpo/q_t": 0.3560681939125061, "grad_norm": 30.310239791870117, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.6555777192115784, "logits/rejected": -0.6643766164779663, "logps/chosen": -67.39103698730469, "logps/ref_chosen": -53.285308837890625, "logps/ref_rejected": -111.54470825195312, "logps/rejected": -151.70994567871094, "loss": 0.9857, "margin_dpo/margin_mean": 26.05950927734375, "margin_dpo/margin_std": 36.949066162109375, "step": 90 }, { "KL/chosen_KL_mean": -14.317174911499023, "KL/mean": -25.541458129882812, "KL/rejected_KL_mean": -36.76573944091797, "KL/std": 24.45753288269043, "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.028681177645921707, "fcm_dpo/delta": -0.2515973150730133, "fcm_dpo/margin": 22.448570251464844, "fcm_dpo/q_t": 0.3691549599170685, "grad_norm": 29.94892692565918, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.6794449090957642, "logits/rejected": -0.643684983253479, "logps/chosen": -76.12013244628906, "logps/ref_chosen": -61.802955627441406, "logps/ref_rejected": -87.87395477294922, "logps/rejected": -124.63969421386719, "loss": 1.0094, "margin_dpo/margin_mean": 22.448570251464844, "margin_dpo/margin_std": 31.180084228515625, "step": 91 }, { "KL/chosen_KL_mean": -12.907651901245117, "KL/mean": -24.19264030456543, "KL/rejected_KL_mean": -35.477630615234375, "KL/std": 25.02846908569336, "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.028016380965709686, "fcm_dpo/delta": -0.23956279456615448, "fcm_dpo/margin": 22.56998062133789, "fcm_dpo/q_t": 0.3695027232170105, "grad_norm": 27.0372371673584, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.7169160842895508, "logits/rejected": -0.693396270275116, "logps/chosen": -64.54841613769531, "logps/ref_chosen": -51.640769958496094, "logps/ref_rejected": -77.88117980957031, "logps/rejected": -113.35881042480469, "loss": 1.0017, "margin_dpo/margin_mean": 22.56998062133789, "margin_dpo/margin_std": 30.188648223876953, "step": 92 }, { "KL/chosen_KL_mean": -14.480842590332031, "KL/mean": -25.734779357910156, "KL/rejected_KL_mean": -36.98870849609375, "KL/std": 23.70608901977539, "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.027309108525514603, "fcm_dpo/delta": -0.22143405675888062, "fcm_dpo/margin": 22.507863998413086, "fcm_dpo/q_t": 0.3679991662502289, "grad_norm": 27.40256118774414, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.738601803779602, "logits/rejected": -0.6867475509643555, "logps/chosen": -67.01008605957031, "logps/ref_chosen": -52.529239654541016, "logps/ref_rejected": -77.16075134277344, "logps/rejected": -114.14945983886719, "loss": 0.9724, "margin_dpo/margin_mean": 22.507863998413086, "margin_dpo/margin_std": 26.947500228881836, "step": 93 }, { "KL/chosen_KL_mean": -15.58909797668457, "KL/mean": -28.327407836914062, "KL/rejected_KL_mean": -41.06570816040039, "KL/std": 26.550617218017578, "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.026492692530155182, "fcm_dpo/delta": -0.28588011860847473, "fcm_dpo/margin": 25.476613998413086, "fcm_dpo/q_t": 0.35494130849838257, "grad_norm": 27.439083099365234, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.6954045295715332, "logits/rejected": -0.6782183647155762, "logps/chosen": -76.81170654296875, "logps/ref_chosen": -61.22261047363281, "logps/ref_rejected": -99.59902954101562, "logps/rejected": -140.66473388671875, "loss": 0.9351, "margin_dpo/margin_mean": 25.476612091064453, "margin_dpo/margin_std": 27.425670623779297, "step": 94 }, { "KL/chosen_KL_mean": -14.52149772644043, "KL/mean": -24.788345336914062, "KL/rejected_KL_mean": -35.055198669433594, "KL/std": 24.6375732421875, "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.026066306978464127, "fcm_dpo/delta": -0.13932469487190247, "fcm_dpo/margin": 20.533700942993164, "fcm_dpo/q_t": 0.3804051876068115, "grad_norm": 26.025171279907227, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.7163432836532593, "logits/rejected": -0.68684983253479, "logps/chosen": -67.0451431274414, "logps/ref_chosen": -52.523643493652344, "logps/ref_rejected": -75.8803482055664, "logps/rejected": -110.935546875, "loss": 1.0224, "margin_dpo/margin_mean": 20.533700942993164, "margin_dpo/margin_std": 25.788597106933594, "step": 95 }, { "KL/chosen_KL_mean": -13.763811111450195, "KL/mean": -27.650653839111328, "KL/rejected_KL_mean": -41.537498474121094, "KL/std": 28.800777435302734, "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.025319525972008705, "fcm_dpo/delta": -0.31601524353027344, "fcm_dpo/margin": 27.773691177368164, "fcm_dpo/q_t": 0.35219550132751465, "grad_norm": 26.12571144104004, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7276456356048584, "logits/rejected": -0.6974596977233887, "logps/chosen": -75.92078399658203, "logps/ref_chosen": -62.15697479248047, "logps/ref_rejected": -96.59601593017578, "logps/rejected": -138.13351440429688, "loss": 0.9254, "margin_dpo/margin_mean": 27.773693084716797, "margin_dpo/margin_std": 31.26250457763672, "step": 96 }, { "KL/chosen_KL_mean": -15.030601501464844, "KL/mean": -25.972335815429688, "KL/rejected_KL_mean": -36.91407012939453, "KL/std": 23.70777702331543, "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.025046605616807938, "fcm_dpo/delta": -0.15248806774616241, "fcm_dpo/margin": 21.88347625732422, "fcm_dpo/q_t": 0.37621963024139404, "grad_norm": 27.10018539428711, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.6382083892822266, "logits/rejected": -0.5908419489860535, "logps/chosen": -69.67697143554688, "logps/ref_chosen": -54.646366119384766, "logps/ref_rejected": -76.96475219726562, "logps/rejected": -113.87882232666016, "loss": 0.9984, "margin_dpo/margin_mean": 21.88347625732422, "margin_dpo/margin_std": 23.601760864257812, "step": 97 }, { "KL/chosen_KL_mean": -18.270347595214844, "KL/mean": -30.684240341186523, "KL/rejected_KL_mean": -43.09813690185547, "KL/std": 26.8064022064209, "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.02441706880927086, "fcm_dpo/delta": -0.21288006007671356, "fcm_dpo/margin": 24.82779312133789, "fcm_dpo/q_t": 0.36630043387413025, "grad_norm": 27.9980525970459, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.6928224563598633, "logits/rejected": -0.6749119758605957, "logps/chosen": -83.52897644042969, "logps/ref_chosen": -65.25862884521484, "logps/ref_rejected": -96.5274887084961, "logps/rejected": -139.62562561035156, "loss": 1.0038, "margin_dpo/margin_mean": 24.82779312133789, "margin_dpo/margin_std": 31.736007690429688, "step": 98 }, { "KL/chosen_KL_mean": -14.74056625366211, "KL/mean": -29.577295303344727, "KL/rejected_KL_mean": -44.41402053833008, "KL/std": 29.210227966308594, "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.0237110685557127, "fcm_dpo/delta": -0.31514137983322144, "fcm_dpo/margin": 29.67345428466797, "fcm_dpo/q_t": 0.3556155562400818, "grad_norm": 24.59937858581543, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.657272458076477, "logits/rejected": -0.6558362245559692, "logps/chosen": -60.379051208496094, "logps/ref_chosen": -45.638484954833984, "logps/ref_rejected": -86.43793487548828, "logps/rejected": -130.85195922851562, "loss": 0.9386, "margin_dpo/margin_mean": 29.6734561920166, "margin_dpo/margin_std": 35.876800537109375, "step": 99 }, { "KL/chosen_KL_mean": -16.435449600219727, "KL/mean": -25.48604393005371, "KL/rejected_KL_mean": -34.53663635253906, "KL/std": 23.471981048583984, "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.02355112135410309, "fcm_dpo/delta": -0.026980634778738022, "fcm_dpo/margin": 18.101184844970703, "fcm_dpo/q_t": 0.40622931718826294, "grad_norm": 27.36819839477539, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6757396459579468, "logits/rejected": -0.6324243545532227, "logps/chosen": -74.02943420410156, "logps/ref_chosen": -57.59397888183594, "logps/ref_rejected": -74.06021118164062, "logps/rejected": -108.59684753417969, "loss": 1.1247, "margin_dpo/margin_mean": 18.101184844970703, "margin_dpo/margin_std": 30.202056884765625, "step": 100 }, { "KL/chosen_KL_mean": -20.00939178466797, "KL/mean": -30.913528442382812, "KL/rejected_KL_mean": -41.817665100097656, "KL/std": 28.39974021911621, "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.02322225831449032, "fcm_dpo/delta": -0.109965018928051, "fcm_dpo/margin": 21.80826759338379, "fcm_dpo/q_t": 0.39490771293640137, "grad_norm": 27.217756271362305, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.6777423620223999, "logits/rejected": -0.6437240242958069, "logps/chosen": -81.65824890136719, "logps/ref_chosen": -61.64885330200195, "logps/ref_rejected": -83.18968200683594, "logps/rejected": -125.00735473632812, "loss": 1.0806, "margin_dpo/margin_mean": 21.808269500732422, "margin_dpo/margin_std": 34.3305778503418, "step": 101 }, { "KL/chosen_KL_mean": -22.16390037536621, "KL/mean": -30.27094268798828, "KL/rejected_KL_mean": -38.377986907958984, "KL/std": 27.00689697265625, "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.023358814418315887, "fcm_dpo/delta": 0.020977407693862915, "fcm_dpo/margin": 16.214086532592773, "fcm_dpo/q_t": 0.4182248115539551, "grad_norm": 30.8178653717041, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.7105182409286499, "logits/rejected": -0.6647392511367798, "logps/chosen": -86.24278259277344, "logps/ref_chosen": -64.0788803100586, "logps/ref_rejected": -68.18707275390625, "logps/rejected": -106.5650634765625, "loss": 1.1713, "margin_dpo/margin_mean": 16.214086532592773, "margin_dpo/margin_std": 31.576683044433594, "step": 102 }, { "KL/chosen_KL_mean": -16.63315200805664, "KL/mean": -32.39291763305664, "KL/rejected_KL_mean": -48.15269088745117, "KL/std": 29.840614318847656, "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.02269843965768814, "fcm_dpo/delta": -0.32748711109161377, "fcm_dpo/margin": 31.519527435302734, "fcm_dpo/q_t": 0.34971487522125244, "grad_norm": 25.520038604736328, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6867813467979431, "logits/rejected": -0.6520103216171265, "logps/chosen": -77.93243408203125, "logps/ref_chosen": -61.299278259277344, "logps/ref_rejected": -93.57270812988281, "logps/rejected": -141.72540283203125, "loss": 0.9196, "margin_dpo/margin_mean": 31.5195255279541, "margin_dpo/margin_std": 35.207923889160156, "step": 103 }, { "KL/chosen_KL_mean": -18.6593074798584, "KL/mean": -33.599281311035156, "KL/rejected_KL_mean": -48.53925323486328, "KL/std": 33.02933120727539, "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.022114867344498634, "fcm_dpo/delta": -0.2693977355957031, "fcm_dpo/margin": 29.879947662353516, "fcm_dpo/q_t": 0.36667945981025696, "grad_norm": 25.26581382751465, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.7141636610031128, "logits/rejected": -0.6804061532020569, "logps/chosen": -73.0320816040039, "logps/ref_chosen": -54.372772216796875, "logps/ref_rejected": -89.5647201538086, "logps/rejected": -138.10397338867188, "loss": 0.988, "margin_dpo/margin_mean": 29.879947662353516, "margin_dpo/margin_std": 39.69654846191406, "step": 104 }, { "KL/chosen_KL_mean": -17.245506286621094, "KL/mean": -38.19187545776367, "KL/rejected_KL_mean": -59.13824462890625, "KL/std": 32.81090545654297, "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.021134644746780396, "fcm_dpo/delta": -0.5075461268424988, "fcm_dpo/margin": 41.89273452758789, "fcm_dpo/q_t": 0.31017786264419556, "grad_norm": 25.08208656311035, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.7256340980529785, "logits/rejected": -0.7023687958717346, "logps/chosen": -71.88445281982422, "logps/ref_chosen": -54.638946533203125, "logps/ref_rejected": -97.97351837158203, "logps/rejected": -157.11175537109375, "loss": 0.7999, "margin_dpo/margin_mean": 41.89273452758789, "margin_dpo/margin_std": 35.73217010498047, "step": 105 }, { "KL/chosen_KL_mean": -19.42215919494629, "KL/mean": -32.675418853759766, "KL/rejected_KL_mean": -45.928688049316406, "KL/std": 28.108577728271484, "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.020594779402017593, "fcm_dpo/delta": -0.15016362071037292, "fcm_dpo/margin": 26.506515502929688, "fcm_dpo/q_t": 0.37773293256759644, "grad_norm": 24.609127044677734, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.662516713142395, "logits/rejected": -0.6267033815383911, "logps/chosen": -74.25505065917969, "logps/ref_chosen": -54.83289337158203, "logps/ref_rejected": -85.22461700439453, "logps/rejected": -131.15330505371094, "loss": 1.0027, "margin_dpo/margin_mean": 26.506515502929688, "margin_dpo/margin_std": 30.649978637695312, "step": 106 }, { "KL/chosen_KL_mean": -23.350513458251953, "KL/mean": -38.28211975097656, "KL/rejected_KL_mean": -53.213722229003906, "KL/std": 33.904205322265625, "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.02017119899392128, "fcm_dpo/delta": -0.20935019850730896, "fcm_dpo/margin": 29.863208770751953, "fcm_dpo/q_t": 0.367611289024353, "grad_norm": 24.215755462646484, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.685881495475769, "logits/rejected": -0.6361589431762695, "logps/chosen": -93.0583267211914, "logps/ref_chosen": -69.70780944824219, "logps/ref_rejected": -94.73950958251953, "logps/rejected": -147.95323181152344, "loss": 0.9882, "margin_dpo/margin_mean": 29.86320686340332, "margin_dpo/margin_std": 35.658416748046875, "step": 107 }, { "KL/chosen_KL_mean": -21.30466651916504, "KL/mean": -38.65107727050781, "KL/rejected_KL_mean": -55.99748992919922, "KL/std": 37.09898376464844, "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.01967495307326317, "fcm_dpo/delta": -0.2925887405872345, "fcm_dpo/margin": 34.69282531738281, "fcm_dpo/q_t": 0.36451563239097595, "grad_norm": 24.30539894104004, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.6991877555847168, "logits/rejected": -0.6731090545654297, "logps/chosen": -77.3145523071289, "logps/ref_chosen": -56.0098876953125, "logps/ref_rejected": -95.79601287841797, "logps/rejected": -151.7935028076172, "loss": 0.9954, "margin_dpo/margin_mean": 34.69282531738281, "margin_dpo/margin_std": 47.79713439941406, "step": 108 }, { "KL/chosen_KL_mean": -19.659822463989258, "KL/mean": -38.451622009277344, "KL/rejected_KL_mean": -57.24341583251953, "KL/std": 36.00904846191406, "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.019026821479201317, "fcm_dpo/delta": -0.32746434211730957, "fcm_dpo/margin": 37.583595275878906, "fcm_dpo/q_t": 0.34695935249328613, "grad_norm": 24.934322357177734, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.6155581474304199, "logits/rejected": -0.5928876996040344, "logps/chosen": -82.5453109741211, "logps/ref_chosen": -62.88549041748047, "logps/ref_rejected": -98.68573760986328, "logps/rejected": -155.9291534423828, "loss": 0.923, "margin_dpo/margin_mean": 37.58359909057617, "margin_dpo/margin_std": 40.89533996582031, "step": 109 }, { "KL/chosen_KL_mean": -19.98564910888672, "KL/mean": -37.01270294189453, "KL/rejected_KL_mean": -54.039756774902344, "KL/std": 35.79534912109375, "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.018443116918206215, "fcm_dpo/delta": -0.2388836145401001, "fcm_dpo/margin": 34.054107666015625, "fcm_dpo/q_t": 0.3669545650482178, "grad_norm": 22.622568130493164, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.6140676736831665, "logits/rejected": -0.5725805163383484, "logps/chosen": -78.73933410644531, "logps/ref_chosen": -58.753684997558594, "logps/ref_rejected": -79.75001525878906, "logps/rejected": -133.78976440429688, "loss": 0.9958, "margin_dpo/margin_mean": 34.054107666015625, "margin_dpo/margin_std": 42.70184326171875, "step": 110 }, { "KL/chosen_KL_mean": -23.330896377563477, "KL/mean": -42.295196533203125, "KL/rejected_KL_mean": -61.259498596191406, "KL/std": 40.184547424316406, "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.01799844577908516, "fcm_dpo/delta": -0.29396289587020874, "fcm_dpo/margin": 37.92860412597656, "fcm_dpo/q_t": 0.35755258798599243, "grad_norm": 26.07427978515625, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.6798889636993408, "logits/rejected": -0.6572076082229614, "logps/chosen": -91.95500183105469, "logps/ref_chosen": -68.62410736083984, "logps/ref_rejected": -98.42886352539062, "logps/rejected": -159.6883544921875, "loss": 0.979, "margin_dpo/margin_mean": 37.92860794067383, "margin_dpo/margin_std": 48.289306640625, "step": 111 }, { "KL/chosen_KL_mean": -21.652713775634766, "KL/mean": -34.00275421142578, "KL/rejected_KL_mean": -46.35279083251953, "KL/std": 28.837787628173828, "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.017818715423345566, "fcm_dpo/delta": -0.04140431433916092, "fcm_dpo/margin": 24.700084686279297, "fcm_dpo/q_t": 0.40135613083839417, "grad_norm": 22.577350616455078, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.6432993412017822, "logits/rejected": -0.5998743772506714, "logps/chosen": -71.90235900878906, "logps/ref_chosen": -50.24964141845703, "logps/ref_rejected": -64.77442932128906, "logps/rejected": -111.1272201538086, "loss": 1.0813, "margin_dpo/margin_mean": 24.700084686279297, "margin_dpo/margin_std": 34.30575180053711, "step": 112 }, { "KL/chosen_KL_mean": -26.370677947998047, "KL/mean": -39.9312744140625, "KL/rejected_KL_mean": -53.49187469482422, "KL/std": 28.95420265197754, "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.017736122012138367, "fcm_dpo/delta": -0.08302031457424164, "fcm_dpo/margin": 27.121187210083008, "fcm_dpo/q_t": 0.39081066846847534, "grad_norm": 22.69746208190918, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.6275451183319092, "logits/rejected": -0.5815028548240662, "logps/chosen": -93.08363342285156, "logps/ref_chosen": -66.71295166015625, "logps/ref_rejected": -77.96870422363281, "logps/rejected": -131.4605712890625, "loss": 1.0368, "margin_dpo/margin_mean": 27.121185302734375, "margin_dpo/margin_std": 32.2321891784668, "step": 113 }, { "KL/chosen_KL_mean": -24.161130905151367, "KL/mean": -45.25044250488281, "KL/rejected_KL_mean": -66.33973693847656, "KL/std": 44.154300689697266, "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.017194289714097977, "fcm_dpo/delta": -0.3392394781112671, "fcm_dpo/margin": 42.178619384765625, "fcm_dpo/q_t": 0.3535785377025604, "grad_norm": 23.69581413269043, "learning_rate": 4.936501251103751e-07, "logits/chosen": -0.6525863409042358, "logits/rejected": -0.6185038089752197, "logps/chosen": -81.94621276855469, "logps/ref_chosen": -57.78507995605469, "logps/ref_rejected": -87.10966491699219, "logps/rejected": -153.44940185546875, "loss": 0.9378, "margin_dpo/margin_mean": 42.178619384765625, "margin_dpo/margin_std": 53.02991485595703, "step": 114 }, { "KL/chosen_KL_mean": -32.144317626953125, "KL/mean": -46.77862548828125, "KL/rejected_KL_mean": -61.412940979003906, "KL/std": 41.4689826965332, "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.01700768992304802, "fcm_dpo/delta": -0.1002306193113327, "fcm_dpo/margin": 29.26862335205078, "fcm_dpo/q_t": 0.3986138701438904, "grad_norm": 32.29568862915039, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.6790868043899536, "logits/rejected": -0.6547123193740845, "logps/chosen": -97.72695922851562, "logps/ref_chosen": -65.5826416015625, "logps/ref_rejected": -98.56552124023438, "logps/rejected": -159.97845458984375, "loss": 1.1207, "margin_dpo/margin_mean": 29.26862335205078, "margin_dpo/margin_std": 53.58454895019531, "step": 115 }, { "KL/chosen_KL_mean": -24.143800735473633, "KL/mean": -39.930686950683594, "KL/rejected_KL_mean": -55.71758270263672, "KL/std": 35.130592346191406, "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.016804486513137817, "fcm_dpo/delta": -0.13403692841529846, "fcm_dpo/margin": 31.573772430419922, "fcm_dpo/q_t": 0.38285309076309204, "grad_norm": 24.514631271362305, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.64164799451828, "logits/rejected": -0.6261519193649292, "logps/chosen": -75.54411315917969, "logps/ref_chosen": -51.40031433105469, "logps/ref_rejected": -80.5218505859375, "logps/rejected": -136.23944091796875, "loss": 1.0119, "margin_dpo/margin_mean": 31.573772430419922, "margin_dpo/margin_std": 37.68627166748047, "step": 116 }, { "KL/chosen_KL_mean": -30.229568481445312, "KL/mean": -43.88397216796875, "KL/rejected_KL_mean": -57.53838348388672, "KL/std": 37.56050491333008, "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.01664135418832302, "fcm_dpo/delta": -0.055726565420627594, "fcm_dpo/margin": 27.30881118774414, "fcm_dpo/q_t": 0.39999547600746155, "grad_norm": 29.87828254699707, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.6171888113021851, "logits/rejected": -0.5780812501907349, "logps/chosen": -99.5279769897461, "logps/ref_chosen": -69.29840850830078, "logps/ref_rejected": -66.583984375, "logps/rejected": -124.12237548828125, "loss": 1.0906, "margin_dpo/margin_mean": 27.30881118774414, "margin_dpo/margin_std": 42.070045471191406, "step": 117 }, { "KL/chosen_KL_mean": -24.80445098876953, "KL/mean": -40.591796875, "KL/rejected_KL_mean": -56.37914276123047, "KL/std": 33.962181091308594, "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.016470149159431458, "fcm_dpo/delta": -0.12322327494621277, "fcm_dpo/margin": 31.574687957763672, "fcm_dpo/q_t": 0.38508450984954834, "grad_norm": 23.370370864868164, "learning_rate": 4.924492340087524e-07, "logits/chosen": -0.6490514278411865, "logits/rejected": -0.6261621117591858, "logps/chosen": -80.4454345703125, "logps/ref_chosen": -55.6409797668457, "logps/ref_rejected": -75.66905975341797, "logps/rejected": -132.04820251464844, "loss": 1.0199, "margin_dpo/margin_mean": 31.574687957763672, "margin_dpo/margin_std": 38.370567321777344, "step": 118 }, { "KL/chosen_KL_mean": -33.29602813720703, "KL/mean": -48.99560546875, "KL/rejected_KL_mean": -64.69517517089844, "KL/std": 37.841270446777344, "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.016222678124904633, "fcm_dpo/delta": -0.11308414489030838, "fcm_dpo/margin": 31.399154663085938, "fcm_dpo/q_t": 0.39160895347595215, "grad_norm": 25.578641891479492, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.6672037839889526, "logits/rejected": -0.6525957584381104, "logps/chosen": -106.80622863769531, "logps/ref_chosen": -73.51019287109375, "logps/ref_rejected": -102.977294921875, "logps/rejected": -167.67247009277344, "loss": 1.0591, "margin_dpo/margin_mean": 31.399154663085938, "margin_dpo/margin_std": 45.50007247924805, "step": 119 }, { "KL/chosen_KL_mean": -34.85655975341797, "KL/mean": -56.261322021484375, "KL/rejected_KL_mean": -77.66609191894531, "KL/std": 47.26749801635742, "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.015881016850471497, "fcm_dpo/delta": -0.2897739112377167, "fcm_dpo/margin": 42.80952453613281, "fcm_dpo/q_t": 0.3552402853965759, "grad_norm": 24.126750946044922, "learning_rate": 4.918104238142103e-07, "logits/chosen": -0.6502152681350708, "logits/rejected": -0.6147547960281372, "logps/chosen": -111.63739013671875, "logps/ref_chosen": -76.78083801269531, "logps/ref_rejected": -108.02374267578125, "logps/rejected": -185.6898193359375, "loss": 0.9457, "margin_dpo/margin_mean": 42.80952453613281, "margin_dpo/margin_std": 49.247589111328125, "step": 120 }, { "KL/chosen_KL_mean": -33.858795166015625, "KL/mean": -57.48423767089844, "KL/rejected_KL_mean": -81.10968017578125, "KL/std": 46.71514892578125, "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.015335135161876678, "fcm_dpo/delta": -0.33797329664230347, "fcm_dpo/margin": 47.250885009765625, "fcm_dpo/q_t": 0.35153231024742126, "grad_norm": 25.32863998413086, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.612008810043335, "logits/rejected": -0.6054136753082275, "logps/chosen": -95.64868927001953, "logps/ref_chosen": -61.789894104003906, "logps/ref_rejected": -109.99456787109375, "logps/rejected": -191.104248046875, "loss": 0.9492, "margin_dpo/margin_mean": 47.250885009765625, "margin_dpo/margin_std": 58.144596099853516, "step": 121 }, { "KL/chosen_KL_mean": -31.168033599853516, "KL/mean": -60.511810302734375, "KL/rejected_KL_mean": -89.85560607910156, "KL/std": 53.64841842651367, "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.014709394425153732, "fcm_dpo/delta": -0.4840208888053894, "fcm_dpo/margin": 58.68756103515625, "fcm_dpo/q_t": 0.3186884820461273, "grad_norm": 25.857219696044922, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.5904278755187988, "logits/rejected": -0.6040619015693665, "logps/chosen": -78.07024383544922, "logps/ref_chosen": -46.9022102355957, "logps/ref_rejected": -106.71418762207031, "logps/rejected": -196.56979370117188, "loss": 0.8285, "margin_dpo/margin_mean": 58.68756103515625, "margin_dpo/margin_std": 54.87202072143555, "step": 122 }, { "KL/chosen_KL_mean": -33.60801696777344, "KL/mean": -53.090492248535156, "KL/rejected_KL_mean": -72.57296752929688, "KL/std": 46.223365783691406, "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.014308318495750427, "fcm_dpo/delta": -0.16279909014701843, "fcm_dpo/margin": 38.964935302734375, "fcm_dpo/q_t": 0.38503628969192505, "grad_norm": 23.776098251342773, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.5932154655456543, "logits/rejected": -0.5714948773384094, "logps/chosen": -94.9466552734375, "logps/ref_chosen": -61.33863830566406, "logps/ref_rejected": -87.775390625, "logps/rejected": -160.34835815429688, "loss": 1.0682, "margin_dpo/margin_mean": 38.964935302734375, "margin_dpo/margin_std": 61.33662414550781, "step": 123 }, { "KL/chosen_KL_mean": -37.506778717041016, "KL/mean": -65.6749267578125, "KL/rejected_KL_mean": -93.84307861328125, "KL/std": 61.69428253173828, "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.013852743431925774, "fcm_dpo/delta": -0.396537721157074, "fcm_dpo/margin": 56.33629608154297, "fcm_dpo/q_t": 0.35203444957733154, "grad_norm": 27.827972412109375, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.5845953226089478, "logits/rejected": -0.5840995907783508, "logps/chosen": -108.95510864257812, "logps/ref_chosen": -71.44833374023438, "logps/ref_rejected": -117.58056640625, "logps/rejected": -211.42364501953125, "loss": 0.9658, "margin_dpo/margin_mean": 56.33629608154297, "margin_dpo/margin_std": 77.30990600585938, "step": 124 }, { "KL/chosen_KL_mean": -31.203296661376953, "KL/mean": -56.6873779296875, "KL/rejected_KL_mean": -82.17146301269531, "KL/std": 51.79102325439453, "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.013450113125145435, "fcm_dpo/delta": -0.29537516832351685, "fcm_dpo/margin": 50.968162536621094, "fcm_dpo/q_t": 0.35877731442451477, "grad_norm": 22.051082611083984, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.5767650604248047, "logits/rejected": -0.5755800008773804, "logps/chosen": -81.34024047851562, "logps/ref_chosen": -50.136940002441406, "logps/ref_rejected": -83.98861694335938, "logps/rejected": -166.1600799560547, "loss": 0.9621, "margin_dpo/margin_mean": 50.968162536621094, "margin_dpo/margin_std": 62.841583251953125, "step": 125 }, { "KL/chosen_KL_mean": -34.27449035644531, "KL/mean": -57.64450454711914, "KL/rejected_KL_mean": -81.01451110839844, "KL/std": 46.706634521484375, "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.013104382902383804, "fcm_dpo/delta": -0.21946457028388977, "fcm_dpo/margin": 46.74002456665039, "fcm_dpo/q_t": 0.369617223739624, "grad_norm": 23.553150177001953, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.5703192949295044, "logits/rejected": -0.5591797232627869, "logps/chosen": -89.94155883789062, "logps/ref_chosen": -55.66706848144531, "logps/ref_rejected": -98.1297607421875, "logps/rejected": -179.14427185058594, "loss": 0.9888, "margin_dpo/margin_mean": 46.74002456665039, "margin_dpo/margin_std": 58.35154724121094, "step": 126 }, { "KL/chosen_KL_mean": -32.794464111328125, "KL/mean": -50.73387145996094, "KL/rejected_KL_mean": -68.67326354980469, "KL/std": 40.8072509765625, "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.012982767075300217, "fcm_dpo/delta": -0.0673581212759018, "fcm_dpo/margin": 35.87880325317383, "fcm_dpo/q_t": 0.3947966694831848, "grad_norm": 23.849796295166016, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.579119086265564, "logits/rejected": -0.5568236112594604, "logps/chosen": -89.3491439819336, "logps/ref_chosen": -56.55467987060547, "logps/ref_rejected": -76.7957763671875, "logps/rejected": -145.46905517578125, "loss": 1.073, "margin_dpo/margin_mean": 35.87880325317383, "margin_dpo/margin_std": 50.529823303222656, "step": 127 }, { "KL/chosen_KL_mean": -35.758731842041016, "KL/mean": -54.428401947021484, "KL/rejected_KL_mean": -73.09808349609375, "KL/std": 48.978797912597656, "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.012904481962323189, "fcm_dpo/delta": -0.08387961983680725, "fcm_dpo/margin": 37.3393440246582, "fcm_dpo/q_t": 0.3962070941925049, "grad_norm": 35.115657806396484, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5600550174713135, "logits/rejected": -0.5325770378112793, "logps/chosen": -93.87968444824219, "logps/ref_chosen": -58.12095642089844, "logps/ref_rejected": -76.43896484375, "logps/rejected": -149.53704833984375, "loss": 1.0968, "margin_dpo/margin_mean": 37.3393440246582, "margin_dpo/margin_std": 60.14480972290039, "step": 128 }, { "KL/chosen_KL_mean": -44.086055755615234, "KL/mean": -63.96501922607422, "KL/rejected_KL_mean": -83.84398651123047, "KL/std": 53.29756164550781, "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.012750450521707535, "fcm_dpo/delta": -0.10966508090496063, "fcm_dpo/margin": 39.75792694091797, "fcm_dpo/q_t": 0.396596223115921, "grad_norm": 24.342992782592773, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.5686518549919128, "logits/rejected": -0.5536836385726929, "logps/chosen": -111.00242614746094, "logps/ref_chosen": -66.91637420654297, "logps/ref_rejected": -96.6422119140625, "logps/rejected": -180.4862060546875, "loss": 1.1002, "margin_dpo/margin_mean": 39.75792694091797, "margin_dpo/margin_std": 68.46754455566406, "step": 129 }, { "KL/chosen_KL_mean": -33.01520919799805, "KL/mean": -62.57740020751953, "KL/rejected_KL_mean": -92.13957977294922, "KL/std": 57.29778289794922, "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.012459097430109978, "fcm_dpo/delta": -0.34889647364616394, "fcm_dpo/margin": 59.1243782043457, "fcm_dpo/q_t": 0.34880900382995605, "grad_norm": 21.42711639404297, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.5372148752212524, "logits/rejected": -0.5315117835998535, "logps/chosen": -77.68206024169922, "logps/ref_chosen": -44.66685104370117, "logps/ref_rejected": -82.78165435791016, "logps/rejected": -174.92123413085938, "loss": 0.9393, "margin_dpo/margin_mean": 59.1243782043457, "margin_dpo/margin_std": 68.27285766601562, "step": 130 }, { "KL/chosen_KL_mean": -28.63924217224121, "KL/mean": -57.72308349609375, "KL/rejected_KL_mean": -86.80691528320312, "KL/std": 54.85826110839844, "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.011984329670667648, "fcm_dpo/delta": -0.30810654163360596, "fcm_dpo/margin": 58.167686462402344, "fcm_dpo/q_t": 0.3473639190196991, "grad_norm": 27.132959365844727, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.5333915948867798, "logits/rejected": -0.5392862558364868, "logps/chosen": -73.56382751464844, "logps/ref_chosen": -44.924591064453125, "logps/ref_rejected": -88.44401550292969, "logps/rejected": -175.2509307861328, "loss": 0.9189, "margin_dpo/margin_mean": 58.167686462402344, "margin_dpo/margin_std": 60.25544738769531, "step": 131 }, { "KL/chosen_KL_mean": -38.82572937011719, "KL/mean": -63.184059143066406, "KL/rejected_KL_mean": -87.54237365722656, "KL/std": 55.58538818359375, "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.011730792000889778, "fcm_dpo/delta": -0.1767834722995758, "fcm_dpo/margin": 48.716651916503906, "fcm_dpo/q_t": 0.3818228244781494, "grad_norm": 23.049833297729492, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.5593503713607788, "logits/rejected": -0.5511766672134399, "logps/chosen": -97.82681274414062, "logps/ref_chosen": -59.00108337402344, "logps/ref_rejected": -87.89215087890625, "logps/rejected": -175.43453979492188, "loss": 1.034, "margin_dpo/margin_mean": 48.716651916503906, "margin_dpo/margin_std": 71.09864807128906, "step": 132 }, { "KL/chosen_KL_mean": -45.97046661376953, "KL/mean": -67.64411163330078, "KL/rejected_KL_mean": -89.3177490234375, "KL/std": 51.025794982910156, "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.011588573455810547, "fcm_dpo/delta": -0.1051153689622879, "fcm_dpo/margin": 43.3472900390625, "fcm_dpo/q_t": 0.3930418789386749, "grad_norm": 26.683420181274414, "learning_rate": 4.870401618977415e-07, "logits/chosen": -0.5510296821594238, "logits/rejected": -0.5348105430603027, "logps/chosen": -112.57496643066406, "logps/ref_chosen": -66.60449981689453, "logps/ref_rejected": -96.33355712890625, "logps/rejected": -185.65130615234375, "loss": 1.0613, "margin_dpo/margin_mean": 43.3472900390625, "margin_dpo/margin_std": 63.578800201416016, "step": 133 }, { "KL/chosen_KL_mean": -36.70982360839844, "KL/mean": -60.16697311401367, "KL/rejected_KL_mean": -83.62411499023438, "KL/std": 50.47258758544922, "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.011467832140624523, "fcm_dpo/delta": -0.14169654250144958, "fcm_dpo/margin": 46.91429901123047, "fcm_dpo/q_t": 0.38246801495552063, "grad_norm": 22.107864379882812, "learning_rate": 4.866292092063986e-07, "logits/chosen": -0.5395177006721497, "logits/rejected": -0.5207037329673767, "logps/chosen": -88.77908325195312, "logps/ref_chosen": -52.06925582885742, "logps/ref_rejected": -87.6545181274414, "logps/rejected": -171.27862548828125, "loss": 1.0101, "margin_dpo/margin_mean": 46.91429901123047, "margin_dpo/margin_std": 57.2902946472168, "step": 134 }, { "KL/chosen_KL_mean": -41.4932861328125, "KL/mean": -73.72633361816406, "KL/rejected_KL_mean": -105.95936584472656, "KL/std": 65.24856567382812, "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.011130950413644314, "fcm_dpo/delta": -0.3293672502040863, "fcm_dpo/margin": 64.46609497070312, "fcm_dpo/q_t": 0.35333138704299927, "grad_norm": 23.30121612548828, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.5058168768882751, "logits/rejected": -0.5387102961540222, "logps/chosen": -91.84715270996094, "logps/ref_chosen": -50.353858947753906, "logps/ref_rejected": -115.97975158691406, "logps/rejected": -221.93911743164062, "loss": 0.9469, "margin_dpo/margin_mean": 64.46609497070312, "margin_dpo/margin_std": 78.12184143066406, "step": 135 }, { "KL/chosen_KL_mean": -48.40357208251953, "KL/mean": -70.24093627929688, "KL/rejected_KL_mean": -92.07830810546875, "KL/std": 58.511390686035156, "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.010931683704257011, "fcm_dpo/delta": -0.08014538139104843, "fcm_dpo/margin": 43.674739837646484, "fcm_dpo/q_t": 0.4048768877983093, "grad_norm": 23.915916442871094, "learning_rate": 4.857886086178193e-07, "logits/chosen": -0.5324341654777527, "logits/rejected": -0.5198547840118408, "logps/chosen": -113.47608184814453, "logps/ref_chosen": -65.072509765625, "logps/ref_rejected": -96.32122802734375, "logps/rejected": -188.3995361328125, "loss": 1.1016, "margin_dpo/margin_mean": 43.67473602294922, "margin_dpo/margin_std": 73.63079833984375, "step": 136 }, { "KL/chosen_KL_mean": -48.21935272216797, "KL/mean": -82.04722595214844, "KL/rejected_KL_mean": -115.87509155273438, "KL/std": 78.75631713867188, "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.010671773925423622, "fcm_dpo/delta": -0.3343368172645569, "fcm_dpo/margin": 67.65573120117188, "fcm_dpo/q_t": 0.35995662212371826, "grad_norm": 21.44792938232422, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.49152958393096924, "logits/rejected": -0.5171319246292114, "logps/chosen": -96.97846984863281, "logps/ref_chosen": -48.759117126464844, "logps/ref_rejected": -113.86376953125, "logps/rejected": -229.73886108398438, "loss": 0.9823, "margin_dpo/margin_mean": 67.65573120117188, "margin_dpo/margin_std": 93.46538543701172, "step": 137 }, { "KL/chosen_KL_mean": -49.967342376708984, "KL/mean": -75.7919692993164, "KL/rejected_KL_mean": -101.61659240722656, "KL/std": 60.058433532714844, "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.010484375059604645, "fcm_dpo/delta": -0.14534850418567657, "fcm_dpo/margin": 51.649253845214844, "fcm_dpo/q_t": 0.37925344705581665, "grad_norm": 23.9995174407959, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.4553467631340027, "logits/rejected": -0.4406394362449646, "logps/chosen": -110.48699188232422, "logps/ref_chosen": -60.519649505615234, "logps/ref_rejected": -93.19694519042969, "logps/rejected": -194.81353759765625, "loss": 1.0095, "margin_dpo/margin_mean": 51.649253845214844, "margin_dpo/margin_std": 61.569252014160156, "step": 138 }, { "KL/chosen_KL_mean": -40.474876403808594, "KL/mean": -71.97026062011719, "KL/rejected_KL_mean": -103.46565246582031, "KL/std": 57.427764892578125, "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.010253118351101875, "fcm_dpo/delta": -0.25380727648735046, "fcm_dpo/margin": 62.99077224731445, "fcm_dpo/q_t": 0.3614668846130371, "grad_norm": 20.68395233154297, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.5000328421592712, "logits/rejected": -0.48780351877212524, "logps/chosen": -87.36625671386719, "logps/ref_chosen": -46.89138412475586, "logps/ref_rejected": -79.72798156738281, "logps/rejected": -183.19363403320312, "loss": 0.9557, "margin_dpo/margin_mean": 62.99076843261719, "margin_dpo/margin_std": 72.46928405761719, "step": 139 }, { "KL/chosen_KL_mean": -49.33660125732422, "KL/mean": -78.02191925048828, "KL/rejected_KL_mean": -106.70724487304688, "KL/std": 64.60366821289062, "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.010033691301941872, "fcm_dpo/delta": -0.1809324026107788, "fcm_dpo/margin": 57.37063980102539, "fcm_dpo/q_t": 0.37695828080177307, "grad_norm": 23.712533950805664, "learning_rate": 4.840329401637809e-07, "logits/chosen": -0.4792005717754364, "logits/rejected": -0.46222788095474243, "logps/chosen": -108.31130981445312, "logps/ref_chosen": -58.97471618652344, "logps/ref_rejected": -83.28410339355469, "logps/rejected": -189.99134826660156, "loss": 1.0174, "margin_dpo/margin_mean": 57.370643615722656, "margin_dpo/margin_std": 76.40518188476562, "step": 140 }, { "KL/chosen_KL_mean": -53.7377815246582, "KL/mean": -81.11244201660156, "KL/rejected_KL_mean": -108.48710632324219, "KL/std": 67.55751037597656, "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.009888198226690292, "fcm_dpo/delta": -0.14525547623634338, "fcm_dpo/margin": 54.74932098388672, "fcm_dpo/q_t": 0.3810844421386719, "grad_norm": 26.525779724121094, "learning_rate": 4.83578576263792e-07, "logits/chosen": -0.4832468628883362, "logits/rejected": -0.46669474244117737, "logps/chosen": -128.81344604492188, "logps/ref_chosen": -75.07566833496094, "logps/ref_rejected": -98.1922607421875, "logps/rejected": -206.6793670654297, "loss": 1.0432, "margin_dpo/margin_mean": 54.74932098388672, "margin_dpo/margin_std": 76.88491821289062, "step": 141 }, { "KL/chosen_KL_mean": -56.92424774169922, "KL/mean": -88.21382141113281, "KL/rejected_KL_mean": -119.50340270996094, "KL/std": 76.11981201171875, "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.009715566411614418, "fcm_dpo/delta": -0.2142324447631836, "fcm_dpo/margin": 62.57915496826172, "fcm_dpo/q_t": 0.3745476007461548, "grad_norm": 27.3022403717041, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.4715278446674347, "logits/rejected": -0.4681225121021271, "logps/chosen": -114.95217895507812, "logps/ref_chosen": -58.027931213378906, "logps/ref_rejected": -94.58222961425781, "logps/rejected": -214.08563232421875, "loss": 1.0324, "margin_dpo/margin_mean": 62.57914733886719, "margin_dpo/margin_std": 89.07696533203125, "step": 142 }, { "KL/chosen_KL_mean": -61.330406188964844, "KL/mean": -81.28429412841797, "KL/rejected_KL_mean": -101.2381820678711, "KL/std": 67.98274230957031, "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.009628836065530777, "fcm_dpo/delta": 0.015870332717895508, "fcm_dpo/margin": 39.907772064208984, "fcm_dpo/q_t": 0.4193767309188843, "grad_norm": 26.948379516601562, "learning_rate": 4.826513955607734e-07, "logits/chosen": -0.42380428314208984, "logits/rejected": -0.4101778268814087, "logps/chosen": -118.9268569946289, "logps/ref_chosen": -57.59645080566406, "logps/ref_rejected": -78.99957275390625, "logps/rejected": -180.23776245117188, "loss": 1.1652, "margin_dpo/margin_mean": 39.907772064208984, "margin_dpo/margin_std": 78.14988708496094, "step": 143 }, { "KL/chosen_KL_mean": -52.056514739990234, "KL/mean": -77.15028381347656, "KL/rejected_KL_mean": -102.24404907226562, "KL/std": 56.15443420410156, "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.009588293731212616, "fcm_dpo/delta": -0.08328879624605179, "fcm_dpo/margin": 50.187538146972656, "fcm_dpo/q_t": 0.39288330078125, "grad_norm": 22.615461349487305, "learning_rate": 4.821786031898176e-07, "logits/chosen": -0.45128071308135986, "logits/rejected": -0.42970043420791626, "logps/chosen": -111.96287536621094, "logps/ref_chosen": -59.90636444091797, "logps/ref_rejected": -82.00025939941406, "logps/rejected": -184.24432373046875, "loss": 1.0523, "margin_dpo/margin_mean": 50.187538146972656, "margin_dpo/margin_std": 65.81391143798828, "step": 144 }, { "KL/chosen_KL_mean": -52.327789306640625, "KL/mean": -79.01912689208984, "KL/rejected_KL_mean": -105.71046447753906, "KL/std": 58.76500701904297, "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.009492561221122742, "fcm_dpo/delta": -0.10956543684005737, "fcm_dpo/margin": 53.38267517089844, "fcm_dpo/q_t": 0.3878716230392456, "grad_norm": 26.634552001953125, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.4835849404335022, "logits/rejected": -0.46149128675460815, "logps/chosen": -108.92845153808594, "logps/ref_chosen": -56.60066604614258, "logps/ref_rejected": -77.86631774902344, "logps/rejected": -183.5767822265625, "loss": 1.0402, "margin_dpo/margin_mean": 53.38267517089844, "margin_dpo/margin_std": 69.78041076660156, "step": 145 }, { "KL/chosen_KL_mean": -71.99324035644531, "KL/mean": -92.98985290527344, "KL/rejected_KL_mean": -113.98645782470703, "KL/std": 62.72893524169922, "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.009455716237425804, "fcm_dpo/delta": 0.002865489572286606, "fcm_dpo/margin": 41.99322509765625, "fcm_dpo/q_t": 0.41135260462760925, "grad_norm": 31.551198959350586, "learning_rate": 4.812146767012779e-07, "logits/chosen": -0.4793580174446106, "logits/rejected": -0.44883793592453003, "logps/chosen": -137.99368286132812, "logps/ref_chosen": -66.00045013427734, "logps/ref_rejected": -81.70278930664062, "logps/rejected": -195.68923950195312, "loss": 1.1451, "margin_dpo/margin_mean": 41.99322509765625, "margin_dpo/margin_std": 74.72547912597656, "step": 146 }, { "KL/chosen_KL_mean": -52.66679000854492, "KL/mean": -81.16831970214844, "KL/rejected_KL_mean": -109.66986083984375, "KL/std": 64.48698425292969, "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.009364601224660873, "fcm_dpo/delta": -0.13770800828933716, "fcm_dpo/margin": 57.00305938720703, "fcm_dpo/q_t": 0.38518407940864563, "grad_norm": 22.39113426208496, "learning_rate": 4.807235679840536e-07, "logits/chosen": -0.4767155647277832, "logits/rejected": -0.4504894018173218, "logps/chosen": -106.07228088378906, "logps/ref_chosen": -53.405487060546875, "logps/ref_rejected": -71.39060974121094, "logps/rejected": -181.0604705810547, "loss": 1.042, "margin_dpo/margin_mean": 57.0030632019043, "margin_dpo/margin_std": 79.139892578125, "step": 147 }, { "KL/chosen_KL_mean": -51.239707946777344, "KL/mean": -74.8543701171875, "KL/rejected_KL_mean": -98.46903991699219, "KL/std": 62.515411376953125, "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.009360980242490768, "fcm_dpo/delta": -0.043911248445510864, "fcm_dpo/margin": 47.22932434082031, "fcm_dpo/q_t": 0.40432801842689514, "grad_norm": 21.143596649169922, "learning_rate": 4.802263794862384e-07, "logits/chosen": -0.5241053104400635, "logits/rejected": -0.5143436789512634, "logps/chosen": -116.17678833007812, "logps/ref_chosen": -64.93708038330078, "logps/ref_rejected": -103.09384155273438, "logps/rejected": -201.56288146972656, "loss": 1.0885, "margin_dpo/margin_mean": 47.22932434082031, "margin_dpo/margin_std": 65.89462280273438, "step": 148 }, { "KL/chosen_KL_mean": -49.8416633605957, "KL/mean": -79.41673278808594, "KL/rejected_KL_mean": -108.9917984008789, "KL/std": 57.800533294677734, "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.009160520508885384, "fcm_dpo/delta": -0.14721731841564178, "fcm_dpo/margin": 59.15013122558594, "fcm_dpo/q_t": 0.3789201080799103, "grad_norm": 20.35516357421875, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.5416568517684937, "logits/rejected": -0.5244969129562378, "logps/chosen": -108.3154296875, "logps/ref_chosen": -58.47376251220703, "logps/ref_rejected": -99.31474304199219, "logps/rejected": -208.30654907226562, "loss": 1.0075, "margin_dpo/margin_mean": 59.150123596191406, "margin_dpo/margin_std": 67.78451538085938, "step": 149 }, { "KL/chosen_KL_mean": -44.17631912231445, "KL/mean": -72.84591674804688, "KL/rejected_KL_mean": -101.5155029296875, "KL/std": 67.9078598022461, "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.009045644663274288, "fcm_dpo/delta": -0.12300632148981094, "fcm_dpo/margin": 57.33917999267578, "fcm_dpo/q_t": 0.3912937641143799, "grad_norm": 20.950302124023438, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.472980797290802, "logits/rejected": -0.4736313223838806, "logps/chosen": -89.88212585449219, "logps/ref_chosen": -45.705810546875, "logps/ref_rejected": -83.34759521484375, "logps/rejected": -184.86309814453125, "loss": 1.0437, "margin_dpo/margin_mean": 57.33917999267578, "margin_dpo/margin_std": 80.5770034790039, "step": 150 }, { "KL/chosen_KL_mean": -54.050907135009766, "KL/mean": -83.80128479003906, "KL/rejected_KL_mean": -113.55165100097656, "KL/std": 63.99048614501953, "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.008962772786617279, "fcm_dpo/delta": -0.13693660497665405, "fcm_dpo/margin": 59.50074768066406, "fcm_dpo/q_t": 0.38013043999671936, "grad_norm": 24.177379608154297, "learning_rate": 4.786984671220053e-07, "logits/chosen": -0.5812417268753052, "logits/rejected": -0.5512596368789673, "logps/chosen": -124.62174224853516, "logps/ref_chosen": -70.57083129882812, "logps/ref_rejected": -100.46382141113281, "logps/rejected": -214.01547241210938, "loss": 1.0088, "margin_dpo/margin_mean": 59.50074768066406, "margin_dpo/margin_std": 69.27177429199219, "step": 151 }, { "KL/chosen_KL_mean": -47.178958892822266, "KL/mean": -82.26052856445312, "KL/rejected_KL_mean": -117.34210968017578, "KL/std": 66.30995178222656, "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.008797157555818558, "fcm_dpo/delta": -0.22390428185462952, "fcm_dpo/margin": 70.16314697265625, "fcm_dpo/q_t": 0.3667450547218323, "grad_norm": 22.298110961914062, "learning_rate": 4.78177092112495e-07, "logits/chosen": -0.5535380840301514, "logits/rejected": -0.5499355792999268, "logps/chosen": -107.34334564208984, "logps/ref_chosen": -60.16438674926758, "logps/ref_rejected": -106.14045715332031, "logps/rejected": -223.48257446289062, "loss": 0.9731, "margin_dpo/margin_mean": 70.16314697265625, "margin_dpo/margin_std": 80.07929992675781, "step": 152 }, { "KL/chosen_KL_mean": -48.634864807128906, "KL/mean": -78.62088012695312, "KL/rejected_KL_mean": -108.60690307617188, "KL/std": 71.24372863769531, "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.008663169108331203, "fcm_dpo/delta": -0.12265150249004364, "fcm_dpo/margin": 59.97203063964844, "fcm_dpo/q_t": 0.38973453640937805, "grad_norm": 17.89531898498535, "learning_rate": 4.776497044244016e-07, "logits/chosen": -0.5327095985412598, "logits/rejected": -0.5256816744804382, "logps/chosen": -104.95014190673828, "logps/ref_chosen": -56.315277099609375, "logps/ref_rejected": -85.65583801269531, "logps/rejected": -194.26272583007812, "loss": 1.0512, "margin_dpo/margin_mean": 59.97203063964844, "margin_dpo/margin_std": 86.57862854003906, "step": 153 }, { "KL/chosen_KL_mean": -57.993751525878906, "KL/mean": -87.43338775634766, "KL/rejected_KL_mean": -116.87301635742188, "KL/std": 72.61052703857422, "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.008578930050134659, "fcm_dpo/delta": -0.10782384127378464, "fcm_dpo/margin": 58.87928009033203, "fcm_dpo/q_t": 0.3921113610267639, "grad_norm": 21.391340255737305, "learning_rate": 4.771163179548808e-07, "logits/chosen": -0.4942118525505066, "logits/rejected": -0.492919921875, "logps/chosen": -120.73632049560547, "logps/ref_chosen": -62.74256896972656, "logps/ref_rejected": -104.24420166015625, "logps/rejected": -221.11721801757812, "loss": 1.0787, "margin_dpo/margin_mean": 58.87928009033203, "margin_dpo/margin_std": 90.07198333740234, "step": 154 }, { "KL/chosen_KL_mean": -52.702293395996094, "KL/mean": -82.81259155273438, "KL/rejected_KL_mean": -112.92288208007812, "KL/std": 68.29090118408203, "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.0084763802587986, "fcm_dpo/delta": -0.11326827853918076, "fcm_dpo/margin": 60.2205924987793, "fcm_dpo/q_t": 0.38809964060783386, "grad_norm": 21.07434844970703, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -0.5273807048797607, "logits/rejected": -0.5057998895645142, "logps/chosen": -113.35547637939453, "logps/ref_chosen": -60.65318298339844, "logps/ref_rejected": -77.49220275878906, "logps/rejected": -190.4150848388672, "loss": 1.0476, "margin_dpo/margin_mean": 60.22058868408203, "margin_dpo/margin_std": 82.90054321289062, "step": 155 }, { "KL/chosen_KL_mean": -74.92457580566406, "KL/mean": -92.70379638671875, "KL/rejected_KL_mean": -110.48301696777344, "KL/std": 70.93622589111328, "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.008492467924952507, "fcm_dpo/delta": 0.09958405047655106, "fcm_dpo/margin": 35.558448791503906, "fcm_dpo/q_t": 0.43309691548347473, "grad_norm": 32.95204162597656, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -0.4810073971748352, "logits/rejected": -0.468288779258728, "logps/chosen": -144.41644287109375, "logps/ref_chosen": -69.49188232421875, "logps/ref_rejected": -77.16929626464844, "logps/rejected": -187.65231323242188, "loss": 1.252, "margin_dpo/margin_mean": 35.55845260620117, "margin_dpo/margin_std": 93.71145629882812, "step": 156 }, { "KL/chosen_KL_mean": -68.62439727783203, "KL/mean": -105.9276123046875, "KL/rejected_KL_mean": -143.2308349609375, "KL/std": 78.11910247802734, "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.008352255448698997, "fcm_dpo/delta": -0.23134897649288177, "fcm_dpo/margin": 74.60643768310547, "fcm_dpo/q_t": 0.3620302677154541, "grad_norm": 25.482498168945312, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -0.43237775564193726, "logits/rejected": -0.4356487989425659, "logps/chosen": -129.99282836914062, "logps/ref_chosen": -61.368438720703125, "logps/ref_rejected": -107.64636993408203, "logps/rejected": -250.877197265625, "loss": 0.9816, "margin_dpo/margin_mean": 74.6064453125, "margin_dpo/margin_std": 87.3945541381836, "step": 157 }, { "KL/chosen_KL_mean": -68.26705932617188, "KL/mean": -107.6445083618164, "KL/rejected_KL_mean": -147.02197265625, "KL/std": 97.27963256835938, "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.008176269009709358, "fcm_dpo/delta": -0.2519432008266449, "fcm_dpo/margin": 78.75489807128906, "fcm_dpo/q_t": 0.37355363368988037, "grad_norm": 22.437089920043945, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -0.41110551357269287, "logits/rejected": -0.4253670275211334, "logps/chosen": -125.87997436523438, "logps/ref_chosen": -57.612918853759766, "logps/ref_rejected": -113.6946792602539, "logps/rejected": -260.7166442871094, "loss": 1.0189, "margin_dpo/margin_mean": 78.75489807128906, "margin_dpo/margin_std": 116.05311584472656, "step": 158 }, { "KL/chosen_KL_mean": -77.33633422851562, "KL/mean": -105.33822631835938, "KL/rejected_KL_mean": -133.34011840820312, "KL/std": 85.79032897949219, "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.008105665445327759, "fcm_dpo/delta": -0.05528353154659271, "fcm_dpo/margin": 56.00377655029297, "fcm_dpo/q_t": 0.40175575017929077, "grad_norm": 24.443878173828125, "learning_rate": 4.743599013306165e-07, "logits/chosen": -0.46176522970199585, "logits/rejected": -0.42901933193206787, "logps/chosen": -158.89666748046875, "logps/ref_chosen": -81.56034851074219, "logps/ref_rejected": -88.89871215820312, "logps/rejected": -222.23883056640625, "loss": 1.1099, "margin_dpo/margin_mean": 56.00377655029297, "margin_dpo/margin_std": 92.10934448242188, "step": 159 }, { "KL/chosen_KL_mean": -79.1042251586914, "KL/mean": -115.31195068359375, "KL/rejected_KL_mean": -151.51966857910156, "KL/std": 91.41381072998047, "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.007949615828692913, "fcm_dpo/delta": -0.18148598074913025, "fcm_dpo/margin": 72.41543579101562, "fcm_dpo/q_t": 0.38441920280456543, "grad_norm": 28.788623809814453, "learning_rate": 4.737908228387656e-07, "logits/chosen": -0.4013976454734802, "logits/rejected": -0.38853079080581665, "logps/chosen": -144.83511352539062, "logps/ref_chosen": -65.73088073730469, "logps/ref_rejected": -97.21781921386719, "logps/rejected": -248.73748779296875, "loss": 1.0617, "margin_dpo/margin_mean": 72.41543579101562, "margin_dpo/margin_std": 113.82682037353516, "step": 160 }, { "KL/chosen_KL_mean": -68.4842529296875, "KL/mean": -100.51941680908203, "KL/rejected_KL_mean": -132.55458068847656, "KL/std": 72.564453125, "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.007867861539125443, "fcm_dpo/delta": -0.10678651928901672, "fcm_dpo/margin": 64.07034301757812, "fcm_dpo/q_t": 0.3917103111743927, "grad_norm": 26.057464599609375, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -0.41672807931900024, "logits/rejected": -0.41233956813812256, "logps/chosen": -120.92071533203125, "logps/ref_chosen": -52.43647003173828, "logps/ref_rejected": -83.43095397949219, "logps/rejected": -215.98553466796875, "loss": 1.0604, "margin_dpo/margin_mean": 64.07034301757812, "margin_dpo/margin_std": 92.08967590332031, "step": 161 }, { "KL/chosen_KL_mean": -64.87035369873047, "KL/mean": -97.34904479980469, "KL/rejected_KL_mean": -129.82774353027344, "KL/std": 81.12202453613281, "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.0077486904338002205, "fcm_dpo/delta": -0.1073184385895729, "fcm_dpo/margin": 64.9573974609375, "fcm_dpo/q_t": 0.39209944009780884, "grad_norm": 23.717235565185547, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.46293187141418457, "logits/rejected": -0.44232678413391113, "logps/chosen": -127.48094177246094, "logps/ref_chosen": -62.6105842590332, "logps/ref_rejected": -89.39057922363281, "logps/rejected": -219.21832275390625, "loss": 1.0674, "margin_dpo/margin_mean": 64.9573974609375, "margin_dpo/margin_std": 94.93345642089844, "step": 162 }, { "KL/chosen_KL_mean": -73.33845520019531, "KL/mean": -104.88301086425781, "KL/rejected_KL_mean": -136.42755126953125, "KL/std": 80.35075378417969, "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.007712847553193569, "fcm_dpo/delta": -0.0887412428855896, "fcm_dpo/margin": 63.08909606933594, "fcm_dpo/q_t": 0.3963692784309387, "grad_norm": 25.23561668395996, "learning_rate": 4.720482655449212e-07, "logits/chosen": -0.3967311382293701, "logits/rejected": -0.37458351254463196, "logps/chosen": -128.36007690429688, "logps/ref_chosen": -55.021629333496094, "logps/ref_rejected": -75.418212890625, "logps/rejected": -211.84576416015625, "loss": 1.0769, "margin_dpo/margin_mean": 63.08909606933594, "margin_dpo/margin_std": 95.95596313476562, "step": 163 }, { "KL/chosen_KL_mean": -66.4931411743164, "KL/mean": -104.46180725097656, "KL/rejected_KL_mean": -142.43045043945312, "KL/std": 79.09434509277344, "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.007562342099845409, "fcm_dpo/delta": -0.18068404495716095, "fcm_dpo/margin": 75.93731689453125, "fcm_dpo/q_t": 0.3738352656364441, "grad_norm": 23.940841674804688, "learning_rate": 4.714556901942599e-07, "logits/chosen": -0.3866020441055298, "logits/rejected": -0.3667601943016052, "logps/chosen": -122.1338119506836, "logps/ref_chosen": -55.64066696166992, "logps/ref_rejected": -79.66463470458984, "logps/rejected": -222.0950927734375, "loss": 0.9952, "margin_dpo/margin_mean": 75.93731689453125, "margin_dpo/margin_std": 90.33500671386719, "step": 164 }, { "KL/chosen_KL_mean": -70.58739471435547, "KL/mean": -95.22163391113281, "KL/rejected_KL_mean": -119.85586547851562, "KL/std": 65.43978881835938, "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.007562550250440836, "fcm_dpo/delta": 0.027895405888557434, "fcm_dpo/margin": 49.26846694946289, "fcm_dpo/q_t": 0.41699641942977905, "grad_norm": 28.192373275756836, "learning_rate": 4.708572792802069e-07, "logits/chosen": -0.4508388042449951, "logits/rejected": -0.41956329345703125, "logps/chosen": -131.89808654785156, "logps/ref_chosen": -61.310691833496094, "logps/ref_rejected": -73.67060852050781, "logps/rejected": -193.52647399902344, "loss": 1.1451, "margin_dpo/margin_mean": 49.26846694946289, "margin_dpo/margin_std": 85.33627319335938, "step": 165 }, { "KL/chosen_KL_mean": -62.29485321044922, "KL/mean": -107.13589477539062, "KL/rejected_KL_mean": -151.9769287109375, "KL/std": 95.33642578125, "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.007406267803162336, "fcm_dpo/delta": -0.27418437600135803, "fcm_dpo/margin": 89.68207550048828, "fcm_dpo/q_t": 0.3678908348083496, "grad_norm": 19.755449295043945, "learning_rate": 4.702530485714461e-07, "logits/chosen": -0.38571715354919434, "logits/rejected": -0.395542174577713, "logps/chosen": -113.27845764160156, "logps/ref_chosen": -50.98360061645508, "logps/ref_rejected": -98.09512329101562, "logps/rejected": -250.07205200195312, "loss": 0.9781, "margin_dpo/margin_mean": 89.68207550048828, "margin_dpo/margin_std": 119.04489135742188, "step": 166 }, { "KL/chosen_KL_mean": -62.64759826660156, "KL/mean": -109.76869201660156, "KL/rejected_KL_mean": -156.8897705078125, "KL/std": 87.04228210449219, "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.007213749922811985, "fcm_dpo/delta": -0.2897433936595917, "fcm_dpo/margin": 94.24217987060547, "fcm_dpo/q_t": 0.3522365093231201, "grad_norm": 21.15924072265625, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.3807927370071411, "logits/rejected": -0.3821834325790405, "logps/chosen": -113.07169342041016, "logps/ref_chosen": -50.424095153808594, "logps/ref_rejected": -96.03042602539062, "logps/rejected": -252.92019653320312, "loss": 0.9275, "margin_dpo/margin_mean": 94.24217224121094, "margin_dpo/margin_std": 99.01016235351562, "step": 167 }, { "KL/chosen_KL_mean": -68.97935485839844, "KL/mean": -104.1439208984375, "KL/rejected_KL_mean": -139.30848693847656, "KL/std": 81.96823120117188, "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.007100098766386509, "fcm_dpo/delta": -0.10204099118709564, "fcm_dpo/margin": 70.32914733886719, "fcm_dpo/q_t": 0.3902486562728882, "grad_norm": 21.94905662536621, "learning_rate": 4.690271916109034e-07, "logits/chosen": -0.405794233083725, "logits/rejected": -0.39295631647109985, "logps/chosen": -118.44217681884766, "logps/ref_chosen": -49.462825775146484, "logps/ref_rejected": -75.30855560302734, "logps/rejected": -214.61705017089844, "loss": 1.0391, "margin_dpo/margin_mean": 70.32914733886719, "margin_dpo/margin_std": 90.95465850830078, "step": 168 }, { "KL/chosen_KL_mean": -69.83270263671875, "KL/mean": -100.67288208007812, "KL/rejected_KL_mean": -131.5130615234375, "KL/std": 81.13442993164062, "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.007038552314043045, "fcm_dpo/delta": -0.03588036447763443, "fcm_dpo/margin": 61.68037414550781, "fcm_dpo/q_t": 0.4094877243041992, "grad_norm": 21.894031524658203, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -0.42251360416412354, "logits/rejected": -0.403925359249115, "logps/chosen": -129.63613891601562, "logps/ref_chosen": -59.803443908691406, "logps/ref_rejected": -83.34574890136719, "logps/rejected": -214.85882568359375, "loss": 1.1309, "margin_dpo/margin_mean": 61.68037414550781, "margin_dpo/margin_std": 110.4356460571289, "step": 169 }, { "KL/chosen_KL_mean": -62.970611572265625, "KL/mean": -99.88447570800781, "KL/rejected_KL_mean": -136.79833984375, "KL/std": 76.65866088867188, "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.0069792973808944225, "fcm_dpo/delta": -0.11909201741218567, "fcm_dpo/margin": 73.82772827148438, "fcm_dpo/q_t": 0.3844184875488281, "grad_norm": 18.95234489440918, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -0.3876168131828308, "logits/rejected": -0.37476158142089844, "logps/chosen": -112.4423828125, "logps/ref_chosen": -49.471771240234375, "logps/ref_rejected": -75.91734313964844, "logps/rejected": -212.71568298339844, "loss": 1.0306, "margin_dpo/margin_mean": 73.82772827148438, "margin_dpo/margin_std": 91.29777526855469, "step": 170 }, { "KL/chosen_KL_mean": -92.04759979248047, "KL/mean": -121.734619140625, "KL/rejected_KL_mean": -151.42166137695312, "KL/std": 87.79399108886719, "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.006971217226237059, "fcm_dpo/delta": -0.01420888677239418, "fcm_dpo/margin": 59.374053955078125, "fcm_dpo/q_t": 0.41368788480758667, "grad_norm": 30.997112274169922, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -0.3963302969932556, "logits/rejected": -0.3728923797607422, "logps/chosen": -176.54690551757812, "logps/ref_chosen": -84.49931335449219, "logps/ref_rejected": -109.38209533691406, "logps/rejected": -260.80377197265625, "loss": 1.1435, "margin_dpo/margin_mean": 59.374053955078125, "margin_dpo/margin_std": 111.98719787597656, "step": 171 }, { "KL/chosen_KL_mean": -83.34523010253906, "KL/mean": -114.48237609863281, "KL/rejected_KL_mean": -145.6195068359375, "KL/std": 86.62696838378906, "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.00694639515131712, "fcm_dpo/delta": -0.03337009996175766, "fcm_dpo/margin": 62.2743034362793, "fcm_dpo/q_t": 0.4046783447265625, "grad_norm": 21.863208770751953, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -0.44137507677078247, "logits/rejected": -0.4202975630760193, "logps/chosen": -151.9991455078125, "logps/ref_chosen": -68.65391540527344, "logps/ref_rejected": -85.43667602539062, "logps/rejected": -231.05618286132812, "loss": 1.1028, "margin_dpo/margin_mean": 62.2743034362793, "margin_dpo/margin_std": 97.67938232421875, "step": 172 }, { "KL/chosen_KL_mean": -75.67850494384766, "KL/mean": -107.38485717773438, "KL/rejected_KL_mean": -139.09121704101562, "KL/std": 80.97750854492188, "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.006938562728464603, "fcm_dpo/delta": -0.04095254838466644, "fcm_dpo/margin": 63.4127082824707, "fcm_dpo/q_t": 0.40140169858932495, "grad_norm": 22.291126251220703, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -0.4557979702949524, "logits/rejected": -0.4270375072956085, "logps/chosen": -138.72938537597656, "logps/ref_chosen": -63.050880432128906, "logps/ref_rejected": -78.68392181396484, "logps/rejected": -217.775146484375, "loss": 1.0729, "margin_dpo/margin_mean": 63.41270446777344, "margin_dpo/margin_std": 85.16667175292969, "step": 173 }, { "KL/chosen_KL_mean": -73.96533203125, "KL/mean": -110.90480041503906, "KL/rejected_KL_mean": -147.84425354003906, "KL/std": 88.3280258178711, "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.006883748807013035, "fcm_dpo/delta": -0.1114603653550148, "fcm_dpo/margin": 73.87892150878906, "fcm_dpo/q_t": 0.3911077380180359, "grad_norm": 23.334630966186523, "learning_rate": 4.652116329460919e-07, "logits/chosen": -0.3489905595779419, "logits/rejected": -0.3669482469558716, "logps/chosen": -127.32830810546875, "logps/ref_chosen": -53.36296844482422, "logps/ref_rejected": -101.91120910644531, "logps/rejected": -249.75546264648438, "loss": 1.0572, "margin_dpo/margin_mean": 73.87892150878906, "margin_dpo/margin_std": 104.1011962890625, "step": 174 }, { "KL/chosen_KL_mean": -69.70106506347656, "KL/mean": -120.88041687011719, "KL/rejected_KL_mean": -172.05975341796875, "KL/std": 95.63670349121094, "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.00668959878385067, "fcm_dpo/delta": -0.29501020908355713, "fcm_dpo/margin": 102.35872650146484, "fcm_dpo/q_t": 0.34925341606140137, "grad_norm": 25.57117462158203, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.3187200427055359, "logits/rejected": -0.30461329221725464, "logps/chosen": -115.11882019042969, "logps/ref_chosen": -45.417762756347656, "logps/ref_rejected": -89.50579833984375, "logps/rejected": -261.5655517578125, "loss": 0.9085, "margin_dpo/margin_mean": 102.35871887207031, "margin_dpo/margin_std": 99.24336242675781, "step": 175 }, { "KL/chosen_KL_mean": -75.16090393066406, "KL/mean": -117.7568359375, "KL/rejected_KL_mean": -160.35279846191406, "KL/std": 92.66831970214844, "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.0065691666677594185, "fcm_dpo/delta": -0.16410863399505615, "fcm_dpo/margin": 85.19189453125, "fcm_dpo/q_t": 0.380332350730896, "grad_norm": 22.5208683013916, "learning_rate": 4.638942309888058e-07, "logits/chosen": -0.3513135313987732, "logits/rejected": -0.369649738073349, "logps/chosen": -125.61373901367188, "logps/ref_chosen": -50.452842712402344, "logps/ref_rejected": -95.5589599609375, "logps/rejected": -255.91177368164062, "loss": 1.0084, "margin_dpo/margin_mean": 85.19189453125, "margin_dpo/margin_std": 107.12543487548828, "step": 176 }, { "KL/chosen_KL_mean": -87.45088195800781, "KL/mean": -129.8768310546875, "KL/rejected_KL_mean": -172.30276489257812, "KL/std": 101.22750854492188, "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.006470865570008755, "fcm_dpo/delta": -0.15313297510147095, "fcm_dpo/margin": 84.85188293457031, "fcm_dpo/q_t": 0.3814983367919922, "grad_norm": 33.10342025756836, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -0.38505709171295166, "logits/rejected": -0.37769830226898193, "logps/chosen": -148.66734313964844, "logps/ref_chosen": -61.216468811035156, "logps/ref_rejected": -95.89378356933594, "logps/rejected": -268.196533203125, "loss": 1.0131, "margin_dpo/margin_mean": 84.85188293457031, "margin_dpo/margin_std": 107.24163818359375, "step": 177 }, { "KL/chosen_KL_mean": -96.93707275390625, "KL/mean": -150.7945098876953, "KL/rejected_KL_mean": -204.65194702148438, "KL/std": 121.38895416259766, "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.006285130977630615, "fcm_dpo/delta": -0.28755688667297363, "fcm_dpo/margin": 107.7148666381836, "fcm_dpo/q_t": 0.36004963517189026, "grad_norm": 27.839982986450195, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -0.27259236574172974, "logits/rejected": -0.26497703790664673, "logps/chosen": -155.20187377929688, "logps/ref_chosen": -58.26478958129883, "logps/ref_rejected": -105.3653335571289, "logps/rejected": -310.01727294921875, "loss": 0.9589, "margin_dpo/margin_mean": 107.7148666381836, "margin_dpo/margin_std": 132.7176971435547, "step": 178 }, { "KL/chosen_KL_mean": -97.64852905273438, "KL/mean": -136.50204467773438, "KL/rejected_KL_mean": -175.3555908203125, "KL/std": 101.76957702636719, "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.006197200156748295, "fcm_dpo/delta": -0.08410830795764923, "fcm_dpo/margin": 77.70704650878906, "fcm_dpo/q_t": 0.39651763439178467, "grad_norm": 37.91785430908203, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -0.29223281145095825, "logits/rejected": -0.29845482110977173, "logps/chosen": -158.70684814453125, "logps/ref_chosen": -61.05832290649414, "logps/ref_rejected": -90.52782440185547, "logps/rejected": -265.8834228515625, "loss": 1.0988, "margin_dpo/margin_mean": 77.70704650878906, "margin_dpo/margin_std": 125.48446655273438, "step": 179 }, { "KL/chosen_KL_mean": -83.23408508300781, "KL/mean": -131.74658203125, "KL/rejected_KL_mean": -180.2590789794922, "KL/std": 91.78398132324219, "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.006104484666138887, "fcm_dpo/delta": -0.19836077094078064, "fcm_dpo/margin": 97.02499389648438, "fcm_dpo/q_t": 0.3690943717956543, "grad_norm": 21.05206871032715, "learning_rate": 4.611919330113591e-07, "logits/chosen": -0.2990095019340515, "logits/rejected": -0.29073792695999146, "logps/chosen": -137.57681274414062, "logps/ref_chosen": -54.34272003173828, "logps/ref_rejected": -98.21183776855469, "logps/rejected": -278.4709167480469, "loss": 0.9838, "margin_dpo/margin_mean": 97.02499389648438, "margin_dpo/margin_std": 113.17401123046875, "step": 180 }, { "KL/chosen_KL_mean": -71.93513488769531, "KL/mean": -102.62535095214844, "KL/rejected_KL_mean": -133.31556701660156, "KL/std": 82.91169738769531, "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.006095272954553366, "fcm_dpo/delta": 0.02632547914981842, "fcm_dpo/margin": 61.38043212890625, "fcm_dpo/q_t": 0.41590094566345215, "grad_norm": 20.209400177001953, "learning_rate": 4.605024008834863e-07, "logits/chosen": -0.37073230743408203, "logits/rejected": -0.34592998027801514, "logps/chosen": -126.93559265136719, "logps/ref_chosen": -55.000457763671875, "logps/ref_rejected": -61.656166076660156, "logps/rejected": -194.97174072265625, "loss": 1.1374, "margin_dpo/margin_mean": 61.38043212890625, "margin_dpo/margin_std": 102.93232727050781, "step": 181 }, { "KL/chosen_KL_mean": -68.23200225830078, "KL/mean": -119.69134521484375, "KL/rejected_KL_mean": -171.15066528320312, "KL/std": 100.23439025878906, "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.005980286747217178, "fcm_dpo/delta": -0.2230871319770813, "fcm_dpo/margin": 102.91866302490234, "fcm_dpo/q_t": 0.3642627000808716, "grad_norm": 20.144750595092773, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.31923383474349976, "logits/rejected": -0.32709991931915283, "logps/chosen": -109.33985900878906, "logps/ref_chosen": -41.107852935791016, "logps/ref_rejected": -89.5215835571289, "logps/rejected": -260.6722412109375, "loss": 0.9789, "margin_dpo/margin_mean": 102.91866302490234, "margin_dpo/margin_std": 120.07926940917969, "step": 182 }, { "KL/chosen_KL_mean": -99.28335571289062, "KL/mean": -128.1656036376953, "KL/rejected_KL_mean": -157.0478515625, "KL/std": 81.12559509277344, "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.00591336889192462, "fcm_dpo/delta": -0.07965878397226334, "fcm_dpo/margin": 57.76450729370117, "fcm_dpo/q_t": 0.42185482382774353, "grad_norm": 21.069660186767578, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -0.30978119373321533, "logits/rejected": -0.29306042194366455, "logps/chosen": -156.80792236328125, "logps/ref_chosen": -57.52456283569336, "logps/ref_rejected": -75.97572326660156, "logps/rejected": -233.02359008789062, "loss": 1.1494, "margin_dpo/margin_mean": 57.764503479003906, "margin_dpo/margin_std": 89.65037536621094, "step": 183 }, { "KL/chosen_KL_mean": -80.81704711914062, "KL/mean": -110.47319030761719, "KL/rejected_KL_mean": -140.12933349609375, "KL/std": 76.08272552490234, "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.005951171740889549, "fcm_dpo/delta": 0.04741087928414345, "fcm_dpo/margin": 59.312294006347656, "fcm_dpo/q_t": 0.4209594130516052, "grad_norm": 21.374256134033203, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -0.41061311960220337, "logits/rejected": -0.4004828929901123, "logps/chosen": -139.36199951171875, "logps/ref_chosen": -58.544952392578125, "logps/ref_rejected": -76.63406372070312, "logps/rejected": -216.76339721679688, "loss": 1.1399, "margin_dpo/margin_mean": 59.312294006347656, "margin_dpo/margin_std": 94.55437469482422, "step": 184 }, { "KL/chosen_KL_mean": -88.34465026855469, "KL/mean": -112.53829193115234, "KL/rejected_KL_mean": -136.73193359375, "KL/std": 88.54534912109375, "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.005989473778754473, "fcm_dpo/delta": 0.1118379533290863, "fcm_dpo/margin": 48.38727951049805, "fcm_dpo/q_t": 0.4377959966659546, "grad_norm": 20.70436668395996, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -0.3213365077972412, "logits/rejected": -0.2884293794631958, "logps/chosen": -150.37049865722656, "logps/ref_chosen": -62.025848388671875, "logps/ref_rejected": -73.7625961303711, "logps/rejected": -210.49453735351562, "loss": 1.2074, "margin_dpo/margin_mean": 48.38727569580078, "margin_dpo/margin_std": 105.9352798461914, "step": 185 }, { "KL/chosen_KL_mean": -81.42449188232422, "KL/mean": -126.01622009277344, "KL/rejected_KL_mean": -170.60794067382812, "KL/std": 91.03404235839844, "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.005945336539298296, "fcm_dpo/delta": -0.13385680317878723, "fcm_dpo/margin": 89.18345642089844, "fcm_dpo/q_t": 0.3814866244792938, "grad_norm": 34.85935974121094, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -0.41816607117652893, "logits/rejected": -0.3946566581726074, "logps/chosen": -150.7779541015625, "logps/ref_chosen": -69.35346984863281, "logps/ref_rejected": -88.07244873046875, "logps/rejected": -258.6803894042969, "loss": 1.0069, "margin_dpo/margin_mean": 89.18345642089844, "margin_dpo/margin_std": 102.38252258300781, "step": 186 }, { "KL/chosen_KL_mean": -77.34542846679688, "KL/mean": -114.29693603515625, "KL/rejected_KL_mean": -151.24844360351562, "KL/std": 86.13810729980469, "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.005925598554313183, "fcm_dpo/delta": -0.038912609219551086, "fcm_dpo/margin": 73.90303039550781, "fcm_dpo/q_t": 0.4008300006389618, "grad_norm": 29.05039405822754, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -0.3810155987739563, "logits/rejected": -0.37561988830566406, "logps/chosen": -130.1018829345703, "logps/ref_chosen": -52.7564582824707, "logps/ref_rejected": -81.96910095214844, "logps/rejected": -233.21754455566406, "loss": 1.0696, "margin_dpo/margin_mean": 73.90303039550781, "margin_dpo/margin_std": 94.80323791503906, "step": 187 }, { "KL/chosen_KL_mean": -73.07807922363281, "KL/mean": -118.4848403930664, "KL/rejected_KL_mean": -163.8916015625, "KL/std": 96.97216796875, "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.005849046632647514, "fcm_dpo/delta": -0.13469749689102173, "fcm_dpo/margin": 90.81352233886719, "fcm_dpo/q_t": 0.3821578025817871, "grad_norm": 35.57075881958008, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -0.35656416416168213, "logits/rejected": -0.36152058839797974, "logps/chosen": -122.49356842041016, "logps/ref_chosen": -49.415489196777344, "logps/ref_rejected": -89.54043579101562, "logps/rejected": -253.43203735351562, "loss": 1.0084, "margin_dpo/margin_mean": 90.81352233886719, "margin_dpo/margin_std": 105.5975341796875, "step": 188 }, { "KL/chosen_KL_mean": -88.35533142089844, "KL/mean": -125.64856719970703, "KL/rejected_KL_mean": -162.94180297851562, "KL/std": 98.73692321777344, "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.005813870579004288, "fcm_dpo/delta": -0.0343741774559021, "fcm_dpo/margin": 74.58646392822266, "fcm_dpo/q_t": 0.4062124490737915, "grad_norm": 29.83676528930664, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -0.3425024151802063, "logits/rejected": -0.32220566272735596, "logps/chosen": -140.75428771972656, "logps/ref_chosen": -52.39896011352539, "logps/ref_rejected": -72.16735076904297, "logps/rejected": -235.10916137695312, "loss": 1.1033, "margin_dpo/margin_mean": 74.58646392822266, "margin_dpo/margin_std": 120.13783264160156, "step": 189 }, { "KL/chosen_KL_mean": -93.82635498046875, "KL/mean": -138.6332550048828, "KL/rejected_KL_mean": -183.44015502929688, "KL/std": 105.86849975585938, "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.0057814717292785645, "fcm_dpo/delta": -0.12157705426216125, "fcm_dpo/margin": 89.61378479003906, "fcm_dpo/q_t": 0.3877827227115631, "grad_norm": 21.399682998657227, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -0.3890034854412079, "logits/rejected": -0.37827688455581665, "logps/chosen": -158.50942993164062, "logps/ref_chosen": -64.68305969238281, "logps/ref_rejected": -102.55052185058594, "logps/rejected": -285.99066162109375, "loss": 1.0533, "margin_dpo/margin_mean": 89.61378479003906, "margin_dpo/margin_std": 122.82894897460938, "step": 190 }, { "KL/chosen_KL_mean": -86.75233459472656, "KL/mean": -150.7803955078125, "KL/rejected_KL_mean": -214.80844116210938, "KL/std": 124.46101379394531, "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.005586773157119751, "fcm_dpo/delta": -0.32792675495147705, "fcm_dpo/margin": 128.05612182617188, "fcm_dpo/q_t": 0.34851568937301636, "grad_norm": 24.331701278686523, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -0.3571997880935669, "logits/rejected": -0.3361099660396576, "logps/chosen": -155.4112091064453, "logps/ref_chosen": -68.65887451171875, "logps/ref_rejected": -110.1396713256836, "logps/rejected": -324.9481201171875, "loss": 0.912, "margin_dpo/margin_mean": 128.05612182617188, "margin_dpo/margin_std": 137.22451782226562, "step": 191 }, { "KL/chosen_KL_mean": -114.01398468017578, "KL/mean": -156.28001403808594, "KL/rejected_KL_mean": -198.54605102539062, "KL/std": 106.98095703125, "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.005530376918613911, "fcm_dpo/delta": -0.06909237802028656, "fcm_dpo/margin": 84.53204345703125, "fcm_dpo/q_t": 0.39953508973121643, "grad_norm": 28.160974502563477, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -0.31193774938583374, "logits/rejected": -0.31198960542678833, "logps/chosen": -183.74090576171875, "logps/ref_chosen": -69.72691345214844, "logps/ref_rejected": -103.32135009765625, "logps/rejected": -301.86737060546875, "loss": 1.089, "margin_dpo/margin_mean": 84.53204345703125, "margin_dpo/margin_std": 131.72988891601562, "step": 192 }, { "KL/chosen_KL_mean": -114.42250061035156, "KL/mean": -138.80294799804688, "KL/rejected_KL_mean": -163.18338012695312, "KL/std": 97.2500228881836, "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.005518913269042969, "fcm_dpo/delta": 0.021666744723916054, "fcm_dpo/margin": 48.76087951660156, "fcm_dpo/q_t": 0.43647685647010803, "grad_norm": 31.76787567138672, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -0.3469330072402954, "logits/rejected": -0.33776986598968506, "logps/chosen": -174.61300659179688, "logps/ref_chosen": -60.19049835205078, "logps/ref_rejected": -76.40755462646484, "logps/rejected": -239.5909423828125, "loss": 1.2443, "margin_dpo/margin_mean": 48.76087951660156, "margin_dpo/margin_std": 123.41360473632812, "step": 193 }, { "KL/chosen_KL_mean": -71.24337768554688, "KL/mean": -112.21340942382812, "KL/rejected_KL_mean": -153.18345642089844, "KL/std": 83.32481384277344, "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.0054954723455011845, "fcm_dpo/delta": -0.051623422652482986, "fcm_dpo/margin": 81.94007873535156, "fcm_dpo/q_t": 0.395707905292511, "grad_norm": 20.26607894897461, "learning_rate": 4.510405240853854e-07, "logits/chosen": -0.2368777096271515, "logits/rejected": -0.21953874826431274, "logps/chosen": -109.08374786376953, "logps/ref_chosen": -37.84037399291992, "logps/ref_rejected": -60.684783935546875, "logps/rejected": -213.8682403564453, "loss": 1.0437, "margin_dpo/margin_mean": 81.94007873535156, "margin_dpo/margin_std": 90.14231872558594, "step": 194 }, { "KL/chosen_KL_mean": -112.94318389892578, "KL/mean": -156.35760498046875, "KL/rejected_KL_mean": -199.77203369140625, "KL/std": 102.81307983398438, "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.0054633598774671555, "fcm_dpo/delta": -0.07621531188488007, "fcm_dpo/margin": 86.82884216308594, "fcm_dpo/q_t": 0.3926421105861664, "grad_norm": 26.608348846435547, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -0.27807819843292236, "logits/rejected": -0.29660797119140625, "logps/chosen": -167.83474731445312, "logps/ref_chosen": -54.891571044921875, "logps/ref_rejected": -96.77095794677734, "logps/rejected": -296.54296875, "loss": 1.0427, "margin_dpo/margin_mean": 86.82884216308594, "margin_dpo/margin_std": 105.82039642333984, "step": 195 }, { "KL/chosen_KL_mean": -87.1651382446289, "KL/mean": -137.2007598876953, "KL/rejected_KL_mean": -187.23635864257812, "KL/std": 106.05245208740234, "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.005381887778639793, "fcm_dpo/delta": -0.14310941100120544, "fcm_dpo/margin": 100.07121276855469, "fcm_dpo/q_t": 0.3824601173400879, "grad_norm": 20.205522537231445, "learning_rate": 4.495043068200599e-07, "logits/chosen": -0.3048306703567505, "logits/rejected": -0.2887340188026428, "logps/chosen": -140.41038513183594, "logps/ref_chosen": -53.245243072509766, "logps/ref_rejected": -76.05294799804688, "logps/rejected": -263.289306640625, "loss": 1.0251, "margin_dpo/margin_mean": 100.07121276855469, "margin_dpo/margin_std": 125.76506042480469, "step": 196 }, { "KL/chosen_KL_mean": -87.905029296875, "KL/mean": -126.29278564453125, "KL/rejected_KL_mean": -164.6805419921875, "KL/std": 89.6479263305664, "epoch": 0.28928046989721, "fcm_dpo/beta": 0.005379960872232914, "fcm_dpo/delta": -0.01339656300842762, "fcm_dpo/margin": 76.7755126953125, "fcm_dpo/q_t": 0.40633296966552734, "grad_norm": 25.854190826416016, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -0.2854430675506592, "logits/rejected": -0.2793597877025604, "logps/chosen": -148.3253631591797, "logps/ref_chosen": -60.42033386230469, "logps/ref_rejected": -77.20890808105469, "logps/rejected": -241.8894500732422, "loss": 1.0832, "margin_dpo/margin_mean": 76.7755126953125, "margin_dpo/margin_std": 100.7091064453125, "step": 197 }, { "KL/chosen_KL_mean": -97.10411071777344, "KL/mean": -145.3472900390625, "KL/rejected_KL_mean": -193.59048461914062, "KL/std": 110.99395751953125, "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.005326003301888704, "fcm_dpo/delta": -0.11684248596429825, "fcm_dpo/margin": 96.48635864257812, "fcm_dpo/q_t": 0.38827261328697205, "grad_norm": 22.704126358032227, "learning_rate": 4.479470611971645e-07, "logits/chosen": -0.32224997878074646, "logits/rejected": -0.32293713092803955, "logps/chosen": -152.14028930664062, "logps/ref_chosen": -55.03618621826172, "logps/ref_rejected": -97.24325561523438, "logps/rejected": -290.833740234375, "loss": 1.0307, "margin_dpo/margin_mean": 96.48635864257812, "margin_dpo/margin_std": 124.41645050048828, "step": 198 }, { "KL/chosen_KL_mean": -93.67039489746094, "KL/mean": -140.52859497070312, "KL/rejected_KL_mean": -187.3867950439453, "KL/std": 99.46040344238281, "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.0052545820362865925, "fcm_dpo/delta": -0.09539773315191269, "fcm_dpo/margin": 93.71640014648438, "fcm_dpo/q_t": 0.38956761360168457, "grad_norm": 34.478763580322266, "learning_rate": 4.471606039587695e-07, "logits/chosen": -0.3307437300682068, "logits/rejected": -0.31390073895454407, "logps/chosen": -150.4992218017578, "logps/ref_chosen": -56.828826904296875, "logps/ref_rejected": -84.64820861816406, "logps/rejected": -272.0350036621094, "loss": 1.0441, "margin_dpo/margin_mean": 93.71640014648438, "margin_dpo/margin_std": 118.5833969116211, "step": 199 }, { "KL/chosen_KL_mean": -91.72415161132812, "KL/mean": -139.52906799316406, "KL/rejected_KL_mean": -187.333984375, "KL/std": 108.85971069335938, "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.0052094850689172745, "fcm_dpo/delta": -0.10084635764360428, "fcm_dpo/margin": 95.60984802246094, "fcm_dpo/q_t": 0.3912206292152405, "grad_norm": 20.905105590820312, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -0.29428964853286743, "logits/rejected": -0.2763988971710205, "logps/chosen": -144.79119873046875, "logps/ref_chosen": -53.06706237792969, "logps/ref_rejected": -80.60843658447266, "logps/rejected": -267.9424133300781, "loss": 1.0584, "margin_dpo/margin_mean": 95.60984802246094, "margin_dpo/margin_std": 135.0447998046875, "step": 200 }, { "KL/chosen_KL_mean": -94.4271469116211, "KL/mean": -142.72763061523438, "KL/rejected_KL_mean": -191.028076171875, "KL/std": 113.81238555908203, "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.005169622600078583, "fcm_dpo/delta": -0.1018773764371872, "fcm_dpo/margin": 96.60094451904297, "fcm_dpo/q_t": 0.389159232378006, "grad_norm": 26.09699249267578, "learning_rate": 4.455721242469372e-07, "logits/chosen": -0.34645718336105347, "logits/rejected": -0.34164953231811523, "logps/chosen": -169.82937622070312, "logps/ref_chosen": -75.4022216796875, "logps/ref_rejected": -114.80821990966797, "logps/rejected": -305.8363037109375, "loss": 1.0446, "margin_dpo/margin_mean": 96.60094451904297, "margin_dpo/margin_std": 127.17774963378906, "step": 201 }, { "KL/chosen_KL_mean": -98.94967651367188, "KL/mean": -132.004150390625, "KL/rejected_KL_mean": -165.05862426757812, "KL/std": 95.95150756835938, "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.0051773954182863235, "fcm_dpo/delta": 0.05868521332740784, "fcm_dpo/margin": 66.10896301269531, "fcm_dpo/q_t": 0.42411088943481445, "grad_norm": 20.637813568115234, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -0.30686646699905396, "logits/rejected": -0.31850093603134155, "logps/chosen": -149.05099487304688, "logps/ref_chosen": -50.101318359375, "logps/ref_rejected": -86.98503112792969, "logps/rejected": -252.04367065429688, "loss": 1.1671, "margin_dpo/margin_mean": 66.10896301269531, "margin_dpo/margin_std": 123.44126892089844, "step": 202 }, { "KL/chosen_KL_mean": -98.62371826171875, "KL/mean": -140.30938720703125, "KL/rejected_KL_mean": -181.99505615234375, "KL/std": 101.09996795654297, "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.005167345050722361, "fcm_dpo/delta": -0.03151167184114456, "fcm_dpo/margin": 83.371337890625, "fcm_dpo/q_t": 0.4019864797592163, "grad_norm": 25.103071212768555, "learning_rate": 4.439630306414758e-07, "logits/chosen": -0.31344640254974365, "logits/rejected": -0.30354416370391846, "logps/chosen": -159.23341369628906, "logps/ref_chosen": -60.60969543457031, "logps/ref_rejected": -85.89596557617188, "logps/rejected": -267.8910217285156, "loss": 1.0711, "margin_dpo/margin_mean": 83.371337890625, "margin_dpo/margin_std": 107.46673583984375, "step": 203 }, { "KL/chosen_KL_mean": -110.24484252929688, "KL/mean": -148.1464385986328, "KL/rejected_KL_mean": -186.04803466796875, "KL/std": 108.95531463623047, "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.005168107338249683, "fcm_dpo/delta": 0.008405912667512894, "fcm_dpo/margin": 75.80317687988281, "fcm_dpo/q_t": 0.41412895917892456, "grad_norm": 25.396324157714844, "learning_rate": 4.431508065452897e-07, "logits/chosen": -0.40366190671920776, "logits/rejected": -0.3658892512321472, "logps/chosen": -190.40980529785156, "logps/ref_chosen": -80.16496276855469, "logps/ref_rejected": -87.69590759277344, "logps/rejected": -273.74395751953125, "loss": 1.1248, "margin_dpo/margin_mean": 75.80317687988281, "margin_dpo/margin_std": 123.09947204589844, "step": 204 }, { "KL/chosen_KL_mean": -108.8935775756836, "KL/mean": -159.8162841796875, "KL/rejected_KL_mean": -210.739013671875, "KL/std": 114.94783020019531, "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.005101551301777363, "fcm_dpo/delta": -0.12394419312477112, "fcm_dpo/margin": 101.84541320800781, "fcm_dpo/q_t": 0.38280677795410156, "grad_norm": 25.676708221435547, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -0.2786010801792145, "logits/rejected": -0.2449745088815689, "logps/chosen": -168.2783203125, "logps/ref_chosen": -59.384735107421875, "logps/ref_rejected": -85.12505340576172, "logps/rejected": -295.86407470703125, "loss": 1.0303, "margin_dpo/margin_mean": 101.84541320800781, "margin_dpo/margin_std": 124.0643310546875, "step": 205 }, { "KL/chosen_KL_mean": -103.83800506591797, "KL/mean": -158.62493896484375, "KL/rejected_KL_mean": -213.41188049316406, "KL/std": 111.25010681152344, "epoch": 0.302496328928047, "fcm_dpo/beta": 0.005038215778768063, "fcm_dpo/delta": -0.15676981210708618, "fcm_dpo/margin": 109.57386779785156, "fcm_dpo/q_t": 0.374674528837204, "grad_norm": 31.18449592590332, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.2506366968154907, "logits/rejected": -0.25460052490234375, "logps/chosen": -150.80250549316406, "logps/ref_chosen": -46.964500427246094, "logps/ref_rejected": -98.9534912109375, "logps/rejected": -312.3653564453125, "loss": 0.9924, "margin_dpo/margin_mean": 109.57386016845703, "margin_dpo/margin_std": 120.0478515625, "step": 206 }, { "KL/chosen_KL_mean": -93.25586700439453, "KL/mean": -158.32984924316406, "KL/rejected_KL_mean": -223.40382385253906, "KL/std": 127.83676147460938, "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.004938926547765732, "fcm_dpo/delta": -0.25064384937286377, "fcm_dpo/margin": 130.14794921875, "fcm_dpo/q_t": 0.36273083090782166, "grad_norm": 25.51757049560547, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -0.3203808069229126, "logits/rejected": -0.29214242100715637, "logps/chosen": -149.3121337890625, "logps/ref_chosen": -56.05625915527344, "logps/ref_rejected": -84.44779968261719, "logps/rejected": -307.85162353515625, "loss": 0.955, "margin_dpo/margin_mean": 130.14796447753906, "margin_dpo/margin_std": 148.46749877929688, "step": 207 }, { "KL/chosen_KL_mean": -143.90771484375, "KL/mean": -192.93313598632812, "KL/rejected_KL_mean": -241.9585418701172, "KL/std": 121.01240539550781, "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.004868675023317337, "fcm_dpo/delta": -0.07939005643129349, "fcm_dpo/margin": 98.05082702636719, "fcm_dpo/q_t": 0.39406818151474, "grad_norm": 27.979568481445312, "learning_rate": 4.398512291636768e-07, "logits/chosen": -0.35994622111320496, "logits/rejected": -0.3440352976322174, "logps/chosen": -210.975341796875, "logps/ref_chosen": -67.06761169433594, "logps/ref_rejected": -94.28689575195312, "logps/rejected": -336.24542236328125, "loss": 1.08, "margin_dpo/margin_mean": 98.05082702636719, "margin_dpo/margin_std": 146.68267822265625, "step": 208 }, { "KL/chosen_KL_mean": -118.966064453125, "KL/mean": -162.17567443847656, "KL/rejected_KL_mean": -205.38528442382812, "KL/std": 105.17567443847656, "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.004856686107814312, "fcm_dpo/delta": -0.02013271115720272, "fcm_dpo/margin": 86.41921997070312, "fcm_dpo/q_t": 0.40564462542533875, "grad_norm": 29.448095321655273, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -0.24659614264965057, "logits/rejected": -0.23566904664039612, "logps/chosen": -175.14776611328125, "logps/ref_chosen": -56.18169403076172, "logps/ref_rejected": -80.94152069091797, "logps/rejected": -286.3267822265625, "loss": 1.1089, "margin_dpo/margin_mean": 86.41921997070312, "margin_dpo/margin_std": 132.27151489257812, "step": 209 }, { "KL/chosen_KL_mean": -103.5418472290039, "KL/mean": -154.00094604492188, "KL/rejected_KL_mean": -204.46005249023438, "KL/std": 109.31126403808594, "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.004831655882298946, "fcm_dpo/delta": -0.0897798091173172, "fcm_dpo/margin": 100.91820526123047, "fcm_dpo/q_t": 0.3923318386077881, "grad_norm": 25.811580657958984, "learning_rate": 4.381713366536311e-07, "logits/chosen": -0.2856072783470154, "logits/rejected": -0.2767780125141144, "logps/chosen": -149.91366577148438, "logps/ref_chosen": -46.371822357177734, "logps/ref_rejected": -76.68162536621094, "logps/rejected": -281.14166259765625, "loss": 1.0508, "margin_dpo/margin_mean": 100.91820526123047, "margin_dpo/margin_std": 132.74383544921875, "step": 210 }, { "KL/chosen_KL_mean": -145.32662963867188, "KL/mean": -188.92605590820312, "KL/rejected_KL_mean": -232.5254669189453, "KL/std": 120.26460266113281, "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.004806230776011944, "fcm_dpo/delta": -0.019507823511958122, "fcm_dpo/margin": 87.1988296508789, "fcm_dpo/q_t": 0.40984851121902466, "grad_norm": 27.256078720092773, "learning_rate": 4.373239415645323e-07, "logits/chosen": -0.35860946774482727, "logits/rejected": -0.31967782974243164, "logps/chosen": -224.25897216796875, "logps/ref_chosen": -78.93235778808594, "logps/ref_rejected": -86.82098388671875, "logps/rejected": -319.346435546875, "loss": 1.1129, "margin_dpo/margin_mean": 87.19883728027344, "margin_dpo/margin_std": 139.08685302734375, "step": 211 }, { "KL/chosen_KL_mean": -117.86914825439453, "KL/mean": -179.65806579589844, "KL/rejected_KL_mean": -241.44700622558594, "KL/std": 132.9044952392578, "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.004715276416391134, "fcm_dpo/delta": -0.18955346941947937, "fcm_dpo/margin": 123.57785034179688, "fcm_dpo/q_t": 0.372633695602417, "grad_norm": 25.3454647064209, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -0.34758710861206055, "logits/rejected": -0.33722245693206787, "logps/chosen": -176.066162109375, "logps/ref_chosen": -58.19701385498047, "logps/ref_rejected": -103.05785369873047, "logps/rejected": -344.5048522949219, "loss": 1.0062, "margin_dpo/margin_mean": 123.57785034179688, "margin_dpo/margin_std": 148.7064208984375, "step": 212 }, { "KL/chosen_KL_mean": -108.09127807617188, "KL/mean": -165.18792724609375, "KL/rejected_KL_mean": -222.28460693359375, "KL/std": 112.09000396728516, "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.004654336720705032, "fcm_dpo/delta": -0.13551779091358185, "fcm_dpo/margin": 114.19332885742188, "fcm_dpo/q_t": 0.38082438707351685, "grad_norm": 22.01609992980957, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -0.3316152095794678, "logits/rejected": -0.3008173108100891, "logps/chosen": -175.60398864746094, "logps/ref_chosen": -67.51271057128906, "logps/ref_rejected": -93.91471862792969, "logps/rejected": -316.1993408203125, "loss": 1.0141, "margin_dpo/margin_mean": 114.19332885742188, "margin_dpo/margin_std": 134.8514404296875, "step": 213 }, { "KL/chosen_KL_mean": -97.71200561523438, "KL/mean": -148.03985595703125, "KL/rejected_KL_mean": -198.36767578125, "KL/std": 111.36173248291016, "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.004625334870070219, "fcm_dpo/delta": -0.0671173483133316, "fcm_dpo/margin": 100.65567016601562, "fcm_dpo/q_t": 0.3962777256965637, "grad_norm": 21.930564880371094, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -0.2828127145767212, "logits/rejected": -0.2885119915008545, "logps/chosen": -139.31689453125, "logps/ref_chosen": -41.604888916015625, "logps/ref_rejected": -77.51741027832031, "logps/rejected": -275.88507080078125, "loss": 1.0601, "margin_dpo/margin_mean": 100.65567016601562, "margin_dpo/margin_std": 133.34600830078125, "step": 214 }, { "KL/chosen_KL_mean": -108.55481719970703, "KL/mean": -162.83993530273438, "KL/rejected_KL_mean": -217.1250762939453, "KL/std": 115.2216796875, "epoch": 0.315712187958884, "fcm_dpo/beta": 0.0045753284357488155, "fcm_dpo/delta": -0.09945982694625854, "fcm_dpo/margin": 108.57025909423828, "fcm_dpo/q_t": 0.38728252053260803, "grad_norm": 26.600690841674805, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -0.30017614364624023, "logits/rejected": -0.2858251929283142, "logps/chosen": -161.83407592773438, "logps/ref_chosen": -53.279266357421875, "logps/ref_rejected": -89.96464538574219, "logps/rejected": -307.0897216796875, "loss": 1.0211, "margin_dpo/margin_mean": 108.57025909423828, "margin_dpo/margin_std": 121.32503509521484, "step": 215 }, { "KL/chosen_KL_mean": -110.91032409667969, "KL/mean": -164.44400024414062, "KL/rejected_KL_mean": -217.9776611328125, "KL/std": 116.61207580566406, "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.004546986427158117, "fcm_dpo/delta": -0.08895479142665863, "fcm_dpo/margin": 107.06733703613281, "fcm_dpo/q_t": 0.391222208738327, "grad_norm": 28.687511444091797, "learning_rate": 4.330133748510036e-07, "logits/chosen": -0.31943368911743164, "logits/rejected": -0.30127909779548645, "logps/chosen": -159.79812622070312, "logps/ref_chosen": -48.887794494628906, "logps/ref_rejected": -77.19892883300781, "logps/rejected": -295.17657470703125, "loss": 1.0535, "margin_dpo/margin_mean": 107.06732940673828, "margin_dpo/margin_std": 142.55877685546875, "step": 216 }, { "KL/chosen_KL_mean": -110.0457763671875, "KL/mean": -171.7996826171875, "KL/rejected_KL_mean": -233.55361938476562, "KL/std": 119.768310546875, "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.004474131390452385, "fcm_dpo/delta": -0.15707343816757202, "fcm_dpo/margin": 123.50784301757812, "fcm_dpo/q_t": 0.376447856426239, "grad_norm": 21.844968795776367, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -0.32586967945098877, "logits/rejected": -0.3191707134246826, "logps/chosen": -159.89108276367188, "logps/ref_chosen": -49.845306396484375, "logps/ref_rejected": -100.07832336425781, "logps/rejected": -333.6319580078125, "loss": 0.9878, "margin_dpo/margin_mean": 123.50784301757812, "margin_dpo/margin_std": 134.11033630371094, "step": 217 }, { "KL/chosen_KL_mean": -115.66281127929688, "KL/mean": -165.30850219726562, "KL/rejected_KL_mean": -214.9542236328125, "KL/std": 118.33514404296875, "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.00444306805729866, "fcm_dpo/delta": -0.04212786629796028, "fcm_dpo/margin": 99.2914047241211, "fcm_dpo/q_t": 0.40180084109306335, "grad_norm": 22.195404052734375, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -0.3458363711833954, "logits/rejected": -0.33776670694351196, "logps/chosen": -174.239501953125, "logps/ref_chosen": -58.576683044433594, "logps/ref_rejected": -87.84639739990234, "logps/rejected": -302.80059814453125, "loss": 1.0785, "margin_dpo/margin_mean": 99.2914047241211, "margin_dpo/margin_std": 137.7327423095703, "step": 218 }, { "KL/chosen_KL_mean": -124.46105194091797, "KL/mean": -168.70489501953125, "KL/rejected_KL_mean": -212.94874572753906, "KL/std": 120.42536163330078, "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.00445276964455843, "fcm_dpo/delta": 0.005898263305425644, "fcm_dpo/margin": 88.48770141601562, "fcm_dpo/q_t": 0.4121992588043213, "grad_norm": 22.444704055786133, "learning_rate": 4.303689819449636e-07, "logits/chosen": -0.3449735641479492, "logits/rejected": -0.33431658148765564, "logps/chosen": -185.54490661621094, "logps/ref_chosen": -61.083858489990234, "logps/ref_rejected": -85.83042907714844, "logps/rejected": -298.7791748046875, "loss": 1.1278, "margin_dpo/margin_mean": 88.4876937866211, "margin_dpo/margin_std": 145.41778564453125, "step": 219 }, { "KL/chosen_KL_mean": -146.02963256835938, "KL/mean": -183.63616943359375, "KL/rejected_KL_mean": -221.24269104003906, "KL/std": 107.1483154296875, "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.004458419047296047, "fcm_dpo/delta": 0.06577261537313461, "fcm_dpo/margin": 75.21304321289062, "fcm_dpo/q_t": 0.42131632566452026, "grad_norm": 22.67669677734375, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -0.31147876381874084, "logits/rejected": -0.2866259217262268, "logps/chosen": -216.06092834472656, "logps/ref_chosen": -70.03128051757812, "logps/ref_rejected": -87.68551635742188, "logps/rejected": -308.92822265625, "loss": 1.1337, "margin_dpo/margin_mean": 75.21304321289062, "margin_dpo/margin_std": 106.4920654296875, "step": 220 }, { "KL/chosen_KL_mean": -136.16143798828125, "KL/mean": -213.4010009765625, "KL/rejected_KL_mean": -290.6405944824219, "KL/std": 141.66920471191406, "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.004385126754641533, "fcm_dpo/delta": -0.2867964208126068, "fcm_dpo/margin": 154.47915649414062, "fcm_dpo/q_t": 0.3487062454223633, "grad_norm": 29.397729873657227, "learning_rate": 4.285822501755485e-07, "logits/chosen": -0.2791799306869507, "logits/rejected": -0.28570184111595154, "logps/chosen": -188.31613159179688, "logps/ref_chosen": -52.15470886230469, "logps/ref_rejected": -106.46768188476562, "logps/rejected": -397.1082763671875, "loss": 0.9085, "margin_dpo/margin_mean": 154.47915649414062, "margin_dpo/margin_std": 143.16314697265625, "step": 221 }, { "KL/chosen_KL_mean": -145.255126953125, "KL/mean": -204.62652587890625, "KL/rejected_KL_mean": -263.99798583984375, "KL/std": 128.56329345703125, "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.004318445920944214, "fcm_dpo/delta": -0.11569039523601532, "fcm_dpo/margin": 118.74282836914062, "fcm_dpo/q_t": 0.3848910629749298, "grad_norm": 26.170949935913086, "learning_rate": 4.276818137766118e-07, "logits/chosen": -0.33129405975341797, "logits/rejected": -0.32808414101600647, "logps/chosen": -206.22622680664062, "logps/ref_chosen": -60.971099853515625, "logps/ref_rejected": -100.00115203857422, "logps/rejected": -363.9991149902344, "loss": 1.0234, "margin_dpo/margin_mean": 118.74282836914062, "margin_dpo/margin_std": 141.51174926757812, "step": 222 }, { "KL/chosen_KL_mean": -161.82325744628906, "KL/mean": -213.307861328125, "KL/rejected_KL_mean": -264.79248046875, "KL/std": 131.9065399169922, "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.004278381355106831, "fcm_dpo/delta": -0.04155343025922775, "fcm_dpo/margin": 102.96919250488281, "fcm_dpo/q_t": 0.40245696902275085, "grad_norm": 34.5849723815918, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -0.2450229823589325, "logits/rejected": -0.23633867502212524, "logps/chosen": -214.46383666992188, "logps/ref_chosen": -52.64057540893555, "logps/ref_rejected": -82.82502746582031, "logps/rejected": -347.61749267578125, "loss": 1.1098, "margin_dpo/margin_mean": 102.96919250488281, "margin_dpo/margin_std": 167.57089233398438, "step": 223 }, { "KL/chosen_KL_mean": -148.254150390625, "KL/mean": -204.7643280029297, "KL/rejected_KL_mean": -261.2745361328125, "KL/std": 144.42315673828125, "epoch": 0.328928046989721, "fcm_dpo/beta": 0.004234839230775833, "fcm_dpo/delta": -0.08161487430334091, "fcm_dpo/margin": 113.02037048339844, "fcm_dpo/q_t": 0.39719724655151367, "grad_norm": 37.836830139160156, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -0.26897698640823364, "logits/rejected": -0.24652792513370514, "logps/chosen": -196.84954833984375, "logps/ref_chosen": -48.59541320800781, "logps/ref_rejected": -77.11648559570312, "logps/rejected": -338.3910217285156, "loss": 1.0801, "margin_dpo/margin_mean": 113.02037048339844, "margin_dpo/margin_std": 168.1210479736328, "step": 224 }, { "KL/chosen_KL_mean": -170.34971618652344, "KL/mean": -240.19332885742188, "KL/rejected_KL_mean": -310.03692626953125, "KL/std": 145.15887451171875, "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.0041815126314759254, "fcm_dpo/delta": -0.1897869110107422, "fcm_dpo/margin": 139.68719482421875, "fcm_dpo/q_t": 0.37290918827056885, "grad_norm": 30.018829345703125, "learning_rate": 4.249525076191759e-07, "logits/chosen": -0.2817176878452301, "logits/rejected": -0.2691669464111328, "logps/chosen": -228.35018920898438, "logps/ref_chosen": -58.000465393066406, "logps/ref_rejected": -99.90291595458984, "logps/rejected": -409.9398193359375, "loss": 1.0032, "margin_dpo/margin_mean": 139.6872100830078, "margin_dpo/margin_std": 175.74925231933594, "step": 225 }, { "KL/chosen_KL_mean": -143.76260375976562, "KL/mean": -200.83154296875, "KL/rejected_KL_mean": -257.90045166015625, "KL/std": 143.9960479736328, "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.004126027226448059, "fcm_dpo/delta": -0.07347771525382996, "fcm_dpo/margin": 114.13786315917969, "fcm_dpo/q_t": 0.3982742428779602, "grad_norm": 30.518173217773438, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -0.35959500074386597, "logits/rejected": -0.3191676139831543, "logps/chosen": -202.66140747070312, "logps/ref_chosen": -58.898799896240234, "logps/ref_rejected": -78.68775939941406, "logps/rejected": -336.5882263183594, "loss": 1.0742, "margin_dpo/margin_mean": 114.13786315917969, "margin_dpo/margin_std": 164.3631591796875, "step": 226 }, { "KL/chosen_KL_mean": -155.82469177246094, "KL/mean": -225.405029296875, "KL/rejected_KL_mean": -294.9853515625, "KL/std": 156.4764404296875, "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.0040718745440244675, "fcm_dpo/delta": -0.1720867156982422, "fcm_dpo/margin": 139.16065979003906, "fcm_dpo/q_t": 0.37408214807510376, "grad_norm": 28.692975997924805, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -0.3732532262802124, "logits/rejected": -0.35874879360198975, "logps/chosen": -214.89688110351562, "logps/ref_chosen": -59.072181701660156, "logps/ref_rejected": -99.41236877441406, "logps/rejected": -394.3977355957031, "loss": 0.9953, "margin_dpo/margin_mean": 139.16065979003906, "margin_dpo/margin_std": 163.25885009765625, "step": 227 }, { "KL/chosen_KL_mean": -154.69204711914062, "KL/mean": -203.87847900390625, "KL/rejected_KL_mean": -253.0648956298828, "KL/std": 127.062255859375, "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.004066440276801586, "fcm_dpo/delta": -5.455687642097473e-05, "fcm_dpo/margin": 98.37286376953125, "fcm_dpo/q_t": 0.40937620401382446, "grad_norm": 25.941659927368164, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -0.3328084647655487, "logits/rejected": -0.31316232681274414, "logps/chosen": -220.5833282470703, "logps/ref_chosen": -65.89128875732422, "logps/ref_rejected": -91.04875183105469, "logps/rejected": -344.1136474609375, "loss": 1.1158, "margin_dpo/margin_mean": 98.37286376953125, "margin_dpo/margin_std": 152.87173461914062, "step": 228 }, { "KL/chosen_KL_mean": -146.78775024414062, "KL/mean": -200.69815063476562, "KL/rejected_KL_mean": -254.60853576660156, "KL/std": 140.32000732421875, "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.004051582887768745, "fcm_dpo/delta": -0.03766883909702301, "fcm_dpo/margin": 107.82078552246094, "fcm_dpo/q_t": 0.4005894660949707, "grad_norm": 39.35184860229492, "learning_rate": 4.212490049118951e-07, "logits/chosen": -0.409065306186676, "logits/rejected": -0.3768808841705322, "logps/chosen": -217.494140625, "logps/ref_chosen": -70.70637512207031, "logps/ref_rejected": -84.52741241455078, "logps/rejected": -339.1359558105469, "loss": 1.082, "margin_dpo/margin_mean": 107.82078552246094, "margin_dpo/margin_std": 151.07818603515625, "step": 229 }, { "KL/chosen_KL_mean": -113.96424102783203, "KL/mean": -189.5445556640625, "KL/rejected_KL_mean": -265.1248779296875, "KL/std": 134.1195068359375, "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.003982962109148502, "fcm_dpo/delta": -0.2084825038909912, "fcm_dpo/margin": 151.16061401367188, "fcm_dpo/q_t": 0.3635350465774536, "grad_norm": 26.65117645263672, "learning_rate": 4.203117865141635e-07, "logits/chosen": -0.28100255131721497, "logits/rejected": -0.2840505540370941, "logps/chosen": -153.24624633789062, "logps/ref_chosen": -39.282005310058594, "logps/ref_rejected": -85.62191009521484, "logps/rejected": -350.74676513671875, "loss": 0.9531, "margin_dpo/margin_mean": 151.16061401367188, "margin_dpo/margin_std": 151.04591369628906, "step": 230 }, { "KL/chosen_KL_mean": -125.48028564453125, "KL/mean": -177.44976806640625, "KL/rejected_KL_mean": -229.41925048828125, "KL/std": 115.8082275390625, "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.003960724454373121, "fcm_dpo/delta": -0.011919192038476467, "fcm_dpo/margin": 103.93896484375, "fcm_dpo/q_t": 0.4071752727031708, "grad_norm": 28.911128997802734, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -0.3797636032104492, "logits/rejected": -0.35482439398765564, "logps/chosen": -188.75672912597656, "logps/ref_chosen": -63.27644348144531, "logps/ref_rejected": -74.1239013671875, "logps/rejected": -303.54315185546875, "loss": 1.0837, "margin_dpo/margin_mean": 103.93896484375, "margin_dpo/margin_std": 136.3351287841797, "step": 231 }, { "KL/chosen_KL_mean": -152.2584228515625, "KL/mean": -195.20050048828125, "KL/rejected_KL_mean": -238.14256286621094, "KL/std": 134.2640838623047, "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.0039717936888337135, "fcm_dpo/delta": 0.059911780059337616, "fcm_dpo/margin": 85.88412475585938, "fcm_dpo/q_t": 0.4243953824043274, "grad_norm": 27.307865142822266, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -0.3296153247356415, "logits/rejected": -0.3048354983329773, "logps/chosen": -223.00718688964844, "logps/ref_chosen": -70.74876403808594, "logps/ref_rejected": -83.97706604003906, "logps/rejected": -322.11962890625, "loss": 1.1477, "margin_dpo/margin_mean": 85.88412475585938, "margin_dpo/margin_std": 142.83050537109375, "step": 232 }, { "KL/chosen_KL_mean": -135.91783142089844, "KL/mean": -204.65484619140625, "KL/rejected_KL_mean": -273.3918762207031, "KL/std": 146.017333984375, "epoch": 0.342143906020558, "fcm_dpo/beta": 0.003943287767469883, "fcm_dpo/delta": -0.14597077667713165, "fcm_dpo/margin": 137.4740447998047, "fcm_dpo/q_t": 0.38463860750198364, "grad_norm": 24.649152755737305, "learning_rate": 4.174733034541245e-07, "logits/chosen": -0.34792935848236084, "logits/rejected": -0.3480525016784668, "logps/chosen": -190.80076599121094, "logps/ref_chosen": -54.8829345703125, "logps/ref_rejected": -107.4800796508789, "logps/rejected": -380.8719482421875, "loss": 1.0429, "margin_dpo/margin_mean": 137.47402954101562, "margin_dpo/margin_std": 192.8949737548828, "step": 233 }, { "KL/chosen_KL_mean": -127.90108489990234, "KL/mean": -198.03463745117188, "KL/rejected_KL_mean": -268.168212890625, "KL/std": 129.72488403320312, "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.0038681491278111935, "fcm_dpo/delta": -0.14766976237297058, "fcm_dpo/margin": 140.26710510253906, "fcm_dpo/q_t": 0.3764752745628357, "grad_norm": 24.525766372680664, "learning_rate": 4.165182829193126e-07, "logits/chosen": -0.3588540554046631, "logits/rejected": -0.3863643705844879, "logps/chosen": -171.99560546875, "logps/ref_chosen": -44.094520568847656, "logps/ref_rejected": -100.00663757324219, "logps/rejected": -368.1748352050781, "loss": 0.9883, "margin_dpo/margin_mean": 140.26710510253906, "margin_dpo/margin_std": 142.52896118164062, "step": 234 }, { "KL/chosen_KL_mean": -163.20513916015625, "KL/mean": -208.35916137695312, "KL/rejected_KL_mean": -253.51319885253906, "KL/std": 124.54771423339844, "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.0038875495083630085, "fcm_dpo/delta": 0.049665797501802444, "fcm_dpo/margin": 90.30805969238281, "fcm_dpo/q_t": 0.42041072249412537, "grad_norm": 26.740400314331055, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -0.3990609049797058, "logits/rejected": -0.37867674231529236, "logps/chosen": -225.44305419921875, "logps/ref_chosen": -62.237911224365234, "logps/ref_rejected": -90.39506530761719, "logps/rejected": -343.90826416015625, "loss": 1.1485, "margin_dpo/margin_mean": 90.30805969238281, "margin_dpo/margin_std": 152.52548217773438, "step": 235 }, { "KL/chosen_KL_mean": -124.98179626464844, "KL/mean": -199.99957275390625, "KL/rejected_KL_mean": -275.0173645019531, "KL/std": 138.9434051513672, "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.0038370825350284576, "fcm_dpo/delta": -0.18083232641220093, "fcm_dpo/margin": 150.0355682373047, "fcm_dpo/q_t": 0.3679265081882477, "grad_norm": 33.604164123535156, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -0.37230125069618225, "logits/rejected": -0.3716471195220947, "logps/chosen": -174.32315063476562, "logps/ref_chosen": -49.34136199951172, "logps/ref_rejected": -103.51162719726562, "logps/rejected": -378.52899169921875, "loss": 0.9526, "margin_dpo/margin_mean": 150.0355682373047, "margin_dpo/margin_std": 135.6669464111328, "step": 236 }, { "KL/chosen_KL_mean": -173.83096313476562, "KL/mean": -229.819580078125, "KL/rejected_KL_mean": -285.8081970214844, "KL/std": 140.05715942382812, "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.003812191542237997, "fcm_dpo/delta": -0.027462899684906006, "fcm_dpo/margin": 111.97723388671875, "fcm_dpo/q_t": 0.40431371331214905, "grad_norm": 30.839101791381836, "learning_rate": 4.136269950853473e-07, "logits/chosen": -0.3602313995361328, "logits/rejected": -0.35376355051994324, "logps/chosen": -227.99908447265625, "logps/ref_chosen": -54.168121337890625, "logps/ref_rejected": -94.78036499023438, "logps/rejected": -380.58856201171875, "loss": 1.0919, "margin_dpo/margin_mean": 111.97724151611328, "margin_dpo/margin_std": 162.9444580078125, "step": 237 }, { "KL/chosen_KL_mean": -161.74685668945312, "KL/mean": -218.63809204101562, "KL/rejected_KL_mean": -275.5293273925781, "KL/std": 138.28067016601562, "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.0037880742456763983, "fcm_dpo/delta": -0.03207089379429817, "fcm_dpo/margin": 113.78250122070312, "fcm_dpo/q_t": 0.4048614501953125, "grad_norm": 25.45376968383789, "learning_rate": 4.126545750510605e-07, "logits/chosen": -0.3611815571784973, "logits/rejected": -0.3783150911331177, "logps/chosen": -215.719970703125, "logps/ref_chosen": -53.973121643066406, "logps/ref_rejected": -89.41795349121094, "logps/rejected": -364.947265625, "loss": 1.0809, "margin_dpo/margin_mean": 113.7824935913086, "margin_dpo/margin_std": 156.38768005371094, "step": 238 }, { "KL/chosen_KL_mean": -173.99893188476562, "KL/mean": -237.2444305419922, "KL/rejected_KL_mean": -300.4898986816406, "KL/std": 130.01622009277344, "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.0037591117434203625, "fcm_dpo/delta": -0.07816985249519348, "fcm_dpo/margin": 126.49100494384766, "fcm_dpo/q_t": 0.39054858684539795, "grad_norm": 27.46272087097168, "learning_rate": 4.116778689174514e-07, "logits/chosen": -0.3645399808883667, "logits/rejected": -0.34787994623184204, "logps/chosen": -232.09674072265625, "logps/ref_chosen": -58.09782409667969, "logps/ref_rejected": -93.59294128417969, "logps/rejected": -394.0828552246094, "loss": 1.0356, "margin_dpo/margin_mean": 126.49099731445312, "margin_dpo/margin_std": 139.7375030517578, "step": 239 }, { "KL/chosen_KL_mean": -181.10501098632812, "KL/mean": -235.21560668945312, "KL/rejected_KL_mean": -289.326171875, "KL/std": 143.65615844726562, "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.003757942235097289, "fcm_dpo/delta": -0.006928959861397743, "fcm_dpo/margin": 108.22117614746094, "fcm_dpo/q_t": 0.40816569328308105, "grad_norm": 32.250770568847656, "learning_rate": 4.106969024216348e-07, "logits/chosen": -0.35770127177238464, "logits/rejected": -0.3304799795150757, "logps/chosen": -241.719482421875, "logps/ref_chosen": -60.6144905090332, "logps/ref_rejected": -74.1185302734375, "logps/rejected": -363.4447021484375, "loss": 1.1151, "margin_dpo/margin_mean": 108.2211685180664, "margin_dpo/margin_std": 168.62789916992188, "step": 240 }, { "KL/chosen_KL_mean": -149.57997131347656, "KL/mean": -231.7826690673828, "KL/rejected_KL_mean": -313.9853820800781, "KL/std": 156.34739685058594, "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0037074610590934753, "fcm_dpo/delta": -0.21590769290924072, "fcm_dpo/margin": 164.4053955078125, "fcm_dpo/q_t": 0.3671102821826935, "grad_norm": 24.17862892150879, "learning_rate": 4.097117014129903e-07, "logits/chosen": -0.4300222396850586, "logits/rejected": -0.4012630581855774, "logps/chosen": -215.67105102539062, "logps/ref_chosen": -66.091064453125, "logps/ref_rejected": -88.06088256835938, "logps/rejected": -402.0462646484375, "loss": 0.9689, "margin_dpo/margin_mean": 164.4053955078125, "margin_dpo/margin_std": 181.3541717529297, "step": 241 }, { "KL/chosen_KL_mean": -168.50222778320312, "KL/mean": -227.4334716796875, "KL/rejected_KL_mean": -286.3647155761719, "KL/std": 131.76930236816406, "epoch": 0.355359765051395, "fcm_dpo/beta": 0.0036807293072342873, "fcm_dpo/delta": -0.03467430919408798, "fcm_dpo/margin": 117.86245727539062, "fcm_dpo/q_t": 0.4024999737739563, "grad_norm": 26.07633399963379, "learning_rate": 4.087222918524807e-07, "logits/chosen": -0.35617491602897644, "logits/rejected": -0.33020225167274475, "logps/chosen": -236.3661651611328, "logps/ref_chosen": -67.86392974853516, "logps/ref_rejected": -83.36033630371094, "logps/rejected": -369.72503662109375, "loss": 1.0726, "margin_dpo/margin_mean": 117.86245727539062, "margin_dpo/margin_std": 153.87033081054688, "step": 242 }, { "KL/chosen_KL_mean": -169.4001922607422, "KL/mean": -241.8457794189453, "KL/rejected_KL_mean": -314.2913818359375, "KL/std": 151.13641357421875, "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.003632163628935814, "fcm_dpo/delta": -0.12977714836597443, "fcm_dpo/margin": 144.89117431640625, "fcm_dpo/q_t": 0.3812503218650818, "grad_norm": 23.20541763305664, "learning_rate": 4.07728699811968e-07, "logits/chosen": -0.3599437475204468, "logits/rejected": -0.32367074489593506, "logps/chosen": -232.48443603515625, "logps/ref_chosen": -63.0842399597168, "logps/ref_rejected": -76.33563232421875, "logps/rejected": -390.62701416015625, "loss": 1.0077, "margin_dpo/margin_mean": 144.89117431640625, "margin_dpo/margin_std": 165.2962646484375, "step": 243 }, { "KL/chosen_KL_mean": -147.78445434570312, "KL/mean": -222.11541748046875, "KL/rejected_KL_mean": -296.4464111328125, "KL/std": 144.9271240234375, "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.0035813949070870876, "fcm_dpo/delta": -0.13626864552497864, "fcm_dpo/margin": 148.6619415283203, "fcm_dpo/q_t": 0.3784053921699524, "grad_norm": 27.900978088378906, "learning_rate": 4.067309514735267e-07, "logits/chosen": -0.42759430408477783, "logits/rejected": -0.41793277859687805, "logps/chosen": -208.92514038085938, "logps/ref_chosen": -61.140689849853516, "logps/ref_rejected": -94.89193725585938, "logps/rejected": -391.33831787109375, "loss": 0.9878, "margin_dpo/margin_mean": 148.66195678710938, "margin_dpo/margin_std": 146.8434295654297, "step": 244 }, { "KL/chosen_KL_mean": -168.794677734375, "KL/mean": -229.46194458007812, "KL/rejected_KL_mean": -290.12921142578125, "KL/std": 146.95806884765625, "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.003548422595486045, "fcm_dpo/delta": -0.03223326802253723, "fcm_dpo/margin": 121.33451843261719, "fcm_dpo/q_t": 0.4033067524433136, "grad_norm": 29.785207748413086, "learning_rate": 4.057290731287531e-07, "logits/chosen": -0.4032437801361084, "logits/rejected": -0.3697701692581177, "logps/chosen": -236.05697631835938, "logps/ref_chosen": -67.26228332519531, "logps/ref_rejected": -87.64010620117188, "logps/rejected": -377.769287109375, "loss": 1.0876, "margin_dpo/margin_mean": 121.33451843261719, "margin_dpo/margin_std": 165.6243438720703, "step": 245 }, { "KL/chosen_KL_mean": -169.9764404296875, "KL/mean": -230.96017456054688, "KL/rejected_KL_mean": -291.94390869140625, "KL/std": 162.06460571289062, "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.003547412808984518, "fcm_dpo/delta": -0.0335458442568779, "fcm_dpo/margin": 121.96745300292969, "fcm_dpo/q_t": 0.4048032760620117, "grad_norm": 29.99637222290039, "learning_rate": 4.047230911780736e-07, "logits/chosen": -0.4445304274559021, "logits/rejected": -0.4077053964138031, "logps/chosen": -236.6734161376953, "logps/ref_chosen": -66.69696807861328, "logps/ref_rejected": -84.34634399414062, "logps/rejected": -376.2902526855469, "loss": 1.0863, "margin_dpo/margin_mean": 121.96745300292969, "margin_dpo/margin_std": 176.64109802246094, "step": 246 }, { "KL/chosen_KL_mean": -202.48690795898438, "KL/mean": -290.2227478027344, "KL/rejected_KL_mean": -377.9586181640625, "KL/std": 172.92800903320312, "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.0034898994490504265, "fcm_dpo/delta": -0.21926915645599365, "fcm_dpo/margin": 175.4716796875, "fcm_dpo/q_t": 0.3639224171638489, "grad_norm": 31.32638931274414, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -0.34930846095085144, "logits/rejected": -0.3457414507865906, "logps/chosen": -259.0922546386719, "logps/ref_chosen": -56.6053466796875, "logps/ref_rejected": -106.29326629638672, "logps/rejected": -484.2518615722656, "loss": 0.9621, "margin_dpo/margin_mean": 175.4716796875, "margin_dpo/margin_std": 188.98797607421875, "step": 247 }, { "KL/chosen_KL_mean": -176.61392211914062, "KL/mean": -252.390625, "KL/rejected_KL_mean": -328.1673278808594, "KL/std": 141.5336151123047, "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.003434403333812952, "fcm_dpo/delta": -0.12410237640142441, "fcm_dpo/margin": 151.55343627929688, "fcm_dpo/q_t": 0.3788222670555115, "grad_norm": 26.931861877441406, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -0.35363906621932983, "logits/rejected": -0.37153977155685425, "logps/chosen": -220.65713500976562, "logps/ref_chosen": -44.043216705322266, "logps/ref_rejected": -91.85687255859375, "logps/rejected": -420.02423095703125, "loss": 0.9855, "margin_dpo/margin_mean": 151.55343627929688, "margin_dpo/margin_std": 138.50741577148438, "step": 248 }, { "KL/chosen_KL_mean": -223.4722137451172, "KL/mean": -264.677734375, "KL/rejected_KL_mean": -305.8832702636719, "KL/std": 150.7882080078125, "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.0034578610211610794, "fcm_dpo/delta": 0.11675933003425598, "fcm_dpo/margin": 82.41104888916016, "fcm_dpo/q_t": 0.43573471903800964, "grad_norm": 46.4980583190918, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -0.3905929923057556, "logits/rejected": -0.3669302463531494, "logps/chosen": -285.91455078125, "logps/ref_chosen": -62.442352294921875, "logps/ref_rejected": -80.46806335449219, "logps/rejected": -386.351318359375, "loss": 1.2111, "margin_dpo/margin_mean": 82.41105651855469, "margin_dpo/margin_std": 178.0140380859375, "step": 249 }, { "KL/chosen_KL_mean": -191.8778076171875, "KL/mean": -268.19073486328125, "KL/rejected_KL_mean": -344.503662109375, "KL/std": 147.71640014648438, "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.003444279544055462, "fcm_dpo/delta": -0.12898439168930054, "fcm_dpo/margin": 152.62586975097656, "fcm_dpo/q_t": 0.3783637285232544, "grad_norm": 38.140594482421875, "learning_rate": 4.006586590948141e-07, "logits/chosen": -0.37574708461761475, "logits/rejected": -0.3200438618659973, "logps/chosen": -257.5145263671875, "logps/ref_chosen": -65.63668823242188, "logps/ref_rejected": -73.87184143066406, "logps/rejected": -418.3755187988281, "loss": 0.9902, "margin_dpo/margin_mean": 152.62586975097656, "margin_dpo/margin_std": 148.07838439941406, "step": 250 }, { "KL/chosen_KL_mean": -210.373291015625, "KL/mean": -262.0963134765625, "KL/rejected_KL_mean": -313.8193664550781, "KL/std": 158.57400512695312, "epoch": 0.368575624082232, "fcm_dpo/beta": 0.003431498073041439, "fcm_dpo/delta": 0.045787617564201355, "fcm_dpo/margin": 103.44607543945312, "fcm_dpo/q_t": 0.4200136065483093, "grad_norm": 31.18090057373047, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -0.3738080859184265, "logits/rejected": -0.34553277492523193, "logps/chosen": -267.5559997558594, "logps/ref_chosen": -57.182716369628906, "logps/ref_rejected": -77.66343688964844, "logps/rejected": -391.4827880859375, "loss": 1.1472, "margin_dpo/margin_mean": 103.44607543945312, "margin_dpo/margin_std": 171.2592010498047, "step": 251 }, { "KL/chosen_KL_mean": -202.80035400390625, "KL/mean": -273.161865234375, "KL/rejected_KL_mean": -343.52337646484375, "KL/std": 140.35821533203125, "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.003416198305785656, "fcm_dpo/delta": -0.08289383351802826, "fcm_dpo/margin": 140.72303771972656, "fcm_dpo/q_t": 0.38559865951538086, "grad_norm": 29.2058048248291, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -0.413099467754364, "logits/rejected": -0.3722429573535919, "logps/chosen": -274.4859924316406, "logps/ref_chosen": -71.68563842773438, "logps/ref_rejected": -84.75799560546875, "logps/rejected": -428.2813720703125, "loss": 1.0164, "margin_dpo/margin_mean": 140.72303771972656, "margin_dpo/margin_std": 137.57774353027344, "step": 252 }, { "KL/chosen_KL_mean": -172.536865234375, "KL/mean": -240.89328002929688, "KL/rejected_KL_mean": -309.2496643066406, "KL/std": 148.74118041992188, "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.0034063609782606363, "fcm_dpo/delta": -0.06741885095834732, "fcm_dpo/margin": 136.71279907226562, "fcm_dpo/q_t": 0.392569899559021, "grad_norm": 23.644466400146484, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -0.414895236492157, "logits/rejected": -0.4082959294319153, "logps/chosen": -241.67080688476562, "logps/ref_chosen": -69.1339340209961, "logps/ref_rejected": -98.70252990722656, "logps/rejected": -407.95220947265625, "loss": 1.0485, "margin_dpo/margin_mean": 136.71279907226562, "margin_dpo/margin_std": 162.5023193359375, "step": 253 }, { "KL/chosen_KL_mean": -175.3002166748047, "KL/mean": -234.9326171875, "KL/rejected_KL_mean": -294.5650329589844, "KL/std": 157.5005645751953, "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.003387467935681343, "fcm_dpo/delta": -0.0041029490530490875, "fcm_dpo/margin": 119.26481628417969, "fcm_dpo/q_t": 0.4124619662761688, "grad_norm": 26.61585235595703, "learning_rate": 3.965307091713037e-07, "logits/chosen": -0.377765953540802, "logits/rejected": -0.3579869270324707, "logps/chosen": -229.45521545410156, "logps/ref_chosen": -54.154998779296875, "logps/ref_rejected": -90.30764770507812, "logps/rejected": -384.8726806640625, "loss": 1.1203, "margin_dpo/margin_mean": 119.26481628417969, "margin_dpo/margin_std": 196.88693237304688, "step": 254 }, { "KL/chosen_KL_mean": -181.1912841796875, "KL/mean": -242.2811279296875, "KL/rejected_KL_mean": -303.3709716796875, "KL/std": 136.550048828125, "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.003378466470167041, "fcm_dpo/delta": -0.013259213417768478, "fcm_dpo/margin": 122.17970275878906, "fcm_dpo/q_t": 0.4052705764770508, "grad_norm": 23.39793586730957, "learning_rate": 3.954890003969163e-07, "logits/chosen": -0.36594948172569275, "logits/rejected": -0.35779714584350586, "logps/chosen": -238.33294677734375, "logps/ref_chosen": -57.14167022705078, "logps/ref_rejected": -90.2085952758789, "logps/rejected": -393.57958984375, "loss": 1.0967, "margin_dpo/margin_mean": 122.17970275878906, "margin_dpo/margin_std": 172.21868896484375, "step": 255 }, { "KL/chosen_KL_mean": -156.4071044921875, "KL/mean": -223.8316650390625, "KL/rejected_KL_mean": -291.2562561035156, "KL/std": 148.6212158203125, "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.003370137419551611, "fcm_dpo/delta": -0.05573172867298126, "fcm_dpo/margin": 134.84915161132812, "fcm_dpo/q_t": 0.3962138891220093, "grad_norm": 22.66250228881836, "learning_rate": 3.944434578520628e-07, "logits/chosen": -0.3113047480583191, "logits/rejected": -0.31630462408065796, "logps/chosen": -211.57058715820312, "logps/ref_chosen": -55.163490295410156, "logps/ref_rejected": -92.56291961669922, "logps/rejected": -383.81915283203125, "loss": 1.0567, "margin_dpo/margin_mean": 134.84913635253906, "margin_dpo/margin_std": 168.68287658691406, "step": 256 }, { "KL/chosen_KL_mean": -151.5054931640625, "KL/mean": -221.162109375, "KL/rejected_KL_mean": -290.8187255859375, "KL/std": 151.1150665283203, "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.0033370412420481443, "fcm_dpo/delta": -0.06718017160892487, "fcm_dpo/margin": 139.313232421875, "fcm_dpo/q_t": 0.39490461349487305, "grad_norm": 20.624574661254883, "learning_rate": 3.933941090877615e-07, "logits/chosen": -0.32282236218452454, "logits/rejected": -0.3079400360584259, "logps/chosen": -200.92919921875, "logps/ref_chosen": -49.42369842529297, "logps/ref_rejected": -79.53791809082031, "logps/rejected": -370.35662841796875, "loss": 1.0528, "margin_dpo/margin_mean": 139.313232421875, "margin_dpo/margin_std": 168.8369598388672, "step": 257 }, { "KL/chosen_KL_mean": -196.22232055664062, "KL/mean": -263.02752685546875, "KL/rejected_KL_mean": -329.832763671875, "KL/std": 156.14871215820312, "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.0033341706730425358, "fcm_dpo/delta": -0.0465044304728508, "fcm_dpo/margin": 133.61041259765625, "fcm_dpo/q_t": 0.3972247242927551, "grad_norm": 24.023584365844727, "learning_rate": 3.923409817553284e-07, "logits/chosen": -0.316849946975708, "logits/rejected": -0.3123030960559845, "logps/chosen": -255.6064453125, "logps/ref_chosen": -59.384124755859375, "logps/ref_rejected": -95.99010467529297, "logps/rejected": -425.8228454589844, "loss": 1.0785, "margin_dpo/margin_mean": 133.61041259765625, "margin_dpo/margin_std": 185.94827270507812, "step": 258 }, { "KL/chosen_KL_mean": -187.66934204101562, "KL/mean": -247.58529663085938, "KL/rejected_KL_mean": -307.50128173828125, "KL/std": 151.50286865234375, "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.0033223240170627832, "fcm_dpo/delta": 0.001799125224351883, "fcm_dpo/margin": 119.83191680908203, "fcm_dpo/q_t": 0.409615159034729, "grad_norm": 26.094667434692383, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -0.3730711042881012, "logits/rejected": -0.36661672592163086, "logps/chosen": -240.4976806640625, "logps/ref_chosen": -52.828346252441406, "logps/ref_rejected": -89.191650390625, "logps/rejected": -396.69293212890625, "loss": 1.1018, "margin_dpo/margin_mean": 119.83192443847656, "margin_dpo/margin_std": 169.11459350585938, "step": 259 }, { "KL/chosen_KL_mean": -195.3187255859375, "KL/mean": -271.7413330078125, "KL/rejected_KL_mean": -348.1639404296875, "KL/std": 158.65463256835938, "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.003307567210868001, "fcm_dpo/delta": -0.10822096467018127, "fcm_dpo/margin": 152.84521484375, "fcm_dpo/q_t": 0.3863574266433716, "grad_norm": 33.96175003051758, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -0.35657697916030884, "logits/rejected": -0.3744693994522095, "logps/chosen": -242.73638916015625, "logps/ref_chosen": -47.41767501831055, "logps/ref_rejected": -95.08978271484375, "logps/rejected": -443.25372314453125, "loss": 1.013, "margin_dpo/margin_mean": 152.84521484375, "margin_dpo/margin_std": 169.8836212158203, "step": 260 }, { "KL/chosen_KL_mean": -197.50112915039062, "KL/mean": -269.7708435058594, "KL/rejected_KL_mean": -342.04052734375, "KL/std": 168.48382568359375, "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.0032701774034649134, "fcm_dpo/delta": -0.07457753270864487, "fcm_dpo/margin": 144.5394287109375, "fcm_dpo/q_t": 0.39346715807914734, "grad_norm": 23.455717086791992, "learning_rate": 3.891592063515376e-07, "logits/chosen": -0.3204799294471741, "logits/rejected": -0.31587928533554077, "logps/chosen": -250.5325164794922, "logps/ref_chosen": -53.03137969970703, "logps/ref_rejected": -88.51494598388672, "logps/rejected": -430.55548095703125, "loss": 1.0506, "margin_dpo/margin_mean": 144.5394287109375, "margin_dpo/margin_std": 183.651611328125, "step": 261 }, { "KL/chosen_KL_mean": -230.0545654296875, "KL/mean": -290.9325256347656, "KL/rejected_KL_mean": -351.8105163574219, "KL/std": 155.808349609375, "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.0032733541447669268, "fcm_dpo/delta": 0.0013803504407405853, "fcm_dpo/margin": 121.75595092773438, "fcm_dpo/q_t": 0.4075956344604492, "grad_norm": 29.473356246948242, "learning_rate": 3.880912432401264e-07, "logits/chosen": -0.3197232484817505, "logits/rejected": -0.28875744342803955, "logps/chosen": -289.6746826171875, "logps/ref_chosen": -59.620140075683594, "logps/ref_rejected": -86.41853332519531, "logps/rejected": -438.22906494140625, "loss": 1.086, "margin_dpo/margin_mean": 121.75595092773438, "margin_dpo/margin_std": 153.61898803710938, "step": 262 }, { "KL/chosen_KL_mean": -204.87518310546875, "KL/mean": -291.5097351074219, "KL/rejected_KL_mean": -378.1442565917969, "KL/std": 180.58358764648438, "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.003221530467271805, "fcm_dpo/delta": -0.16330862045288086, "fcm_dpo/margin": 173.26904296875, "fcm_dpo/q_t": 0.3738434314727783, "grad_norm": 23.88220977783203, "learning_rate": 3.870196412960302e-07, "logits/chosen": -0.3573540151119232, "logits/rejected": -0.3329017460346222, "logps/chosen": -264.296142578125, "logps/ref_chosen": -59.42094421386719, "logps/ref_rejected": -96.85720825195312, "logps/rejected": -475.00146484375, "loss": 0.9867, "margin_dpo/margin_mean": 173.26905822753906, "margin_dpo/margin_std": 186.5802764892578, "step": 263 }, { "KL/chosen_KL_mean": -210.99795532226562, "KL/mean": -283.88690185546875, "KL/rejected_KL_mean": -356.77581787109375, "KL/std": 164.59521484375, "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.0031880121678113937, "fcm_dpo/delta": -0.06703611463308334, "fcm_dpo/margin": 145.77786254882812, "fcm_dpo/q_t": 0.39508941769599915, "grad_norm": 27.110586166381836, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -0.374176949262619, "logits/rejected": -0.3629017472267151, "logps/chosen": -273.72003173828125, "logps/ref_chosen": -62.722084045410156, "logps/ref_rejected": -93.85620880126953, "logps/rejected": -450.6320495605469, "loss": 1.0561, "margin_dpo/margin_mean": 145.77786254882812, "margin_dpo/margin_std": 182.66200256347656, "step": 264 }, { "KL/chosen_KL_mean": -237.04635620117188, "KL/mean": -308.437744140625, "KL/rejected_KL_mean": -379.8291015625, "KL/std": 194.23110961914062, "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.0031799268908798695, "fcm_dpo/delta": -0.0553123913705349, "fcm_dpo/margin": 142.78271484375, "fcm_dpo/q_t": 0.40076446533203125, "grad_norm": 29.41632843017578, "learning_rate": 3.848656339557562e-07, "logits/chosen": -0.33580052852630615, "logits/rejected": -0.31790751218795776, "logps/chosen": -299.017822265625, "logps/ref_chosen": -61.971466064453125, "logps/ref_rejected": -88.02059936523438, "logps/rejected": -467.8497009277344, "loss": 1.0958, "margin_dpo/margin_mean": 142.78274536132812, "margin_dpo/margin_std": 224.72232055664062, "step": 265 }, { "KL/chosen_KL_mean": -247.69691467285156, "KL/mean": -305.4302978515625, "KL/rejected_KL_mean": -363.1636962890625, "KL/std": 160.6911163330078, "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.0031850021332502365, "fcm_dpo/delta": 0.03283064067363739, "fcm_dpo/margin": 115.46676635742188, "fcm_dpo/q_t": 0.4174157381057739, "grad_norm": 63.25857925415039, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -0.36059173941612244, "logits/rejected": -0.3145703673362732, "logps/chosen": -314.79656982421875, "logps/ref_chosen": -67.09967041015625, "logps/ref_rejected": -67.97122192382812, "logps/rejected": -431.1349182128906, "loss": 1.1341, "margin_dpo/margin_mean": 115.46676635742188, "margin_dpo/margin_std": 187.57647705078125, "step": 266 }, { "KL/chosen_KL_mean": -224.26531982421875, "KL/mean": -294.53692626953125, "KL/rejected_KL_mean": -364.8084716796875, "KL/std": 172.51197814941406, "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.00316983787342906, "fcm_dpo/delta": -0.046759672462940216, "fcm_dpo/margin": 140.54315185546875, "fcm_dpo/q_t": 0.39952901005744934, "grad_norm": 43.019325256347656, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -0.3794315457344055, "logits/rejected": -0.342626690864563, "logps/chosen": -293.236083984375, "logps/ref_chosen": -68.97075653076172, "logps/ref_rejected": -90.16844940185547, "logps/rejected": -454.9769287109375, "loss": 1.0756, "margin_dpo/margin_mean": 140.5431365966797, "margin_dpo/margin_std": 194.42465209960938, "step": 267 }, { "KL/chosen_KL_mean": -234.8605194091797, "KL/mean": -304.49365234375, "KL/rejected_KL_mean": -374.12677001953125, "KL/std": 169.82725524902344, "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.0031569194979965687, "fcm_dpo/delta": -0.04073052108287811, "fcm_dpo/margin": 139.26629638671875, "fcm_dpo/q_t": 0.402080237865448, "grad_norm": 35.164588928222656, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -0.36576220393180847, "logits/rejected": -0.3701912760734558, "logps/chosen": -290.7608337402344, "logps/ref_chosen": -55.90031051635742, "logps/ref_rejected": -101.64763641357422, "logps/rejected": -475.7744140625, "loss": 1.0845, "margin_dpo/margin_mean": 139.26629638671875, "margin_dpo/margin_std": 201.86578369140625, "step": 268 }, { "KL/chosen_KL_mean": -243.96658325195312, "KL/mean": -328.11651611328125, "KL/rejected_KL_mean": -412.26641845703125, "KL/std": 171.32447814941406, "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.0031313346698880196, "fcm_dpo/delta": -0.13035503029823303, "fcm_dpo/margin": 168.2998504638672, "fcm_dpo/q_t": 0.38259977102279663, "grad_norm": 30.891639709472656, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -0.4082416594028473, "logits/rejected": -0.3874931335449219, "logps/chosen": -314.00616455078125, "logps/ref_chosen": -70.03955841064453, "logps/ref_rejected": -107.34937286376953, "logps/rejected": -519.6158447265625, "loss": 1.0322, "margin_dpo/margin_mean": 168.29983520507812, "margin_dpo/margin_std": 217.715576171875, "step": 269 }, { "KL/chosen_KL_mean": -209.37474060058594, "KL/mean": -271.95452880859375, "KL/rejected_KL_mean": -334.53436279296875, "KL/std": 150.77197265625, "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.0031246920116245747, "fcm_dpo/delta": 0.008997350931167603, "fcm_dpo/margin": 125.15963745117188, "fcm_dpo/q_t": 0.41077637672424316, "grad_norm": 45.283546447753906, "learning_rate": 3.794189242333106e-07, "logits/chosen": -0.4233609139919281, "logits/rejected": -0.41205936670303345, "logps/chosen": -278.908203125, "logps/ref_chosen": -69.53347778320312, "logps/ref_rejected": -109.92864990234375, "logps/rejected": -444.4630126953125, "loss": 1.1118, "margin_dpo/margin_mean": 125.15963745117188, "margin_dpo/margin_std": 185.95555114746094, "step": 270 }, { "KL/chosen_KL_mean": -188.9541473388672, "KL/mean": -265.62396240234375, "KL/rejected_KL_mean": -342.2938232421875, "KL/std": 149.54208374023438, "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0030992082320153713, "fcm_dpo/delta": -0.07721711695194244, "fcm_dpo/margin": 153.33963012695312, "fcm_dpo/q_t": 0.39120548963546753, "grad_norm": 27.018428802490234, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -0.3589969277381897, "logits/rejected": -0.33432579040527344, "logps/chosen": -245.71871948242188, "logps/ref_chosen": -56.76456832885742, "logps/ref_rejected": -92.51383972167969, "logps/rejected": -434.8076477050781, "loss": 1.0279, "margin_dpo/margin_mean": 153.3396453857422, "margin_dpo/margin_std": 167.06747436523438, "step": 271 }, { "KL/chosen_KL_mean": -173.87306213378906, "KL/mean": -266.062744140625, "KL/rejected_KL_mean": -358.25244140625, "KL/std": 168.98089599609375, "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.003064734861254692, "fcm_dpo/delta": -0.16969643533229828, "fcm_dpo/margin": 184.37937927246094, "fcm_dpo/q_t": 0.3702283501625061, "grad_norm": 34.31309509277344, "learning_rate": 3.772161666010912e-07, "logits/chosen": -0.31992781162261963, "logits/rejected": -0.3296176791191101, "logps/chosen": -223.37022399902344, "logps/ref_chosen": -49.497154235839844, "logps/ref_rejected": -105.54279327392578, "logps/rejected": -463.79522705078125, "loss": 0.9675, "margin_dpo/margin_mean": 184.37937927246094, "margin_dpo/margin_std": 175.53350830078125, "step": 272 }, { "KL/chosen_KL_mean": -202.81802368164062, "KL/mean": -292.4516296386719, "KL/rejected_KL_mean": -382.08526611328125, "KL/std": 168.8096923828125, "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.0030071022920310497, "fcm_dpo/delta": -0.14330768585205078, "fcm_dpo/margin": 179.26719665527344, "fcm_dpo/q_t": 0.37718284130096436, "grad_norm": 23.817079544067383, "learning_rate": 3.761097448550755e-07, "logits/chosen": -0.3086550533771515, "logits/rejected": -0.28471726179122925, "logps/chosen": -265.7934265136719, "logps/ref_chosen": -62.97539520263672, "logps/ref_rejected": -92.49858093261719, "logps/rejected": -474.5838317871094, "loss": 0.9925, "margin_dpo/margin_mean": 179.26719665527344, "margin_dpo/margin_std": 188.98374938964844, "step": 273 }, { "KL/chosen_KL_mean": -239.08636474609375, "KL/mean": -309.47210693359375, "KL/rejected_KL_mean": -379.85784912109375, "KL/std": 157.9844970703125, "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.003000047989189625, "fcm_dpo/delta": -0.02283497154712677, "fcm_dpo/margin": 140.77145385742188, "fcm_dpo/q_t": 0.4034059941768646, "grad_norm": 29.533851623535156, "learning_rate": 3.75e-07, "logits/chosen": -0.2807186245918274, "logits/rejected": -0.260738730430603, "logps/chosen": -294.75408935546875, "logps/ref_chosen": -55.66770935058594, "logps/ref_rejected": -77.33308410644531, "logps/rejected": -457.19091796875, "loss": 1.077, "margin_dpo/margin_mean": 140.77145385742188, "margin_dpo/margin_std": 181.21035766601562, "step": 274 }, { "KL/chosen_KL_mean": -190.72259521484375, "KL/mean": -263.8143005371094, "KL/rejected_KL_mean": -336.906005859375, "KL/std": 159.49850463867188, "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.002990099135786295, "fcm_dpo/delta": -0.03795165196061134, "fcm_dpo/margin": 146.18341064453125, "fcm_dpo/q_t": 0.39963993430137634, "grad_norm": 28.2340145111084, "learning_rate": 3.738869612786737e-07, "logits/chosen": -0.3173510432243347, "logits/rejected": -0.3210340142250061, "logps/chosen": -239.31729125976562, "logps/ref_chosen": -48.594703674316406, "logps/ref_rejected": -93.30369567871094, "logps/rejected": -430.209716796875, "loss": 1.0664, "margin_dpo/margin_mean": 146.18341064453125, "margin_dpo/margin_std": 182.6461181640625, "step": 275 }, { "KL/chosen_KL_mean": -207.98233032226562, "KL/mean": -282.283203125, "KL/rejected_KL_mean": -356.5841064453125, "KL/std": 165.94070434570312, "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.002971104346215725, "fcm_dpo/delta": -0.04252216964960098, "fcm_dpo/margin": 148.60177612304688, "fcm_dpo/q_t": 0.4004755914211273, "grad_norm": 30.543088912963867, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -0.31873783469200134, "logits/rejected": -0.29504579305648804, "logps/chosen": -264.55975341796875, "logps/ref_chosen": -56.57740783691406, "logps/ref_rejected": -70.36566925048828, "logps/rejected": -426.94976806640625, "loss": 1.0728, "margin_dpo/margin_mean": 148.60177612304688, "margin_dpo/margin_std": 199.07870483398438, "step": 276 }, { "KL/chosen_KL_mean": -230.2869110107422, "KL/mean": -309.2855529785156, "KL/rejected_KL_mean": -388.2841796875, "KL/std": 167.18478393554688, "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.0029588094912469387, "fcm_dpo/delta": -0.0690733790397644, "fcm_dpo/margin": 157.99728393554688, "fcm_dpo/q_t": 0.39369434118270874, "grad_norm": 26.234636306762695, "learning_rate": 3.71651119641714e-07, "logits/chosen": -0.29924240708351135, "logits/rejected": -0.2815479636192322, "logps/chosen": -286.5584716796875, "logps/ref_chosen": -56.27156066894531, "logps/ref_rejected": -92.88127136230469, "logps/rejected": -481.16546630859375, "loss": 1.0493, "margin_dpo/margin_mean": 157.99728393554688, "margin_dpo/margin_std": 193.92010498046875, "step": 277 }, { "KL/chosen_KL_mean": -206.88705444335938, "KL/mean": -296.92340087890625, "KL/rejected_KL_mean": -386.95977783203125, "KL/std": 179.54661560058594, "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.0029181232675909996, "fcm_dpo/delta": -0.12909561395645142, "fcm_dpo/margin": 180.07273864746094, "fcm_dpo/q_t": 0.38142508268356323, "grad_norm": 31.345380783081055, "learning_rate": 3.705283756425872e-07, "logits/chosen": -0.2834761142730713, "logits/rejected": -0.28856098651885986, "logps/chosen": -259.8289794921875, "logps/ref_chosen": -52.94194030761719, "logps/ref_rejected": -91.25357818603516, "logps/rejected": -478.21337890625, "loss": 1.004, "margin_dpo/margin_mean": 180.07275390625, "margin_dpo/margin_std": 198.38058471679688, "step": 278 }, { "KL/chosen_KL_mean": -245.1502227783203, "KL/mean": -334.9271240234375, "KL/rejected_KL_mean": -424.7039794921875, "KL/std": 189.08995056152344, "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.0028769150376319885, "fcm_dpo/delta": -0.12047116458415985, "fcm_dpo/margin": 179.55380249023438, "fcm_dpo/q_t": 0.38520491123199463, "grad_norm": 30.17645263671875, "learning_rate": 3.6940245560867e-07, "logits/chosen": -0.23838657140731812, "logits/rejected": -0.23660916090011597, "logps/chosen": -293.7915344238281, "logps/ref_chosen": -48.641319274902344, "logps/ref_rejected": -87.8514404296875, "logps/rejected": -512.555419921875, "loss": 1.0304, "margin_dpo/margin_mean": 179.55380249023438, "margin_dpo/margin_std": 223.5946044921875, "step": 279 }, { "KL/chosen_KL_mean": -248.6533203125, "KL/mean": -343.9190673828125, "KL/rejected_KL_mean": -439.184814453125, "KL/std": 181.25201416015625, "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.002845579292625189, "fcm_dpo/delta": -0.14617213606834412, "fcm_dpo/margin": 190.531494140625, "fcm_dpo/q_t": 0.3750941753387451, "grad_norm": 27.4671688079834, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -0.271923303604126, "logits/rejected": -0.2730262279510498, "logps/chosen": -307.450439453125, "logps/ref_chosen": -58.797122955322266, "logps/ref_rejected": -98.61885070800781, "logps/rejected": -537.8037109375, "loss": 0.9844, "margin_dpo/margin_mean": 190.53150939941406, "margin_dpo/margin_std": 191.6612548828125, "step": 280 }, { "KL/chosen_KL_mean": -226.9546356201172, "KL/mean": -309.49078369140625, "KL/rejected_KL_mean": -392.0268859863281, "KL/std": 174.78067016601562, "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.0028265006840229034, "fcm_dpo/delta": -0.06817743927240372, "fcm_dpo/margin": 165.07223510742188, "fcm_dpo/q_t": 0.39088958501815796, "grad_norm": 26.444442749023438, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -0.3203830122947693, "logits/rejected": -0.2967671751976013, "logps/chosen": -282.44317626953125, "logps/ref_chosen": -55.488521575927734, "logps/ref_rejected": -80.88258361816406, "logps/rejected": -472.9094543457031, "loss": 1.0406, "margin_dpo/margin_mean": 165.07223510742188, "margin_dpo/margin_std": 188.06488037109375, "step": 281 }, { "KL/chosen_KL_mean": -251.03106689453125, "KL/mean": -316.7037658691406, "KL/rejected_KL_mean": -382.37646484375, "KL/std": 178.154541015625, "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.0028289747424423695, "fcm_dpo/delta": 0.028688903898000717, "fcm_dpo/margin": 131.34539794921875, "fcm_dpo/q_t": 0.41740673780441284, "grad_norm": 27.60743522644043, "learning_rate": 3.660059364023408e-07, "logits/chosen": -0.3705713748931885, "logits/rejected": -0.34871238470077515, "logps/chosen": -324.1011962890625, "logps/ref_chosen": -73.07014465332031, "logps/ref_rejected": -95.35098266601562, "logps/rejected": -477.7274169921875, "loss": 1.1241, "margin_dpo/margin_mean": 131.34539794921875, "margin_dpo/margin_std": 205.223876953125, "step": 282 }, { "KL/chosen_KL_mean": -265.2757263183594, "KL/mean": -361.2529296875, "KL/rejected_KL_mean": -457.23016357421875, "KL/std": 214.1881103515625, "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.002800577785819769, "fcm_dpo/delta": -0.14126329123973846, "fcm_dpo/margin": 191.95440673828125, "fcm_dpo/q_t": 0.37666594982147217, "grad_norm": 34.12551498413086, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -0.3212432265281677, "logits/rejected": -0.32373398542404175, "logps/chosen": -327.1741638183594, "logps/ref_chosen": -61.89844512939453, "logps/ref_rejected": -96.98655700683594, "logps/rejected": -554.2166748046875, "loss": 0.9907, "margin_dpo/margin_mean": 191.9544219970703, "margin_dpo/margin_std": 196.35537719726562, "step": 283 }, { "KL/chosen_KL_mean": -247.26083374023438, "KL/mean": -337.656982421875, "KL/rejected_KL_mean": -428.0531005859375, "KL/std": 198.7021484375, "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.0027607777155935764, "fcm_dpo/delta": -0.10182726383209229, "fcm_dpo/margin": 180.79226684570312, "fcm_dpo/q_t": 0.38608792424201965, "grad_norm": 41.67982864379883, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -0.3618922829627991, "logits/rejected": -0.3465798795223236, "logps/chosen": -305.6963806152344, "logps/ref_chosen": -58.4355354309082, "logps/ref_rejected": -93.46926879882812, "logps/rejected": -521.5223999023438, "loss": 1.0107, "margin_dpo/margin_mean": 180.79226684570312, "margin_dpo/margin_std": 191.53631591796875, "step": 284 }, { "KL/chosen_KL_mean": -280.42071533203125, "KL/mean": -361.66033935546875, "KL/rejected_KL_mean": -442.8999938964844, "KL/std": 175.34622192382812, "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.002754541113972664, "fcm_dpo/delta": -0.048833686858415604, "fcm_dpo/margin": 162.47930908203125, "fcm_dpo/q_t": 0.3953213095664978, "grad_norm": 26.926042556762695, "learning_rate": 3.625819059005228e-07, "logits/chosen": -0.3326127529144287, "logits/rejected": -0.31620490550994873, "logps/chosen": -346.65289306640625, "logps/ref_chosen": -66.23219299316406, "logps/ref_rejected": -99.1268310546875, "logps/rejected": -542.02685546875, "loss": 1.0516, "margin_dpo/margin_mean": 162.47930908203125, "margin_dpo/margin_std": 184.34237670898438, "step": 285 }, { "KL/chosen_KL_mean": -277.4112548828125, "KL/mean": -369.2188720703125, "KL/rejected_KL_mean": -461.0264892578125, "KL/std": 199.40582275390625, "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.0027244749944657087, "fcm_dpo/delta": -0.10277135670185089, "fcm_dpo/margin": 183.615234375, "fcm_dpo/q_t": 0.38644248247146606, "grad_norm": 28.425804138183594, "learning_rate": 3.614345889441346e-07, "logits/chosen": -0.3559093475341797, "logits/rejected": -0.3347648084163666, "logps/chosen": -350.36224365234375, "logps/ref_chosen": -72.95100402832031, "logps/ref_rejected": -88.58845520019531, "logps/rejected": -549.6149291992188, "loss": 1.0286, "margin_dpo/margin_mean": 183.615234375, "margin_dpo/margin_std": 217.96173095703125, "step": 286 }, { "KL/chosen_KL_mean": -261.5997314453125, "KL/mean": -335.04229736328125, "KL/rejected_KL_mean": -408.4848327636719, "KL/std": 173.95217895507812, "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.002716212999075651, "fcm_dpo/delta": 0.0010463353246450424, "fcm_dpo/margin": 146.88510131835938, "fcm_dpo/q_t": 0.4084421694278717, "grad_norm": 26.484556198120117, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -0.3276909589767456, "logits/rejected": -0.30225396156311035, "logps/chosen": -323.140869140625, "logps/ref_chosen": -61.54115295410156, "logps/ref_rejected": -77.69607543945312, "logps/rejected": -486.180908203125, "loss": 1.0875, "margin_dpo/margin_mean": 146.88510131835938, "margin_dpo/margin_std": 189.57159423828125, "step": 287 }, { "KL/chosen_KL_mean": -250.09185791015625, "KL/mean": -342.5625, "KL/rejected_KL_mean": -435.0331726074219, "KL/std": 174.55943298339844, "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.0026842262595891953, "fcm_dpo/delta": -0.09998185932636261, "fcm_dpo/margin": 184.9413299560547, "fcm_dpo/q_t": 0.38463830947875977, "grad_norm": 21.34337043762207, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -0.3534676432609558, "logits/rejected": -0.3342832326889038, "logps/chosen": -306.7530517578125, "logps/ref_chosen": -56.661224365234375, "logps/ref_rejected": -87.33570098876953, "logps/rejected": -522.368896484375, "loss": 1.0087, "margin_dpo/margin_mean": 184.94134521484375, "margin_dpo/margin_std": 177.844970703125, "step": 288 }, { "KL/chosen_KL_mean": -247.0579833984375, "KL/mean": -350.28240966796875, "KL/rejected_KL_mean": -453.50677490234375, "KL/std": 198.4517822265625, "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.002661726903170347, "fcm_dpo/delta": -0.15359723567962646, "fcm_dpo/margin": 206.44882202148438, "fcm_dpo/q_t": 0.3754338324069977, "grad_norm": 26.822708129882812, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -0.2866683602333069, "logits/rejected": -0.28309381008148193, "logps/chosen": -292.28839111328125, "logps/ref_chosen": -45.23039245605469, "logps/ref_rejected": -87.64266967773438, "logps/rejected": -541.1494140625, "loss": 0.9846, "margin_dpo/margin_mean": 206.44879150390625, "margin_dpo/margin_std": 213.51644897460938, "step": 289 }, { "KL/chosen_KL_mean": -268.5120849609375, "KL/mean": -371.7317810058594, "KL/rejected_KL_mean": -474.95147705078125, "KL/std": 210.44107055664062, "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.002616926096379757, "fcm_dpo/delta": -0.14428411424160004, "fcm_dpo/margin": 206.43942260742188, "fcm_dpo/q_t": 0.3804370164871216, "grad_norm": 24.522186279296875, "learning_rate": 3.568162605525952e-07, "logits/chosen": -0.2609245181083679, "logits/rejected": -0.2813086211681366, "logps/chosen": -323.98358154296875, "logps/ref_chosen": -55.47149658203125, "logps/ref_rejected": -116.70857238769531, "logps/rejected": -591.6600341796875, "loss": 1.0094, "margin_dpo/margin_mean": 206.43942260742188, "margin_dpo/margin_std": 247.92044067382812, "step": 290 }, { "KL/chosen_KL_mean": -230.06326293945312, "KL/mean": -323.6925048828125, "KL/rejected_KL_mean": -417.32177734375, "KL/std": 181.36814880371094, "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.0025994423776865005, "fcm_dpo/delta": -0.08899101614952087, "fcm_dpo/margin": 187.25848388671875, "fcm_dpo/q_t": 0.3880588114261627, "grad_norm": 31.68715476989746, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -0.3303547501564026, "logits/rejected": -0.31436848640441895, "logps/chosen": -293.3236083984375, "logps/ref_chosen": -63.26036834716797, "logps/ref_rejected": -89.29708862304688, "logps/rejected": -506.61883544921875, "loss": 1.0307, "margin_dpo/margin_mean": 187.25848388671875, "margin_dpo/margin_std": 211.5198974609375, "step": 291 }, { "KL/chosen_KL_mean": -261.43878173828125, "KL/mean": -362.51861572265625, "KL/rejected_KL_mean": -463.598388671875, "KL/std": 217.88809204101562, "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.0025680752005428076, "fcm_dpo/delta": -0.12225315719842911, "fcm_dpo/margin": 202.15960693359375, "fcm_dpo/q_t": 0.382857084274292, "grad_norm": 29.966228485107422, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -0.3206707537174225, "logits/rejected": -0.3061414361000061, "logps/chosen": -315.3572998046875, "logps/ref_chosen": -53.91852951049805, "logps/ref_rejected": -89.96138000488281, "logps/rejected": -553.559814453125, "loss": 1.0308, "margin_dpo/margin_mean": 202.15960693359375, "margin_dpo/margin_std": 255.03884887695312, "step": 292 }, { "KL/chosen_KL_mean": -266.5294189453125, "KL/mean": -346.7402038574219, "KL/rejected_KL_mean": -426.95098876953125, "KL/std": 211.74429321289062, "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.002555608283728361, "fcm_dpo/delta": -0.010212033987045288, "fcm_dpo/margin": 160.42156982421875, "fcm_dpo/q_t": 0.40869617462158203, "grad_norm": 45.290653228759766, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -0.3368123769760132, "logits/rejected": -0.32695645093917847, "logps/chosen": -326.90545654296875, "logps/ref_chosen": -60.376033782958984, "logps/ref_rejected": -77.85244750976562, "logps/rejected": -504.8034362792969, "loss": 1.1009, "margin_dpo/margin_mean": 160.4215850830078, "margin_dpo/margin_std": 237.15798950195312, "step": 293 }, { "KL/chosen_KL_mean": -252.91036987304688, "KL/mean": -340.8631591796875, "KL/rejected_KL_mean": -428.81591796875, "KL/std": 192.0465087890625, "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.0025399108417332172, "fcm_dpo/delta": -0.047990452498197556, "fcm_dpo/margin": 175.9055633544922, "fcm_dpo/q_t": 0.400339812040329, "grad_norm": 29.165407180786133, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -0.3062417805194855, "logits/rejected": -0.3089578151702881, "logps/chosen": -300.9979248046875, "logps/ref_chosen": -48.0875358581543, "logps/ref_rejected": -81.89698791503906, "logps/rejected": -510.7129211425781, "loss": 1.0685, "margin_dpo/margin_mean": 175.9055633544922, "margin_dpo/margin_std": 233.61227416992188, "step": 294 }, { "KL/chosen_KL_mean": -301.30224609375, "KL/mean": -397.0389404296875, "KL/rejected_KL_mean": -492.775634765625, "KL/std": 212.322265625, "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.002525617368519306, "fcm_dpo/delta": -0.08562206476926804, "fcm_dpo/margin": 191.47335815429688, "fcm_dpo/q_t": 0.3909651041030884, "grad_norm": 27.463563919067383, "learning_rate": 3.509802512179737e-07, "logits/chosen": -0.2896770238876343, "logits/rejected": -0.290955126285553, "logps/chosen": -351.2268981933594, "logps/ref_chosen": -49.92467498779297, "logps/ref_rejected": -87.45632934570312, "logps/rejected": -580.23193359375, "loss": 1.0469, "margin_dpo/margin_mean": 191.47335815429688, "margin_dpo/margin_std": 241.90957641601562, "step": 295 }, { "KL/chosen_KL_mean": -359.9180603027344, "KL/mean": -426.7522277832031, "KL/rejected_KL_mean": -493.5863952636719, "KL/std": 202.1461181640625, "epoch": 0.434654919236417, "fcm_dpo/beta": 0.002509201178327203, "fcm_dpo/delta": -0.03729523718357086, "fcm_dpo/margin": 133.6683349609375, "fcm_dpo/q_t": 0.4235140383243561, "grad_norm": 45.111167907714844, "learning_rate": 3.498049431928577e-07, "logits/chosen": -0.3604152202606201, "logits/rejected": -0.3418993353843689, "logps/chosen": -425.4093017578125, "logps/ref_chosen": -65.49124145507812, "logps/ref_rejected": -93.08908081054688, "logps/rejected": -586.6754760742188, "loss": 1.1758, "margin_dpo/margin_mean": 133.6683349609375, "margin_dpo/margin_std": 252.33218383789062, "step": 296 }, { "KL/chosen_KL_mean": -313.9520568847656, "KL/mean": -391.02191162109375, "KL/rejected_KL_mean": -468.09173583984375, "KL/std": 195.4499969482422, "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.002508984412997961, "fcm_dpo/delta": 0.013493766076862812, "fcm_dpo/margin": 154.13967895507812, "fcm_dpo/q_t": 0.41029807925224304, "grad_norm": 49.82147979736328, "learning_rate": 3.486270052146694e-07, "logits/chosen": -0.3424445390701294, "logits/rejected": -0.3403598368167877, "logps/chosen": -370.42901611328125, "logps/ref_chosen": -56.476951599121094, "logps/ref_rejected": -95.1385498046875, "logps/rejected": -563.2302856445312, "loss": 1.0935, "margin_dpo/margin_mean": 154.13967895507812, "margin_dpo/margin_std": 194.82879638671875, "step": 297 }, { "KL/chosen_KL_mean": -310.0024719238281, "KL/mean": -410.1289978027344, "KL/rejected_KL_mean": -510.2554931640625, "KL/std": 235.77732849121094, "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.0024994537234306335, "fcm_dpo/delta": -0.10309357941150665, "fcm_dpo/margin": 200.2530517578125, "fcm_dpo/q_t": 0.3903202712535858, "grad_norm": 27.238431930541992, "learning_rate": 3.474464683231698e-07, "logits/chosen": -0.3699042797088623, "logits/rejected": -0.3843574523925781, "logps/chosen": -377.32763671875, "logps/ref_chosen": -67.32516479492188, "logps/ref_rejected": -116.66217041015625, "logps/rejected": -626.917724609375, "loss": 1.0457, "margin_dpo/margin_mean": 200.25302124023438, "margin_dpo/margin_std": 266.128662109375, "step": 298 }, { "KL/chosen_KL_mean": -259.36639404296875, "KL/mean": -347.03094482421875, "KL/rejected_KL_mean": -434.695556640625, "KL/std": 185.60708618164062, "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.0024865521118044853, "fcm_dpo/delta": -0.03695205599069595, "fcm_dpo/margin": 175.32911682128906, "fcm_dpo/q_t": 0.40157145261764526, "grad_norm": 34.25696563720703, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.36456483602523804, "logits/rejected": -0.36391931772232056, "logps/chosen": -308.3284912109375, "logps/ref_chosen": -48.96209716796875, "logps/ref_rejected": -84.32823944091797, "logps/rejected": -519.0238037109375, "loss": 1.0759, "margin_dpo/margin_mean": 175.3291015625, "margin_dpo/margin_std": 231.766357421875, "step": 299 }, { "KL/chosen_KL_mean": -315.7066650390625, "KL/mean": -422.8812255859375, "KL/rejected_KL_mean": -530.0557861328125, "KL/std": 226.07566833496094, "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.0024577926378697157, "fcm_dpo/delta": -0.1301524043083191, "fcm_dpo/margin": 214.34909057617188, "fcm_dpo/q_t": 0.3810431957244873, "grad_norm": 35.657649993896484, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -0.35023725032806396, "logits/rejected": -0.34927883744239807, "logps/chosen": -374.7803955078125, "logps/ref_chosen": -59.07371139526367, "logps/ref_rejected": -95.9664535522461, "logps/rejected": -626.022216796875, "loss": 1.0246, "margin_dpo/margin_mean": 214.34910583496094, "margin_dpo/margin_std": 262.06231689453125, "step": 300 }, { "KL/chosen_KL_mean": -261.52374267578125, "KL/mean": -361.13104248046875, "KL/rejected_KL_mean": -460.73834228515625, "KL/std": 200.12277221679688, "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.002426966093480587, "fcm_dpo/delta": -0.0857255756855011, "fcm_dpo/margin": 199.21456909179688, "fcm_dpo/q_t": 0.3913128077983856, "grad_norm": 29.40239906311035, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -0.38150107860565186, "logits/rejected": -0.3794858753681183, "logps/chosen": -318.77313232421875, "logps/ref_chosen": -57.249366760253906, "logps/ref_rejected": -92.35354614257812, "logps/rejected": -553.0919189453125, "loss": 1.0345, "margin_dpo/margin_mean": 199.21456909179688, "margin_dpo/margin_std": 237.46759033203125, "step": 301 }, { "KL/chosen_KL_mean": -216.02481079101562, "KL/mean": -303.01495361328125, "KL/rejected_KL_mean": -390.005126953125, "KL/std": 180.0056915283203, "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.0024160263128578663, "fcm_dpo/delta": -0.020986871793866158, "fcm_dpo/margin": 173.98028564453125, "fcm_dpo/q_t": 0.4036945402622223, "grad_norm": 19.06124496459961, "learning_rate": 3.426989547989902e-07, "logits/chosen": -0.3539729118347168, "logits/rejected": -0.3578333854675293, "logps/chosen": -267.2228088378906, "logps/ref_chosen": -51.197994232177734, "logps/ref_rejected": -97.22636413574219, "logps/rejected": -487.2314758300781, "loss": 1.0715, "margin_dpo/margin_mean": 173.98028564453125, "margin_dpo/margin_std": 212.90484619140625, "step": 302 }, { "KL/chosen_KL_mean": -238.58770751953125, "KL/mean": -316.30780029296875, "KL/rejected_KL_mean": -394.02789306640625, "KL/std": 194.1252899169922, "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.002420827280730009, "fcm_dpo/delta": 0.024122366681694984, "fcm_dpo/margin": 155.44015502929688, "fcm_dpo/q_t": 0.4143924117088318, "grad_norm": 24.42957305908203, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -0.37989115715026855, "logits/rejected": -0.3611317574977875, "logps/chosen": -305.3016357421875, "logps/ref_chosen": -66.71394348144531, "logps/ref_rejected": -86.94542694091797, "logps/rejected": -480.97332763671875, "loss": 1.1184, "margin_dpo/margin_mean": 155.440185546875, "margin_dpo/margin_std": 229.31715393066406, "step": 303 }, { "KL/chosen_KL_mean": -208.25198364257812, "KL/mean": -296.2002258300781, "KL/rejected_KL_mean": -384.1484375, "KL/std": 177.62567138671875, "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.002425071783363819, "fcm_dpo/delta": -0.027229398488998413, "fcm_dpo/margin": 175.89645385742188, "fcm_dpo/q_t": 0.39837926626205444, "grad_norm": 29.601119995117188, "learning_rate": 3.403104165467883e-07, "logits/chosen": -0.3989083766937256, "logits/rejected": -0.3849946856498718, "logps/chosen": -280.20269775390625, "logps/ref_chosen": -71.95069885253906, "logps/ref_rejected": -90.47203063964844, "logps/rejected": -474.6204833984375, "loss": 1.0409, "margin_dpo/margin_mean": 175.89645385742188, "margin_dpo/margin_std": 156.81117248535156, "step": 304 }, { "KL/chosen_KL_mean": -231.21951293945312, "KL/mean": -307.9289855957031, "KL/rejected_KL_mean": -384.6384582519531, "KL/std": 201.5638885498047, "epoch": 0.447870778267254, "fcm_dpo/beta": 0.002417425625026226, "fcm_dpo/delta": 0.02943410724401474, "fcm_dpo/margin": 153.41891479492188, "fcm_dpo/q_t": 0.41627079248428345, "grad_norm": 27.138019561767578, "learning_rate": 3.391125620245535e-07, "logits/chosen": -0.41401392221450806, "logits/rejected": -0.39217787981033325, "logps/chosen": -298.0147705078125, "logps/ref_chosen": -66.79523468017578, "logps/ref_rejected": -92.75459289550781, "logps/rejected": -477.39306640625, "loss": 1.1181, "margin_dpo/margin_mean": 153.41891479492188, "margin_dpo/margin_std": 221.77337646484375, "step": 305 }, { "KL/chosen_KL_mean": -229.625244140625, "KL/mean": -305.220947265625, "KL/rejected_KL_mean": -380.8166198730469, "KL/std": 177.791015625, "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.0024337819777429104, "fcm_dpo/delta": 0.03258597105741501, "fcm_dpo/margin": 151.19137573242188, "fcm_dpo/q_t": 0.4141908884048462, "grad_norm": 26.584671020507812, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -0.35081660747528076, "logits/rejected": -0.3195233941078186, "logps/chosen": -299.30914306640625, "logps/ref_chosen": -69.68389892578125, "logps/ref_rejected": -85.15919494628906, "logps/rejected": -465.9757995605469, "loss": 1.0999, "margin_dpo/margin_mean": 151.19139099121094, "margin_dpo/margin_std": 187.10317993164062, "step": 306 }, { "KL/chosen_KL_mean": -215.2943572998047, "KL/mean": -296.1317138671875, "KL/rejected_KL_mean": -376.96905517578125, "KL/std": 165.89390563964844, "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.0024325139820575714, "fcm_dpo/delta": 0.00681762769818306, "fcm_dpo/margin": 161.6746826171875, "fcm_dpo/q_t": 0.4085942506790161, "grad_norm": 25.906414031982422, "learning_rate": 3.367098400098881e-07, "logits/chosen": -0.3687888979911804, "logits/rejected": -0.34732580184936523, "logps/chosen": -285.45977783203125, "logps/ref_chosen": -70.16542053222656, "logps/ref_rejected": -86.97230529785156, "logps/rejected": -463.94134521484375, "loss": 1.0872, "margin_dpo/margin_mean": 161.6746826171875, "margin_dpo/margin_std": 200.76708984375, "step": 307 }, { "KL/chosen_KL_mean": -225.435546875, "KL/mean": -316.33099365234375, "KL/rejected_KL_mean": -407.2264404296875, "KL/std": 194.58316040039062, "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.0024283677339553833, "fcm_dpo/delta": -0.04238416999578476, "fcm_dpo/margin": 181.79086303710938, "fcm_dpo/q_t": 0.3963702917098999, "grad_norm": 29.603750228881836, "learning_rate": 3.355050358314172e-07, "logits/chosen": -0.34480029344558716, "logits/rejected": -0.3308371305465698, "logps/chosen": -280.6805419921875, "logps/ref_chosen": -55.2449951171875, "logps/ref_rejected": -79.37226104736328, "logps/rejected": -486.59869384765625, "loss": 1.0408, "margin_dpo/margin_mean": 181.79086303710938, "margin_dpo/margin_std": 184.86737060546875, "step": 308 }, { "KL/chosen_KL_mean": -229.71560668945312, "KL/mean": -321.12689208984375, "KL/rejected_KL_mean": -412.5382080078125, "KL/std": 200.42385864257812, "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.0024233111180365086, "fcm_dpo/delta": -0.04414871707558632, "fcm_dpo/margin": 182.82260131835938, "fcm_dpo/q_t": 0.3967331647872925, "grad_norm": 40.203651428222656, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -0.30514609813690186, "logits/rejected": -0.2906920909881592, "logps/chosen": -278.6746826171875, "logps/ref_chosen": -48.959083557128906, "logps/ref_rejected": -82.34072875976562, "logps/rejected": -494.8789367675781, "loss": 1.0527, "margin_dpo/margin_mean": 182.82260131835938, "margin_dpo/margin_std": 206.30902099609375, "step": 309 }, { "KL/chosen_KL_mean": -269.08026123046875, "KL/mean": -356.98431396484375, "KL/rejected_KL_mean": -444.88836669921875, "KL/std": 182.12594604492188, "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.00241499999538064, "fcm_dpo/delta": -0.02522626519203186, "fcm_dpo/margin": 175.80810546875, "fcm_dpo/q_t": 0.3999808430671692, "grad_norm": 25.993465423583984, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -0.37713727355003357, "logits/rejected": -0.3597584366798401, "logps/chosen": -331.8220520019531, "logps/ref_chosen": -62.74177932739258, "logps/ref_rejected": -79.9302978515625, "logps/rejected": -524.8186645507812, "loss": 1.0501, "margin_dpo/margin_mean": 175.80810546875, "margin_dpo/margin_std": 173.99118041992188, "step": 310 }, { "KL/chosen_KL_mean": -291.9866943359375, "KL/mean": -389.12078857421875, "KL/rejected_KL_mean": -486.2549133300781, "KL/std": 237.48434448242188, "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.0023961286060512066, "fcm_dpo/delta": -0.06706373393535614, "fcm_dpo/margin": 194.26821899414062, "fcm_dpo/q_t": 0.39537882804870605, "grad_norm": 25.610626220703125, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -0.33371835947036743, "logits/rejected": -0.3037863075733185, "logps/chosen": -345.0146484375, "logps/ref_chosen": -53.02798080444336, "logps/ref_rejected": -77.43820190429688, "logps/rejected": -563.693115234375, "loss": 1.0552, "margin_dpo/margin_mean": 194.26821899414062, "margin_dpo/margin_std": 249.4111328125, "step": 311 }, { "KL/chosen_KL_mean": -284.6394348144531, "KL/mean": -373.61962890625, "KL/rejected_KL_mean": -462.599853515625, "KL/std": 207.23092651367188, "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.002392760245129466, "fcm_dpo/delta": -0.0265023373067379, "fcm_dpo/margin": 177.96038818359375, "fcm_dpo/q_t": 0.40469247102737427, "grad_norm": 26.777978897094727, "learning_rate": 3.306636061080487e-07, "logits/chosen": -0.29555267095565796, "logits/rejected": -0.2810285985469818, "logps/chosen": -334.0316467285156, "logps/ref_chosen": -49.39221954345703, "logps/ref_rejected": -75.79280853271484, "logps/rejected": -538.3926391601562, "loss": 1.0824, "margin_dpo/margin_mean": 177.96038818359375, "margin_dpo/margin_std": 244.01686096191406, "step": 312 }, { "KL/chosen_KL_mean": -274.753662109375, "KL/mean": -365.9559326171875, "KL/rejected_KL_mean": -457.1582336425781, "KL/std": 222.33108520507812, "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.0023880554363131523, "fcm_dpo/delta": -0.03676465526223183, "fcm_dpo/margin": 182.40457153320312, "fcm_dpo/q_t": 0.4009496569633484, "grad_norm": 25.21873664855957, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -0.35764968395233154, "logits/rejected": -0.3531670570373535, "logps/chosen": -324.90643310546875, "logps/ref_chosen": -50.152740478515625, "logps/ref_rejected": -86.40620422363281, "logps/rejected": -543.564453125, "loss": 1.0789, "margin_dpo/margin_mean": 182.40457153320312, "margin_dpo/margin_std": 242.63485717773438, "step": 313 }, { "KL/chosen_KL_mean": -293.580810546875, "KL/mean": -380.2122802734375, "KL/rejected_KL_mean": -466.84375, "KL/std": 214.81533813476562, "epoch": 0.461086637298091, "fcm_dpo/beta": 0.0023697358556091785, "fcm_dpo/delta": -0.011065401136875153, "fcm_dpo/margin": 173.26296997070312, "fcm_dpo/q_t": 0.4087506830692291, "grad_norm": 21.553348541259766, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -0.3941684067249298, "logits/rejected": -0.39253175258636475, "logps/chosen": -350.818359375, "logps/ref_chosen": -57.237579345703125, "logps/ref_rejected": -97.5965347290039, "logps/rejected": -564.4403076171875, "loss": 1.1109, "margin_dpo/margin_mean": 173.26296997070312, "margin_dpo/margin_std": 267.32855224609375, "step": 314 }, { "KL/chosen_KL_mean": -272.3924865722656, "KL/mean": -351.9873962402344, "KL/rejected_KL_mean": -431.582275390625, "KL/std": 185.17767333984375, "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.0023773042485117912, "fcm_dpo/delta": 0.02197282202541828, "fcm_dpo/margin": 159.18983459472656, "fcm_dpo/q_t": 0.4126739203929901, "grad_norm": 25.598684310913086, "learning_rate": 3.270101039870797e-07, "logits/chosen": -0.27878469228744507, "logits/rejected": -0.28102076053619385, "logps/chosen": -321.46209716796875, "logps/ref_chosen": -49.06958770751953, "logps/ref_rejected": -85.68087768554688, "logps/rejected": -517.26318359375, "loss": 1.0996, "margin_dpo/margin_mean": 159.18984985351562, "margin_dpo/margin_std": 206.75869750976562, "step": 315 }, { "KL/chosen_KL_mean": -262.9434509277344, "KL/mean": -372.22076416015625, "KL/rejected_KL_mean": -481.4980773925781, "KL/std": 218.68289184570312, "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.0023532514460384846, "fcm_dpo/delta": -0.11787203699350357, "fcm_dpo/margin": 218.55462646484375, "fcm_dpo/q_t": 0.38381052017211914, "grad_norm": 26.231597900390625, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -0.35521891713142395, "logits/rejected": -0.35822099447250366, "logps/chosen": -317.2041931152344, "logps/ref_chosen": -54.26074981689453, "logps/ref_rejected": -101.2814712524414, "logps/rejected": -582.779541015625, "loss": 1.0101, "margin_dpo/margin_mean": 218.55462646484375, "margin_dpo/margin_std": 240.36514282226562, "step": 316 }, { "KL/chosen_KL_mean": -265.1885681152344, "KL/mean": -386.59716796875, "KL/rejected_KL_mean": -508.0057373046875, "KL/std": 205.4729461669922, "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.0023275299463421106, "fcm_dpo/delta": -0.1697898656129837, "fcm_dpo/margin": 242.81719970703125, "fcm_dpo/q_t": 0.37069547176361084, "grad_norm": 29.441675186157227, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -0.361335813999176, "logits/rejected": -0.34446266293525696, "logps/chosen": -321.28277587890625, "logps/ref_chosen": -56.094207763671875, "logps/ref_rejected": -100.69905090332031, "logps/rejected": -608.704833984375, "loss": 0.9674, "margin_dpo/margin_mean": 242.81719970703125, "margin_dpo/margin_std": 227.80020141601562, "step": 317 }, { "KL/chosen_KL_mean": -276.64666748046875, "KL/mean": -368.34423828125, "KL/rejected_KL_mean": -460.04180908203125, "KL/std": 210.08200073242188, "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.0023121268022805452, "fcm_dpo/delta": -0.024627672508358955, "fcm_dpo/margin": 183.39511108398438, "fcm_dpo/q_t": 0.40299540758132935, "grad_norm": 28.099123001098633, "learning_rate": 3.233383385962115e-07, "logits/chosen": -0.4194433093070984, "logits/rejected": -0.38650479912757874, "logps/chosen": -341.2923583984375, "logps/ref_chosen": -64.64569854736328, "logps/ref_rejected": -82.76425170898438, "logps/rejected": -542.8060302734375, "loss": 1.0716, "margin_dpo/margin_mean": 183.3951416015625, "margin_dpo/margin_std": 228.771728515625, "step": 318 }, { "KL/chosen_KL_mean": -248.31747436523438, "KL/mean": -355.7877197265625, "KL/rejected_KL_mean": -463.2579345703125, "KL/std": 224.69723510742188, "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.0022909443359822035, "fcm_dpo/delta": -0.09473305940628052, "fcm_dpo/margin": 214.94046020507812, "fcm_dpo/q_t": 0.38672956824302673, "grad_norm": 27.515636444091797, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -0.365750253200531, "logits/rejected": -0.37738245725631714, "logps/chosen": -297.70123291015625, "logps/ref_chosen": -49.383758544921875, "logps/ref_rejected": -113.90650939941406, "logps/rejected": -577.1644287109375, "loss": 1.0225, "margin_dpo/margin_mean": 214.94046020507812, "margin_dpo/margin_std": 239.97752380371094, "step": 319 }, { "KL/chosen_KL_mean": -263.43206787109375, "KL/mean": -381.2987365722656, "KL/rejected_KL_mean": -499.1654052734375, "KL/std": 234.49795532226562, "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.0022584707476198673, "fcm_dpo/delta": -0.13631628453731537, "fcm_dpo/margin": 235.73333740234375, "fcm_dpo/q_t": 0.3790479302406311, "grad_norm": 35.816165924072266, "learning_rate": 3.208807785813777e-07, "logits/chosen": -0.35946518182754517, "logits/rejected": -0.36113548278808594, "logps/chosen": -322.93695068359375, "logps/ref_chosen": -59.50489044189453, "logps/ref_rejected": -97.66717529296875, "logps/rejected": -596.83251953125, "loss": 0.9952, "margin_dpo/margin_mean": 235.73333740234375, "margin_dpo/margin_std": 250.2997589111328, "step": 320 }, { "KL/chosen_KL_mean": -316.8851318359375, "KL/mean": -419.2160339355469, "KL/rejected_KL_mean": -521.546875, "KL/std": 239.943115234375, "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.002237518085166812, "fcm_dpo/delta": -0.059808555990457535, "fcm_dpo/margin": 204.66180419921875, "fcm_dpo/q_t": 0.39591050148010254, "grad_norm": 24.665813446044922, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -0.3208162784576416, "logits/rejected": -0.3067053556442261, "logps/chosen": -378.433837890625, "logps/ref_chosen": -61.548683166503906, "logps/ref_rejected": -91.64103698730469, "logps/rejected": -613.18798828125, "loss": 1.0571, "margin_dpo/margin_mean": 204.6617889404297, "margin_dpo/margin_std": 253.84730529785156, "step": 321 }, { "KL/chosen_KL_mean": -280.55859375, "KL/mean": -392.18109130859375, "KL/rejected_KL_mean": -503.8035583496094, "KL/std": 217.13014221191406, "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.0022206236608326435, "fcm_dpo/delta": -0.09844300150871277, "fcm_dpo/margin": 223.24496459960938, "fcm_dpo/q_t": 0.3849971294403076, "grad_norm": 25.391277313232422, "learning_rate": 3.184157475180207e-07, "logits/chosen": -0.33665627241134644, "logits/rejected": -0.33168351650238037, "logps/chosen": -337.8486328125, "logps/ref_chosen": -57.29003143310547, "logps/ref_rejected": -95.74992370605469, "logps/rejected": -599.553466796875, "loss": 1.0101, "margin_dpo/margin_mean": 223.24496459960938, "margin_dpo/margin_std": 225.46527099609375, "step": 322 }, { "KL/chosen_KL_mean": -303.197509765625, "KL/mean": -397.31976318359375, "KL/rejected_KL_mean": -491.4420166015625, "KL/std": 215.2032470703125, "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.0022140834480524063, "fcm_dpo/delta": -0.017185624688863754, "fcm_dpo/margin": 188.2445068359375, "fcm_dpo/q_t": 0.4033932685852051, "grad_norm": 35.27677536010742, "learning_rate": 3.171805115074251e-07, "logits/chosen": -0.3552204370498657, "logits/rejected": -0.3479047417640686, "logps/chosen": -354.4314880371094, "logps/ref_chosen": -51.23395919799805, "logps/ref_rejected": -75.06192016601562, "logps/rejected": -566.50390625, "loss": 1.0792, "margin_dpo/margin_mean": 188.2445068359375, "margin_dpo/margin_std": 236.12969970703125, "step": 323 }, { "KL/chosen_KL_mean": -351.04327392578125, "KL/mean": -444.7715759277344, "KL/rejected_KL_mean": -538.4998779296875, "KL/std": 236.59896850585938, "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.0022205086424946785, "fcm_dpo/delta": -0.01711501181125641, "fcm_dpo/margin": 187.4566192626953, "fcm_dpo/q_t": 0.4070439636707306, "grad_norm": 41.267127990722656, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -0.3706352710723877, "logits/rejected": -0.35114365816116333, "logps/chosen": -416.1784362792969, "logps/ref_chosen": -65.13516998291016, "logps/ref_rejected": -86.47750854492188, "logps/rejected": -624.9774169921875, "loss": 1.1032, "margin_dpo/margin_mean": 187.4566192626953, "margin_dpo/margin_std": 269.1782531738281, "step": 324 }, { "KL/chosen_KL_mean": -282.0893249511719, "KL/mean": -358.64837646484375, "KL/rejected_KL_mean": -435.2073974609375, "KL/std": 205.12814331054688, "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.0022224104031920433, "fcm_dpo/delta": 0.06069522351026535, "fcm_dpo/margin": 153.11805725097656, "fcm_dpo/q_t": 0.4203560948371887, "grad_norm": 36.374202728271484, "learning_rate": 3.147047612756302e-07, "logits/chosen": -0.4578849673271179, "logits/rejected": -0.43039295077323914, "logps/chosen": -338.304931640625, "logps/ref_chosen": -56.215599060058594, "logps/ref_rejected": -70.08592987060547, "logps/rejected": -505.2933044433594, "loss": 1.1272, "margin_dpo/margin_mean": 153.11807250976562, "margin_dpo/margin_std": 203.81394958496094, "step": 325 }, { "KL/chosen_KL_mean": -320.332763671875, "KL/mean": -397.1983642578125, "KL/rejected_KL_mean": -474.06396484375, "KL/std": 202.8357696533203, "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.0022304444573819637, "fcm_dpo/delta": 0.0580889992415905, "fcm_dpo/margin": 153.73118591308594, "fcm_dpo/q_t": 0.41890084743499756, "grad_norm": 45.461204528808594, "learning_rate": 3.134643122927519e-07, "logits/chosen": -0.44517433643341064, "logits/rejected": -0.4201740622520447, "logps/chosen": -393.0577087402344, "logps/ref_chosen": -72.72496032714844, "logps/ref_rejected": -79.8467788696289, "logps/rejected": -553.9107666015625, "loss": 1.1188, "margin_dpo/margin_mean": 153.731201171875, "margin_dpo/margin_std": 192.40139770507812, "step": 326 }, { "KL/chosen_KL_mean": -286.7748107910156, "KL/mean": -405.4556884765625, "KL/rejected_KL_mean": -524.1365356445312, "KL/std": 219.92962646484375, "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.0022127358242869377, "fcm_dpo/delta": -0.1288842260837555, "fcm_dpo/margin": 237.36172485351562, "fcm_dpo/q_t": 0.3784708082675934, "grad_norm": 38.16813659667969, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -0.4000493884086609, "logits/rejected": -0.38037049770355225, "logps/chosen": -355.90924072265625, "logps/ref_chosen": -69.13441467285156, "logps/ref_rejected": -111.93377685546875, "logps/rejected": -636.0703125, "loss": 0.9955, "margin_dpo/margin_mean": 237.36172485351562, "margin_dpo/margin_std": 241.5782470703125, "step": 327 }, { "KL/chosen_KL_mean": -305.38720703125, "KL/mean": -413.01141357421875, "KL/rejected_KL_mean": -520.6356201171875, "KL/std": 244.02554321289062, "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0021971408277750015, "fcm_dpo/delta": -0.07468114793300629, "fcm_dpo/margin": 215.24847412109375, "fcm_dpo/q_t": 0.39337801933288574, "grad_norm": 28.978845596313477, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -0.369107723236084, "logits/rejected": -0.367361843585968, "logps/chosen": -365.07440185546875, "logps/ref_chosen": -59.68719482421875, "logps/ref_rejected": -90.85499572753906, "logps/rejected": -611.4906616210938, "loss": 1.0456, "margin_dpo/margin_mean": 215.24847412109375, "margin_dpo/margin_std": 267.3666687011719, "step": 328 }, { "KL/chosen_KL_mean": -340.37298583984375, "KL/mean": -448.5474853515625, "KL/rejected_KL_mean": -556.7220458984375, "KL/std": 253.79428100585938, "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0021742871031165123, "fcm_dpo/delta": -0.07264669984579086, "fcm_dpo/margin": 216.34902954101562, "fcm_dpo/q_t": 0.39304083585739136, "grad_norm": 33.51917266845703, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -0.37974807620048523, "logits/rejected": -0.3639347553253174, "logps/chosen": -405.61920166015625, "logps/ref_chosen": -65.2461929321289, "logps/ref_rejected": -100.69770812988281, "logps/rejected": -657.4197387695312, "loss": 1.0604, "margin_dpo/margin_mean": 216.34902954101562, "margin_dpo/margin_std": 278.5408630371094, "step": 329 }, { "KL/chosen_KL_mean": -288.67608642578125, "KL/mean": -402.93505859375, "KL/rejected_KL_mean": -517.1940307617188, "KL/std": 241.49658203125, "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.0021610576659440994, "fcm_dpo/delta": -0.09621290862560272, "fcm_dpo/margin": 228.5179443359375, "fcm_dpo/q_t": 0.38732674717903137, "grad_norm": 28.922462463378906, "learning_rate": 3.084861204504122e-07, "logits/chosen": -0.36659538745880127, "logits/rejected": -0.3661782145500183, "logps/chosen": -335.6744384765625, "logps/ref_chosen": -46.998348236083984, "logps/ref_rejected": -86.87684631347656, "logps/rejected": -604.0708618164062, "loss": 1.0234, "margin_dpo/margin_mean": 228.5179443359375, "margin_dpo/margin_std": 258.7063903808594, "step": 330 }, { "KL/chosen_KL_mean": -310.08087158203125, "KL/mean": -423.1429443359375, "KL/rejected_KL_mean": -536.2050170898438, "KL/std": 198.00454711914062, "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.002141648903489113, "fcm_dpo/delta": -0.08637881278991699, "fcm_dpo/margin": 226.1241455078125, "fcm_dpo/q_t": 0.3862583339214325, "grad_norm": 28.123804092407227, "learning_rate": 3.072376374875335e-07, "logits/chosen": -0.4031466245651245, "logits/rejected": -0.395566463470459, "logps/chosen": -360.6051025390625, "logps/ref_chosen": -50.52424621582031, "logps/ref_rejected": -89.01544189453125, "logps/rejected": -625.220458984375, "loss": 0.9994, "margin_dpo/margin_mean": 226.1241455078125, "margin_dpo/margin_std": 195.76531982421875, "step": 331 }, { "KL/chosen_KL_mean": -316.3252258300781, "KL/mean": -394.60198974609375, "KL/rejected_KL_mean": -472.87872314453125, "KL/std": 199.60614013671875, "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.002148597501218319, "fcm_dpo/delta": 0.06467652320861816, "fcm_dpo/margin": 156.5535125732422, "fcm_dpo/q_t": 0.42228198051452637, "grad_norm": 25.59779930114746, "learning_rate": 3.059876462596758e-07, "logits/chosen": -0.3974687159061432, "logits/rejected": -0.37797918915748596, "logps/chosen": -365.5054931640625, "logps/ref_chosen": -49.18028259277344, "logps/ref_rejected": -76.48515319824219, "logps/rejected": -549.3638916015625, "loss": 1.1305, "margin_dpo/margin_mean": 156.55352783203125, "margin_dpo/margin_std": 220.91140747070312, "step": 332 }, { "KL/chosen_KL_mean": -325.0455627441406, "KL/mean": -431.7874450683594, "KL/rejected_KL_mean": -538.529296875, "KL/std": 234.52767944335938, "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.0021342344116419554, "fcm_dpo/delta": -0.057633012533187866, "fcm_dpo/margin": 213.48370361328125, "fcm_dpo/q_t": 0.3971046805381775, "grad_norm": 30.5307674407959, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -0.4062424302101135, "logits/rejected": -0.3987647593021393, "logps/chosen": -388.80133056640625, "logps/ref_chosen": -63.75574493408203, "logps/ref_rejected": -95.04411315917969, "logps/rejected": -633.5733642578125, "loss": 1.0699, "margin_dpo/margin_mean": 213.48370361328125, "margin_dpo/margin_std": 282.26739501953125, "step": 333 }, { "KL/chosen_KL_mean": -314.6381530761719, "KL/mean": -410.3465270996094, "KL/rejected_KL_mean": -506.054931640625, "KL/std": 256.6167907714844, "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.0021365054417401552, "fcm_dpo/delta": -0.00915764831006527, "fcm_dpo/margin": 191.416748046875, "fcm_dpo/q_t": 0.4075648784637451, "grad_norm": 27.895448684692383, "learning_rate": 3.034832708016243e-07, "logits/chosen": -0.4134640693664551, "logits/rejected": -0.40898776054382324, "logps/chosen": -381.617919921875, "logps/ref_chosen": -66.97975158691406, "logps/ref_rejected": -95.31692504882812, "logps/rejected": -601.371826171875, "loss": 1.1071, "margin_dpo/margin_mean": 191.416748046875, "margin_dpo/margin_std": 287.8053283691406, "step": 334 }, { "KL/chosen_KL_mean": -340.8347473144531, "KL/mean": -420.40362548828125, "KL/rejected_KL_mean": -499.97247314453125, "KL/std": 239.92379760742188, "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.00215163454413414, "fcm_dpo/delta": 0.058172550052404404, "fcm_dpo/margin": 159.1376953125, "fcm_dpo/q_t": 0.4224546253681183, "grad_norm": 31.30469512939453, "learning_rate": 3.022289525640531e-07, "logits/chosen": -0.47105371952056885, "logits/rejected": -0.44997382164001465, "logps/chosen": -403.3772277832031, "logps/ref_chosen": -62.54248046875, "logps/ref_rejected": -87.61770629882812, "logps/rejected": -587.5902099609375, "loss": 1.147, "margin_dpo/margin_mean": 159.1376953125, "margin_dpo/margin_std": 256.5185241699219, "step": 335 }, { "KL/chosen_KL_mean": -337.67694091796875, "KL/mean": -456.9693603515625, "KL/rejected_KL_mean": -576.26171875, "KL/std": 266.134033203125, "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.0021344278939068317, "fcm_dpo/delta": -0.1120191439986229, "fcm_dpo/margin": 238.58480834960938, "fcm_dpo/q_t": 0.3868195414543152, "grad_norm": 38.22102737426758, "learning_rate": 3.009732580450086e-07, "logits/chosen": -0.4278082847595215, "logits/rejected": -0.4280624985694885, "logps/chosen": -392.20806884765625, "logps/ref_chosen": -54.53115463256836, "logps/ref_rejected": -104.40424346923828, "logps/rejected": -680.666015625, "loss": 1.0426, "margin_dpo/margin_mean": 238.58480834960938, "margin_dpo/margin_std": 310.658447265625, "step": 336 }, { "KL/chosen_KL_mean": -302.09124755859375, "KL/mean": -415.0406494140625, "KL/rejected_KL_mean": -527.9901123046875, "KL/std": 218.13992309570312, "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.0021110186353325844, "fcm_dpo/delta": -0.0788479745388031, "fcm_dpo/margin": 225.89886474609375, "fcm_dpo/q_t": 0.39037024974823, "grad_norm": 39.11850357055664, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -0.4359098970890045, "logits/rejected": -0.4225747585296631, "logps/chosen": -367.21990966796875, "logps/ref_chosen": -65.12869262695312, "logps/ref_rejected": -101.72701263427734, "logps/rejected": -629.7171020507812, "loss": 1.0344, "margin_dpo/margin_mean": 225.89886474609375, "margin_dpo/margin_std": 259.23974609375, "step": 337 }, { "KL/chosen_KL_mean": -288.96722412109375, "KL/mean": -404.56036376953125, "KL/rejected_KL_mean": -520.1534423828125, "KL/std": 224.17669677734375, "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.002094029448926449, "fcm_dpo/delta": -0.08624735474586487, "fcm_dpo/margin": 231.18624877929688, "fcm_dpo/q_t": 0.3876437246799469, "grad_norm": 39.132667541503906, "learning_rate": 2.984578725527675e-07, "logits/chosen": -0.4282386302947998, "logits/rejected": -0.42397069931030273, "logps/chosen": -347.3899230957031, "logps/ref_chosen": -58.422706604003906, "logps/ref_rejected": -89.06854248046875, "logps/rejected": -609.2219848632812, "loss": 1.016, "margin_dpo/margin_mean": 231.18624877929688, "margin_dpo/margin_std": 235.46348571777344, "step": 338 }, { "KL/chosen_KL_mean": -302.4131774902344, "KL/mean": -405.36090087890625, "KL/rejected_KL_mean": -508.3086242675781, "KL/std": 225.501708984375, "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.00209475914016366, "fcm_dpo/delta": -0.032657869160175323, "fcm_dpo/margin": 205.89544677734375, "fcm_dpo/q_t": 0.4000750184059143, "grad_norm": 39.0589485168457, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -0.4808998703956604, "logits/rejected": -0.4646437168121338, "logps/chosen": -362.40850830078125, "logps/ref_chosen": -59.99531555175781, "logps/ref_rejected": -103.9109115600586, "logps/rejected": -612.219482421875, "loss": 1.0614, "margin_dpo/margin_mean": 205.89544677734375, "margin_dpo/margin_std": 227.13043212890625, "step": 339 }, { "KL/chosen_KL_mean": -306.40838623046875, "KL/mean": -403.55328369140625, "KL/rejected_KL_mean": -500.6982116699219, "KL/std": 214.56753540039062, "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.0020795511081814766, "fcm_dpo/delta": -0.004235394299030304, "fcm_dpo/margin": 194.28976440429688, "fcm_dpo/q_t": 0.40789303183555603, "grad_norm": 32.46416091918945, "learning_rate": 2.959373794541426e-07, "logits/chosen": -0.42205601930618286, "logits/rejected": -0.4012700319290161, "logps/chosen": -359.2386474609375, "logps/ref_chosen": -52.83022689819336, "logps/ref_rejected": -73.10723114013672, "logps/rejected": -573.805419921875, "loss": 1.0937, "margin_dpo/margin_mean": 194.28977966308594, "margin_dpo/margin_std": 265.2552185058594, "step": 340 }, { "KL/chosen_KL_mean": -299.121826171875, "KL/mean": -406.19964599609375, "KL/rejected_KL_mean": -513.2774658203125, "KL/std": 231.76580810546875, "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.0020710937678813934, "fcm_dpo/delta": -0.044832952320575714, "fcm_dpo/margin": 214.15560913085938, "fcm_dpo/q_t": 0.398419588804245, "grad_norm": 32.579837799072266, "learning_rate": 2.946753005532965e-07, "logits/chosen": -0.42187175154685974, "logits/rejected": -0.4231615364551544, "logps/chosen": -347.0216369628906, "logps/ref_chosen": -47.899803161621094, "logps/ref_rejected": -101.80987548828125, "logps/rejected": -615.0873413085938, "loss": 1.0562, "margin_dpo/margin_mean": 214.15560913085938, "margin_dpo/margin_std": 249.23519897460938, "step": 341 }, { "KL/chosen_KL_mean": -285.2144470214844, "KL/mean": -383.55755615234375, "KL/rejected_KL_mean": -481.900634765625, "KL/std": 211.0296173095703, "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.0020756039302796125, "fcm_dpo/delta": -0.008514024317264557, "fcm_dpo/margin": 196.68621826171875, "fcm_dpo/q_t": 0.40535086393356323, "grad_norm": 29.37921905517578, "learning_rate": 2.934120444167326e-07, "logits/chosen": -0.41295289993286133, "logits/rejected": -0.3835510313510895, "logps/chosen": -357.2110900878906, "logps/ref_chosen": -71.99664306640625, "logps/ref_rejected": -92.58959197998047, "logps/rejected": -574.490234375, "loss": 1.0793, "margin_dpo/margin_mean": 196.68621826171875, "margin_dpo/margin_std": 243.93203735351562, "step": 342 }, { "KL/chosen_KL_mean": -275.50653076171875, "KL/mean": -393.09033203125, "KL/rejected_KL_mean": -510.6741027832031, "KL/std": 219.71334838867188, "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.0020575113594532013, "fcm_dpo/delta": -0.08598469197750092, "fcm_dpo/margin": 235.1676025390625, "fcm_dpo/q_t": 0.38554584980010986, "grad_norm": 25.824504852294922, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -0.4432937502861023, "logits/rejected": -0.4439564347267151, "logps/chosen": -329.91217041015625, "logps/ref_chosen": -54.405616760253906, "logps/ref_rejected": -111.04142761230469, "logps/rejected": -621.715576171875, "loss": 0.9987, "margin_dpo/margin_mean": 235.16761779785156, "margin_dpo/margin_std": 193.387451171875, "step": 343 }, { "KL/chosen_KL_mean": -278.17584228515625, "KL/mean": -382.2356262207031, "KL/rejected_KL_mean": -486.29541015625, "KL/std": 239.05442810058594, "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.00205832626670599, "fcm_dpo/delta": -0.029630951583385468, "fcm_dpo/margin": 208.11956787109375, "fcm_dpo/q_t": 0.40320706367492676, "grad_norm": 28.157310485839844, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -0.421103835105896, "logits/rejected": -0.42124348878860474, "logps/chosen": -332.1405029296875, "logps/ref_chosen": -53.96466827392578, "logps/ref_rejected": -90.62336730957031, "logps/rejected": -576.9188232421875, "loss": 1.066, "margin_dpo/margin_mean": 208.11956787109375, "margin_dpo/margin_std": 241.80572509765625, "step": 344 }, { "KL/chosen_KL_mean": -327.5186767578125, "KL/mean": -443.9566650390625, "KL/rejected_KL_mean": -560.3946533203125, "KL/std": 230.4110107421875, "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.0020382110960781574, "fcm_dpo/delta": -0.0764290988445282, "fcm_dpo/margin": 232.87599182128906, "fcm_dpo/q_t": 0.3901875615119934, "grad_norm": 24.22003173828125, "learning_rate": 2.896155456223163e-07, "logits/chosen": -0.43897050619125366, "logits/rejected": -0.4347718358039856, "logps/chosen": -389.2043762207031, "logps/ref_chosen": -61.685699462890625, "logps/ref_rejected": -99.49041748046875, "logps/rejected": -659.8851318359375, "loss": 1.0293, "margin_dpo/margin_mean": 232.87600708007812, "margin_dpo/margin_std": 254.49136352539062, "step": 345 }, { "KL/chosen_KL_mean": -339.9031677246094, "KL/mean": -448.6493835449219, "KL/rejected_KL_mean": -557.3956298828125, "KL/std": 233.10879516601562, "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.0020257416181266308, "fcm_dpo/delta": -0.04150301218032837, "fcm_dpo/margin": 217.49244689941406, "fcm_dpo/q_t": 0.3968901038169861, "grad_norm": 27.246360778808594, "learning_rate": 2.883479137196714e-07, "logits/chosen": -0.3788645267486572, "logits/rejected": -0.3643970489501953, "logps/chosen": -395.159423828125, "logps/ref_chosen": -55.256263732910156, "logps/ref_rejected": -77.41532135009766, "logps/rejected": -634.8109130859375, "loss": 1.0528, "margin_dpo/margin_mean": 217.49244689941406, "margin_dpo/margin_std": 247.9464111328125, "step": 346 }, { "KL/chosen_KL_mean": -344.7647705078125, "KL/mean": -448.3440246582031, "KL/rejected_KL_mean": -551.9232177734375, "KL/std": 247.40902709960938, "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.002020738087594509, "fcm_dpo/delta": -0.019013745710253716, "fcm_dpo/margin": 207.15847778320312, "fcm_dpo/q_t": 0.4043384790420532, "grad_norm": 29.887693405151367, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -0.3656623959541321, "logits/rejected": -0.35582268238067627, "logps/chosen": -402.3310241699219, "logps/ref_chosen": -57.56623840332031, "logps/ref_rejected": -92.35509490966797, "logps/rejected": -644.2783203125, "loss": 1.0832, "margin_dpo/margin_mean": 207.15847778320312, "margin_dpo/margin_std": 276.2279052734375, "step": 347 }, { "KL/chosen_KL_mean": -311.39959716796875, "KL/mean": -401.85382080078125, "KL/rejected_KL_mean": -492.3080749511719, "KL/std": 215.942626953125, "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.002018729457631707, "fcm_dpo/delta": 0.03512571379542351, "fcm_dpo/margin": 180.908447265625, "fcm_dpo/q_t": 0.4162946939468384, "grad_norm": 29.292268753051758, "learning_rate": 2.858096518347179e-07, "logits/chosen": -0.3954634666442871, "logits/rejected": -0.3925415873527527, "logps/chosen": -367.7173156738281, "logps/ref_chosen": -56.31770324707031, "logps/ref_rejected": -89.13836669921875, "logps/rejected": -581.4464111328125, "loss": 1.1176, "margin_dpo/margin_mean": 180.90846252441406, "margin_dpo/margin_std": 251.56272888183594, "step": 348 }, { "KL/chosen_KL_mean": -302.69281005859375, "KL/mean": -409.18768310546875, "KL/rejected_KL_mean": -515.6825561523438, "KL/std": 244.12184143066406, "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.002026638016104698, "fcm_dpo/delta": -0.03247387707233429, "fcm_dpo/margin": 212.98977661132812, "fcm_dpo/q_t": 0.40324437618255615, "grad_norm": 24.500629425048828, "learning_rate": 2.845390887379706e-07, "logits/chosen": -0.37826380133628845, "logits/rejected": -0.37416696548461914, "logps/chosen": -360.71832275390625, "logps/ref_chosen": -58.025516510009766, "logps/ref_rejected": -97.50515747070312, "logps/rejected": -613.187744140625, "loss": 1.0867, "margin_dpo/margin_mean": 212.98977661132812, "margin_dpo/margin_std": 299.3753662109375, "step": 349 }, { "KL/chosen_KL_mean": -322.9959716796875, "KL/mean": -425.35693359375, "KL/rejected_KL_mean": -527.7179565429688, "KL/std": 237.5716552734375, "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.002013745252043009, "fcm_dpo/delta": -0.012782763689756393, "fcm_dpo/margin": 204.7219696044922, "fcm_dpo/q_t": 0.40567225217819214, "grad_norm": 32.45606994628906, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -0.4236910343170166, "logits/rejected": -0.4241228699684143, "logps/chosen": -387.32647705078125, "logps/ref_chosen": -64.33049011230469, "logps/ref_rejected": -89.87164306640625, "logps/rejected": -617.589599609375, "loss": 1.1016, "margin_dpo/margin_mean": 204.72198486328125, "margin_dpo/margin_std": 298.5418701171875, "step": 350 }, { "KL/chosen_KL_mean": -301.8697509765625, "KL/mean": -426.64111328125, "KL/rejected_KL_mean": -551.4124755859375, "KL/std": 277.66021728515625, "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.0019994243048131466, "fcm_dpo/delta": -0.10174670815467834, "fcm_dpo/margin": 249.542724609375, "fcm_dpo/q_t": 0.39052557945251465, "grad_norm": 26.109012603759766, "learning_rate": 2.819952656376487e-07, "logits/chosen": -0.4383613169193268, "logits/rejected": -0.434881329536438, "logps/chosen": -362.5418701171875, "logps/ref_chosen": -60.6721305847168, "logps/ref_rejected": -101.5654296875, "logps/rejected": -652.9779052734375, "loss": 1.047, "margin_dpo/margin_mean": 249.542724609375, "margin_dpo/margin_std": 331.46405029296875, "step": 351 }, { "KL/chosen_KL_mean": -353.83587646484375, "KL/mean": -431.2773132324219, "KL/rejected_KL_mean": -508.7187194824219, "KL/std": 258.7840576171875, "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.002007380360737443, "fcm_dpo/delta": 0.09052670747041702, "fcm_dpo/margin": 154.8828887939453, "fcm_dpo/q_t": 0.4296043813228607, "grad_norm": 35.03353500366211, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -0.4138913154602051, "logits/rejected": -0.381538987159729, "logps/chosen": -424.7793273925781, "logps/ref_chosen": -70.9434585571289, "logps/ref_rejected": -76.6419677734375, "logps/rejected": -585.3607177734375, "loss": 1.1807, "margin_dpo/margin_mean": 154.8828887939453, "margin_dpo/margin_std": 290.50494384765625, "step": 352 }, { "KL/chosen_KL_mean": -340.01251220703125, "KL/mean": -443.328857421875, "KL/rejected_KL_mean": -546.6452026367188, "KL/std": 257.87725830078125, "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.002007407834753394, "fcm_dpo/delta": -0.015510857105255127, "fcm_dpo/margin": 206.63265991210938, "fcm_dpo/q_t": 0.40632784366607666, "grad_norm": 29.06142234802246, "learning_rate": 2.794480701395219e-07, "logits/chosen": -0.46683937311172485, "logits/rejected": -0.4540403485298157, "logps/chosen": -398.4078674316406, "logps/ref_chosen": -58.39533996582031, "logps/ref_rejected": -80.33553314208984, "logps/rejected": -626.980712890625, "loss": 1.0954, "margin_dpo/margin_mean": 206.63265991210938, "margin_dpo/margin_std": 289.2551574707031, "step": 353 }, { "KL/chosen_KL_mean": -282.2817687988281, "KL/mean": -397.5120544433594, "KL/rejected_KL_mean": -512.7423095703125, "KL/std": 236.1700439453125, "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.0020015356130898, "fcm_dpo/delta": -0.062798872590065, "fcm_dpo/margin": 230.4605712890625, "fcm_dpo/q_t": 0.39260250329971313, "grad_norm": 30.49873161315918, "learning_rate": 2.781732916288303e-07, "logits/chosen": -0.3843461275100708, "logits/rejected": -0.36748528480529785, "logps/chosen": -342.08477783203125, "logps/ref_chosen": -59.80299377441406, "logps/ref_rejected": -88.75750732421875, "logps/rejected": -601.4998779296875, "loss": 1.0285, "margin_dpo/margin_mean": 230.4605712890625, "margin_dpo/margin_std": 233.89678955078125, "step": 354 }, { "KL/chosen_KL_mean": -279.9733581542969, "KL/mean": -393.3092041015625, "KL/rejected_KL_mean": -506.6451110839844, "KL/std": 233.89459228515625, "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.001989755779504776, "fcm_dpo/delta": -0.05232168734073639, "fcm_dpo/margin": 226.67178344726562, "fcm_dpo/q_t": 0.39414530992507935, "grad_norm": 42.037132263183594, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -0.4780992865562439, "logits/rejected": -0.45962148904800415, "logps/chosen": -334.10186767578125, "logps/ref_chosen": -54.12849807739258, "logps/ref_rejected": -82.40606689453125, "logps/rejected": -589.0511474609375, "loss": 1.0331, "margin_dpo/margin_mean": 226.67176818847656, "margin_dpo/margin_std": 225.0481414794922, "step": 355 }, { "KL/chosen_KL_mean": -339.6640319824219, "KL/mean": -402.67962646484375, "KL/rejected_KL_mean": -465.6952819824219, "KL/std": 238.7618865966797, "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.001991572557017207, "fcm_dpo/delta": 0.020430708304047585, "fcm_dpo/margin": 126.03125, "fcm_dpo/q_t": 0.44326043128967285, "grad_norm": 35.44585418701172, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -0.43858081102371216, "logits/rejected": -0.4161129593849182, "logps/chosen": -404.33782958984375, "logps/ref_chosen": -64.6738052368164, "logps/ref_rejected": -75.89926147460938, "logps/rejected": -541.5945434570312, "loss": 1.2348, "margin_dpo/margin_mean": 126.03125762939453, "margin_dpo/margin_std": 294.35003662109375, "step": 356 }, { "KL/chosen_KL_mean": -307.77178955078125, "KL/mean": -404.3070068359375, "KL/rejected_KL_mean": -500.8422546386719, "KL/std": 227.6900634765625, "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.001992112258449197, "fcm_dpo/delta": 0.015565093606710434, "fcm_dpo/margin": 193.07044982910156, "fcm_dpo/q_t": 0.40995845198631287, "grad_norm": 27.028295516967773, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -0.4511488080024719, "logits/rejected": -0.43502819538116455, "logps/chosen": -360.49761962890625, "logps/ref_chosen": -52.725799560546875, "logps/ref_rejected": -86.84115600585938, "logps/rejected": -587.6834106445312, "loss": 1.0897, "margin_dpo/margin_mean": 193.07044982910156, "margin_dpo/margin_std": 232.12307739257812, "step": 357 }, { "KL/chosen_KL_mean": -285.88421630859375, "KL/mean": -376.27392578125, "KL/rejected_KL_mean": -466.66363525390625, "KL/std": 219.6377716064453, "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.0020017188508063555, "fcm_dpo/delta": 0.0388420931994915, "fcm_dpo/margin": 180.77944946289062, "fcm_dpo/q_t": 0.4163302183151245, "grad_norm": 24.81241798400879, "learning_rate": 2.730670898658255e-07, "logits/chosen": -0.4996035397052765, "logits/rejected": -0.4798714816570282, "logps/chosen": -349.08966064453125, "logps/ref_chosen": -63.20543670654297, "logps/ref_rejected": -88.373291015625, "logps/rejected": -555.0369873046875, "loss": 1.1092, "margin_dpo/margin_mean": 180.77944946289062, "margin_dpo/margin_std": 236.979248046875, "step": 358 }, { "KL/chosen_KL_mean": -329.12579345703125, "KL/mean": -436.353271484375, "KL/rejected_KL_mean": -543.5807495117188, "KL/std": 229.46609497070312, "epoch": 0.527165932452276, "fcm_dpo/beta": 0.0019993209280073643, "fcm_dpo/delta": -0.02941155806183815, "fcm_dpo/margin": 214.4549560546875, "fcm_dpo/q_t": 0.4025813341140747, "grad_norm": 51.26544952392578, "learning_rate": 2.717889356869146e-07, "logits/chosen": -0.4144907593727112, "logits/rejected": -0.3954973518848419, "logps/chosen": -385.4959716796875, "logps/ref_chosen": -56.370216369628906, "logps/ref_rejected": -82.17375183105469, "logps/rejected": -625.7545166015625, "loss": 1.076, "margin_dpo/margin_mean": 214.4549560546875, "margin_dpo/margin_std": 278.1531982421875, "step": 359 }, { "KL/chosen_KL_mean": -320.970703125, "KL/mean": -405.2179870605469, "KL/rejected_KL_mean": -489.4652404785156, "KL/std": 195.44993591308594, "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.002009341726079583, "fcm_dpo/delta": 0.06243222951889038, "fcm_dpo/margin": 168.49453735351562, "fcm_dpo/q_t": 0.4197266101837158, "grad_norm": 39.978214263916016, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -0.41467490792274475, "logits/rejected": -0.39842575788497925, "logps/chosen": -372.43109130859375, "logps/ref_chosen": -51.460384368896484, "logps/ref_rejected": -69.83892059326172, "logps/rejected": -559.30419921875, "loss": 1.1129, "margin_dpo/margin_mean": 168.49453735351562, "margin_dpo/margin_std": 194.20201110839844, "step": 360 }, { "KL/chosen_KL_mean": -342.26776123046875, "KL/mean": -424.4127197265625, "KL/rejected_KL_mean": -506.5577087402344, "KL/std": 229.03176879882812, "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.0020234398543834686, "fcm_dpo/delta": 0.06860539317131042, "fcm_dpo/margin": 164.2899627685547, "fcm_dpo/q_t": 0.42270296812057495, "grad_norm": 31.659528732299805, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -0.46579843759536743, "logits/rejected": -0.45978784561157227, "logps/chosen": -396.13726806640625, "logps/ref_chosen": -53.86951446533203, "logps/ref_rejected": -90.7692642211914, "logps/rejected": -597.3270263671875, "loss": 1.1397, "margin_dpo/margin_mean": 164.2899627685547, "margin_dpo/margin_std": 245.25772094726562, "step": 361 }, { "KL/chosen_KL_mean": -297.3023376464844, "KL/mean": -432.37384033203125, "KL/rejected_KL_mean": -567.4454345703125, "KL/std": 242.36074829101562, "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.0019996415358036757, "fcm_dpo/delta": -0.14447136223316193, "fcm_dpo/margin": 270.14306640625, "fcm_dpo/q_t": 0.3760761320590973, "grad_norm": 30.424165725708008, "learning_rate": 2.679511629948319e-07, "logits/chosen": -0.47015565633773804, "logits/rejected": -0.47379571199417114, "logps/chosen": -355.94140625, "logps/ref_chosen": -58.639060974121094, "logps/ref_rejected": -105.58195495605469, "logps/rejected": -673.02734375, "loss": 0.9747, "margin_dpo/margin_mean": 270.14306640625, "margin_dpo/margin_std": 252.4774169921875, "step": 362 }, { "KL/chosen_KL_mean": -270.51788330078125, "KL/mean": -407.674560546875, "KL/rejected_KL_mean": -544.8311767578125, "KL/std": 241.01730346679688, "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.0019766553305089474, "fcm_dpo/delta": -0.1460692584514618, "fcm_dpo/margin": 274.31329345703125, "fcm_dpo/q_t": 0.3755940794944763, "grad_norm": 27.271324157714844, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -0.4525992274284363, "logits/rejected": -0.43701645731925964, "logps/chosen": -315.0762939453125, "logps/ref_chosen": -44.558380126953125, "logps/ref_rejected": -74.69496154785156, "logps/rejected": -619.526123046875, "loss": 0.9773, "margin_dpo/margin_mean": 274.31329345703125, "margin_dpo/margin_std": 263.22528076171875, "step": 363 }, { "KL/chosen_KL_mean": -289.7828369140625, "KL/mean": -389.1034240722656, "KL/rejected_KL_mean": -488.4240417480469, "KL/std": 230.4542236328125, "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.0019719074480235577, "fcm_dpo/delta": 0.008304569870233536, "fcm_dpo/margin": 198.64120483398438, "fcm_dpo/q_t": 0.40914398431777954, "grad_norm": 25.84320640563965, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -0.4640185236930847, "logits/rejected": -0.46951600909233093, "logps/chosen": -338.67742919921875, "logps/ref_chosen": -48.894622802734375, "logps/ref_rejected": -91.395751953125, "logps/rejected": -579.81982421875, "loss": 1.0964, "margin_dpo/margin_mean": 198.64120483398438, "margin_dpo/margin_std": 262.0460205078125, "step": 364 }, { "KL/chosen_KL_mean": -278.07470703125, "KL/mean": -384.00201416015625, "KL/rejected_KL_mean": -489.9293518066406, "KL/std": 242.47592163085938, "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.0019668107852339745, "fcm_dpo/delta": -0.01703200675547123, "fcm_dpo/margin": 211.85464477539062, "fcm_dpo/q_t": 0.40431296825408936, "grad_norm": 24.915437698364258, "learning_rate": 2.641091331089811e-07, "logits/chosen": -0.4698528051376343, "logits/rejected": -0.4782125949859619, "logps/chosen": -329.56744384765625, "logps/ref_chosen": -51.49274444580078, "logps/ref_rejected": -92.70166778564453, "logps/rejected": -582.6309814453125, "loss": 1.0661, "margin_dpo/margin_mean": 211.8546600341797, "margin_dpo/margin_std": 245.96029663085938, "step": 365 }, { "KL/chosen_KL_mean": -254.35284423828125, "KL/mean": -363.2435302734375, "KL/rejected_KL_mean": -472.13421630859375, "KL/std": 230.414306640625, "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.001956676598638296, "fcm_dpo/delta": -0.02698526531457901, "fcm_dpo/margin": 217.7813720703125, "fcm_dpo/q_t": 0.40227261185646057, "grad_norm": 23.5860652923584, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -0.44011300802230835, "logits/rejected": -0.4551384449005127, "logps/chosen": -299.07342529296875, "logps/ref_chosen": -44.7205696105957, "logps/ref_rejected": -83.31040954589844, "logps/rejected": -555.4446411132812, "loss": 1.0726, "margin_dpo/margin_mean": 217.7813720703125, "margin_dpo/margin_std": 272.1701354980469, "step": 366 }, { "KL/chosen_KL_mean": -265.6231689453125, "KL/mean": -356.1582336425781, "KL/rejected_KL_mean": -446.69329833984375, "KL/std": 205.83560180664062, "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.0019680976402014494, "fcm_dpo/delta": 0.04436366260051727, "fcm_dpo/margin": 181.07012939453125, "fcm_dpo/q_t": 0.4157208800315857, "grad_norm": 19.829448699951172, "learning_rate": 2.615458646614349e-07, "logits/chosen": -0.4592265188694, "logits/rejected": -0.4390316903591156, "logps/chosen": -324.0285949707031, "logps/ref_chosen": -58.405418395996094, "logps/ref_rejected": -76.75132751464844, "logps/rejected": -523.444580078125, "loss": 1.1049, "margin_dpo/margin_mean": 181.07012939453125, "margin_dpo/margin_std": 216.0451202392578, "step": 367 }, { "KL/chosen_KL_mean": -253.53057861328125, "KL/mean": -390.93402099609375, "KL/rejected_KL_mean": -528.3375244140625, "KL/std": 229.09591674804688, "epoch": 0.540381791483113, "fcm_dpo/beta": 0.0019493200816214085, "fcm_dpo/delta": -0.13935965299606323, "fcm_dpo/margin": 274.8069152832031, "fcm_dpo/q_t": 0.3731291890144348, "grad_norm": 39.253082275390625, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -0.46787190437316895, "logits/rejected": -0.47481292486190796, "logps/chosen": -297.98309326171875, "logps/ref_chosen": -44.452518463134766, "logps/ref_rejected": -98.55526733398438, "logps/rejected": -626.8927612304688, "loss": 0.9579, "margin_dpo/margin_mean": 274.8069152832031, "margin_dpo/margin_std": 203.40640258789062, "step": 368 }, { "KL/chosen_KL_mean": -335.7747497558594, "KL/mean": -411.5692138671875, "KL/rejected_KL_mean": -487.3636474609375, "KL/std": 234.81884765625, "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.0019605052657425404, "fcm_dpo/delta": 0.10399104654788971, "fcm_dpo/margin": 151.58892822265625, "fcm_dpo/q_t": 0.4309307336807251, "grad_norm": 35.00344467163086, "learning_rate": 2.589813792448196e-07, "logits/chosen": -0.5002185702323914, "logits/rejected": -0.4782012701034546, "logps/chosen": -407.15625, "logps/ref_chosen": -71.38150024414062, "logps/ref_rejected": -91.29582214355469, "logps/rejected": -578.6594848632812, "loss": 1.1739, "margin_dpo/margin_mean": 151.58892822265625, "margin_dpo/margin_std": 259.8010559082031, "step": 369 }, { "KL/chosen_KL_mean": -357.89031982421875, "KL/mean": -425.114990234375, "KL/rejected_KL_mean": -492.33966064453125, "KL/std": 241.58839416503906, "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.0019837317522615194, "fcm_dpo/delta": 0.1349019706249237, "fcm_dpo/margin": 134.4493408203125, "fcm_dpo/q_t": 0.4381292462348938, "grad_norm": 32.33375549316406, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -0.5110623836517334, "logits/rejected": -0.5016048550605774, "logps/chosen": -429.4978332519531, "logps/ref_chosen": -71.60749816894531, "logps/ref_rejected": -97.25978088378906, "logps/rejected": -589.5994873046875, "loss": 1.2033, "margin_dpo/margin_mean": 134.4493408203125, "margin_dpo/margin_std": 263.70660400390625, "step": 370 }, { "KL/chosen_KL_mean": -349.0395202636719, "KL/mean": -454.69659423828125, "KL/rejected_KL_mean": -560.3536987304688, "KL/std": 257.37115478515625, "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.0019879695028066635, "fcm_dpo/delta": -0.020561737939715385, "fcm_dpo/margin": 211.31417846679688, "fcm_dpo/q_t": 0.40404176712036133, "grad_norm": 22.81478500366211, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -0.5306122303009033, "logits/rejected": -0.5140925645828247, "logps/chosen": -418.4540100097656, "logps/ref_chosen": -69.41448974609375, "logps/ref_rejected": -99.17217254638672, "logps/rejected": -659.52587890625, "loss": 1.0828, "margin_dpo/margin_mean": 211.31417846679688, "margin_dpo/margin_std": 280.86492919921875, "step": 371 }, { "KL/chosen_KL_mean": -343.93743896484375, "KL/mean": -468.98333740234375, "KL/rejected_KL_mean": -594.0292358398438, "KL/std": 296.0002136230469, "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.0019684848375618458, "fcm_dpo/delta": -0.09475220739841461, "fcm_dpo/margin": 250.091796875, "fcm_dpo/q_t": 0.39228248596191406, "grad_norm": 26.199745178222656, "learning_rate": 2.551329606220976e-07, "logits/chosen": -0.522529661655426, "logits/rejected": -0.49960562586784363, "logps/chosen": -405.75543212890625, "logps/ref_chosen": -61.8179931640625, "logps/ref_rejected": -78.53948974609375, "logps/rejected": -672.5687255859375, "loss": 1.0493, "margin_dpo/margin_mean": 250.09181213378906, "margin_dpo/margin_std": 338.2940673828125, "step": 372 }, { "KL/chosen_KL_mean": -372.826171875, "KL/mean": -492.62713623046875, "KL/rejected_KL_mean": -612.4281005859375, "KL/std": 283.1507873535156, "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.001959893386811018, "fcm_dpo/delta": -0.07133302092552185, "fcm_dpo/margin": 239.6019287109375, "fcm_dpo/q_t": 0.3923265337944031, "grad_norm": 32.04021072387695, "learning_rate": 2.538498388222517e-07, "logits/chosen": -0.5104029774665833, "logits/rejected": -0.48797744512557983, "logps/chosen": -437.04327392578125, "logps/ref_chosen": -64.21713256835938, "logps/ref_rejected": -85.95960998535156, "logps/rejected": -698.3876953125, "loss": 1.0425, "margin_dpo/margin_mean": 239.6019287109375, "margin_dpo/margin_std": 276.7124328613281, "step": 373 }, { "KL/chosen_KL_mean": -337.2335205078125, "KL/mean": -444.0258483886719, "KL/rejected_KL_mean": -550.8182373046875, "KL/std": 301.60516357421875, "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.001944505493156612, "fcm_dpo/delta": -0.01592355966567993, "fcm_dpo/margin": 213.58465576171875, "fcm_dpo/q_t": 0.4093152582645416, "grad_norm": 35.01904296875, "learning_rate": 2.525666155755725e-07, "logits/chosen": -0.5831848382949829, "logits/rejected": -0.563025951385498, "logps/chosen": -407.8836975097656, "logps/ref_chosen": -70.65018463134766, "logps/ref_rejected": -93.64016723632812, "logps/rejected": -644.4583740234375, "loss": 1.1086, "margin_dpo/margin_mean": 213.5846710205078, "margin_dpo/margin_std": 333.40771484375, "step": 374 }, { "KL/chosen_KL_mean": -351.09576416015625, "KL/mean": -455.89691162109375, "KL/rejected_KL_mean": -560.697998046875, "KL/std": 252.77169799804688, "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.00194136006757617, "fcm_dpo/delta": -0.0074718669056892395, "fcm_dpo/margin": 209.60220336914062, "fcm_dpo/q_t": 0.4071958065032959, "grad_norm": 37.43479537963867, "learning_rate": 2.512833246961859e-07, "logits/chosen": -0.5338048934936523, "logits/rejected": -0.531802773475647, "logps/chosen": -411.176025390625, "logps/ref_chosen": -60.080223083496094, "logps/ref_rejected": -88.93830871582031, "logps/rejected": -649.6363525390625, "loss": 1.1005, "margin_dpo/margin_mean": 209.6022186279297, "margin_dpo/margin_std": 296.93292236328125, "step": 375 }, { "KL/chosen_KL_mean": -351.7929992675781, "KL/mean": -482.906494140625, "KL/rejected_KL_mean": -614.02001953125, "KL/std": 278.260009765625, "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.0019302696455270052, "fcm_dpo/delta": -0.10899202525615692, "fcm_dpo/margin": 262.2270202636719, "fcm_dpo/q_t": 0.38526269793510437, "grad_norm": 26.96346092224121, "learning_rate": 2.5e-07, "logits/chosen": -0.5299756526947021, "logits/rejected": -0.5187512040138245, "logps/chosen": -414.45330810546875, "logps/ref_chosen": -62.660308837890625, "logps/ref_rejected": -105.52660369873047, "logps/rejected": -719.546630859375, "loss": 1.03, "margin_dpo/margin_mean": 262.2270202636719, "margin_dpo/margin_std": 318.892822265625, "step": 376 }, { "KL/chosen_KL_mean": -351.45703125, "KL/mean": -473.6996765136719, "KL/rejected_KL_mean": -595.9423828125, "KL/std": 293.1138000488281, "epoch": 0.55359765051395, "fcm_dpo/beta": 0.001917961984872818, "fcm_dpo/delta": -0.07053986191749573, "fcm_dpo/margin": 244.4853515625, "fcm_dpo/q_t": 0.39490044116973877, "grad_norm": 32.56324005126953, "learning_rate": 2.487166753038141e-07, "logits/chosen": -0.4774741530418396, "logits/rejected": -0.4759942889213562, "logps/chosen": -405.9357604980469, "logps/ref_chosen": -54.478736877441406, "logps/ref_rejected": -98.70335388183594, "logps/rejected": -694.645751953125, "loss": 1.0543, "margin_dpo/margin_mean": 244.4853515625, "margin_dpo/margin_std": 313.40069580078125, "step": 377 }, { "KL/chosen_KL_mean": -332.24822998046875, "KL/mean": -464.07470703125, "KL/rejected_KL_mean": -595.9012451171875, "KL/std": 259.45196533203125, "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.001897640759125352, "fcm_dpo/delta": -0.10289917886257172, "fcm_dpo/margin": 263.6529541015625, "fcm_dpo/q_t": 0.38391435146331787, "grad_norm": 40.62555694580078, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -0.5073676109313965, "logits/rejected": -0.5229573249816895, "logps/chosen": -377.2687683105469, "logps/ref_chosen": -45.02053451538086, "logps/ref_rejected": -88.0469741821289, "logps/rejected": -683.9481811523438, "loss": 1.0141, "margin_dpo/margin_mean": 263.6529541015625, "margin_dpo/margin_std": 282.84375, "step": 378 }, { "KL/chosen_KL_mean": -369.21722412109375, "KL/mean": -490.90283203125, "KL/rejected_KL_mean": -612.58837890625, "KL/std": 263.14385986328125, "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.0018807162996381521, "fcm_dpo/delta": -0.059339869767427444, "fcm_dpo/margin": 243.37112426757812, "fcm_dpo/q_t": 0.3961556851863861, "grad_norm": 41.72347640991211, "learning_rate": 2.461501611777483e-07, "logits/chosen": -0.5051323771476746, "logits/rejected": -0.5261910557746887, "logps/chosen": -422.39935302734375, "logps/ref_chosen": -53.182098388671875, "logps/ref_rejected": -114.3001708984375, "logps/rejected": -726.8885498046875, "loss": 1.0571, "margin_dpo/margin_mean": 243.3711395263672, "margin_dpo/margin_std": 304.7427978515625, "step": 379 }, { "KL/chosen_KL_mean": -356.805419921875, "KL/mean": -492.5494384765625, "KL/rejected_KL_mean": -628.29345703125, "KL/std": 293.99835205078125, "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.001867371378466487, "fcm_dpo/delta": -0.10970066487789154, "fcm_dpo/margin": 271.48809814453125, "fcm_dpo/q_t": 0.3830481171607971, "grad_norm": 29.717735290527344, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -0.45388370752334595, "logits/rejected": -0.480474054813385, "logps/chosen": -408.158447265625, "logps/ref_chosen": -51.3530387878418, "logps/ref_rejected": -104.19169616699219, "logps/rejected": -732.4851684570312, "loss": 1.0195, "margin_dpo/margin_mean": 271.4880676269531, "margin_dpo/margin_std": 310.1554260253906, "step": 380 }, { "KL/chosen_KL_mean": -361.4216613769531, "KL/mean": -462.06500244140625, "KL/rejected_KL_mean": -562.7083129882812, "KL/std": 248.3142852783203, "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.001862676115706563, "fcm_dpo/delta": 0.02552897296845913, "fcm_dpo/margin": 201.28662109375, "fcm_dpo/q_t": 0.4147588610649109, "grad_norm": 27.22147560119629, "learning_rate": 2.435840528363426e-07, "logits/chosen": -0.4980400800704956, "logits/rejected": -0.4761255383491516, "logps/chosen": -419.2247314453125, "logps/ref_chosen": -57.80306625366211, "logps/ref_rejected": -79.21940612792969, "logps/rejected": -641.927734375, "loss": 1.1335, "margin_dpo/margin_mean": 201.28662109375, "margin_dpo/margin_std": 328.384033203125, "step": 381 }, { "KL/chosen_KL_mean": -360.59393310546875, "KL/mean": -480.1690673828125, "KL/rejected_KL_mean": -599.7442626953125, "KL/std": 239.404296875, "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.0018593824934214354, "fcm_dpo/delta": -0.045690521597862244, "fcm_dpo/margin": 239.15036010742188, "fcm_dpo/q_t": 0.3968258202075958, "grad_norm": 37.34941101074219, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -0.5110651254653931, "logits/rejected": -0.5134783983230591, "logps/chosen": -426.61419677734375, "logps/ref_chosen": -66.02030181884766, "logps/ref_rejected": -110.71016693115234, "logps/rejected": -710.4544677734375, "loss": 1.0446, "margin_dpo/margin_mean": 239.15036010742188, "margin_dpo/margin_std": 259.5975646972656, "step": 382 }, { "KL/chosen_KL_mean": -368.52093505859375, "KL/mean": -489.22088623046875, "KL/rejected_KL_mean": -609.9208984375, "KL/std": 261.40411376953125, "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.0018501915037631989, "fcm_dpo/delta": -0.04771873354911804, "fcm_dpo/margin": 241.3999481201172, "fcm_dpo/q_t": 0.39832523465156555, "grad_norm": 27.045928955078125, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -0.49125421047210693, "logits/rejected": -0.4994109869003296, "logps/chosen": -418.91241455078125, "logps/ref_chosen": -50.39148712158203, "logps/ref_rejected": -93.71589660644531, "logps/rejected": -703.63671875, "loss": 1.0802, "margin_dpo/margin_mean": 241.39993286132812, "margin_dpo/margin_std": 340.89404296875, "step": 383 }, { "KL/chosen_KL_mean": -365.1363525390625, "KL/mean": -463.38427734375, "KL/rejected_KL_mean": -561.6322021484375, "KL/std": 249.699462890625, "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.0018579517491161823, "fcm_dpo/delta": 0.0352373830974102, "fcm_dpo/margin": 196.4958038330078, "fcm_dpo/q_t": 0.4140353202819824, "grad_norm": 22.924989700317383, "learning_rate": 2.397362428170992e-07, "logits/chosen": -0.5382051467895508, "logits/rejected": -0.529058575630188, "logps/chosen": -417.1824645996094, "logps/ref_chosen": -52.046104431152344, "logps/ref_rejected": -85.76089477539062, "logps/rejected": -647.39306640625, "loss": 1.1015, "margin_dpo/margin_mean": 196.4958038330078, "margin_dpo/margin_std": 234.65902709960938, "step": 384 }, { "KL/chosen_KL_mean": -334.7351379394531, "KL/mean": -454.896240234375, "KL/rejected_KL_mean": -575.0573120117188, "KL/std": 213.41387939453125, "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.0018485202454030514, "fcm_dpo/delta": -0.04525205120444298, "fcm_dpo/margin": 240.32217407226562, "fcm_dpo/q_t": 0.3952626585960388, "grad_norm": 33.12544250488281, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -0.531487762928009, "logits/rejected": -0.4987449645996094, "logps/chosen": -400.28729248046875, "logps/ref_chosen": -65.55215454101562, "logps/ref_rejected": -77.82792663574219, "logps/rejected": -652.88525390625, "loss": 1.0361, "margin_dpo/margin_mean": 240.32217407226562, "margin_dpo/margin_std": 236.79721069335938, "step": 385 }, { "KL/chosen_KL_mean": -352.7705078125, "KL/mean": -480.95391845703125, "KL/rejected_KL_mean": -609.1373291015625, "KL/std": 269.5135498046875, "epoch": 0.566813509544787, "fcm_dpo/beta": 0.0018375307554379106, "fcm_dpo/delta": -0.07277083396911621, "fcm_dpo/margin": 256.3668212890625, "fcm_dpo/q_t": 0.39270809292793274, "grad_norm": 27.93240737915039, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -0.5182179808616638, "logits/rejected": -0.510960578918457, "logps/chosen": -410.9923400878906, "logps/ref_chosen": -58.22185516357422, "logps/ref_rejected": -92.32742309570312, "logps/rejected": -701.4647216796875, "loss": 1.0448, "margin_dpo/margin_mean": 256.3668212890625, "margin_dpo/margin_std": 312.1371154785156, "step": 386 }, { "KL/chosen_KL_mean": -390.4794006347656, "KL/mean": -495.60986328125, "KL/rejected_KL_mean": -600.7403564453125, "KL/std": 248.39511108398438, "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.0018315855413675308, "fcm_dpo/delta": 0.015043869614601135, "fcm_dpo/margin": 210.26095581054688, "fcm_dpo/q_t": 0.4094206690788269, "grad_norm": 41.249908447265625, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -0.6175644397735596, "logits/rejected": -0.5995555520057678, "logps/chosen": -456.89886474609375, "logps/ref_chosen": -66.41944885253906, "logps/ref_rejected": -92.16915893554688, "logps/rejected": -692.9094848632812, "loss": 1.0887, "margin_dpo/margin_mean": 210.26095581054688, "margin_dpo/margin_std": 248.58956909179688, "step": 387 }, { "KL/chosen_KL_mean": -377.8406677246094, "KL/mean": -515.5494384765625, "KL/rejected_KL_mean": -653.2581787109375, "KL/std": 286.0020751953125, "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.0018191644921898842, "fcm_dpo/delta": -0.10386500507593155, "fcm_dpo/margin": 275.41748046875, "fcm_dpo/q_t": 0.3879193663597107, "grad_norm": 30.190383911132812, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -0.49368101358413696, "logits/rejected": -0.5035091042518616, "logps/chosen": -427.9701232910156, "logps/ref_chosen": -50.129459381103516, "logps/ref_rejected": -104.43305969238281, "logps/rejected": -757.6912231445312, "loss": 1.0263, "margin_dpo/margin_mean": 275.41748046875, "margin_dpo/margin_std": 328.736328125, "step": 388 }, { "KL/chosen_KL_mean": -415.5103454589844, "KL/mean": -533.9041748046875, "KL/rejected_KL_mean": -652.2980346679688, "KL/std": 284.1199951171875, "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.001811205642297864, "fcm_dpo/delta": -0.029575031250715256, "fcm_dpo/margin": 236.78768920898438, "fcm_dpo/q_t": 0.40235158801078796, "grad_norm": 33.09611892700195, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -0.5412299633026123, "logits/rejected": -0.5279366970062256, "logps/chosen": -473.41693115234375, "logps/ref_chosen": -57.906593322753906, "logps/ref_rejected": -77.91454315185547, "logps/rejected": -730.2125854492188, "loss": 1.0798, "margin_dpo/margin_mean": 236.78770446777344, "margin_dpo/margin_std": 318.11456298828125, "step": 389 }, { "KL/chosen_KL_mean": -415.3945007324219, "KL/mean": -532.8773193359375, "KL/rejected_KL_mean": -650.360107421875, "KL/std": 292.0013427734375, "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.001804722473025322, "fcm_dpo/delta": -0.024779455736279488, "fcm_dpo/margin": 234.965576171875, "fcm_dpo/q_t": 0.40768325328826904, "grad_norm": 31.104320526123047, "learning_rate": 2.320488370051681e-07, "logits/chosen": -0.521344780921936, "logits/rejected": -0.5126668810844421, "logps/chosen": -464.62042236328125, "logps/ref_chosen": -49.22591781616211, "logps/ref_rejected": -85.5281982421875, "logps/rejected": -735.8883056640625, "loss": 1.1072, "margin_dpo/margin_mean": 234.965576171875, "margin_dpo/margin_std": 377.9402160644531, "step": 390 }, { "KL/chosen_KL_mean": -402.7524719238281, "KL/mean": -475.3613586425781, "KL/rejected_KL_mean": -547.97021484375, "KL/std": 272.24322509765625, "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.0018261673394590616, "fcm_dpo/delta": 0.13658833503723145, "fcm_dpo/margin": 145.21774291992188, "fcm_dpo/q_t": 0.43881237506866455, "grad_norm": 44.82551193237305, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -0.5299597978591919, "logits/rejected": -0.5175088047981262, "logps/chosen": -467.0821228027344, "logps/ref_chosen": -64.32965087890625, "logps/ref_rejected": -86.73820495605469, "logps/rejected": -634.7084350585938, "loss": 1.1992, "margin_dpo/margin_mean": 145.2177276611328, "margin_dpo/margin_std": 272.92193603515625, "step": 391 }, { "KL/chosen_KL_mean": -344.9163513183594, "KL/mean": -484.80401611328125, "KL/rejected_KL_mean": -624.6917114257812, "KL/std": 282.89739990234375, "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.0018200704362243414, "fcm_dpo/delta": -0.11199337244033813, "fcm_dpo/margin": 279.7752990722656, "fcm_dpo/q_t": 0.383048415184021, "grad_norm": 29.697546005249023, "learning_rate": 2.294897926507156e-07, "logits/chosen": -0.5169392824172974, "logits/rejected": -0.5098834037780762, "logps/chosen": -398.42034912109375, "logps/ref_chosen": -53.50397872924805, "logps/ref_rejected": -102.34584045410156, "logps/rejected": -727.0375366210938, "loss": 0.9991, "margin_dpo/margin_mean": 279.77532958984375, "margin_dpo/margin_std": 275.7098083496094, "step": 392 }, { "KL/chosen_KL_mean": -343.71734619140625, "KL/mean": -456.7199401855469, "KL/rejected_KL_mean": -569.7225341796875, "KL/std": 284.97540283203125, "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0018092221580445766, "fcm_dpo/delta": -0.009102771058678627, "fcm_dpo/margin": 226.00515747070312, "fcm_dpo/q_t": 0.41020962595939636, "grad_norm": 47.57746124267578, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -0.5029352307319641, "logits/rejected": -0.504021406173706, "logps/chosen": -390.1912841796875, "logps/ref_chosen": -46.473915100097656, "logps/ref_rejected": -71.96885681152344, "logps/rejected": -641.69140625, "loss": 1.106, "margin_dpo/margin_mean": 226.00515747070312, "margin_dpo/margin_std": 352.2773742675781, "step": 393 }, { "KL/chosen_KL_mean": -391.92578125, "KL/mean": -508.56024169921875, "KL/rejected_KL_mean": -625.1947021484375, "KL/std": 295.7176818847656, "epoch": 0.57856093979442, "fcm_dpo/beta": 0.0018048053607344627, "fcm_dpo/delta": -0.02154848352074623, "fcm_dpo/margin": 233.2689208984375, "fcm_dpo/q_t": 0.4036337435245514, "grad_norm": 27.146669387817383, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -0.5260942578315735, "logits/rejected": -0.524019181728363, "logps/chosen": -444.83734130859375, "logps/ref_chosen": -52.91154861450195, "logps/ref_rejected": -90.8226318359375, "logps/rejected": -716.017333984375, "loss": 1.0749, "margin_dpo/margin_mean": 233.2689208984375, "margin_dpo/margin_std": 294.8382568359375, "step": 394 }, { "KL/chosen_KL_mean": -382.04046630859375, "KL/mean": -507.2308654785156, "KL/rejected_KL_mean": -632.4212646484375, "KL/std": 293.729736328125, "epoch": 0.580029368575624, "fcm_dpo/beta": 0.001797014381736517, "fcm_dpo/delta": -0.051209770143032074, "fcm_dpo/margin": 250.38079833984375, "fcm_dpo/q_t": 0.39935189485549927, "grad_norm": 38.09134292602539, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -0.5735211372375488, "logits/rejected": -0.5697096586227417, "logps/chosen": -444.5865478515625, "logps/ref_chosen": -62.546112060546875, "logps/ref_rejected": -83.78262329101562, "logps/rejected": -716.203857421875, "loss": 1.0749, "margin_dpo/margin_mean": 250.38082885742188, "margin_dpo/margin_std": 346.9578857421875, "step": 395 }, { "KL/chosen_KL_mean": -390.10418701171875, "KL/mean": -504.8359375, "KL/rejected_KL_mean": -619.5677490234375, "KL/std": 280.2290344238281, "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.00179403240326792, "fcm_dpo/delta": -0.01193785946816206, "fcm_dpo/margin": 229.46353149414062, "fcm_dpo/q_t": 0.4044559597969055, "grad_norm": 26.76568031311035, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -0.5892349481582642, "logits/rejected": -0.5674149394035339, "logps/chosen": -459.10009765625, "logps/ref_chosen": -68.99594116210938, "logps/ref_rejected": -88.64665985107422, "logps/rejected": -708.21435546875, "loss": 1.0683, "margin_dpo/margin_mean": 229.4635467529297, "margin_dpo/margin_std": 264.5689392089844, "step": 396 }, { "KL/chosen_KL_mean": -362.35675048828125, "KL/mean": -515.5724487304688, "KL/rejected_KL_mean": -668.7880859375, "KL/std": 288.4814453125, "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.0017716542351990938, "fcm_dpo/delta": -0.14731790125370026, "fcm_dpo/margin": 306.431396484375, "fcm_dpo/q_t": 0.3749983310699463, "grad_norm": 41.37638854980469, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -0.5667599439620972, "logits/rejected": -0.5667216181755066, "logps/chosen": -423.6339111328125, "logps/ref_chosen": -61.27716827392578, "logps/ref_rejected": -103.11612701416016, "logps/rejected": -771.9042358398438, "loss": 0.9805, "margin_dpo/margin_mean": 306.4313659667969, "margin_dpo/margin_std": 299.587646484375, "step": 397 }, { "KL/chosen_KL_mean": -423.58306884765625, "KL/mean": -554.8473510742188, "KL/rejected_KL_mean": -686.1116943359375, "KL/std": 320.2618713378906, "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.0017602917505428195, "fcm_dpo/delta": -0.0635901689529419, "fcm_dpo/margin": 262.528564453125, "fcm_dpo/q_t": 0.3985213041305542, "grad_norm": 32.223777770996094, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -0.6214488744735718, "logits/rejected": -0.6171579957008362, "logps/chosen": -491.734619140625, "logps/ref_chosen": -68.15155029296875, "logps/ref_rejected": -108.52360534667969, "logps/rejected": -794.63525390625, "loss": 1.0673, "margin_dpo/margin_mean": 262.528564453125, "margin_dpo/margin_std": 365.87701416015625, "step": 398 }, { "KL/chosen_KL_mean": -380.365478515625, "KL/mean": -508.25189208984375, "KL/rejected_KL_mean": -636.1383056640625, "KL/std": 293.50164794921875, "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0017485737334936857, "fcm_dpo/delta": -0.048456672579050064, "fcm_dpo/margin": 255.77288818359375, "fcm_dpo/q_t": 0.40013986825942993, "grad_norm": 38.44267272949219, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -0.5989868640899658, "logits/rejected": -0.5554165840148926, "logps/chosen": -441.2552795410156, "logps/ref_chosen": -60.889801025390625, "logps/ref_rejected": -77.965576171875, "logps/rejected": -714.1038818359375, "loss": 1.08, "margin_dpo/margin_mean": 255.77285766601562, "margin_dpo/margin_std": 361.84649658203125, "step": 399 }, { "KL/chosen_KL_mean": -368.0055236816406, "KL/mean": -543.0501098632812, "KL/rejected_KL_mean": -718.0947265625, "KL/std": 286.2700500488281, "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.0017187241464853287, "fcm_dpo/delta": -0.20846638083457947, "fcm_dpo/margin": 350.08917236328125, "fcm_dpo/q_t": 0.3612917959690094, "grad_norm": 26.391523361206055, "learning_rate": 2.192779273338215e-07, "logits/chosen": -0.5936537981033325, "logits/rejected": -0.5919471383094788, "logps/chosen": -431.64910888671875, "logps/ref_chosen": -63.64359664916992, "logps/ref_rejected": -105.252685546875, "logps/rejected": -823.347412109375, "loss": 0.9419, "margin_dpo/margin_mean": 350.08917236328125, "margin_dpo/margin_std": 319.36767578125, "step": 400 }, { "KL/chosen_KL_mean": -435.6270446777344, "KL/mean": -533.0802612304688, "KL/rejected_KL_mean": -630.533447265625, "KL/std": 307.6553039550781, "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.0017190496437251568, "fcm_dpo/delta": 0.06607242673635483, "fcm_dpo/margin": 194.9063720703125, "fcm_dpo/q_t": 0.426999032497406, "grad_norm": 35.03347396850586, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -0.591301441192627, "logits/rejected": -0.5788199305534363, "logps/chosen": -492.79010009765625, "logps/ref_chosen": -57.16303253173828, "logps/ref_rejected": -83.79249572753906, "logps/rejected": -714.325927734375, "loss": 1.1903, "margin_dpo/margin_mean": 194.9063720703125, "margin_dpo/margin_std": 405.22808837890625, "step": 401 }, { "KL/chosen_KL_mean": -349.45294189453125, "KL/mean": -544.983642578125, "KL/rejected_KL_mean": -740.514404296875, "KL/std": 371.795654296875, "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.001689492492005229, "fcm_dpo/delta": -0.26964718103408813, "fcm_dpo/margin": 391.06146240234375, "fcm_dpo/q_t": 0.3538634181022644, "grad_norm": 27.208879470825195, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -0.6506165266036987, "logits/rejected": -0.6465029120445251, "logps/chosen": -400.1933288574219, "logps/ref_chosen": -50.74037170410156, "logps/ref_rejected": -81.0460433959961, "logps/rejected": -821.5604248046875, "loss": 0.917, "margin_dpo/margin_mean": 391.06146240234375, "margin_dpo/margin_std": 385.4092712402344, "step": 402 }, { "KL/chosen_KL_mean": -376.1459655761719, "KL/mean": -522.2297973632812, "KL/rejected_KL_mean": -668.3135375976562, "KL/std": 303.7095031738281, "epoch": 0.591776798825257, "fcm_dpo/beta": 0.0016665069852024317, "fcm_dpo/delta": -0.08910196274518967, "fcm_dpo/margin": 292.1676330566406, "fcm_dpo/q_t": 0.388282835483551, "grad_norm": 30.522640228271484, "learning_rate": 2.154609112620295e-07, "logits/chosen": -0.6762701869010925, "logits/rejected": -0.6811692714691162, "logps/chosen": -423.29327392578125, "logps/ref_chosen": -47.14731216430664, "logps/ref_rejected": -77.2666015625, "logps/rejected": -745.5802001953125, "loss": 1.0256, "margin_dpo/margin_mean": 292.1676025390625, "margin_dpo/margin_std": 321.37841796875, "step": 403 }, { "KL/chosen_KL_mean": -453.879638671875, "KL/mean": -607.360107421875, "KL/rejected_KL_mean": -760.840576171875, "KL/std": 370.8985595703125, "epoch": 0.593245227606461, "fcm_dpo/beta": 0.0016497753094881773, "fcm_dpo/delta": -0.1091829165816307, "fcm_dpo/margin": 306.96087646484375, "fcm_dpo/q_t": 0.3911134600639343, "grad_norm": 31.827224731445312, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -0.6803830862045288, "logits/rejected": -0.6808692216873169, "logps/chosen": -501.75494384765625, "logps/ref_chosen": -47.875274658203125, "logps/ref_rejected": -77.15499877929688, "logps/rejected": -837.9955444335938, "loss": 1.081, "margin_dpo/margin_mean": 306.96087646484375, "margin_dpo/margin_std": 484.5469970703125, "step": 404 }, { "KL/chosen_KL_mean": -506.3373718261719, "KL/mean": -623.0357666015625, "KL/rejected_KL_mean": -739.7340698242188, "KL/std": 341.96575927734375, "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.0016373193357139826, "fcm_dpo/delta": -0.09679268300533295, "fcm_dpo/margin": 233.39666748046875, "fcm_dpo/q_t": 0.4155503511428833, "grad_norm": 44.04417419433594, "learning_rate": 2.129207286861638e-07, "logits/chosen": -0.6529548168182373, "logits/rejected": -0.6453539729118347, "logps/chosen": -571.500244140625, "logps/ref_chosen": -65.16290283203125, "logps/ref_rejected": -87.18678283691406, "logps/rejected": -826.9208984375, "loss": 1.1604, "margin_dpo/margin_mean": 233.3966827392578, "margin_dpo/margin_std": 420.79632568359375, "step": 405 }, { "KL/chosen_KL_mean": -504.4444580078125, "KL/mean": -670.9434814453125, "KL/rejected_KL_mean": -837.4425048828125, "KL/std": 409.6977233886719, "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.0016175368800759315, "fcm_dpo/delta": -0.14244601130485535, "fcm_dpo/margin": 332.998046875, "fcm_dpo/q_t": 0.38683146238327026, "grad_norm": 31.830657958984375, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -0.7117749452590942, "logits/rejected": -0.7342991232872009, "logps/chosen": -554.1852416992188, "logps/ref_chosen": -49.740814208984375, "logps/ref_rejected": -92.07862854003906, "logps/rejected": -929.5211181640625, "loss": 1.041, "margin_dpo/margin_mean": 332.998046875, "margin_dpo/margin_std": 462.95513916015625, "step": 406 }, { "KL/chosen_KL_mean": -487.5089111328125, "KL/mean": -582.2493896484375, "KL/rejected_KL_mean": -676.9898681640625, "KL/std": 341.89263916015625, "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.0016208180459216237, "fcm_dpo/delta": 0.0940016433596611, "fcm_dpo/margin": 189.48098754882812, "fcm_dpo/q_t": 0.42886239290237427, "grad_norm": 40.08480453491211, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -0.7205852270126343, "logits/rejected": -0.6887321472167969, "logps/chosen": -543.839599609375, "logps/ref_chosen": -56.33069610595703, "logps/ref_rejected": -77.51209259033203, "logps/rejected": -754.501953125, "loss": 1.2139, "margin_dpo/margin_mean": 189.48097229003906, "margin_dpo/margin_std": 426.8612060546875, "step": 407 }, { "KL/chosen_KL_mean": -488.0704345703125, "KL/mean": -594.4813232421875, "KL/rejected_KL_mean": -700.8922729492188, "KL/std": 310.1326904296875, "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.0016301569994539022, "fcm_dpo/delta": 0.05386827886104584, "fcm_dpo/margin": 212.82177734375, "fcm_dpo/q_t": 0.4206947088241577, "grad_norm": 41.006221771240234, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -0.6842703819274902, "logits/rejected": -0.6612120866775513, "logps/chosen": -557.8597412109375, "logps/ref_chosen": -69.789306640625, "logps/ref_rejected": -90.09693908691406, "logps/rejected": -790.9891967773438, "loss": 1.1492, "margin_dpo/margin_mean": 212.82179260253906, "margin_dpo/margin_std": 353.79248046875, "step": 408 }, { "KL/chosen_KL_mean": -453.1000671386719, "KL/mean": -564.703125, "KL/rejected_KL_mean": -676.30615234375, "KL/std": 381.3478088378906, "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.0016393279656767845, "fcm_dpo/delta": 0.03426938131451607, "fcm_dpo/margin": 223.20606994628906, "fcm_dpo/q_t": 0.41481828689575195, "grad_norm": 59.193416595458984, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -0.6385992765426636, "logits/rejected": -0.6249116063117981, "logps/chosen": -520.41748046875, "logps/ref_chosen": -67.31744384765625, "logps/ref_rejected": -84.904296875, "logps/rejected": -761.21044921875, "loss": 1.1496, "margin_dpo/margin_mean": 223.20608520507812, "margin_dpo/margin_std": 382.4623107910156, "step": 409 }, { "KL/chosen_KL_mean": -416.6119384765625, "KL/mean": -551.2454833984375, "KL/rejected_KL_mean": -685.8790893554688, "KL/std": 340.37841796875, "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.0016306547913700342, "fcm_dpo/delta": -0.039969172328710556, "fcm_dpo/margin": 269.26708984375, "fcm_dpo/q_t": 0.39879608154296875, "grad_norm": 33.50337219238281, "learning_rate": 2.065879555832674e-07, "logits/chosen": -0.6794567108154297, "logits/rejected": -0.6865516901016235, "logps/chosen": -468.0772705078125, "logps/ref_chosen": -51.465354919433594, "logps/ref_rejected": -83.198974609375, "logps/rejected": -769.0780639648438, "loss": 1.0792, "margin_dpo/margin_mean": 269.26708984375, "margin_dpo/margin_std": 368.21893310546875, "step": 410 }, { "KL/chosen_KL_mean": -423.6837158203125, "KL/mean": -541.8031616210938, "KL/rejected_KL_mean": -659.922607421875, "KL/std": 336.0458984375, "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.0016164029948413372, "fcm_dpo/delta": -0.07978048175573349, "fcm_dpo/margin": 236.23883056640625, "fcm_dpo/q_t": 0.4166368842124939, "grad_norm": 45.770408630371094, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -0.6459128260612488, "logits/rejected": -0.6529507637023926, "logps/chosen": -475.9909973144531, "logps/ref_chosen": -52.30727005004883, "logps/ref_rejected": -80.69495391845703, "logps/rejected": -740.6175537109375, "loss": 1.1373, "margin_dpo/margin_mean": 236.23883056640625, "margin_dpo/margin_std": 390.1459655761719, "step": 411 }, { "KL/chosen_KL_mean": -404.8624267578125, "KL/mean": -531.0133056640625, "KL/rejected_KL_mean": -657.1642456054688, "KL/std": 315.80413818359375, "epoch": 0.604992657856094, "fcm_dpo/beta": 0.0016159769147634506, "fcm_dpo/delta": -0.007876180112361908, "fcm_dpo/margin": 252.3018341064453, "fcm_dpo/q_t": 0.40613073110580444, "grad_norm": 43.324363708496094, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -0.6817220449447632, "logits/rejected": -0.7102159261703491, "logps/chosen": -458.00653076171875, "logps/ref_chosen": -53.144126892089844, "logps/ref_rejected": -100.0608139038086, "logps/rejected": -757.2250366210938, "loss": 1.0908, "margin_dpo/margin_mean": 252.3018341064453, "margin_dpo/margin_std": 339.39776611328125, "step": 412 }, { "KL/chosen_KL_mean": -445.6668701171875, "KL/mean": -565.676025390625, "KL/rejected_KL_mean": -685.6851806640625, "KL/std": 343.90045166015625, "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0016158015932887793, "fcm_dpo/delta": 0.012414616532623768, "fcm_dpo/margin": 240.018310546875, "fcm_dpo/q_t": 0.4103829562664032, "grad_norm": 48.77497482299805, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -0.581852912902832, "logits/rejected": -0.5829395055770874, "logps/chosen": -507.24884033203125, "logps/ref_chosen": -61.58196258544922, "logps/ref_rejected": -99.47340393066406, "logps/rejected": -785.1585693359375, "loss": 1.1251, "margin_dpo/margin_mean": 240.01829528808594, "margin_dpo/margin_std": 382.1087646484375, "step": 413 }, { "KL/chosen_KL_mean": -410.1985778808594, "KL/mean": -545.940673828125, "KL/rejected_KL_mean": -681.6827392578125, "KL/std": 313.23284912109375, "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.0016170567832887173, "fcm_dpo/delta": -0.04011237248778343, "fcm_dpo/margin": 271.4841613769531, "fcm_dpo/q_t": 0.3984266519546509, "grad_norm": 41.59056091308594, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -0.5111421346664429, "logits/rejected": -0.5018640160560608, "logps/chosen": -456.830078125, "logps/ref_chosen": -46.63148498535156, "logps/ref_rejected": -87.64653015136719, "logps/rejected": -769.3292236328125, "loss": 1.0743, "margin_dpo/margin_mean": 271.4841613769531, "margin_dpo/margin_std": 356.08306884765625, "step": 414 }, { "KL/chosen_KL_mean": -406.1727294921875, "KL/mean": -505.6904296875, "KL/rejected_KL_mean": -605.2080688476562, "KL/std": 278.98065185546875, "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.0016174819320440292, "fcm_dpo/delta": 0.07935836911201477, "fcm_dpo/margin": 199.03533935546875, "fcm_dpo/q_t": 0.4250248372554779, "grad_norm": 32.30403518676758, "learning_rate": 2.002837796667909e-07, "logits/chosen": -0.5725187063217163, "logits/rejected": -0.5640978813171387, "logps/chosen": -484.791015625, "logps/ref_chosen": -78.6182861328125, "logps/ref_rejected": -100.47752380371094, "logps/rejected": -705.685546875, "loss": 1.1467, "margin_dpo/margin_mean": 199.0353240966797, "margin_dpo/margin_std": 299.36395263671875, "step": 415 }, { "KL/chosen_KL_mean": -368.3544616699219, "KL/mean": -525.67529296875, "KL/rejected_KL_mean": -682.99609375, "KL/std": 330.7052001953125, "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.001612334861420095, "fcm_dpo/delta": -0.11002575606107712, "fcm_dpo/margin": 314.6416015625, "fcm_dpo/q_t": 0.38389724493026733, "grad_norm": 30.73332405090332, "learning_rate": 1.990267419549914e-07, "logits/chosen": -0.5767337083816528, "logits/rejected": -0.577792763710022, "logps/chosen": -426.63360595703125, "logps/ref_chosen": -58.27912521362305, "logps/ref_rejected": -90.56871795654297, "logps/rejected": -773.5648193359375, "loss": 1.011, "margin_dpo/margin_mean": 314.6416015625, "margin_dpo/margin_std": 339.9158020019531, "step": 416 }, { "KL/chosen_KL_mean": -364.384765625, "KL/mean": -508.7856140136719, "KL/rejected_KL_mean": -653.1865234375, "KL/std": 298.44720458984375, "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.001599437789991498, "fcm_dpo/delta": -0.06335951387882233, "fcm_dpo/margin": 288.8017272949219, "fcm_dpo/q_t": 0.391997754573822, "grad_norm": 47.862770080566406, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -0.5380935668945312, "logits/rejected": -0.5061866641044617, "logps/chosen": -414.58349609375, "logps/ref_chosen": -50.1987190246582, "logps/ref_rejected": -68.15184020996094, "logps/rejected": -721.3383178710938, "loss": 1.0336, "margin_dpo/margin_mean": 288.8017272949219, "margin_dpo/margin_std": 306.3311462402344, "step": 417 }, { "KL/chosen_KL_mean": -390.33111572265625, "KL/mean": -528.9058837890625, "KL/rejected_KL_mean": -667.4805908203125, "KL/std": 329.7533874511719, "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.001596780028194189, "fcm_dpo/delta": -0.04398631677031517, "fcm_dpo/margin": 277.1495361328125, "fcm_dpo/q_t": 0.4014670252799988, "grad_norm": 24.271198272705078, "learning_rate": 1.965167291983757e-07, "logits/chosen": -0.6195993423461914, "logits/rejected": -0.5918015241622925, "logps/chosen": -472.3095703125, "logps/ref_chosen": -81.97846984863281, "logps/ref_rejected": -104.69148254394531, "logps/rejected": -772.172119140625, "loss": 1.0803, "margin_dpo/margin_mean": 277.1495361328125, "margin_dpo/margin_std": 378.1399841308594, "step": 418 }, { "KL/chosen_KL_mean": -359.47064208984375, "KL/mean": -516.6668701171875, "KL/rejected_KL_mean": -673.8631591796875, "KL/std": 296.4808349609375, "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.0015776680083945394, "fcm_dpo/delta": -0.09841260313987732, "fcm_dpo/margin": 314.3925476074219, "fcm_dpo/q_t": 0.38518738746643066, "grad_norm": 39.38912582397461, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -0.5384039282798767, "logits/rejected": -0.5316938161849976, "logps/chosen": -412.4192810058594, "logps/ref_chosen": -52.948646545410156, "logps/ref_rejected": -91.58309936523438, "logps/rejected": -765.4462890625, "loss": 1.0206, "margin_dpo/margin_mean": 314.39251708984375, "margin_dpo/margin_std": 347.848876953125, "step": 419 }, { "KL/chosen_KL_mean": -473.1541748046875, "KL/mean": -576.36474609375, "KL/rejected_KL_mean": -679.5751953125, "KL/std": 321.3580322265625, "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.0015831240452826023, "fcm_dpo/delta": 0.0743355080485344, "fcm_dpo/margin": 206.4210968017578, "fcm_dpo/q_t": 0.4267392158508301, "grad_norm": 55.58370590209961, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -0.5968104600906372, "logits/rejected": -0.5501741170883179, "logps/chosen": -550.924072265625, "logps/ref_chosen": -77.7699203491211, "logps/ref_rejected": -69.31985473632812, "logps/rejected": -748.8951416015625, "loss": 1.1855, "margin_dpo/margin_mean": 206.42111206054688, "margin_dpo/margin_std": 412.8056640625, "step": 420 }, { "KL/chosen_KL_mean": -396.0514831542969, "KL/mean": -511.7779235839844, "KL/rejected_KL_mean": -627.5043334960938, "KL/std": 307.17010498046875, "epoch": 0.618208516886931, "fcm_dpo/beta": 0.0015912302769720554, "fcm_dpo/delta": 0.03213408216834068, "fcm_dpo/margin": 231.45286560058594, "fcm_dpo/q_t": 0.41308170557022095, "grad_norm": 37.79745101928711, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -0.6047526001930237, "logits/rejected": -0.5907719135284424, "logps/chosen": -449.8173522949219, "logps/ref_chosen": -53.765865325927734, "logps/ref_rejected": -89.28144836425781, "logps/rejected": -716.7857666015625, "loss": 1.1119, "margin_dpo/margin_mean": 231.45286560058594, "margin_dpo/margin_std": 305.76007080078125, "step": 421 }, { "KL/chosen_KL_mean": -444.47344970703125, "KL/mean": -577.7868041992188, "KL/rejected_KL_mean": -711.10009765625, "KL/std": 318.5824890136719, "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.0015860656276345253, "fcm_dpo/delta": -0.02341364696621895, "fcm_dpo/margin": 266.6266784667969, "fcm_dpo/q_t": 0.4029381275177002, "grad_norm": 37.30603790283203, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -0.6128122806549072, "logits/rejected": -0.6130838394165039, "logps/chosen": -513.1072387695312, "logps/ref_chosen": -68.6337661743164, "logps/ref_rejected": -87.86351013183594, "logps/rejected": -798.963623046875, "loss": 1.09, "margin_dpo/margin_mean": 266.62664794921875, "margin_dpo/margin_std": 376.72772216796875, "step": 422 }, { "KL/chosen_KL_mean": -442.69097900390625, "KL/mean": -593.7772216796875, "KL/rejected_KL_mean": -744.863525390625, "KL/std": 316.8233947753906, "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.0015762131661176682, "fcm_dpo/delta": -0.07819212973117828, "fcm_dpo/margin": 302.17254638671875, "fcm_dpo/q_t": 0.39091968536376953, "grad_norm": 35.244747161865234, "learning_rate": 1.902669377503756e-07, "logits/chosen": -0.6149768829345703, "logits/rejected": -0.6239042282104492, "logps/chosen": -497.6812744140625, "logps/ref_chosen": -54.99030303955078, "logps/ref_rejected": -86.30654907226562, "logps/rejected": -831.1700439453125, "loss": 1.0466, "margin_dpo/margin_mean": 302.17254638671875, "margin_dpo/margin_std": 374.8946533203125, "step": 423 }, { "KL/chosen_KL_mean": -408.0943908691406, "KL/mean": -545.8341064453125, "KL/rejected_KL_mean": -683.57373046875, "KL/std": 307.42901611328125, "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.0015672901645302773, "fcm_dpo/delta": -0.03262971714138985, "fcm_dpo/margin": 275.4793395996094, "fcm_dpo/q_t": 0.40227210521698, "grad_norm": 37.73208236694336, "learning_rate": 1.890215699729057e-07, "logits/chosen": -0.6178228855133057, "logits/rejected": -0.5884814262390137, "logps/chosen": -464.1063232421875, "logps/ref_chosen": -56.01192092895508, "logps/ref_rejected": -66.47896575927734, "logps/rejected": -750.052734375, "loss": 1.073, "margin_dpo/margin_mean": 275.4793395996094, "margin_dpo/margin_std": 357.9488525390625, "step": 424 }, { "KL/chosen_KL_mean": -495.09136962890625, "KL/mean": -594.1044921875, "KL/rejected_KL_mean": -693.11767578125, "KL/std": 313.45208740234375, "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.001581181539222598, "fcm_dpo/delta": 0.08807256817817688, "fcm_dpo/margin": 198.02630615234375, "fcm_dpo/q_t": 0.4265795946121216, "grad_norm": 40.72890090942383, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -0.6312949657440186, "logits/rejected": -0.6337833404541016, "logps/chosen": -541.9603271484375, "logps/ref_chosen": -46.86899948120117, "logps/ref_rejected": -95.92545318603516, "logps/rejected": -789.0431518554688, "loss": 1.1785, "margin_dpo/margin_mean": 198.02630615234375, "margin_dpo/margin_std": 365.4080505371094, "step": 425 }, { "KL/chosen_KL_mean": -447.80145263671875, "KL/mean": -569.6341552734375, "KL/rejected_KL_mean": -691.466796875, "KL/std": 296.7207946777344, "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.0015872271033003926, "fcm_dpo/delta": 0.013328127562999725, "fcm_dpo/margin": 243.6654052734375, "fcm_dpo/q_t": 0.40938708186149597, "grad_norm": 39.436275482177734, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -0.6700231432914734, "logits/rejected": -0.6314303874969482, "logps/chosen": -524.385009765625, "logps/ref_chosen": -76.58354187011719, "logps/ref_rejected": -81.26658630371094, "logps/rejected": -772.7333984375, "loss": 1.0934, "margin_dpo/margin_mean": 243.6654052734375, "margin_dpo/margin_std": 301.1048583984375, "step": 426 }, { "KL/chosen_KL_mean": -404.7652587890625, "KL/mean": -502.8582458496094, "KL/rejected_KL_mean": -600.9512939453125, "KL/std": 270.48968505859375, "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.0015933769755065441, "fcm_dpo/delta": 0.08883827924728394, "fcm_dpo/margin": 196.18600463867188, "fcm_dpo/q_t": 0.4279603958129883, "grad_norm": 38.65625, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -0.6598665714263916, "logits/rejected": -0.6343759894371033, "logps/chosen": -469.619140625, "logps/ref_chosen": -64.8538818359375, "logps/ref_rejected": -78.5660171508789, "logps/rejected": -679.5172729492188, "loss": 1.1568, "margin_dpo/margin_mean": 196.18600463867188, "margin_dpo/margin_std": 310.1150817871094, "step": 427 }, { "KL/chosen_KL_mean": -541.8001708984375, "KL/mean": -685.3818359375, "KL/rejected_KL_mean": -828.9635009765625, "KL/std": 373.19793701171875, "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.0015891847433522344, "fcm_dpo/delta": -0.05788760259747505, "fcm_dpo/margin": 287.1632080078125, "fcm_dpo/q_t": 0.3983868360519409, "grad_norm": 42.709285736083984, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -0.6648718118667603, "logits/rejected": -0.6649109125137329, "logps/chosen": -604.4368896484375, "logps/ref_chosen": -62.63666534423828, "logps/ref_rejected": -103.28181457519531, "logps/rejected": -932.2452392578125, "loss": 1.0954, "margin_dpo/margin_mean": 287.1632080078125, "margin_dpo/margin_std": 445.24249267578125, "step": 428 }, { "KL/chosen_KL_mean": -558.6399536132812, "KL/mean": -670.6189575195312, "KL/rejected_KL_mean": -782.5979614257812, "KL/std": 353.8562927246094, "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.001599712180905044, "fcm_dpo/delta": 0.04193819314241409, "fcm_dpo/margin": 223.9580078125, "fcm_dpo/q_t": 0.4191283583641052, "grad_norm": 42.31879425048828, "learning_rate": 1.828194884925749e-07, "logits/chosen": -0.7320696711540222, "logits/rejected": -0.7067664861679077, "logps/chosen": -639.8739624023438, "logps/ref_chosen": -81.23401641845703, "logps/ref_rejected": -91.79493713378906, "logps/rejected": -874.3928833007812, "loss": 1.1686, "margin_dpo/margin_mean": 223.9580078125, "margin_dpo/margin_std": 418.2169494628906, "step": 429 }, { "KL/chosen_KL_mean": -485.84906005859375, "KL/mean": -585.9974365234375, "KL/rejected_KL_mean": -686.145751953125, "KL/std": 333.52752685546875, "epoch": 0.631424375917768, "fcm_dpo/beta": 0.0016081253997981548, "fcm_dpo/delta": 0.07896231859922409, "fcm_dpo/margin": 200.2966766357422, "fcm_dpo/q_t": 0.4268134832382202, "grad_norm": 45.6950569152832, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -0.6843982934951782, "logits/rejected": -0.6812521815299988, "logps/chosen": -546.7694091796875, "logps/ref_chosen": -60.920326232910156, "logps/ref_rejected": -104.42280578613281, "logps/rejected": -790.5685424804688, "loss": 1.1699, "margin_dpo/margin_mean": 200.29666137695312, "margin_dpo/margin_std": 362.1824035644531, "step": 430 }, { "KL/chosen_KL_mean": -476.85858154296875, "KL/mean": -639.10986328125, "KL/rejected_KL_mean": -801.361083984375, "KL/std": 353.5206298828125, "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.001593923894688487, "fcm_dpo/delta": -0.12039424479007721, "fcm_dpo/margin": 324.5025329589844, "fcm_dpo/q_t": 0.3837839663028717, "grad_norm": 37.95612716674805, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -0.7060000896453857, "logits/rejected": -0.7101027965545654, "logps/chosen": -534.2073364257812, "logps/ref_chosen": -57.34874725341797, "logps/ref_rejected": -92.84022521972656, "logps/rejected": -894.2013549804688, "loss": 1.0314, "margin_dpo/margin_mean": 324.5025329589844, "margin_dpo/margin_std": 400.6416015625, "step": 431 }, { "KL/chosen_KL_mean": -433.40106201171875, "KL/mean": -608.9974975585938, "KL/rejected_KL_mean": -784.593994140625, "KL/std": 369.924560546875, "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.0015768797602504492, "fcm_dpo/delta": -0.15814030170440674, "fcm_dpo/margin": 351.19293212890625, "fcm_dpo/q_t": 0.37685060501098633, "grad_norm": 34.167293548583984, "learning_rate": 1.791192214186223e-07, "logits/chosen": -0.6776980757713318, "logits/rejected": -0.6659233570098877, "logps/chosen": -504.475830078125, "logps/ref_chosen": -71.07479095458984, "logps/ref_rejected": -98.57952880859375, "logps/rejected": -883.1734619140625, "loss": 0.9869, "margin_dpo/margin_mean": 351.19293212890625, "margin_dpo/margin_std": 373.4866943359375, "step": 432 }, { "KL/chosen_KL_mean": -559.2772827148438, "KL/mean": -664.7277221679688, "KL/rejected_KL_mean": -770.1781616210938, "KL/std": 378.35809326171875, "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.0015741533134132624, "fcm_dpo/delta": 0.06906940788030624, "fcm_dpo/margin": 210.90089416503906, "fcm_dpo/q_t": 0.4204699993133545, "grad_norm": 33.000389099121094, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -0.7252877950668335, "logits/rejected": -0.7109028697013855, "logps/chosen": -617.5504760742188, "logps/ref_chosen": -58.273193359375, "logps/ref_rejected": -95.95089721679688, "logps/rejected": -866.1290283203125, "loss": 1.1957, "margin_dpo/margin_mean": 210.90089416503906, "margin_dpo/margin_std": 443.9112548828125, "step": 433 }, { "KL/chosen_KL_mean": -496.6466369628906, "KL/mean": -609.65185546875, "KL/rejected_KL_mean": -722.6571044921875, "KL/std": 331.86968994140625, "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.0015789458993822336, "fcm_dpo/delta": 0.04392336681485176, "fcm_dpo/margin": 226.01043701171875, "fcm_dpo/q_t": 0.4241020679473877, "grad_norm": 29.258861541748047, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -0.7415832281112671, "logits/rejected": -0.7311919927597046, "logps/chosen": -558.620361328125, "logps/ref_chosen": -61.97370147705078, "logps/ref_rejected": -78.49861145019531, "logps/rejected": -801.1557006835938, "loss": 1.1536, "margin_dpo/margin_mean": 226.01043701171875, "margin_dpo/margin_std": 413.2788391113281, "step": 434 }, { "KL/chosen_KL_mean": -455.0494689941406, "KL/mean": -597.1414184570312, "KL/rejected_KL_mean": -739.2333374023438, "KL/std": 330.17828369140625, "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.0015741032548248768, "fcm_dpo/delta": -0.048613518476486206, "fcm_dpo/margin": 284.1838684082031, "fcm_dpo/q_t": 0.39784517884254456, "grad_norm": 33.527957916259766, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -0.8012436628341675, "logits/rejected": -0.784874677658081, "logps/chosen": -506.551513671875, "logps/ref_chosen": -51.502052307128906, "logps/ref_rejected": -87.56689453125, "logps/rejected": -826.8002319335938, "loss": 1.065, "margin_dpo/margin_mean": 284.183837890625, "margin_dpo/margin_std": 358.4755859375, "step": 435 }, { "KL/chosen_KL_mean": -454.6264343261719, "KL/mean": -576.6045532226562, "KL/rejected_KL_mean": -698.5827026367188, "KL/std": 299.61407470703125, "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.0015738060465082526, "fcm_dpo/delta": 0.016265233978629112, "fcm_dpo/margin": 243.956298828125, "fcm_dpo/q_t": 0.4141663908958435, "grad_norm": 39.384552001953125, "learning_rate": 1.742118314717391e-07, "logits/chosen": -0.702154815196991, "logits/rejected": -0.6535923480987549, "logps/chosen": -526.0301513671875, "logps/ref_chosen": -71.40371704101562, "logps/ref_rejected": -82.72775268554688, "logps/rejected": -781.3104248046875, "loss": 1.1158, "margin_dpo/margin_mean": 243.956298828125, "margin_dpo/margin_std": 367.78271484375, "step": 436 }, { "KL/chosen_KL_mean": -440.7288513183594, "KL/mean": -567.1207885742188, "KL/rejected_KL_mean": -693.5126953125, "KL/std": 293.3643798828125, "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.0015777763910591602, "fcm_dpo/delta": 0.0011859128717333078, "fcm_dpo/margin": 252.7838897705078, "fcm_dpo/q_t": 0.40734195709228516, "grad_norm": 29.819622039794922, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -0.7542532086372375, "logits/rejected": -0.729715883731842, "logps/chosen": -505.4731140136719, "logps/ref_chosen": -64.7442626953125, "logps/ref_rejected": -82.04356384277344, "logps/rejected": -775.5562744140625, "loss": 1.0905, "margin_dpo/margin_mean": 252.78387451171875, "margin_dpo/margin_std": 329.2221984863281, "step": 437 }, { "KL/chosen_KL_mean": -469.1627502441406, "KL/mean": -613.673095703125, "KL/rejected_KL_mean": -758.1834716796875, "KL/std": 323.0591735839844, "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.0015675770118832588, "fcm_dpo/delta": -0.054624516516923904, "fcm_dpo/margin": 289.0207214355469, "fcm_dpo/q_t": 0.39459216594696045, "grad_norm": 32.62807846069336, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -0.7667056322097778, "logits/rejected": -0.7360678911209106, "logps/chosen": -528.181396484375, "logps/ref_chosen": -59.0186653137207, "logps/ref_rejected": -83.07682800292969, "logps/rejected": -841.26025390625, "loss": 1.05, "margin_dpo/margin_mean": 289.0207214355469, "margin_dpo/margin_std": 334.79864501953125, "step": 438 }, { "KL/chosen_KL_mean": -487.53643798828125, "KL/mean": -603.5372314453125, "KL/rejected_KL_mean": -719.5379638671875, "KL/std": 336.40850830078125, "epoch": 0.644640234948605, "fcm_dpo/beta": 0.001555339666083455, "fcm_dpo/delta": -0.08725128322839737, "fcm_dpo/margin": 232.0015411376953, "fcm_dpo/q_t": 0.41888073086738586, "grad_norm": 40.83659744262695, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -0.757964015007019, "logits/rejected": -0.7572835683822632, "logps/chosen": -541.3204956054688, "logps/ref_chosen": -53.78407669067383, "logps/ref_rejected": -83.98545837402344, "logps/rejected": -803.5234375, "loss": 1.1478, "margin_dpo/margin_mean": 232.00155639648438, "margin_dpo/margin_std": 380.4766845703125, "step": 439 }, { "KL/chosen_KL_mean": -491.5701904296875, "KL/mean": -634.1220703125, "KL/rejected_KL_mean": -776.6740112304688, "KL/std": 404.11834716796875, "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.0015516489511355758, "fcm_dpo/delta": -0.043332286179065704, "fcm_dpo/margin": 285.1037902832031, "fcm_dpo/q_t": 0.4026166796684265, "grad_norm": 44.514137268066406, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -0.773739218711853, "logits/rejected": -0.7669236660003662, "logps/chosen": -570.1369018554688, "logps/ref_chosen": -78.56671905517578, "logps/ref_rejected": -96.49775695800781, "logps/rejected": -873.1717529296875, "loss": 1.0713, "margin_dpo/margin_mean": 285.103759765625, "margin_dpo/margin_std": 386.797119140625, "step": 440 }, { "KL/chosen_KL_mean": -591.2857666015625, "KL/mean": -716.7557373046875, "KL/rejected_KL_mean": -842.2255859375, "KL/std": 407.85955810546875, "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.001553539652377367, "fcm_dpo/delta": 0.010096218436956406, "fcm_dpo/margin": 250.93984985351562, "fcm_dpo/q_t": 0.4177462160587311, "grad_norm": 40.89066696166992, "learning_rate": 1.681227682404166e-07, "logits/chosen": -0.7674189805984497, "logits/rejected": -0.7517750263214111, "logps/chosen": -652.1102294921875, "logps/ref_chosen": -60.824440002441406, "logps/ref_rejected": -96.47080993652344, "logps/rejected": -938.6964111328125, "loss": 1.1743, "margin_dpo/margin_mean": 250.93988037109375, "margin_dpo/margin_std": 502.06768798828125, "step": 441 }, { "KL/chosen_KL_mean": -503.0433349609375, "KL/mean": -659.7222900390625, "KL/rejected_KL_mean": -816.4011840820312, "KL/std": 401.5861511230469, "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.0015460660215467215, "fcm_dpo/delta": -0.08682595193386078, "fcm_dpo/margin": 313.3577880859375, "fcm_dpo/q_t": 0.3956022262573242, "grad_norm": 36.8726806640625, "learning_rate": 1.669113001300851e-07, "logits/chosen": -0.7465405464172363, "logits/rejected": -0.7388708591461182, "logps/chosen": -550.0545654296875, "logps/ref_chosen": -47.01121520996094, "logps/ref_rejected": -76.53926086425781, "logps/rejected": -892.9404296875, "loss": 1.0721, "margin_dpo/margin_mean": 313.3577880859375, "margin_dpo/margin_std": 448.83795166015625, "step": 442 }, { "KL/chosen_KL_mean": -538.9011840820312, "KL/mean": -627.173095703125, "KL/rejected_KL_mean": -715.4451904296875, "KL/std": 372.1048583984375, "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.0015383637510240078, "fcm_dpo/delta": 0.010316350497305393, "fcm_dpo/margin": 176.54397583007812, "fcm_dpo/q_t": 0.43916434049606323, "grad_norm": 39.52730178833008, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -0.733747124671936, "logits/rejected": -0.710411548614502, "logps/chosen": -610.1741943359375, "logps/ref_chosen": -71.27301788330078, "logps/ref_rejected": -86.679931640625, "logps/rejected": -802.1251220703125, "loss": 1.239, "margin_dpo/margin_mean": 176.54397583007812, "margin_dpo/margin_std": 431.0137634277344, "step": 443 }, { "KL/chosen_KL_mean": -545.4714965820312, "KL/mean": -706.712890625, "KL/rejected_KL_mean": -867.954345703125, "KL/std": 410.585693359375, "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0015250996220856905, "fcm_dpo/delta": -0.09439031034708023, "fcm_dpo/margin": 322.4828796386719, "fcm_dpo/q_t": 0.39275673031806946, "grad_norm": 35.95709991455078, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -0.6589124202728271, "logits/rejected": -0.6668537259101868, "logps/chosen": -602.6851806640625, "logps/ref_chosen": -57.213706970214844, "logps/ref_rejected": -97.25489807128906, "logps/rejected": -965.209228515625, "loss": 1.059, "margin_dpo/margin_mean": 322.48291015625, "margin_dpo/margin_std": 454.73382568359375, "step": 444 }, { "KL/chosen_KL_mean": -464.7125549316406, "KL/mean": -615.153076171875, "KL/rejected_KL_mean": -765.593505859375, "KL/std": 321.0918273925781, "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.0015192454447969794, "fcm_dpo/delta": -0.058439984917640686, "fcm_dpo/margin": 300.8808898925781, "fcm_dpo/q_t": 0.39640486240386963, "grad_norm": 32.19375228881836, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -0.6859538555145264, "logits/rejected": -0.6752911806106567, "logps/chosen": -532.0123291015625, "logps/ref_chosen": -67.29979705810547, "logps/ref_rejected": -92.68267059326172, "logps/rejected": -858.276123046875, "loss": 1.0589, "margin_dpo/margin_mean": 300.88092041015625, "margin_dpo/margin_std": 378.2691650390625, "step": 445 }, { "KL/chosen_KL_mean": -440.65826416015625, "KL/mean": -595.5625, "KL/rejected_KL_mean": -750.4666748046875, "KL/std": 347.827880859375, "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.0015123614575713873, "fcm_dpo/delta": -0.07043258845806122, "fcm_dpo/margin": 309.8084716796875, "fcm_dpo/q_t": 0.39075666666030884, "grad_norm": 36.1616096496582, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -0.7100153565406799, "logits/rejected": -0.7190457582473755, "logps/chosen": -499.7567138671875, "logps/ref_chosen": -59.098487854003906, "logps/ref_rejected": -101.26419067382812, "logps/rejected": -851.7308349609375, "loss": 1.0343, "margin_dpo/margin_mean": 309.8084716796875, "margin_dpo/margin_std": 329.4242248535156, "step": 446 }, { "KL/chosen_KL_mean": -485.10504150390625, "KL/mean": -651.3726806640625, "KL/rejected_KL_mean": -817.6403198242188, "KL/std": 411.0166015625, "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.0014915233477950096, "fcm_dpo/delta": -0.09860781580209732, "fcm_dpo/margin": 332.5352783203125, "fcm_dpo/q_t": 0.39083898067474365, "grad_norm": 33.39097595214844, "learning_rate": 1.608874379754465e-07, "logits/chosen": -0.7963074445724487, "logits/rejected": -0.8120626211166382, "logps/chosen": -541.1803588867188, "logps/ref_chosen": -56.07533264160156, "logps/ref_rejected": -98.69475555419922, "logps/rejected": -916.3350830078125, "loss": 1.0444, "margin_dpo/margin_mean": 332.5352783203125, "margin_dpo/margin_std": 443.71478271484375, "step": 447 }, { "KL/chosen_KL_mean": -549.9185791015625, "KL/mean": -715.212646484375, "KL/rejected_KL_mean": -880.5067138671875, "KL/std": 387.88446044921875, "epoch": 0.657856093979442, "fcm_dpo/beta": 0.0014809533022344112, "fcm_dpo/delta": -0.09178226441144943, "fcm_dpo/margin": 330.588134765625, "fcm_dpo/q_t": 0.3909049928188324, "grad_norm": 37.11544418334961, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -0.7098146677017212, "logits/rejected": -0.7185966372489929, "logps/chosen": -609.9224243164062, "logps/ref_chosen": -60.00384521484375, "logps/ref_rejected": -102.26465606689453, "logps/rejected": -982.7713623046875, "loss": 1.0476, "margin_dpo/margin_mean": 330.588134765625, "margin_dpo/margin_std": 435.3096923828125, "step": 448 }, { "KL/chosen_KL_mean": -565.3760986328125, "KL/mean": -737.2021484375, "KL/rejected_KL_mean": -909.028076171875, "KL/std": 458.106689453125, "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.001463989494368434, "fcm_dpo/delta": -0.10579749941825867, "fcm_dpo/margin": 343.6519470214844, "fcm_dpo/q_t": 0.3931967616081238, "grad_norm": 33.413673400878906, "learning_rate": 1.584941086944423e-07, "logits/chosen": -0.7576150894165039, "logits/rejected": -0.746334433555603, "logps/chosen": -632.9027099609375, "logps/ref_chosen": -67.52661895751953, "logps/ref_rejected": -88.59690856933594, "logps/rejected": -997.625, "loss": 1.0698, "margin_dpo/margin_mean": 343.6519775390625, "margin_dpo/margin_std": 530.8461303710938, "step": 449 }, { "KL/chosen_KL_mean": -459.6373596191406, "KL/mean": -645.7894897460938, "KL/rejected_KL_mean": -831.9415893554688, "KL/std": 388.321044921875, "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.0014468804001808167, "fcm_dpo/delta": -0.14240342378616333, "fcm_dpo/margin": 372.30426025390625, "fcm_dpo/q_t": 0.37629586458206177, "grad_norm": 43.87886428833008, "learning_rate": 1.573010452010098e-07, "logits/chosen": -0.8462213277816772, "logits/rejected": -0.8595950603485107, "logps/chosen": -516.7454833984375, "logps/ref_chosen": -57.10811996459961, "logps/ref_rejected": -102.75494384765625, "logps/rejected": -934.696533203125, "loss": 0.9856, "margin_dpo/margin_mean": 372.30426025390625, "margin_dpo/margin_std": 375.06005859375, "step": 450 }, { "KL/chosen_KL_mean": -655.2659912109375, "KL/mean": -786.9049072265625, "KL/rejected_KL_mean": -918.5439453125, "KL/std": 500.51873779296875, "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.0014441582607105374, "fcm_dpo/delta": 0.02004837803542614, "fcm_dpo/margin": 263.2779541015625, "fcm_dpo/q_t": 0.4101364016532898, "grad_norm": 50.99306106567383, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -0.9174356460571289, "logits/rejected": -0.8886551856994629, "logps/chosen": -713.7348022460938, "logps/ref_chosen": -58.46883010864258, "logps/ref_rejected": -72.92941284179688, "logps/rejected": -991.4733276367188, "loss": 1.2051, "margin_dpo/margin_mean": 263.2779541015625, "margin_dpo/margin_std": 591.6530151367188, "step": 451 }, { "KL/chosen_KL_mean": -478.00341796875, "KL/mean": -634.8177490234375, "KL/rejected_KL_mean": -791.6319580078125, "KL/std": 377.42657470703125, "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.001435277285054326, "fcm_dpo/delta": -0.05143965408205986, "fcm_dpo/margin": 313.6285400390625, "fcm_dpo/q_t": 0.3956111669540405, "grad_norm": 30.187389373779297, "learning_rate": 1.549222776991186e-07, "logits/chosen": -0.8042168617248535, "logits/rejected": -0.8225170373916626, "logps/chosen": -528.3939819335938, "logps/ref_chosen": -50.39055252075195, "logps/ref_rejected": -97.77142333984375, "logps/rejected": -889.4034423828125, "loss": 1.0508, "margin_dpo/margin_mean": 313.6285400390625, "margin_dpo/margin_std": 363.00054931640625, "step": 452 }, { "KL/chosen_KL_mean": -546.2333984375, "KL/mean": -708.6800537109375, "KL/rejected_KL_mean": -871.1268310546875, "KL/std": 392.1535949707031, "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.0014257467119023204, "fcm_dpo/delta": -0.06497231125831604, "fcm_dpo/margin": 324.8934326171875, "fcm_dpo/q_t": 0.3990872800350189, "grad_norm": 34.372772216796875, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -0.8440671563148499, "logits/rejected": -0.8263705968856812, "logps/chosen": -603.9482421875, "logps/ref_chosen": -57.71485137939453, "logps/ref_rejected": -82.20741271972656, "logps/rejected": -953.334228515625, "loss": 1.0778, "margin_dpo/margin_mean": 324.8934631347656, "margin_dpo/margin_std": 473.4591064453125, "step": 453 }, { "KL/chosen_KL_mean": -651.534912109375, "KL/mean": -822.3595581054688, "KL/rejected_KL_mean": -993.18408203125, "KL/std": 444.50738525390625, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.0014141410356387496, "fcm_dpo/delta": -0.08552803844213486, "fcm_dpo/margin": 341.649169921875, "fcm_dpo/q_t": 0.3941487669944763, "grad_norm": 39.139339447021484, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -0.8544209003448486, "logits/rejected": -0.8400380611419678, "logps/chosen": -712.4805908203125, "logps/ref_chosen": -60.945648193359375, "logps/ref_rejected": -84.95079040527344, "logps/rejected": -1078.1348876953125, "loss": 1.0725, "margin_dpo/margin_mean": 341.649169921875, "margin_dpo/margin_std": 507.084228515625, "step": 454 }, { "KL/chosen_KL_mean": -564.0460205078125, "KL/mean": -756.771484375, "KL/rejected_KL_mean": -949.4969482421875, "KL/std": 434.53131103515625, "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0014031952014192939, "fcm_dpo/delta": -0.14472892880439758, "fcm_dpo/margin": 385.450927734375, "fcm_dpo/q_t": 0.38020026683807373, "grad_norm": 33.70637130737305, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -0.8295320272445679, "logits/rejected": -0.8505617380142212, "logps/chosen": -608.9327392578125, "logps/ref_chosen": -44.88671112060547, "logps/ref_rejected": -115.30147552490234, "logps/rejected": -1064.79833984375, "loss": 1.0249, "margin_dpo/margin_mean": 385.450927734375, "margin_dpo/margin_std": 487.4344482421875, "step": 455 }, { "KL/chosen_KL_mean": -542.165771484375, "KL/mean": -737.29248046875, "KL/rejected_KL_mean": -932.4191284179688, "KL/std": 445.7962341308594, "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.0013802102766931057, "fcm_dpo/delta": -0.14236783981323242, "fcm_dpo/margin": 390.25335693359375, "fcm_dpo/q_t": 0.37905341386795044, "grad_norm": 34.32986831665039, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -0.8049008846282959, "logits/rejected": -0.829048752784729, "logps/chosen": -599.2025756835938, "logps/ref_chosen": -57.036781311035156, "logps/ref_rejected": -105.21784210205078, "logps/rejected": -1037.636962890625, "loss": 0.9897, "margin_dpo/margin_mean": 390.25335693359375, "margin_dpo/margin_std": 416.21002197265625, "step": 456 }, { "KL/chosen_KL_mean": -505.49066162109375, "KL/mean": -710.5162963867188, "KL/rejected_KL_mean": -915.5419921875, "KL/std": 429.17242431640625, "epoch": 0.671071953010279, "fcm_dpo/beta": 0.001355804386548698, "fcm_dpo/delta": -0.1606726348400116, "fcm_dpo/margin": 410.0513000488281, "fcm_dpo/q_t": 0.37336650490760803, "grad_norm": 50.008827209472656, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -0.8348103761672974, "logits/rejected": -0.8382035493850708, "logps/chosen": -559.733154296875, "logps/ref_chosen": -54.24253845214844, "logps/ref_rejected": -85.10956573486328, "logps/rejected": -1000.6514892578125, "loss": 0.9782, "margin_dpo/margin_mean": 410.0513000488281, "margin_dpo/margin_std": 415.6295166015625, "step": 457 }, { "KL/chosen_KL_mean": -571.1600341796875, "KL/mean": -721.54638671875, "KL/rejected_KL_mean": -871.9327392578125, "KL/std": 375.446044921875, "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.0013467208482325077, "fcm_dpo/delta": -0.005669653415679932, "fcm_dpo/margin": 300.77264404296875, "fcm_dpo/q_t": 0.4096975326538086, "grad_norm": 33.263877868652344, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -0.8518056273460388, "logits/rejected": -0.840267539024353, "logps/chosen": -626.56884765625, "logps/ref_chosen": -55.40888214111328, "logps/ref_rejected": -97.68325805664062, "logps/rejected": -969.615966796875, "loss": 1.0987, "margin_dpo/margin_mean": 300.77264404296875, "margin_dpo/margin_std": 419.0684814453125, "step": 458 }, { "KL/chosen_KL_mean": -596.946533203125, "KL/mean": -784.0278930664062, "KL/rejected_KL_mean": -971.109130859375, "KL/std": 449.78326416015625, "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.0013373238034546375, "fcm_dpo/delta": -0.10349556058645248, "fcm_dpo/margin": 374.16259765625, "fcm_dpo/q_t": 0.3898807168006897, "grad_norm": 37.392234802246094, "learning_rate": 1.466771464027316e-07, "logits/chosen": -0.8045568466186523, "logits/rejected": -0.8199119567871094, "logps/chosen": -643.5040283203125, "logps/ref_chosen": -46.55748748779297, "logps/ref_rejected": -86.16854095458984, "logps/rejected": -1057.2777099609375, "loss": 1.0568, "margin_dpo/margin_mean": 374.16259765625, "margin_dpo/margin_std": 516.4439086914062, "step": 459 }, { "KL/chosen_KL_mean": -642.6575317382812, "KL/mean": -851.060791015625, "KL/rejected_KL_mean": -1059.4639892578125, "KL/std": 428.037109375, "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.0013223644345998764, "fcm_dpo/delta": -0.1554030030965805, "fcm_dpo/margin": 416.8063049316406, "fcm_dpo/q_t": 0.37675726413726807, "grad_norm": 36.663360595703125, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -0.8338453769683838, "logits/rejected": -0.8649412393569946, "logps/chosen": -694.29248046875, "logps/ref_chosen": -51.63489532470703, "logps/ref_rejected": -104.11935424804688, "logps/rejected": -1163.583251953125, "loss": 0.9875, "margin_dpo/margin_mean": 416.80633544921875, "margin_dpo/margin_std": 453.399658203125, "step": 460 }, { "KL/chosen_KL_mean": -657.5977172851562, "KL/mean": -817.214599609375, "KL/rejected_KL_mean": -976.83154296875, "KL/std": 429.95391845703125, "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.0013168307486921549, "fcm_dpo/delta": -0.020892973989248276, "fcm_dpo/margin": 319.23382568359375, "fcm_dpo/q_t": 0.40674418210983276, "grad_norm": 33.73992919921875, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -0.858330249786377, "logits/rejected": -0.8671220541000366, "logps/chosen": -712.7796630859375, "logps/ref_chosen": -55.18195724487305, "logps/ref_rejected": -86.47689819335938, "logps/rejected": -1063.3084716796875, "loss": 1.0947, "margin_dpo/margin_mean": 319.2338562011719, "margin_dpo/margin_std": 468.2879333496094, "step": 461 }, { "KL/chosen_KL_mean": -621.3289794921875, "KL/mean": -758.6487426757812, "KL/rejected_KL_mean": -895.9684448242188, "KL/std": 408.9393005371094, "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.001316360547207296, "fcm_dpo/delta": 0.03917883336544037, "fcm_dpo/margin": 274.63946533203125, "fcm_dpo/q_t": 0.4196895658969879, "grad_norm": 41.01453399658203, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -0.9267274141311646, "logits/rejected": -0.9083724617958069, "logps/chosen": -691.2569580078125, "logps/ref_chosen": -69.92803192138672, "logps/ref_rejected": -78.84111022949219, "logps/rejected": -974.8095703125, "loss": 1.1402, "margin_dpo/margin_mean": 274.63946533203125, "margin_dpo/margin_std": 454.64776611328125, "step": 462 }, { "KL/chosen_KL_mean": -614.42333984375, "KL/mean": -786.9886474609375, "KL/rejected_KL_mean": -959.553955078125, "KL/std": 419.7926940917969, "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.001315572764724493, "fcm_dpo/delta": -0.05530533939599991, "fcm_dpo/margin": 345.13055419921875, "fcm_dpo/q_t": 0.39796507358551025, "grad_norm": 32.41996765136719, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -0.8691219687461853, "logits/rejected": -0.8700894117355347, "logps/chosen": -669.69775390625, "logps/ref_chosen": -55.27437210083008, "logps/ref_rejected": -89.02497863769531, "logps/rejected": -1048.578857421875, "loss": 1.0662, "margin_dpo/margin_mean": 345.13055419921875, "margin_dpo/margin_std": 457.62176513671875, "step": 463 }, { "KL/chosen_KL_mean": -600.5834350585938, "KL/mean": -850.0731811523438, "KL/rejected_KL_mean": -1099.56298828125, "KL/std": 488.45526123046875, "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.0012829885818064213, "fcm_dpo/delta": -0.24955548346042633, "fcm_dpo/margin": 498.9794921875, "fcm_dpo/q_t": 0.3563224673271179, "grad_norm": 42.92744827270508, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -0.864883303642273, "logits/rejected": -0.9228291511535645, "logps/chosen": -651.4957275390625, "logps/ref_chosen": -50.91230010986328, "logps/ref_rejected": -102.4893798828125, "logps/rejected": -1202.05224609375, "loss": 0.9282, "margin_dpo/margin_mean": 498.9794921875, "margin_dpo/margin_std": 475.25860595703125, "step": 464 }, { "KL/chosen_KL_mean": -598.503662109375, "KL/mean": -802.850341796875, "KL/rejected_KL_mean": -1007.1970825195312, "KL/std": 488.08880615234375, "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.001269632950425148, "fcm_dpo/delta": -0.12196629494428635, "fcm_dpo/margin": 408.6933898925781, "fcm_dpo/q_t": 0.37997591495513916, "grad_norm": 30.996089935302734, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -0.851067841053009, "logits/rejected": -0.871573805809021, "logps/chosen": -658.6204833984375, "logps/ref_chosen": -60.116851806640625, "logps/ref_rejected": -113.94602966308594, "logps/rejected": -1121.14306640625, "loss": 1.0264, "margin_dpo/margin_mean": 408.6933898925781, "margin_dpo/margin_std": 498.4493408203125, "step": 465 }, { "KL/chosen_KL_mean": -651.2394409179688, "KL/mean": -829.18798828125, "KL/rejected_KL_mean": -1007.1365966796875, "KL/std": 461.8182067871094, "epoch": 0.684287812041116, "fcm_dpo/beta": 0.0012555132852867246, "fcm_dpo/delta": -0.04844123125076294, "fcm_dpo/margin": 355.89715576171875, "fcm_dpo/q_t": 0.39813193678855896, "grad_norm": 34.06509017944336, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -0.8493870496749878, "logits/rejected": -0.8454769849777222, "logps/chosen": -704.1603393554688, "logps/ref_chosen": -52.920921325683594, "logps/ref_rejected": -90.3154296875, "logps/rejected": -1097.4520263671875, "loss": 1.074, "margin_dpo/margin_mean": 355.89715576171875, "margin_dpo/margin_std": 472.82452392578125, "step": 466 }, { "KL/chosen_KL_mean": -826.9212646484375, "KL/mean": -1013.4310302734375, "KL/rejected_KL_mean": -1199.940673828125, "KL/std": 584.6832275390625, "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.0012465352192521095, "fcm_dpo/delta": -0.06744933128356934, "fcm_dpo/margin": 373.0193786621094, "fcm_dpo/q_t": 0.39987167716026306, "grad_norm": 53.18272018432617, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -0.9681617021560669, "logits/rejected": -0.9566558599472046, "logps/chosen": -905.6370849609375, "logps/ref_chosen": -78.7158203125, "logps/ref_rejected": -102.86019897460938, "logps/rejected": -1302.8009033203125, "loss": 1.1327, "margin_dpo/margin_mean": 373.0194091796875, "margin_dpo/margin_std": 667.3912963867188, "step": 467 }, { "KL/chosen_KL_mean": -637.194091796875, "KL/mean": -866.535400390625, "KL/rejected_KL_mean": -1095.876708984375, "KL/std": 503.98370361328125, "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.0012332710903137922, "fcm_dpo/delta": -0.17055264115333557, "fcm_dpo/margin": 458.6826477050781, "fcm_dpo/q_t": 0.37581923604011536, "grad_norm": 52.71803283691406, "learning_rate": 1.362737437810114e-07, "logits/chosen": -0.8924639225006104, "logits/rejected": -0.9000778794288635, "logps/chosen": -707.1294555664062, "logps/ref_chosen": -69.93536376953125, "logps/ref_rejected": -101.02880859375, "logps/rejected": -1196.905517578125, "loss": 0.9999, "margin_dpo/margin_mean": 458.6826171875, "margin_dpo/margin_std": 557.9136962890625, "step": 468 }, { "KL/chosen_KL_mean": -674.091796875, "KL/mean": -917.2705078125, "KL/rejected_KL_mean": -1160.44921875, "KL/std": 503.095947265625, "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.0012071155942976475, "fcm_dpo/delta": -0.1936628669500351, "fcm_dpo/margin": 486.35748291015625, "fcm_dpo/q_t": 0.3694315552711487, "grad_norm": 36.40598678588867, "learning_rate": 1.351323902551631e-07, "logits/chosen": -0.8922240734100342, "logits/rejected": -0.9052385091781616, "logps/chosen": -742.2164916992188, "logps/ref_chosen": -68.12469482421875, "logps/ref_rejected": -104.78640747070312, "logps/rejected": -1265.235595703125, "loss": 0.978, "margin_dpo/margin_mean": 486.35748291015625, "margin_dpo/margin_std": 537.0926513671875, "step": 469 }, { "KL/chosen_KL_mean": -584.605224609375, "KL/mean": -782.6975708007812, "KL/rejected_KL_mean": -980.7899169921875, "KL/std": 488.2651672363281, "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.0012003884185105562, "fcm_dpo/delta": -0.07743942737579346, "fcm_dpo/margin": 396.18463134765625, "fcm_dpo/q_t": 0.392379492521286, "grad_norm": 29.941707611083984, "learning_rate": 1.339940635976592e-07, "logits/chosen": -0.8477715253829956, "logits/rejected": -0.8522796630859375, "logps/chosen": -628.397216796875, "logps/ref_chosen": -43.791927337646484, "logps/ref_rejected": -82.70285034179688, "logps/rejected": -1063.4927978515625, "loss": 1.0593, "margin_dpo/margin_mean": 396.18463134765625, "margin_dpo/margin_std": 526.9808349609375, "step": 470 }, { "KL/chosen_KL_mean": -723.69287109375, "KL/mean": -888.711669921875, "KL/rejected_KL_mean": -1053.73046875, "KL/std": 517.3129272460938, "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.00119293003808707, "fcm_dpo/delta": 0.006166655570268631, "fcm_dpo/margin": 330.03759765625, "fcm_dpo/q_t": 0.4141054153442383, "grad_norm": 44.29683303833008, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -0.9283441305160522, "logits/rejected": -0.9297612905502319, "logps/chosen": -787.0324096679688, "logps/ref_chosen": -63.33952331542969, "logps/ref_rejected": -83.61048126220703, "logps/rejected": -1137.3409423828125, "loss": 1.1301, "margin_dpo/margin_mean": 330.0375671386719, "margin_dpo/margin_std": 555.3355712890625, "step": 471 }, { "KL/chosen_KL_mean": -694.7491455078125, "KL/mean": -900.8193359375, "KL/rejected_KL_mean": -1106.8895263671875, "KL/std": 589.9913940429688, "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.001185831381008029, "fcm_dpo/delta": -0.0912611111998558, "fcm_dpo/margin": 412.1404113769531, "fcm_dpo/q_t": 0.39623403549194336, "grad_norm": 34.139041900634766, "learning_rate": 1.317266107909975e-07, "logits/chosen": -0.9295023679733276, "logits/rejected": -0.9044417142868042, "logps/chosen": -778.415283203125, "logps/ref_chosen": -83.66610717773438, "logps/ref_rejected": -117.20919799804688, "logps/rejected": -1224.0987548828125, "loss": 1.0802, "margin_dpo/margin_mean": 412.140380859375, "margin_dpo/margin_std": 622.5784912109375, "step": 472 }, { "KL/chosen_KL_mean": -844.9259033203125, "KL/mean": -950.9039306640625, "KL/rejected_KL_mean": -1056.8818359375, "KL/std": 620.759521484375, "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0011900889221578836, "fcm_dpo/delta": 0.04035104438662529, "fcm_dpo/margin": 211.95591735839844, "fcm_dpo/q_t": 0.44867947697639465, "grad_norm": 131.445068359375, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -0.8819540739059448, "logits/rejected": -0.851367712020874, "logps/chosen": -908.4229125976562, "logps/ref_chosen": -63.49696731567383, "logps/ref_rejected": -81.14657592773438, "logps/rejected": -1138.0283203125, "loss": 1.3351, "margin_dpo/margin_mean": 211.9559326171875, "margin_dpo/margin_std": 784.6436767578125, "step": 473 }, { "KL/chosen_KL_mean": -676.4264526367188, "KL/mean": -852.287109375, "KL/rejected_KL_mean": -1028.147705078125, "KL/std": 525.454833984375, "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.0011939804535359144, "fcm_dpo/delta": -0.021219126880168915, "fcm_dpo/margin": 351.7213439941406, "fcm_dpo/q_t": 0.4053725600242615, "grad_norm": 47.71774673461914, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -0.8572415113449097, "logits/rejected": -0.8637101650238037, "logps/chosen": -729.0383911132812, "logps/ref_chosen": -52.6119384765625, "logps/ref_rejected": -90.08041381835938, "logps/rejected": -1118.2281494140625, "loss": 1.1248, "margin_dpo/margin_mean": 351.7213439941406, "margin_dpo/margin_std": 567.389892578125, "step": 474 }, { "KL/chosen_KL_mean": -492.31280517578125, "KL/mean": -697.96044921875, "KL/rejected_KL_mean": -903.608154296875, "KL/std": 407.37347412109375, "epoch": 0.697503671071953, "fcm_dpo/beta": 0.0011786803370341659, "fcm_dpo/delta": -0.0870213657617569, "fcm_dpo/margin": 411.29534912109375, "fcm_dpo/q_t": 0.38825133442878723, "grad_norm": 56.91255569458008, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -0.8645976781845093, "logits/rejected": -0.8880435228347778, "logps/chosen": -534.8079833984375, "logps/ref_chosen": -42.49519348144531, "logps/ref_rejected": -90.06294250488281, "logps/rejected": -993.6710815429688, "loss": 1.0216, "margin_dpo/margin_mean": 411.29534912109375, "margin_dpo/margin_std": 438.8066711425781, "step": 475 }, { "KL/chosen_KL_mean": -581.162109375, "KL/mean": -763.0509033203125, "KL/rejected_KL_mean": -944.939697265625, "KL/std": 481.3084411621094, "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.0011740537593141198, "fcm_dpo/delta": -0.027722399681806564, "fcm_dpo/margin": 363.7776184082031, "fcm_dpo/q_t": 0.40231823921203613, "grad_norm": 102.54533386230469, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -0.8250603675842285, "logits/rejected": -0.8420618772506714, "logps/chosen": -624.1114501953125, "logps/ref_chosen": -42.94938278198242, "logps/ref_rejected": -73.71023559570312, "logps/rejected": -1018.64990234375, "loss": 1.0709, "margin_dpo/margin_mean": 363.7776184082031, "margin_dpo/margin_std": 458.6659240722656, "step": 476 }, { "KL/chosen_KL_mean": -576.864990234375, "KL/mean": -748.666015625, "KL/rejected_KL_mean": -920.467041015625, "KL/std": 453.67510986328125, "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.0011754365405067801, "fcm_dpo/delta": -0.004048405215144157, "fcm_dpo/margin": 343.6020812988281, "fcm_dpo/q_t": 0.407484769821167, "grad_norm": 50.94183349609375, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -0.8757802248001099, "logits/rejected": -0.8439153432846069, "logps/chosen": -647.6376342773438, "logps/ref_chosen": -70.77261352539062, "logps/ref_rejected": -76.13737487792969, "logps/rejected": -996.6044311523438, "loss": 1.1156, "margin_dpo/margin_mean": 343.6020812988281, "margin_dpo/margin_std": 538.941650390625, "step": 477 }, { "KL/chosen_KL_mean": -512.1103515625, "KL/mean": -712.9010009765625, "KL/rejected_KL_mean": -913.691650390625, "KL/std": 417.81170654296875, "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.0011684978380799294, "fcm_dpo/delta": -0.07088005542755127, "fcm_dpo/margin": 401.581298828125, "fcm_dpo/q_t": 0.39256751537323, "grad_norm": 44.561859130859375, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -0.7855898141860962, "logits/rejected": -0.8061913251876831, "logps/chosen": -553.5509033203125, "logps/ref_chosen": -41.440513610839844, "logps/ref_rejected": -85.36196899414062, "logps/rejected": -999.0535888671875, "loss": 1.0413, "margin_dpo/margin_mean": 401.581298828125, "margin_dpo/margin_std": 472.5474853515625, "step": 478 }, { "KL/chosen_KL_mean": -679.7722778320312, "KL/mean": -867.8972778320312, "KL/rejected_KL_mean": -1056.022216796875, "KL/std": 525.4520874023438, "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.0011651396052911878, "fcm_dpo/delta": -0.03945866972208023, "fcm_dpo/margin": 376.24993896484375, "fcm_dpo/q_t": 0.4034544825553894, "grad_norm": 36.7021484375, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -0.8868274092674255, "logits/rejected": -0.9125785827636719, "logps/chosen": -733.68017578125, "logps/ref_chosen": -53.907920837402344, "logps/ref_rejected": -95.1163330078125, "logps/rejected": -1151.1385498046875, "loss": 1.1051, "margin_dpo/margin_mean": 376.24993896484375, "margin_dpo/margin_std": 603.150146484375, "step": 479 }, { "KL/chosen_KL_mean": -800.5087890625, "KL/mean": -963.208740234375, "KL/rejected_KL_mean": -1125.90869140625, "KL/std": 507.62554931640625, "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.0011562576983124018, "fcm_dpo/delta": -0.07623150199651718, "fcm_dpo/margin": 325.4000244140625, "fcm_dpo/q_t": 0.4170445203781128, "grad_norm": 49.098358154296875, "learning_rate": 1.227838333989088e-07, "logits/chosen": -0.9548571109771729, "logits/rejected": -0.9503375291824341, "logps/chosen": -859.1915283203125, "logps/ref_chosen": -58.682701110839844, "logps/ref_rejected": -82.93248748779297, "logps/rejected": -1208.8411865234375, "loss": 1.1562, "margin_dpo/margin_mean": 325.4000244140625, "margin_dpo/margin_std": 569.0355224609375, "step": 480 }, { "KL/chosen_KL_mean": -698.514404296875, "KL/mean": -934.3134765625, "KL/rejected_KL_mean": -1170.112548828125, "KL/std": 530.9942016601562, "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.001140213687904179, "fcm_dpo/delta": -0.14147771894931793, "fcm_dpo/margin": 471.5982360839844, "fcm_dpo/q_t": 0.3806016147136688, "grad_norm": 40.23090362548828, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -0.994126558303833, "logits/rejected": -1.0202418565750122, "logps/chosen": -753.4786376953125, "logps/ref_chosen": -54.964271545410156, "logps/ref_rejected": -92.42044067382812, "logps/rejected": -1262.532958984375, "loss": 1.0124, "margin_dpo/margin_mean": 471.5982666015625, "margin_dpo/margin_std": 567.219482421875, "step": 481 }, { "KL/chosen_KL_mean": -790.0252685546875, "KL/mean": -897.4144897460938, "KL/rejected_KL_mean": -1004.8038330078125, "KL/std": 556.745849609375, "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.001139242434874177, "fcm_dpo/delta": 0.06273117661476135, "fcm_dpo/margin": 214.77850341796875, "fcm_dpo/q_t": 0.4425661563873291, "grad_norm": 51.694149017333984, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -0.9190036058425903, "logits/rejected": -0.90464186668396, "logps/chosen": -857.5787353515625, "logps/ref_chosen": -67.553466796875, "logps/ref_rejected": -87.58953857421875, "logps/rejected": -1092.393310546875, "loss": 1.2784, "margin_dpo/margin_mean": 214.7785186767578, "margin_dpo/margin_std": 644.366943359375, "step": 482 }, { "KL/chosen_KL_mean": -724.80859375, "KL/mean": -970.60693359375, "KL/rejected_KL_mean": -1216.4053955078125, "KL/std": 547.3455810546875, "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.0011287643574178219, "fcm_dpo/delta": -0.15940281748771667, "fcm_dpo/margin": 491.59686279296875, "fcm_dpo/q_t": 0.3793754577636719, "grad_norm": 40.69145202636719, "learning_rate": 1.194847979251979e-07, "logits/chosen": -0.9518593549728394, "logits/rejected": -0.9543057680130005, "logps/chosen": -788.1383666992188, "logps/ref_chosen": -63.32981872558594, "logps/ref_rejected": -95.78697204589844, "logps/rejected": -1312.1923828125, "loss": 1.011, "margin_dpo/margin_mean": 491.5968322753906, "margin_dpo/margin_std": 612.5662841796875, "step": 483 }, { "KL/chosen_KL_mean": -609.849609375, "KL/mean": -838.743408203125, "KL/rejected_KL_mean": -1067.6370849609375, "KL/std": 533.2965087890625, "epoch": 0.71071953010279, "fcm_dpo/beta": 0.0011179596185684204, "fcm_dpo/delta": -0.1146901473402977, "fcm_dpo/margin": 457.7874755859375, "fcm_dpo/q_t": 0.3862311840057373, "grad_norm": 33.894779205322266, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -1.0216343402862549, "logits/rejected": -1.041937232017517, "logps/chosen": -668.98779296875, "logps/ref_chosen": -59.13812255859375, "logps/ref_rejected": -84.37144470214844, "logps/rejected": -1152.008544921875, "loss": 1.024, "margin_dpo/margin_mean": 457.7874755859375, "margin_dpo/margin_std": 545.956298828125, "step": 484 }, { "KL/chosen_KL_mean": -657.9251708984375, "KL/mean": -873.3860473632812, "KL/rejected_KL_mean": -1088.846923828125, "KL/std": 551.51806640625, "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.0011059865355491638, "fcm_dpo/delta": -0.07843407988548279, "fcm_dpo/margin": 430.9217529296875, "fcm_dpo/q_t": 0.395374596118927, "grad_norm": 30.792177200317383, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -0.9116197824478149, "logits/rejected": -0.9275870323181152, "logps/chosen": -716.7747802734375, "logps/ref_chosen": -58.849571228027344, "logps/ref_rejected": -103.36408233642578, "logps/rejected": -1192.2109375, "loss": 1.0703, "margin_dpo/margin_mean": 430.9217224121094, "margin_dpo/margin_std": 620.1131591796875, "step": 485 }, { "KL/chosen_KL_mean": -730.796142578125, "KL/mean": -973.44775390625, "KL/rejected_KL_mean": -1216.099365234375, "KL/std": 588.2183837890625, "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.001091314246878028, "fcm_dpo/delta": -0.1332990676164627, "fcm_dpo/margin": 485.30328369140625, "fcm_dpo/q_t": 0.38558250665664673, "grad_norm": 42.85213851928711, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.025226354598999, "logits/rejected": -1.0336995124816895, "logps/chosen": -786.0557861328125, "logps/ref_chosen": -55.25966262817383, "logps/ref_rejected": -92.13936614990234, "logps/rejected": -1308.23876953125, "loss": 1.0581, "margin_dpo/margin_mean": 485.30328369140625, "margin_dpo/margin_std": 714.6268310546875, "step": 486 }, { "KL/chosen_KL_mean": -738.9359130859375, "KL/mean": -917.86376953125, "KL/rejected_KL_mean": -1096.791748046875, "KL/std": 529.3869018554688, "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.0010912182042375207, "fcm_dpo/delta": 0.009562283754348755, "fcm_dpo/margin": 357.8558349609375, "fcm_dpo/q_t": 0.40953782200813293, "grad_norm": 35.427616119384766, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -1.0364776849746704, "logits/rejected": -1.0385382175445557, "logps/chosen": -791.9991455078125, "logps/ref_chosen": -53.06330871582031, "logps/ref_rejected": -92.41883087158203, "logps/rejected": -1189.2105712890625, "loss": 1.1157, "margin_dpo/margin_mean": 357.85589599609375, "margin_dpo/margin_std": 545.7218017578125, "step": 487 }, { "KL/chosen_KL_mean": -610.1397705078125, "KL/mean": -795.30224609375, "KL/rejected_KL_mean": -980.4647216796875, "KL/std": 488.50335693359375, "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.0010911128483712673, "fcm_dpo/delta": -0.00425250269472599, "fcm_dpo/margin": 370.32489013671875, "fcm_dpo/q_t": 0.4075971841812134, "grad_norm": 37.004825592041016, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -0.9542009830474854, "logits/rejected": -0.9616404175758362, "logps/chosen": -662.3679809570312, "logps/ref_chosen": -52.22815704345703, "logps/ref_rejected": -84.00656127929688, "logps/rejected": -1064.4713134765625, "loss": 1.0826, "margin_dpo/margin_mean": 370.32489013671875, "margin_dpo/margin_std": 467.35565185546875, "step": 488 }, { "KL/chosen_KL_mean": -572.46044921875, "KL/mean": -741.5594482421875, "KL/rejected_KL_mean": -910.658447265625, "KL/std": 456.1171875, "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.0010937793413177133, "fcm_dpo/delta": 0.030445974320173264, "fcm_dpo/margin": 338.1979675292969, "fcm_dpo/q_t": 0.41659069061279297, "grad_norm": 35.718448638916016, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -0.9860827922821045, "logits/rejected": -0.9753962755203247, "logps/chosen": -628.4500732421875, "logps/ref_chosen": -55.989627838134766, "logps/ref_rejected": -79.39812469482422, "logps/rejected": -990.0565185546875, "loss": 1.1177, "margin_dpo/margin_mean": 338.19793701171875, "margin_dpo/margin_std": 492.15753173828125, "step": 489 }, { "KL/chosen_KL_mean": -665.18701171875, "KL/mean": -841.7398071289062, "KL/rejected_KL_mean": -1018.2926025390625, "KL/std": 565.1947021484375, "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.0010953254532068968, "fcm_dpo/delta": 0.013375891372561455, "fcm_dpo/margin": 353.1055908203125, "fcm_dpo/q_t": 0.4144512116909027, "grad_norm": 36.92960739135742, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -0.9983633756637573, "logits/rejected": -1.038588523864746, "logps/chosen": -717.5534057617188, "logps/ref_chosen": -52.36639404296875, "logps/ref_rejected": -110.4090576171875, "logps/rejected": -1128.70166015625, "loss": 1.1514, "margin_dpo/margin_mean": 353.1055603027344, "margin_dpo/margin_std": 646.734130859375, "step": 490 }, { "KL/chosen_KL_mean": -616.6781616210938, "KL/mean": -748.5814208984375, "KL/rejected_KL_mean": -880.484619140625, "KL/std": 459.8958740234375, "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.0011040510144084692, "fcm_dpo/delta": 0.11021701991558075, "fcm_dpo/margin": 263.8064880371094, "fcm_dpo/q_t": 0.43390926718711853, "grad_norm": 33.199649810791016, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -0.9711790084838867, "logits/rejected": -0.9616298675537109, "logps/chosen": -676.79443359375, "logps/ref_chosen": -60.11626434326172, "logps/ref_rejected": -73.27278900146484, "logps/rejected": -953.7574462890625, "loss": 1.1829, "margin_dpo/margin_mean": 263.8064880371094, "margin_dpo/margin_std": 483.6207580566406, "step": 491 }, { "KL/chosen_KL_mean": -635.3643798828125, "KL/mean": -769.837890625, "KL/rejected_KL_mean": -904.3115234375, "KL/std": 484.90753173828125, "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0011131614446640015, "fcm_dpo/delta": 0.10221239924430847, "fcm_dpo/margin": 268.947021484375, "fcm_dpo/q_t": 0.43284350633621216, "grad_norm": 43.230899810791016, "learning_rate": 1.097764975115576e-07, "logits/chosen": -1.0389471054077148, "logits/rejected": -1.0215075016021729, "logps/chosen": -689.3585815429688, "logps/ref_chosen": -53.994178771972656, "logps/ref_rejected": -72.65962219238281, "logps/rejected": -976.9710693359375, "loss": 1.2, "margin_dpo/margin_mean": 268.947021484375, "margin_dpo/margin_std": 554.927978515625, "step": 492 }, { "KL/chosen_KL_mean": -699.6456298828125, "KL/mean": -847.1690063476562, "KL/rejected_KL_mean": -994.6923828125, "KL/std": 572.62060546875, "epoch": 0.723935389133627, "fcm_dpo/beta": 0.0011133572552353144, "fcm_dpo/delta": -0.03688516467809677, "fcm_dpo/margin": 295.0467224121094, "fcm_dpo/q_t": 0.42351529002189636, "grad_norm": 34.844478607177734, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -1.1079821586608887, "logits/rejected": -1.0783543586730957, "logps/chosen": -775.1428833007812, "logps/ref_chosen": -75.49723815917969, "logps/ref_rejected": -87.32301330566406, "logps/rejected": -1082.015380859375, "loss": 1.1584, "margin_dpo/margin_mean": 295.04669189453125, "margin_dpo/margin_std": 489.8016357421875, "step": 493 }, { "KL/chosen_KL_mean": -565.2142944335938, "KL/mean": -797.517578125, "KL/rejected_KL_mean": -1029.821044921875, "KL/std": 503.0618896484375, "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.0011037448421120644, "fcm_dpo/delta": -0.11577919125556946, "fcm_dpo/margin": 464.60662841796875, "fcm_dpo/q_t": 0.38258957862854004, "grad_norm": 38.89155960083008, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -0.9311180114746094, "logits/rejected": -0.9682860970497131, "logps/chosen": -606.5736083984375, "logps/ref_chosen": -41.35926818847656, "logps/ref_rejected": -86.09136962890625, "logps/rejected": -1115.912353515625, "loss": 1.0082, "margin_dpo/margin_mean": 464.606689453125, "margin_dpo/margin_std": 502.99725341796875, "step": 494 }, { "KL/chosen_KL_mean": -633.6134033203125, "KL/mean": -830.8568725585938, "KL/rejected_KL_mean": -1028.1004638671875, "KL/std": 498.0575866699219, "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0010981929954141378, "fcm_dpo/delta": -0.033951401710510254, "fcm_dpo/margin": 394.487060546875, "fcm_dpo/q_t": 0.40414437651634216, "grad_norm": 42.01950454711914, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -1.0234172344207764, "logits/rejected": -1.0299253463745117, "logps/chosen": -697.1484375, "logps/ref_chosen": -63.53507995605469, "logps/ref_rejected": -91.42443084716797, "logps/rejected": -1119.52490234375, "loss": 1.0924, "margin_dpo/margin_mean": 394.487060546875, "margin_dpo/margin_std": 591.7906494140625, "step": 495 }, { "KL/chosen_KL_mean": -781.1953735351562, "KL/mean": -882.2847900390625, "KL/rejected_KL_mean": -983.3742065429688, "KL/std": 428.71868896484375, "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.0011013501789420843, "fcm_dpo/delta": 0.08242875337600708, "fcm_dpo/margin": 202.17886352539062, "fcm_dpo/q_t": 0.44761157035827637, "grad_norm": 73.75658416748047, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -1.0234228372573853, "logits/rejected": -0.9919195175170898, "logps/chosen": -853.787353515625, "logps/ref_chosen": -72.5919189453125, "logps/ref_rejected": -84.32933807373047, "logps/rejected": -1067.70361328125, "loss": 1.2357, "margin_dpo/margin_mean": 202.17886352539062, "margin_dpo/margin_std": 450.7599792480469, "step": 496 }, { "KL/chosen_KL_mean": -759.360595703125, "KL/mean": -857.68359375, "KL/rejected_KL_mean": -956.0064697265625, "KL/std": 512.5221557617188, "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.0011091852793470025, "fcm_dpo/delta": 0.029760103672742844, "fcm_dpo/margin": 196.64585876464844, "fcm_dpo/q_t": 0.4496675729751587, "grad_norm": 32.76177978515625, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -0.9546867609024048, "logits/rejected": -0.935001015663147, "logps/chosen": -817.95458984375, "logps/ref_chosen": -58.59397506713867, "logps/ref_rejected": -76.28836822509766, "logps/rejected": -1032.294921875, "loss": 1.2508, "margin_dpo/margin_mean": 196.6458740234375, "margin_dpo/margin_std": 478.19500732421875, "step": 497 }, { "KL/chosen_KL_mean": -722.9497680664062, "KL/mean": -899.7124633789062, "KL/rejected_KL_mean": -1076.47509765625, "KL/std": 522.7940063476562, "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.0011089190375059843, "fcm_dpo/delta": 0.008106292225420475, "fcm_dpo/margin": 353.52532958984375, "fcm_dpo/q_t": 0.4107200503349304, "grad_norm": 31.680055618286133, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -0.9777836799621582, "logits/rejected": -0.9642585515975952, "logps/chosen": -794.1553955078125, "logps/ref_chosen": -71.20565795898438, "logps/ref_rejected": -83.95803833007812, "logps/rejected": -1160.43310546875, "loss": 1.1266, "margin_dpo/margin_mean": 353.5253601074219, "margin_dpo/margin_std": 576.6744384765625, "step": 498 }, { "KL/chosen_KL_mean": -631.90966796875, "KL/mean": -870.5003051757812, "KL/rejected_KL_mean": -1109.091064453125, "KL/std": 562.7127685546875, "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0010997585486620665, "fcm_dpo/delta": -0.1281210333108902, "fcm_dpo/margin": 477.1813049316406, "fcm_dpo/q_t": 0.38377606868743896, "grad_norm": 32.53940963745117, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -0.988601565361023, "logits/rejected": -1.0279262065887451, "logps/chosen": -683.1648559570312, "logps/ref_chosen": -51.25519561767578, "logps/ref_rejected": -101.07870483398438, "logps/rejected": -1210.169677734375, "loss": 1.0295, "margin_dpo/margin_mean": 477.1813049316406, "margin_dpo/margin_std": 616.3780517578125, "step": 499 }, { "KL/chosen_KL_mean": -733.3824462890625, "KL/mean": -922.1387329101562, "KL/rejected_KL_mean": -1110.89501953125, "KL/std": 483.62493896484375, "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.0010943380184471607, "fcm_dpo/delta": -0.013418500311672688, "fcm_dpo/margin": 377.5125427246094, "fcm_dpo/q_t": 0.4065711498260498, "grad_norm": 32.50249099731445, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -0.9592075347900391, "logits/rejected": -0.9868057370185852, "logps/chosen": -790.409912109375, "logps/ref_chosen": -57.027442932128906, "logps/ref_rejected": -93.93421173095703, "logps/rejected": -1204.8291015625, "loss": 1.1156, "margin_dpo/margin_mean": 377.5125732421875, "margin_dpo/margin_std": 596.400634765625, "step": 500 }, { "KL/chosen_KL_mean": -652.4368286132812, "KL/mean": -851.3330688476562, "KL/rejected_KL_mean": -1050.229248046875, "KL/std": 501.94122314453125, "epoch": 0.73568281938326, "fcm_dpo/beta": 0.0010930509306490421, "fcm_dpo/delta": -0.03560645133256912, "fcm_dpo/margin": 397.7924499511719, "fcm_dpo/q_t": 0.40338021516799927, "grad_norm": 31.65900993347168, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -0.9449106454849243, "logits/rejected": -0.9296088218688965, "logps/chosen": -706.79638671875, "logps/ref_chosen": -54.359527587890625, "logps/ref_rejected": -80.15670013427734, "logps/rejected": -1130.385986328125, "loss": 1.1051, "margin_dpo/margin_mean": 397.79241943359375, "margin_dpo/margin_std": 624.0454711914062, "step": 501 }, { "KL/chosen_KL_mean": -590.9495849609375, "KL/mean": -798.0103149414062, "KL/rejected_KL_mean": -1005.0711059570312, "KL/std": 499.19720458984375, "epoch": 0.737151248164464, "fcm_dpo/beta": 0.0010861887130886316, "fcm_dpo/delta": -0.050974659621715546, "fcm_dpo/margin": 414.1214599609375, "fcm_dpo/q_t": 0.401166707277298, "grad_norm": 28.097309112548828, "learning_rate": 9.934134090518592e-08, "logits/chosen": -0.8737450838088989, "logits/rejected": -0.8553862571716309, "logps/chosen": -658.550048828125, "logps/ref_chosen": -67.60050964355469, "logps/ref_rejected": -82.94876098632812, "logps/rejected": -1088.019775390625, "loss": 1.0626, "margin_dpo/margin_mean": 414.1214599609375, "margin_dpo/margin_std": 551.1231689453125, "step": 502 }, { "KL/chosen_KL_mean": -591.0940551757812, "KL/mean": -777.262939453125, "KL/rejected_KL_mean": -963.431884765625, "KL/std": 456.4488220214844, "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.0010829106904566288, "fcm_dpo/delta": -0.003399871289730072, "fcm_dpo/margin": 372.33782958984375, "fcm_dpo/q_t": 0.4094755947589874, "grad_norm": 30.805957794189453, "learning_rate": 9.831921068732571e-08, "logits/chosen": -0.8509905338287354, "logits/rejected": -0.8399511575698853, "logps/chosen": -646.1724853515625, "logps/ref_chosen": -55.078407287597656, "logps/ref_rejected": -82.50544738769531, "logps/rejected": -1045.937255859375, "loss": 1.0941, "margin_dpo/margin_mean": 372.33782958984375, "margin_dpo/margin_std": 513.9088134765625, "step": 503 }, { "KL/chosen_KL_mean": -612.4645385742188, "KL/mean": -844.0081176757812, "KL/rejected_KL_mean": -1075.5516357421875, "KL/std": 534.1448974609375, "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.0010781993623822927, "fcm_dpo/delta": -0.10178365558385849, "fcm_dpo/margin": 463.0871887207031, "fcm_dpo/q_t": 0.38906794786453247, "grad_norm": 28.29525375366211, "learning_rate": 9.730107739932805e-08, "logits/chosen": -0.9321706295013428, "logits/rejected": -0.9592180252075195, "logps/chosen": -672.4302978515625, "logps/ref_chosen": -59.96575164794922, "logps/ref_rejected": -103.76212310791016, "logps/rejected": -1179.313720703125, "loss": 1.0388, "margin_dpo/margin_mean": 463.087158203125, "margin_dpo/margin_std": 572.2286987304688, "step": 504 }, { "KL/chosen_KL_mean": -688.0867919921875, "KL/mean": -798.6611938476562, "KL/rejected_KL_mean": -909.235595703125, "KL/std": 483.4139099121094, "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.0010871076956391335, "fcm_dpo/delta": 0.1614731252193451, "fcm_dpo/margin": 221.14871215820312, "fcm_dpo/q_t": 0.44385814666748047, "grad_norm": 40.906707763671875, "learning_rate": 9.628696786995188e-08, "logits/chosen": -0.9417963027954102, "logits/rejected": -0.9116191267967224, "logps/chosen": -764.24169921875, "logps/ref_chosen": -76.1549072265625, "logps/ref_rejected": -88.58537292480469, "logps/rejected": -997.8209228515625, "loss": 1.2195, "margin_dpo/margin_mean": 221.14871215820312, "margin_dpo/margin_std": 449.42291259765625, "step": 505 }, { "KL/chosen_KL_mean": -552.0079345703125, "KL/mean": -739.86669921875, "KL/rejected_KL_mean": -927.7255249023438, "KL/std": 480.6181640625, "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.001087966375052929, "fcm_dpo/delta": -0.009178068488836288, "fcm_dpo/margin": 375.71759033203125, "fcm_dpo/q_t": 0.40685611963272095, "grad_norm": 41.05369567871094, "learning_rate": 9.527690882192635e-08, "logits/chosen": -0.9096465110778809, "logits/rejected": -0.9178716540336609, "logps/chosen": -600.9683837890625, "logps/ref_chosen": -48.96050262451172, "logps/ref_rejected": -78.41505432128906, "logps/rejected": -1006.1405639648438, "loss": 1.0943, "margin_dpo/margin_mean": 375.71759033203125, "margin_dpo/margin_std": 518.1092529296875, "step": 506 }, { "KL/chosen_KL_mean": -654.116943359375, "KL/mean": -816.158447265625, "KL/rejected_KL_mean": -978.199951171875, "KL/std": 556.29931640625, "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.0010937817860394716, "fcm_dpo/delta": 0.04635169357061386, "fcm_dpo/margin": 324.0830383300781, "fcm_dpo/q_t": 0.4228259325027466, "grad_norm": 30.889352798461914, "learning_rate": 9.427092687124691e-08, "logits/chosen": -0.957282543182373, "logits/rejected": -0.95253586769104, "logps/chosen": -720.9183959960938, "logps/ref_chosen": -66.80149841308594, "logps/ref_rejected": -95.37289428710938, "logps/rejected": -1073.5728759765625, "loss": 1.1562, "margin_dpo/margin_mean": 324.08306884765625, "margin_dpo/margin_std": 591.4808959960938, "step": 507 }, { "KL/chosen_KL_mean": -661.949951171875, "KL/mean": -807.3236083984375, "KL/rejected_KL_mean": -952.697265625, "KL/std": 519.9277954101562, "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.0011030698660761118, "fcm_dpo/delta": 0.08043861389160156, "fcm_dpo/margin": 290.74737548828125, "fcm_dpo/q_t": 0.4288504719734192, "grad_norm": 35.57503890991211, "learning_rate": 9.326904852647344e-08, "logits/chosen": -0.9091905355453491, "logits/rejected": -0.9017305374145508, "logps/chosen": -733.25341796875, "logps/ref_chosen": -71.303466796875, "logps/ref_rejected": -95.6275405883789, "logps/rejected": -1048.3248291015625, "loss": 1.1934, "margin_dpo/margin_mean": 290.74737548828125, "margin_dpo/margin_std": 599.302734375, "step": 508 }, { "KL/chosen_KL_mean": -501.7856750488281, "KL/mean": -661.9292602539062, "KL/rejected_KL_mean": -822.0728149414062, "KL/std": 382.68975830078125, "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.0011099616531282663, "fcm_dpo/delta": 0.04505918547511101, "fcm_dpo/margin": 320.2871398925781, "fcm_dpo/q_t": 0.4187580645084381, "grad_norm": 29.257396697998047, "learning_rate": 9.227130018803195e-08, "logits/chosen": -0.814157247543335, "logits/rejected": -0.7995736598968506, "logps/chosen": -565.6046142578125, "logps/ref_chosen": -63.81895065307617, "logps/ref_rejected": -83.25643920898438, "logps/rejected": -905.3292236328125, "loss": 1.1258, "margin_dpo/margin_mean": 320.287109375, "margin_dpo/margin_std": 466.1398010253906, "step": 509 }, { "KL/chosen_KL_mean": -626.7471923828125, "KL/mean": -820.874755859375, "KL/rejected_KL_mean": -1015.002197265625, "KL/std": 441.17962646484375, "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0011074226349592209, "fcm_dpo/delta": -0.030622189864516258, "fcm_dpo/margin": 388.2550354003906, "fcm_dpo/q_t": 0.3993905484676361, "grad_norm": 29.603723526000977, "learning_rate": 9.127770814751932e-08, "logits/chosen": -0.8510131239891052, "logits/rejected": -0.8649104237556458, "logps/chosen": -678.6256103515625, "logps/ref_chosen": -51.878448486328125, "logps/ref_rejected": -102.7651596069336, "logps/rejected": -1117.767333984375, "loss": 1.0558, "margin_dpo/margin_mean": 388.25506591796875, "margin_dpo/margin_std": 431.11676025390625, "step": 510 }, { "KL/chosen_KL_mean": -593.8864135742188, "KL/mean": -752.0758056640625, "KL/rejected_KL_mean": -910.26513671875, "KL/std": 475.2417907714844, "epoch": 0.750367107195301, "fcm_dpo/beta": 0.0011101202107965946, "fcm_dpo/delta": 0.04963508993387222, "fcm_dpo/margin": 316.3787841796875, "fcm_dpo/q_t": 0.4193497896194458, "grad_norm": 29.786012649536133, "learning_rate": 9.028829858700973e-08, "logits/chosen": -0.9209908246994019, "logits/rejected": -0.9174632430076599, "logps/chosen": -654.12451171875, "logps/ref_chosen": -60.23811721801758, "logps/ref_rejected": -92.85676574707031, "logps/rejected": -1003.1219482421875, "loss": 1.1544, "margin_dpo/margin_mean": 316.3787536621094, "margin_dpo/margin_std": 560.738037109375, "step": 511 }, { "KL/chosen_KL_mean": -489.36114501953125, "KL/mean": -704.290283203125, "KL/rejected_KL_mean": -919.219482421875, "KL/std": 439.2633056640625, "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.001103471266105771, "fcm_dpo/delta": -0.07630272209644318, "fcm_dpo/margin": 429.8582763671875, "fcm_dpo/q_t": 0.3897845149040222, "grad_norm": 37.09547805786133, "learning_rate": 8.930309757836516e-08, "logits/chosen": -0.8986982107162476, "logits/rejected": -0.9108966588973999, "logps/chosen": -544.2666625976562, "logps/ref_chosen": -54.905494689941406, "logps/ref_rejected": -81.87586975097656, "logps/rejected": -1001.0953369140625, "loss": 1.0214, "margin_dpo/margin_mean": 429.8583068847656, "margin_dpo/margin_std": 438.10321044921875, "step": 512 }, { "KL/chosen_KL_mean": -636.2532958984375, "KL/mean": -780.3316650390625, "KL/rejected_KL_mean": -924.4100341796875, "KL/std": 423.2544250488281, "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.00109992700163275, "fcm_dpo/delta": -0.025476014241576195, "fcm_dpo/margin": 288.15679931640625, "fcm_dpo/q_t": 0.4271167516708374, "grad_norm": 44.68075942993164, "learning_rate": 8.832213108254863e-08, "logits/chosen": -0.9700770378112793, "logits/rejected": -0.9396867752075195, "logps/chosen": -701.169677734375, "logps/ref_chosen": -64.91644287109375, "logps/ref_rejected": -76.06245422363281, "logps/rejected": -1000.4725341796875, "loss": 1.1665, "margin_dpo/margin_mean": 288.15679931640625, "margin_dpo/margin_std": 489.13134765625, "step": 513 }, { "KL/chosen_KL_mean": -640.1827392578125, "KL/mean": -787.67919921875, "KL/rejected_KL_mean": -935.1756591796875, "KL/std": 454.5134582519531, "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.00110641831997782, "fcm_dpo/delta": 0.07481080293655396, "fcm_dpo/margin": 294.99298095703125, "fcm_dpo/q_t": 0.42692023515701294, "grad_norm": 36.836143493652344, "learning_rate": 8.734542494893954e-08, "logits/chosen": -0.9246986508369446, "logits/rejected": -0.904691755771637, "logps/chosen": -714.412353515625, "logps/ref_chosen": -74.22957611083984, "logps/ref_rejected": -78.945556640625, "logps/rejected": -1014.1212158203125, "loss": 1.1608, "margin_dpo/margin_mean": 294.99298095703125, "margin_dpo/margin_std": 514.4725341796875, "step": 514 }, { "KL/chosen_KL_mean": -525.1871337890625, "KL/mean": -651.1346435546875, "KL/rejected_KL_mean": -777.08203125, "KL/std": 396.79437255859375, "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.0011181033914908767, "fcm_dpo/delta": 0.12005515396595001, "fcm_dpo/margin": 251.8948974609375, "fcm_dpo/q_t": 0.4334961175918579, "grad_norm": 34.989688873291016, "learning_rate": 8.637300491465272e-08, "logits/chosen": -0.8561180830001831, "logits/rejected": -0.8611321449279785, "logps/chosen": -575.5887451171875, "logps/ref_chosen": -50.40156555175781, "logps/ref_rejected": -87.09774780273438, "logps/rejected": -864.1798095703125, "loss": 1.1867, "margin_dpo/margin_mean": 251.89492797851562, "margin_dpo/margin_std": 460.27935791015625, "step": 515 }, { "KL/chosen_KL_mean": -596.61669921875, "KL/mean": -770.6539306640625, "KL/rejected_KL_mean": -944.6910400390625, "KL/std": 452.91241455078125, "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.0011240649037063122, "fcm_dpo/delta": 0.00880388356745243, "fcm_dpo/margin": 348.07440185546875, "fcm_dpo/q_t": 0.408348023891449, "grad_norm": 32.15882110595703, "learning_rate": 8.540489660386064e-08, "logits/chosen": -0.9794058799743652, "logits/rejected": -0.9973136186599731, "logps/chosen": -661.2662353515625, "logps/ref_chosen": -64.64956665039062, "logps/ref_rejected": -111.72237396240234, "logps/rejected": -1056.4134521484375, "loss": 1.0971, "margin_dpo/margin_mean": 348.0743713378906, "margin_dpo/margin_std": 459.40338134765625, "step": 516 }, { "KL/chosen_KL_mean": -583.4486083984375, "KL/mean": -800.865478515625, "KL/rejected_KL_mean": -1018.2823486328125, "KL/std": 497.50537109375, "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.0011140817077830434, "fcm_dpo/delta": -0.0866817757487297, "fcm_dpo/margin": 434.8336486816406, "fcm_dpo/q_t": 0.39268559217453003, "grad_norm": 31.913042068481445, "learning_rate": 8.444112552711752e-08, "logits/chosen": -0.93073570728302, "logits/rejected": -0.9230749607086182, "logps/chosen": -644.3621826171875, "logps/ref_chosen": -60.913551330566406, "logps/ref_rejected": -89.08308410644531, "logps/rejected": -1107.365478515625, "loss": 1.0456, "margin_dpo/margin_mean": 434.83367919921875, "margin_dpo/margin_std": 564.2802124023438, "step": 517 }, { "KL/chosen_KL_mean": -531.2779541015625, "KL/mean": -708.3522338867188, "KL/rejected_KL_mean": -885.426513671875, "KL/std": 404.1721496582031, "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.0011119421105831861, "fcm_dpo/delta": 0.006271369755268097, "fcm_dpo/margin": 354.1485900878906, "fcm_dpo/q_t": 0.40764957666397095, "grad_norm": 31.667274475097656, "learning_rate": 8.348171708068747e-08, "logits/chosen": -0.9522829055786133, "logits/rejected": -0.9628884792327881, "logps/chosen": -588.7338256835938, "logps/ref_chosen": -57.45589065551758, "logps/ref_rejected": -85.31269836425781, "logps/rejected": -970.7392578125, "loss": 1.0874, "margin_dpo/margin_mean": 354.1485900878906, "margin_dpo/margin_std": 437.1719970703125, "step": 518 }, { "KL/chosen_KL_mean": -544.94873046875, "KL/mean": -669.02685546875, "KL/rejected_KL_mean": -793.10498046875, "KL/std": 379.0975646972656, "epoch": 0.762114537444934, "fcm_dpo/beta": 0.0011145803146064281, "fcm_dpo/delta": 0.018153877928853035, "fcm_dpo/margin": 248.15621948242188, "fcm_dpo/q_t": 0.4351785182952881, "grad_norm": 26.59686851501465, "learning_rate": 8.25266965458755e-08, "logits/chosen": -0.9192925691604614, "logits/rejected": -0.8974975347518921, "logps/chosen": -619.0120849609375, "logps/ref_chosen": -74.06331634521484, "logps/ref_rejected": -104.44416809082031, "logps/rejected": -897.5491943359375, "loss": 1.192, "margin_dpo/margin_mean": 248.15621948242188, "margin_dpo/margin_std": 452.4583435058594, "step": 519 }, { "KL/chosen_KL_mean": -566.6231689453125, "KL/mean": -728.2750854492188, "KL/rejected_KL_mean": -889.9268798828125, "KL/std": 431.6119689941406, "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.0011171854566782713, "fcm_dpo/delta": 0.039489611983299255, "fcm_dpo/margin": 323.3037109375, "fcm_dpo/q_t": 0.41814154386520386, "grad_norm": 34.39997100830078, "learning_rate": 8.15760890883607e-08, "logits/chosen": -0.8914647102355957, "logits/rejected": -0.8902548551559448, "logps/chosen": -636.9230346679688, "logps/ref_chosen": -70.2998275756836, "logps/ref_rejected": -99.98133850097656, "logps/rejected": -989.9082641601562, "loss": 1.1248, "margin_dpo/margin_mean": 323.3037109375, "margin_dpo/margin_std": 469.2210693359375, "step": 520 }, { "KL/chosen_KL_mean": -525.2251586914062, "KL/mean": -698.4849243164062, "KL/rejected_KL_mean": -871.74462890625, "KL/std": 449.91253662109375, "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0011244425550103188, "fcm_dpo/delta": 0.010120447725057602, "fcm_dpo/margin": 346.51953125, "fcm_dpo/q_t": 0.4106011986732483, "grad_norm": 25.767515182495117, "learning_rate": 8.062991975753378e-08, "logits/chosen": -0.9617199897766113, "logits/rejected": -0.9598067402839661, "logps/chosen": -583.3680419921875, "logps/ref_chosen": -58.14292526245117, "logps/ref_rejected": -83.28060913085938, "logps/rejected": -955.0252685546875, "loss": 1.1012, "margin_dpo/margin_mean": 346.51953125, "margin_dpo/margin_std": 464.26416015625, "step": 521 }, { "KL/chosen_KL_mean": -578.2764892578125, "KL/mean": -737.2076416015625, "KL/rejected_KL_mean": -896.1387939453125, "KL/std": 469.0858154296875, "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.001124875620007515, "fcm_dpo/delta": 0.04321271926164627, "fcm_dpo/margin": 317.8622131347656, "fcm_dpo/q_t": 0.4179668128490448, "grad_norm": 27.95970916748047, "learning_rate": 7.968821348583643e-08, "logits/chosen": -0.9487807750701904, "logits/rejected": -0.9443739652633667, "logps/chosen": -624.82421875, "logps/ref_chosen": -46.54766845703125, "logps/ref_rejected": -66.01388549804688, "logps/rejected": -962.1526489257812, "loss": 1.1318, "margin_dpo/margin_mean": 317.86224365234375, "margin_dpo/margin_std": 489.20440673828125, "step": 522 }, { "KL/chosen_KL_mean": -616.9910888671875, "KL/mean": -790.3553466796875, "KL/rejected_KL_mean": -963.7196044921875, "KL/std": 531.9453125, "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.0011264176573604345, "fcm_dpo/delta": 0.00962541438639164, "fcm_dpo/margin": 346.72845458984375, "fcm_dpo/q_t": 0.4115051031112671, "grad_norm": 49.35715103149414, "learning_rate": 7.875099508810484e-08, "logits/chosen": -0.9954909086227417, "logits/rejected": -0.9879505634307861, "logps/chosen": -678.7607421875, "logps/ref_chosen": -61.76960372924805, "logps/ref_rejected": -83.76141357421875, "logps/rejected": -1047.48095703125, "loss": 1.1189, "margin_dpo/margin_mean": 346.72845458984375, "margin_dpo/margin_std": 540.300537109375, "step": 523 }, { "KL/chosen_KL_mean": -613.022705078125, "KL/mean": -792.3504638671875, "KL/rejected_KL_mean": -971.6781005859375, "KL/std": 508.4144287109375, "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.001124032773077488, "fcm_dpo/delta": -0.00345630943775177, "fcm_dpo/margin": 358.6553649902344, "fcm_dpo/q_t": 0.4057971239089966, "grad_norm": 34.90718078613281, "learning_rate": 7.781828926091535e-08, "logits/chosen": -1.0312086343765259, "logits/rejected": -1.0126185417175293, "logps/chosen": -691.0947875976562, "logps/ref_chosen": -78.0720443725586, "logps/ref_rejected": -81.30198669433594, "logps/rejected": -1052.9801025390625, "loss": 1.1073, "margin_dpo/margin_mean": 358.6553649902344, "margin_dpo/margin_std": 517.4310302734375, "step": 524 }, { "KL/chosen_KL_mean": -596.0318603515625, "KL/mean": -821.548828125, "KL/rejected_KL_mean": -1047.0657958984375, "KL/std": 515.7371826171875, "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.0011153742671012878, "fcm_dpo/delta": -0.10615138709545135, "fcm_dpo/margin": 451.0339660644531, "fcm_dpo/q_t": 0.38702392578125, "grad_norm": 34.25413131713867, "learning_rate": 7.689012058193384e-08, "logits/chosen": -0.9680420160293579, "logits/rejected": -1.0015491247177124, "logps/chosen": -646.8596801757812, "logps/ref_chosen": -50.827857971191406, "logps/ref_rejected": -100.05294036865234, "logps/rejected": -1147.11865234375, "loss": 1.026, "margin_dpo/margin_mean": 451.0339660644531, "margin_dpo/margin_std": 533.57666015625, "step": 525 }, { "KL/chosen_KL_mean": -641.201904296875, "KL/mean": -872.0587158203125, "KL/rejected_KL_mean": -1102.91552734375, "KL/std": 532.249267578125, "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.0011071430053561926, "fcm_dpo/delta": -0.1140209287405014, "fcm_dpo/margin": 461.7136535644531, "fcm_dpo/q_t": 0.383984237909317, "grad_norm": 31.422950744628906, "learning_rate": 7.596651350926836e-08, "logits/chosen": -0.9887993335723877, "logits/rejected": -0.9786882400512695, "logps/chosen": -704.369140625, "logps/ref_chosen": -63.167236328125, "logps/ref_rejected": -86.30934143066406, "logps/rejected": -1189.224853515625, "loss": 1.0411, "margin_dpo/margin_mean": 461.713623046875, "margin_dpo/margin_std": 597.789794921875, "step": 526 }, { "KL/chosen_KL_mean": -675.8800048828125, "KL/mean": -825.571533203125, "KL/rejected_KL_mean": -975.262939453125, "KL/std": 547.1878662109375, "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.0011063042329624295, "fcm_dpo/delta": 0.06996496021747589, "fcm_dpo/margin": 299.3829650878906, "fcm_dpo/q_t": 0.4228494465351105, "grad_norm": 37.20889663696289, "learning_rate": 7.504749238082414e-08, "logits/chosen": -1.1690475940704346, "logits/rejected": -1.126373529434204, "logps/chosen": -747.0086669921875, "logps/ref_chosen": -71.12867736816406, "logps/ref_rejected": -78.3425521850586, "logps/rejected": -1053.60546875, "loss": 1.1431, "margin_dpo/margin_mean": 299.3829650878906, "margin_dpo/margin_std": 453.30816650390625, "step": 527 }, { "KL/chosen_KL_mean": -692.7294921875, "KL/mean": -880.4638671875, "KL/rejected_KL_mean": -1068.1982421875, "KL/std": 529.0624389648438, "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0011081206612288952, "fcm_dpo/delta": -0.016417687758803368, "fcm_dpo/margin": 375.46881103515625, "fcm_dpo/q_t": 0.4085617959499359, "grad_norm": 37.37120056152344, "learning_rate": 7.413308141366254e-08, "logits/chosen": -1.0335596799850464, "logits/rejected": -1.0141860246658325, "logps/chosen": -760.8189697265625, "logps/ref_chosen": -68.0894546508789, "logps/ref_rejected": -93.91006469726562, "logps/rejected": -1162.1083984375, "loss": 1.1177, "margin_dpo/margin_mean": 375.46881103515625, "margin_dpo/margin_std": 612.314697265625, "step": 528 }, { "KL/chosen_KL_mean": -838.7152099609375, "KL/mean": -968.595458984375, "KL/rejected_KL_mean": -1098.4757080078125, "KL/std": 483.984130859375, "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.001110037206672132, "fcm_dpo/delta": 0.017718147486448288, "fcm_dpo/margin": 259.760498046875, "fcm_dpo/q_t": 0.4340352416038513, "grad_norm": 38.99552536010742, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.1005570888519287, "logits/rejected": -1.1074516773223877, "logps/chosen": -894.2901611328125, "logps/ref_chosen": -55.57495880126953, "logps/ref_rejected": -89.20909118652344, "logps/rejected": -1187.684814453125, "loss": 1.2326, "margin_dpo/margin_mean": 259.760498046875, "margin_dpo/margin_std": 621.6509399414062, "step": 529 }, { "KL/chosen_KL_mean": -708.62939453125, "KL/mean": -925.3953857421875, "KL/rejected_KL_mean": -1142.161376953125, "KL/std": 603.607666015625, "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.0011045248247683048, "fcm_dpo/delta": -0.0807407796382904, "fcm_dpo/margin": 433.53204345703125, "fcm_dpo/q_t": 0.39779338240623474, "grad_norm": 48.64304733276367, "learning_rate": 7.231818622338822e-08, "logits/chosen": -0.9852283000946045, "logits/rejected": -0.9839862585067749, "logps/chosen": -756.2308349609375, "logps/ref_chosen": -47.601417541503906, "logps/ref_rejected": -87.2845230102539, "logps/rejected": -1229.4459228515625, "loss": 1.1157, "margin_dpo/margin_mean": 433.53204345703125, "margin_dpo/margin_std": 764.8116455078125, "step": 530 }, { "KL/chosen_KL_mean": -784.3384399414062, "KL/mean": -969.3201904296875, "KL/rejected_KL_mean": -1154.302001953125, "KL/std": 622.963623046875, "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.0011020496021956205, "fcm_dpo/delta": -0.007951600477099419, "fcm_dpo/margin": 369.9634704589844, "fcm_dpo/q_t": 0.4096531271934509, "grad_norm": 47.655757904052734, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.1287320852279663, "logits/rejected": -1.113837480545044, "logps/chosen": -839.58447265625, "logps/ref_chosen": -55.246063232421875, "logps/ref_rejected": -70.60598754882812, "logps/rejected": -1224.907958984375, "loss": 1.1196, "margin_dpo/margin_mean": 369.9634704589844, "margin_dpo/margin_std": 602.9121704101562, "step": 531 }, { "KL/chosen_KL_mean": -803.9302368164062, "KL/mean": -986.0310668945312, "KL/rejected_KL_mean": -1168.1319580078125, "KL/std": 576.912353515625, "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.0010961306979879737, "fcm_dpo/delta": 0.0002321675419807434, "fcm_dpo/margin": 364.2017822265625, "fcm_dpo/q_t": 0.4125855565071106, "grad_norm": 69.55883026123047, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.0360071659088135, "logits/rejected": -1.0059716701507568, "logps/chosen": -874.2162475585938, "logps/ref_chosen": -70.28601837158203, "logps/ref_rejected": -86.5913314819336, "logps/rejected": -1254.7232666015625, "loss": 1.1623, "margin_dpo/margin_mean": 364.2017517089844, "margin_dpo/margin_std": 691.018310546875, "step": 532 }, { "KL/chosen_KL_mean": -728.8594360351562, "KL/mean": -868.2634887695312, "KL/rejected_KL_mean": -1007.6676025390625, "KL/std": 510.1298522949219, "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.0010999096557497978, "fcm_dpo/delta": -0.006392236333340406, "fcm_dpo/margin": 278.80816650390625, "fcm_dpo/q_t": 0.43125709891319275, "grad_norm": 32.62123489379883, "learning_rate": 6.963101805503646e-08, "logits/chosen": -1.0549545288085938, "logits/rejected": -1.0321242809295654, "logps/chosen": -793.7145385742188, "logps/ref_chosen": -64.8551025390625, "logps/ref_rejected": -76.58805847167969, "logps/rejected": -1084.255615234375, "loss": 1.2058, "margin_dpo/margin_mean": 278.80816650390625, "margin_dpo/margin_std": 602.3246459960938, "step": 533 }, { "KL/chosen_KL_mean": -744.268798828125, "KL/mean": -940.6812744140625, "KL/rejected_KL_mean": -1137.09375, "KL/std": 581.1688232421875, "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.001092701917514205, "fcm_dpo/delta": -0.030640922486782074, "fcm_dpo/margin": 392.8249206542969, "fcm_dpo/q_t": 0.4050602912902832, "grad_norm": 32.57998275756836, "learning_rate": 6.874476976660184e-08, "logits/chosen": -1.0226364135742188, "logits/rejected": -1.0192043781280518, "logps/chosen": -804.38818359375, "logps/ref_chosen": -60.119388580322266, "logps/ref_rejected": -78.54347229003906, "logps/rejected": -1215.63720703125, "loss": 1.0993, "margin_dpo/margin_mean": 392.824951171875, "margin_dpo/margin_std": 582.9520874023438, "step": 534 }, { "KL/chosen_KL_mean": -627.1175537109375, "KL/mean": -852.0411987304688, "KL/rejected_KL_mean": -1076.96484375, "KL/std": 558.7318115234375, "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0010921892244368792, "fcm_dpo/delta": -0.09375444799661636, "fcm_dpo/margin": 449.84735107421875, "fcm_dpo/q_t": 0.3906930983066559, "grad_norm": 33.22504806518555, "learning_rate": 6.786329772205246e-08, "logits/chosen": -0.9372042417526245, "logits/rejected": -0.939862847328186, "logps/chosen": -681.44775390625, "logps/ref_chosen": -54.330238342285156, "logps/ref_rejected": -96.30763244628906, "logps/rejected": -1173.2724609375, "loss": 1.0486, "margin_dpo/margin_mean": 449.84735107421875, "margin_dpo/margin_std": 577.599609375, "step": 535 }, { "KL/chosen_KL_mean": -568.00146484375, "KL/mean": -827.193603515625, "KL/rejected_KL_mean": -1086.3857421875, "KL/std": 608.2098388671875, "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.0010737399570643902, "fcm_dpo/delta": -0.16100189089775085, "fcm_dpo/margin": 518.3843383789062, "fcm_dpo/q_t": 0.3829028010368347, "grad_norm": 56.74153137207031, "learning_rate": 6.698662514899638e-08, "logits/chosen": -0.9405168294906616, "logits/rejected": -0.9727465510368347, "logps/chosen": -615.08203125, "logps/ref_chosen": -47.08053207397461, "logps/ref_rejected": -89.09783935546875, "logps/rejected": -1175.483642578125, "loss": 1.0262, "margin_dpo/margin_mean": 518.3843383789062, "margin_dpo/margin_std": 723.623046875, "step": 536 }, { "KL/chosen_KL_mean": -583.8455810546875, "KL/mean": -774.345703125, "KL/rejected_KL_mean": -964.8458251953125, "KL/std": 504.427490234375, "epoch": 0.788546255506608, "fcm_dpo/beta": 0.0010713583324104548, "fcm_dpo/delta": -0.008746057748794556, "fcm_dpo/margin": 381.0003662109375, "fcm_dpo/q_t": 0.4074852764606476, "grad_norm": 44.69304275512695, "learning_rate": 6.611477514857114e-08, "logits/chosen": -0.9550840854644775, "logits/rejected": -0.9326896667480469, "logps/chosen": -641.593017578125, "logps/ref_chosen": -57.747467041015625, "logps/ref_rejected": -70.43838500976562, "logps/rejected": -1035.2841796875, "loss": 1.1209, "margin_dpo/margin_mean": 381.0003356933594, "margin_dpo/margin_std": 604.8282470703125, "step": 537 }, { "KL/chosen_KL_mean": -731.8800048828125, "KL/mean": -923.4337158203125, "KL/rejected_KL_mean": -1114.9873046875, "KL/std": 585.79248046875, "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.0010648441966623068, "fcm_dpo/delta": -0.008301455527544022, "fcm_dpo/margin": 383.10736083984375, "fcm_dpo/q_t": 0.4077230989933014, "grad_norm": 41.17292785644531, "learning_rate": 6.524777069483525e-08, "logits/chosen": -0.9506703019142151, "logits/rejected": -0.9313552379608154, "logps/chosen": -798.2959594726562, "logps/ref_chosen": -66.41594696044922, "logps/ref_rejected": -84.22808837890625, "logps/rejected": -1199.2154541015625, "loss": 1.1046, "margin_dpo/margin_mean": 383.1073913574219, "margin_dpo/margin_std": 566.696044921875, "step": 538 }, { "KL/chosen_KL_mean": -605.9473876953125, "KL/mean": -802.4550170898438, "KL/rejected_KL_mean": -998.962646484375, "KL/std": 455.50811767578125, "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.001066508935764432, "fcm_dpo/delta": -0.01959817484021187, "fcm_dpo/margin": 393.01519775390625, "fcm_dpo/q_t": 0.40525415539741516, "grad_norm": 32.76362228393555, "learning_rate": 6.438563463416221e-08, "logits/chosen": -0.982710599899292, "logits/rejected": -0.9756343364715576, "logps/chosen": -664.4402465820312, "logps/ref_chosen": -58.492855072021484, "logps/ref_rejected": -91.85395050048828, "logps/rejected": -1090.816650390625, "loss": 1.08, "margin_dpo/margin_mean": 393.0152282714844, "margin_dpo/margin_std": 511.8883972167969, "step": 539 }, { "KL/chosen_KL_mean": -607.8150024414062, "KL/mean": -850.8222045898438, "KL/rejected_KL_mean": -1093.829345703125, "KL/std": 560.0186767578125, "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.0010573212057352066, "fcm_dpo/delta": -0.11679581552743912, "fcm_dpo/margin": 486.01434326171875, "fcm_dpo/q_t": 0.38948220014572144, "grad_norm": 36.21888732910156, "learning_rate": 6.352838968463919e-08, "logits/chosen": -0.9439930319786072, "logits/rejected": -0.96489417552948, "logps/chosen": -671.2975463867188, "logps/ref_chosen": -63.482513427734375, "logps/ref_rejected": -116.42999267578125, "logps/rejected": -1210.25927734375, "loss": 1.0536, "margin_dpo/margin_mean": 486.0143127441406, "margin_dpo/margin_std": 680.3463134765625, "step": 540 }, { "KL/chosen_KL_mean": -727.278564453125, "KL/mean": -857.4025268554688, "KL/rejected_KL_mean": -987.5263671875, "KL/std": 491.0820007324219, "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.001053128275088966, "fcm_dpo/delta": 0.007406635209918022, "fcm_dpo/margin": 260.24786376953125, "fcm_dpo/q_t": 0.43850696086883545, "grad_norm": 51.57310104370117, "learning_rate": 6.267605843546767e-08, "logits/chosen": -1.0330259799957275, "logits/rejected": -1.0214934349060059, "logps/chosen": -805.5589599609375, "logps/ref_chosen": -78.28036499023438, "logps/ref_rejected": -103.273681640625, "logps/rejected": -1090.800048828125, "loss": 1.2217, "margin_dpo/margin_mean": 260.24786376953125, "margin_dpo/margin_std": 586.0404052734375, "step": 541 }, { "KL/chosen_KL_mean": -627.02099609375, "KL/mean": -874.4854125976562, "KL/rejected_KL_mean": -1121.94970703125, "KL/std": 565.48876953125, "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.0010395846329629421, "fcm_dpo/delta": -0.11870333552360535, "fcm_dpo/margin": 494.9288024902344, "fcm_dpo/q_t": 0.3869064450263977, "grad_norm": 46.05047607421875, "learning_rate": 6.182866334636888e-08, "logits/chosen": -1.0332136154174805, "logits/rejected": -1.0657650232315063, "logps/chosen": -684.5059814453125, "logps/ref_chosen": -57.48497009277344, "logps/ref_rejected": -96.47506713867188, "logps/rejected": -1218.4248046875, "loss": 1.0441, "margin_dpo/margin_mean": 494.9288024902344, "margin_dpo/margin_std": 656.42431640625, "step": 542 }, { "KL/chosen_KL_mean": -702.0233764648438, "KL/mean": -875.46875, "KL/rejected_KL_mean": -1048.9141845703125, "KL/std": 650.103271484375, "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.0010423632338643074, "fcm_dpo/delta": 0.03910157456994057, "fcm_dpo/margin": 346.890869140625, "fcm_dpo/q_t": 0.4295872449874878, "grad_norm": 56.74467468261719, "learning_rate": 6.098622674699147e-08, "logits/chosen": -0.9723141193389893, "logits/rejected": -1.0032031536102295, "logps/chosen": -762.640869140625, "logps/ref_chosen": -60.61750793457031, "logps/ref_rejected": -105.59896850585938, "logps/rejected": -1154.51318359375, "loss": 1.199, "margin_dpo/margin_mean": 346.890869140625, "margin_dpo/margin_std": 787.779541015625, "step": 543 }, { "KL/chosen_KL_mean": -699.0665893554688, "KL/mean": -904.1417236328125, "KL/rejected_KL_mean": -1109.217041015625, "KL/std": 525.4854125976562, "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.0010421974584460258, "fcm_dpo/delta": -0.028073592111468315, "fcm_dpo/margin": 410.15032958984375, "fcm_dpo/q_t": 0.4034753441810608, "grad_norm": 39.14487838745117, "learning_rate": 6.01487708363232e-08, "logits/chosen": -0.9627532958984375, "logits/rejected": -0.9823678135871887, "logps/chosen": -758.7088623046875, "logps/ref_chosen": -59.642303466796875, "logps/ref_rejected": -100.95469665527344, "logps/rejected": -1210.171630859375, "loss": 1.0915, "margin_dpo/margin_mean": 410.15032958984375, "margin_dpo/margin_std": 594.836669921875, "step": 544 }, { "KL/chosen_KL_mean": -665.3682861328125, "KL/mean": -904.0440063476562, "KL/rejected_KL_mean": -1142.7197265625, "KL/std": 538.8303833007812, "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.0010349673684686422, "fcm_dpo/delta": -0.0963834896683693, "fcm_dpo/margin": 477.3514709472656, "fcm_dpo/q_t": 0.38999858498573303, "grad_norm": 32.36417007446289, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -0.8844515085220337, "logits/rejected": -0.9089665412902832, "logps/chosen": -733.016845703125, "logps/ref_chosen": -67.64859771728516, "logps/ref_rejected": -95.90800476074219, "logps/rejected": -1238.6278076171875, "loss": 1.0415, "margin_dpo/margin_mean": 477.3514404296875, "margin_dpo/margin_std": 607.8944091796875, "step": 545 }, { "KL/chosen_KL_mean": -620.0447998046875, "KL/mean": -773.5802001953125, "KL/rejected_KL_mean": -927.1156005859375, "KL/std": 466.5025634765625, "epoch": 0.801762114537445, "fcm_dpo/beta": 0.001038281712681055, "fcm_dpo/delta": 0.0824858769774437, "fcm_dpo/margin": 307.0708923339844, "fcm_dpo/q_t": 0.4243466258049011, "grad_norm": 31.921030044555664, "learning_rate": 5.848888922025552e-08, "logits/chosen": -0.9386703372001648, "logits/rejected": -0.9219088554382324, "logps/chosen": -670.7890014648438, "logps/ref_chosen": -50.744232177734375, "logps/ref_rejected": -81.86622619628906, "logps/rejected": -1008.9818725585938, "loss": 1.1562, "margin_dpo/margin_mean": 307.0708923339844, "margin_dpo/margin_std": 493.91357421875, "step": 546 }, { "KL/chosen_KL_mean": -614.9042358398438, "KL/mean": -828.0643310546875, "KL/rejected_KL_mean": -1041.2242431640625, "KL/std": 532.6348266601562, "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.001038061804138124, "fcm_dpo/delta": -0.04350613057613373, "fcm_dpo/margin": 426.320068359375, "fcm_dpo/q_t": 0.4003497362136841, "grad_norm": 49.95761489868164, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -0.9308812618255615, "logits/rejected": -0.9381631016731262, "logps/chosen": -688.5919189453125, "logps/ref_chosen": -73.6877212524414, "logps/ref_rejected": -90.76136779785156, "logps/rejected": -1131.985595703125, "loss": 1.0764, "margin_dpo/margin_mean": 426.320068359375, "margin_dpo/margin_std": 586.7791748046875, "step": 547 }, { "KL/chosen_KL_mean": -657.8882446289062, "KL/mean": -843.9827270507812, "KL/rejected_KL_mean": -1030.0771484375, "KL/std": 555.6254272460938, "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.0010367643553763628, "fcm_dpo/delta": 0.014406869187951088, "fcm_dpo/margin": 372.18896484375, "fcm_dpo/q_t": 0.4164394736289978, "grad_norm": 30.012165069580078, "learning_rate": 5.684919345471029e-08, "logits/chosen": -1.041344165802002, "logits/rejected": -1.0381653308868408, "logps/chosen": -723.1345825195312, "logps/ref_chosen": -65.24634552001953, "logps/ref_rejected": -94.11807250976562, "logps/rejected": -1124.1953125, "loss": 1.1255, "margin_dpo/margin_mean": 372.18896484375, "margin_dpo/margin_std": 620.2734375, "step": 548 }, { "KL/chosen_KL_mean": -680.7687377929688, "KL/mean": -830.600341796875, "KL/rejected_KL_mean": -980.431884765625, "KL/std": 459.7933349609375, "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.0010372500400990248, "fcm_dpo/delta": -0.002494156127795577, "fcm_dpo/margin": 299.6632080078125, "fcm_dpo/q_t": 0.429217129945755, "grad_norm": 53.25551223754883, "learning_rate": 5.603696935852426e-08, "logits/chosen": -0.9812500476837158, "logits/rejected": -0.9636787176132202, "logps/chosen": -729.9810791015625, "logps/ref_chosen": -49.21235656738281, "logps/ref_rejected": -73.91031646728516, "logps/rejected": -1054.34228515625, "loss": 1.1786, "margin_dpo/margin_mean": 299.6632385253906, "margin_dpo/margin_std": 550.6836547851562, "step": 549 }, { "KL/chosen_KL_mean": -688.97412109375, "KL/mean": -860.2440185546875, "KL/rejected_KL_mean": -1031.513916015625, "KL/std": 518.7420043945312, "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.0010400728788226843, "fcm_dpo/delta": 0.04453900456428528, "fcm_dpo/margin": 342.53973388671875, "fcm_dpo/q_t": 0.41772040724754333, "grad_norm": 38.28087615966797, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -0.9451702833175659, "logits/rejected": -0.9619972705841064, "logps/chosen": -745.7811279296875, "logps/ref_chosen": -56.80695343017578, "logps/ref_rejected": -95.12580871582031, "logps/rejected": -1126.6396484375, "loss": 1.1281, "margin_dpo/margin_mean": 342.5397644042969, "margin_dpo/margin_std": 515.8134155273438, "step": 550 }, { "KL/chosen_KL_mean": -614.6915283203125, "KL/mean": -889.161376953125, "KL/rejected_KL_mean": -1163.6312255859375, "KL/std": 555.507568359375, "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.0010279393754899502, "fcm_dpo/delta": -0.1691897213459015, "fcm_dpo/margin": 548.9395751953125, "fcm_dpo/q_t": 0.3716570734977722, "grad_norm": 40.599369049072266, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -0.935494065284729, "logits/rejected": -0.987216055393219, "logps/chosen": -673.7978515625, "logps/ref_chosen": -59.10633087158203, "logps/ref_rejected": -111.67280578613281, "logps/rejected": -1275.303955078125, "loss": 0.9718, "margin_dpo/margin_mean": 548.939697265625, "margin_dpo/margin_std": 550.9382934570312, "step": 551 }, { "KL/chosen_KL_mean": -578.4395751953125, "KL/mean": -906.6759643554688, "KL/rejected_KL_mean": -1234.912353515625, "KL/std": 645.207763671875, "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.000999854295514524, "fcm_dpo/delta": -0.2669294476509094, "fcm_dpo/margin": 656.4727783203125, "fcm_dpo/q_t": 0.3599150478839874, "grad_norm": 48.74257278442383, "learning_rate": 5.363104864490034e-08, "logits/chosen": -0.9877578020095825, "logits/rejected": -1.0204415321350098, "logps/chosen": -640.794189453125, "logps/ref_chosen": -62.35459899902344, "logps/ref_rejected": -104.56210327148438, "logps/rejected": -1339.474365234375, "loss": 0.9475, "margin_dpo/margin_mean": 656.4727783203125, "margin_dpo/margin_std": 730.9444580078125, "step": 552 }, { "KL/chosen_KL_mean": -671.1646728515625, "KL/mean": -830.6427001953125, "KL/rejected_KL_mean": -990.1207275390625, "KL/std": 539.9063110351562, "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.0010021075140684843, "fcm_dpo/delta": 0.08171496540307999, "fcm_dpo/margin": 318.9561462402344, "fcm_dpo/q_t": 0.4295930862426758, "grad_norm": 27.265270233154297, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -0.9643408060073853, "logits/rejected": -0.9552521109580994, "logps/chosen": -739.4234619140625, "logps/ref_chosen": -68.25881958007812, "logps/ref_rejected": -98.0971450805664, "logps/rejected": -1088.2178955078125, "loss": 1.168, "margin_dpo/margin_mean": 318.9561462402344, "margin_dpo/margin_std": 580.3341064453125, "step": 553 }, { "KL/chosen_KL_mean": -698.4722900390625, "KL/mean": -916.677978515625, "KL/rejected_KL_mean": -1134.8837890625, "KL/std": 573.2615966796875, "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.0010056591127067804, "fcm_dpo/delta": -0.04003161936998367, "fcm_dpo/margin": 436.4113464355469, "fcm_dpo/q_t": 0.40393197536468506, "grad_norm": 90.09304809570312, "learning_rate": 5.205293880283551e-08, "logits/chosen": -0.945639967918396, "logits/rejected": -0.9097083806991577, "logps/chosen": -766.4199829101562, "logps/ref_chosen": -67.94767761230469, "logps/ref_rejected": -89.78272247314453, "logps/rejected": -1224.66650390625, "loss": 1.1214, "margin_dpo/margin_mean": 436.411376953125, "margin_dpo/margin_std": 732.8826293945312, "step": 554 }, { "KL/chosen_KL_mean": -696.947509765625, "KL/mean": -960.1400146484375, "KL/rejected_KL_mean": -1223.3326416015625, "KL/std": 601.5198974609375, "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.0009933036053553224, "fcm_dpo/delta": -0.12606467306613922, "fcm_dpo/margin": 526.385009765625, "fcm_dpo/q_t": 0.3890535533428192, "grad_norm": 34.906185150146484, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.0455199480056763, "logits/rejected": -1.097933053970337, "logps/chosen": -750.2780151367188, "logps/ref_chosen": -53.33049011230469, "logps/ref_rejected": -108.47937774658203, "logps/rejected": -1331.81201171875, "loss": 1.0568, "margin_dpo/margin_mean": 526.385009765625, "margin_dpo/margin_std": 771.40576171875, "step": 555 }, { "KL/chosen_KL_mean": -692.2669677734375, "KL/mean": -844.3529052734375, "KL/rejected_KL_mean": -996.4388427734375, "KL/std": 467.8826599121094, "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.000995925161987543, "fcm_dpo/delta": 0.0985875129699707, "fcm_dpo/margin": 304.17181396484375, "fcm_dpo/q_t": 0.42986422777175903, "grad_norm": 33.946044921875, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.021904468536377, "logits/rejected": -1.0080738067626953, "logps/chosen": -750.9114990234375, "logps/ref_chosen": -58.64447021484375, "logps/ref_rejected": -101.34040832519531, "logps/rejected": -1097.779296875, "loss": 1.1616, "margin_dpo/margin_mean": 304.17181396484375, "margin_dpo/margin_std": 485.27044677734375, "step": 556 }, { "KL/chosen_KL_mean": -720.950439453125, "KL/mean": -956.94775390625, "KL/rejected_KL_mean": -1192.9451904296875, "KL/std": 639.4453735351562, "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.000991870416328311, "fcm_dpo/delta": -0.07001565396785736, "fcm_dpo/margin": 471.9947204589844, "fcm_dpo/q_t": 0.39792197942733765, "grad_norm": 49.5215950012207, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.0259175300598145, "logits/rejected": -1.060760259628296, "logps/chosen": -788.7911376953125, "logps/ref_chosen": -67.84066009521484, "logps/ref_rejected": -109.93965911865234, "logps/rejected": -1302.884765625, "loss": 1.0874, "margin_dpo/margin_mean": 471.9947509765625, "margin_dpo/margin_std": 715.3715209960938, "step": 557 }, { "KL/chosen_KL_mean": -663.1494750976562, "KL/mean": -974.2451171875, "KL/rejected_KL_mean": -1285.3406982421875, "KL/std": 596.8526611328125, "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.0009766137227416039, "fcm_dpo/delta": -0.21403643488883972, "fcm_dpo/margin": 622.1912841796875, "fcm_dpo/q_t": 0.3626197576522827, "grad_norm": 29.86164093017578, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -1.0814244747161865, "logits/rejected": -1.0883204936981201, "logps/chosen": -725.5177001953125, "logps/ref_chosen": -62.36824035644531, "logps/ref_rejected": -102.16102600097656, "logps/rejected": -1387.501708984375, "loss": 0.963, "margin_dpo/margin_mean": 622.1912841796875, "margin_dpo/margin_std": 662.1346435546875, "step": 558 }, { "KL/chosen_KL_mean": -754.6362915039062, "KL/mean": -1022.7030029296875, "KL/rejected_KL_mean": -1290.7696533203125, "KL/std": 619.6690673828125, "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.0009626256651245058, "fcm_dpo/delta": -0.11912199854850769, "fcm_dpo/margin": 536.13330078125, "fcm_dpo/q_t": 0.3861456513404846, "grad_norm": 31.161109924316406, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -1.1892856359481812, "logits/rejected": -1.1905286312103271, "logps/chosen": -815.388671875, "logps/ref_chosen": -60.752323150634766, "logps/ref_rejected": -93.44229125976562, "logps/rejected": -1384.2119140625, "loss": 1.0321, "margin_dpo/margin_mean": 536.13330078125, "margin_dpo/margin_std": 686.558349609375, "step": 559 }, { "KL/chosen_KL_mean": -694.3114013671875, "KL/mean": -896.381591796875, "KL/rejected_KL_mean": -1098.4517822265625, "KL/std": 594.5814208984375, "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.0009576280135661364, "fcm_dpo/delta": 0.012987758964300156, "fcm_dpo/margin": 404.14044189453125, "fcm_dpo/q_t": 0.4146135449409485, "grad_norm": 38.00310134887695, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -0.9253185987472534, "logits/rejected": -0.9051507711410522, "logps/chosen": -752.4151611328125, "logps/ref_chosen": -58.10382080078125, "logps/ref_rejected": -79.99122619628906, "logps/rejected": -1178.4429931640625, "loss": 1.1291, "margin_dpo/margin_mean": 404.14044189453125, "margin_dpo/margin_std": 662.3407592773438, "step": 560 }, { "KL/chosen_KL_mean": -819.3272705078125, "KL/mean": -979.8409423828125, "KL/rejected_KL_mean": -1140.3546142578125, "KL/std": 548.7021484375, "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0009685677941888571, "fcm_dpo/delta": 0.09021740406751633, "fcm_dpo/margin": 321.0272521972656, "fcm_dpo/q_t": 0.4277527630329132, "grad_norm": 40.23988723754883, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.1046230792999268, "logits/rejected": -1.1246148347854614, "logps/chosen": -870.2401733398438, "logps/ref_chosen": -50.912879943847656, "logps/ref_rejected": -99.06856536865234, "logps/rejected": -1239.423095703125, "loss": 1.2007, "margin_dpo/margin_mean": 321.02728271484375, "margin_dpo/margin_std": 687.259521484375, "step": 561 }, { "KL/chosen_KL_mean": -745.23193359375, "KL/mean": -967.99609375, "KL/rejected_KL_mean": -1190.76025390625, "KL/std": 601.22607421875, "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0009640766074880958, "fcm_dpo/delta": -0.030630581080913544, "fcm_dpo/margin": 445.52838134765625, "fcm_dpo/q_t": 0.404424250125885, "grad_norm": 46.153045654296875, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -1.0498061180114746, "logits/rejected": -1.0524308681488037, "logps/chosen": -804.6962890625, "logps/ref_chosen": -59.46440124511719, "logps/ref_rejected": -96.54266357421875, "logps/rejected": -1287.302978515625, "loss": 1.0927, "margin_dpo/margin_mean": 445.5283508300781, "margin_dpo/margin_std": 656.296630859375, "step": 562 }, { "KL/chosen_KL_mean": -846.0560913085938, "KL/mean": -1012.888916015625, "KL/rejected_KL_mean": -1179.7216796875, "KL/std": 638.2447509765625, "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.0009634558809921145, "fcm_dpo/delta": -0.05767164006829262, "fcm_dpo/margin": 333.6656494140625, "fcm_dpo/q_t": 0.42173728346824646, "grad_norm": 55.212547302246094, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.0601184368133545, "logits/rejected": -1.059585690498352, "logps/chosen": -906.664306640625, "logps/ref_chosen": -60.60819625854492, "logps/ref_rejected": -94.56770324707031, "logps/rejected": -1274.2894287109375, "loss": 1.2261, "margin_dpo/margin_mean": 333.6656494140625, "margin_dpo/margin_std": 764.817138671875, "step": 563 }, { "KL/chosen_KL_mean": -765.0787353515625, "KL/mean": -982.6279296875, "KL/rejected_KL_mean": -1200.17724609375, "KL/std": 581.216064453125, "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.0009602903737686574, "fcm_dpo/delta": -0.0182206854224205, "fcm_dpo/margin": 435.0984191894531, "fcm_dpo/q_t": 0.4047633409500122, "grad_norm": 36.67222595214844, "learning_rate": 4.447860229910544e-08, "logits/chosen": -1.1164246797561646, "logits/rejected": -1.107216477394104, "logps/chosen": -839.3470458984375, "logps/ref_chosen": -74.26837921142578, "logps/ref_rejected": -93.23818969726562, "logps/rejected": -1293.415283203125, "loss": 1.0771, "margin_dpo/margin_mean": 435.0984191894531, "margin_dpo/margin_std": 550.7127685546875, "step": 564 }, { "KL/chosen_KL_mean": -791.003173828125, "KL/mean": -1022.08740234375, "KL/rejected_KL_mean": -1253.171630859375, "KL/std": 678.6461181640625, "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.000955872586928308, "fcm_dpo/delta": -0.04271770268678665, "fcm_dpo/margin": 462.16839599609375, "fcm_dpo/q_t": 0.4068571925163269, "grad_norm": 51.650264739990234, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.0443668365478516, "logits/rejected": -1.038867712020874, "logps/chosen": -860.0230712890625, "logps/ref_chosen": -69.0199203491211, "logps/ref_rejected": -85.7789306640625, "logps/rejected": -1338.9505615234375, "loss": 1.1256, "margin_dpo/margin_mean": 462.16839599609375, "margin_dpo/margin_std": 813.2290649414062, "step": 565 }, { "KL/chosen_KL_mean": -779.5007934570312, "KL/mean": -1033.7266845703125, "KL/rejected_KL_mean": -1287.95263671875, "KL/std": 692.0765380859375, "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.0009514681296423078, "fcm_dpo/delta": -0.08593890815973282, "fcm_dpo/margin": 508.45184326171875, "fcm_dpo/q_t": 0.39394253492355347, "grad_norm": 33.00722885131836, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -1.0474693775177002, "logits/rejected": -1.0704981088638306, "logps/chosen": -846.046142578125, "logps/ref_chosen": -66.5453109741211, "logps/ref_rejected": -103.86932373046875, "logps/rejected": -1391.822021484375, "loss": 1.0916, "margin_dpo/margin_mean": 508.4518127441406, "margin_dpo/margin_std": 790.054931640625, "step": 566 }, { "KL/chosen_KL_mean": -752.8310546875, "KL/mean": -941.5557861328125, "KL/rejected_KL_mean": -1130.2806396484375, "KL/std": 474.6645202636719, "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.0009493097895756364, "fcm_dpo/delta": 0.042454224079847336, "fcm_dpo/margin": 377.4496154785156, "fcm_dpo/q_t": 0.416124165058136, "grad_norm": 37.01401138305664, "learning_rate": 4.231101308059165e-08, "logits/chosen": -1.1754206418991089, "logits/rejected": -1.1703362464904785, "logps/chosen": -805.6893310546875, "logps/ref_chosen": -52.85829544067383, "logps/ref_rejected": -85.37095642089844, "logps/rejected": -1215.651611328125, "loss": 1.1146, "margin_dpo/margin_mean": 377.4496154785156, "margin_dpo/margin_std": 508.71038818359375, "step": 567 }, { "KL/chosen_KL_mean": -732.2120361328125, "KL/mean": -992.5909423828125, "KL/rejected_KL_mean": -1252.9698486328125, "KL/std": 562.322265625, "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.0009439606801606715, "fcm_dpo/delta": -0.09389245510101318, "fcm_dpo/margin": 520.7578735351562, "fcm_dpo/q_t": 0.38713839650154114, "grad_norm": 28.33488655090332, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.0602999925613403, "logits/rejected": -1.0991747379302979, "logps/chosen": -777.4044189453125, "logps/ref_chosen": -45.1923828125, "logps/ref_rejected": -89.09236907958984, "logps/rejected": -1342.062255859375, "loss": 1.0249, "margin_dpo/margin_mean": 520.7578125, "margin_dpo/margin_std": 590.0891723632812, "step": 568 }, { "KL/chosen_KL_mean": -826.0765380859375, "KL/mean": -1036.68115234375, "KL/rejected_KL_mean": -1247.285888671875, "KL/std": 716.984130859375, "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.000945072213653475, "fcm_dpo/delta": 0.0015114173293113708, "fcm_dpo/margin": 421.209228515625, "fcm_dpo/q_t": 0.4105045795440674, "grad_norm": 47.668739318847656, "learning_rate": 4.089328585837512e-08, "logits/chosen": -1.1122533082962036, "logits/rejected": -1.1140843629837036, "logps/chosen": -889.797119140625, "logps/ref_chosen": -63.72056198120117, "logps/ref_rejected": -79.10325622558594, "logps/rejected": -1326.3890380859375, "loss": 1.1478, "margin_dpo/margin_mean": 421.2092590332031, "margin_dpo/margin_std": 756.34326171875, "step": 569 }, { "KL/chosen_KL_mean": -775.0736083984375, "KL/mean": -970.983154296875, "KL/rejected_KL_mean": -1166.8927001953125, "KL/std": 585.8502197265625, "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.0009430573554709554, "fcm_dpo/delta": 0.031048066914081573, "fcm_dpo/margin": 391.8191223144531, "fcm_dpo/q_t": 0.4179254174232483, "grad_norm": 31.495166778564453, "learning_rate": 4.019267817841834e-08, "logits/chosen": -1.1752269268035889, "logits/rejected": -1.1550638675689697, "logps/chosen": -836.6881103515625, "logps/ref_chosen": -61.61454391479492, "logps/ref_rejected": -82.14186096191406, "logps/rejected": -1249.03466796875, "loss": 1.1347, "margin_dpo/margin_mean": 391.819091796875, "margin_dpo/margin_std": 633.9605102539062, "step": 570 }, { "KL/chosen_KL_mean": -785.480224609375, "KL/mean": -1001.910888671875, "KL/rejected_KL_mean": -1218.341552734375, "KL/std": 558.670166015625, "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.0009431964717805386, "fcm_dpo/delta": -0.008509235456585884, "fcm_dpo/margin": 432.861328125, "fcm_dpo/q_t": 0.40848302841186523, "grad_norm": 44.37576675415039, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.1002910137176514, "logits/rejected": -1.1096596717834473, "logps/chosen": -838.5343017578125, "logps/ref_chosen": -53.05406188964844, "logps/ref_rejected": -91.33682250976562, "logps/rejected": -1309.678466796875, "loss": 1.1233, "margin_dpo/margin_mean": 432.861328125, "margin_dpo/margin_std": 713.20458984375, "step": 571 }, { "KL/chosen_KL_mean": -822.61279296875, "KL/mean": -1061.8223876953125, "KL/rejected_KL_mean": -1301.031982421875, "KL/std": 679.26806640625, "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.0009392797946929932, "fcm_dpo/delta": -0.05064094066619873, "fcm_dpo/margin": 478.4192810058594, "fcm_dpo/q_t": 0.40520572662353516, "grad_norm": 29.390687942504883, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.128666877746582, "logits/rejected": -1.1588270664215088, "logps/chosen": -871.0720825195312, "logps/ref_chosen": -48.45928955078125, "logps/ref_rejected": -83.55703735351562, "logps/rejected": -1384.589111328125, "loss": 1.1148, "margin_dpo/margin_mean": 478.41925048828125, "margin_dpo/margin_std": 823.0018310546875, "step": 572 }, { "KL/chosen_KL_mean": -781.2991943359375, "KL/mean": -1001.1578979492188, "KL/rejected_KL_mean": -1221.0164794921875, "KL/std": 596.3881225585938, "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.0009392928332090378, "fcm_dpo/delta": -0.013323244638741016, "fcm_dpo/margin": 439.71728515625, "fcm_dpo/q_t": 0.40707600116729736, "grad_norm": 29.680734634399414, "learning_rate": 3.812409996461275e-08, "logits/chosen": -1.162482738494873, "logits/rejected": -1.1760870218276978, "logps/chosen": -832.921875, "logps/ref_chosen": -51.62262725830078, "logps/ref_rejected": -85.32499694824219, "logps/rejected": -1306.3414306640625, "loss": 1.0857, "margin_dpo/margin_mean": 439.71728515625, "margin_dpo/margin_std": 595.1873779296875, "step": 573 }, { "KL/chosen_KL_mean": -724.2657470703125, "KL/mean": -953.5592041015625, "KL/rejected_KL_mean": -1182.8525390625, "KL/std": 532.4700927734375, "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.0009368173778057098, "fcm_dpo/delta": -0.030269186943769455, "fcm_dpo/margin": 458.5869140625, "fcm_dpo/q_t": 0.40228691697120667, "grad_norm": 29.51076889038086, "learning_rate": 3.74457160675965e-08, "logits/chosen": -1.1240806579589844, "logits/rejected": -1.145003318786621, "logps/chosen": -775.3101806640625, "logps/ref_chosen": -51.04446029663086, "logps/ref_rejected": -92.80640411376953, "logps/rejected": -1275.658935546875, "loss": 1.0787, "margin_dpo/margin_mean": 458.5869140625, "margin_dpo/margin_std": 613.4527587890625, "step": 574 }, { "KL/chosen_KL_mean": -811.00634765625, "KL/mean": -1024.9534912109375, "KL/rejected_KL_mean": -1238.9005126953125, "KL/std": 559.50830078125, "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.0009320209501311183, "fcm_dpo/delta": 0.000727042555809021, "fcm_dpo/margin": 427.89422607421875, "fcm_dpo/q_t": 0.4095437228679657, "grad_norm": 42.62068557739258, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.046234130859375, "logits/rejected": -1.0514554977416992, "logps/chosen": -882.7965087890625, "logps/ref_chosen": -71.7901382446289, "logps/ref_rejected": -95.38619995117188, "logps/rejected": -1334.2867431640625, "loss": 1.1328, "margin_dpo/margin_mean": 427.89422607421875, "margin_dpo/margin_std": 713.029541015625, "step": 575 }, { "KL/chosen_KL_mean": -782.13525390625, "KL/mean": -943.5439453125, "KL/rejected_KL_mean": -1104.95263671875, "KL/std": 511.6619567871094, "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.0009402718860656023, "fcm_dpo/delta": 0.09798312187194824, "fcm_dpo/margin": 322.81732177734375, "fcm_dpo/q_t": 0.429963618516922, "grad_norm": 39.150245666503906, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -1.1449861526489258, "logits/rejected": -1.1737542152404785, "logps/chosen": -836.3982543945312, "logps/ref_chosen": -54.262962341308594, "logps/ref_rejected": -100.75428009033203, "logps/rejected": -1205.7069091796875, "loss": 1.1796, "margin_dpo/margin_mean": 322.8173828125, "margin_dpo/margin_std": 582.9664306640625, "step": 576 }, { "KL/chosen_KL_mean": -690.1397094726562, "KL/mean": -899.7841186523438, "KL/rejected_KL_mean": -1109.428466796875, "KL/std": 570.5064086914062, "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.0009443428134545684, "fcm_dpo/delta": 0.004122039768844843, "fcm_dpo/margin": 419.2888488769531, "fcm_dpo/q_t": 0.4105992913246155, "grad_norm": 30.886186599731445, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.102234125137329, "logits/rejected": -1.082305669784546, "logps/chosen": -752.0494384765625, "logps/ref_chosen": -61.909706115722656, "logps/ref_rejected": -84.07069396972656, "logps/rejected": -1193.499267578125, "loss": 1.1191, "margin_dpo/margin_mean": 419.28887939453125, "margin_dpo/margin_std": 626.2908325195312, "step": 577 }, { "KL/chosen_KL_mean": -685.677978515625, "KL/mean": -919.28759765625, "KL/rejected_KL_mean": -1152.897216796875, "KL/std": 582.769287109375, "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.0009395558736287057, "fcm_dpo/delta": -0.04024638235569, "fcm_dpo/margin": 467.21929931640625, "fcm_dpo/q_t": 0.39982476830482483, "grad_norm": 37.12548065185547, "learning_rate": 3.478836705390808e-08, "logits/chosen": -0.9650574922561646, "logits/rejected": -0.9971319437026978, "logps/chosen": -734.941650390625, "logps/ref_chosen": -49.26368713378906, "logps/ref_rejected": -83.4362564086914, "logps/rejected": -1236.33349609375, "loss": 1.0655, "margin_dpo/margin_mean": 467.21929931640625, "margin_dpo/margin_std": 579.5484619140625, "step": 578 }, { "KL/chosen_KL_mean": -784.9281005859375, "KL/mean": -929.6224365234375, "KL/rejected_KL_mean": -1074.316650390625, "KL/std": 574.9615478515625, "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.0009488767245784402, "fcm_dpo/delta": 0.1272491067647934, "fcm_dpo/margin": 289.38848876953125, "fcm_dpo/q_t": 0.4369036555290222, "grad_norm": 43.24967956542969, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.0656390190124512, "logits/rejected": -1.0647817850112915, "logps/chosen": -843.81396484375, "logps/ref_chosen": -58.88581848144531, "logps/ref_rejected": -94.78762817382812, "logps/rejected": -1169.104248046875, "loss": 1.2174, "margin_dpo/margin_mean": 289.3885192871094, "margin_dpo/margin_std": 634.266845703125, "step": 579 }, { "KL/chosen_KL_mean": -582.00390625, "KL/mean": -842.2623901367188, "KL/rejected_KL_mean": -1102.5208740234375, "KL/std": 623.9799194335938, "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.0009446794865652919, "fcm_dpo/delta": -0.09435415267944336, "fcm_dpo/margin": 520.5169677734375, "fcm_dpo/q_t": 0.39273136854171753, "grad_norm": 27.779617309570312, "learning_rate": 3.349364905389032e-08, "logits/chosen": -0.8903079032897949, "logits/rejected": -0.9164822101593018, "logps/chosen": -630.7107543945312, "logps/ref_chosen": -48.70683670043945, "logps/ref_rejected": -81.7583999633789, "logps/rejected": -1184.279296875, "loss": 1.049, "margin_dpo/margin_mean": 520.5169677734375, "margin_dpo/margin_std": 708.4932861328125, "step": 580 }, { "KL/chosen_KL_mean": -791.675048828125, "KL/mean": -977.7584228515625, "KL/rejected_KL_mean": -1163.841796875, "KL/std": 612.6318359375, "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.0009482804453000426, "fcm_dpo/delta": 0.04787049442529678, "fcm_dpo/margin": 372.1667175292969, "fcm_dpo/q_t": 0.4224420487880707, "grad_norm": 41.1692008972168, "learning_rate": 3.285483927764726e-08, "logits/chosen": -1.1497280597686768, "logits/rejected": -1.1572540998458862, "logps/chosen": -853.8973999023438, "logps/ref_chosen": -62.22235107421875, "logps/ref_rejected": -91.73568725585938, "logps/rejected": -1255.577392578125, "loss": 1.154, "margin_dpo/margin_mean": 372.166748046875, "margin_dpo/margin_std": 670.7994995117188, "step": 581 }, { "KL/chosen_KL_mean": -701.05224609375, "KL/mean": -898.874755859375, "KL/rejected_KL_mean": -1096.6973876953125, "KL/std": 523.3825073242188, "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.0009447854827158153, "fcm_dpo/delta": -0.08350235223770142, "fcm_dpo/margin": 395.6450500488281, "fcm_dpo/q_t": 0.4139753580093384, "grad_norm": 35.411434173583984, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.082298994064331, "logits/rejected": -1.106623649597168, "logps/chosen": -759.2808837890625, "logps/ref_chosen": -58.228660583496094, "logps/ref_rejected": -110.06959533691406, "logps/rejected": -1206.7669677734375, "loss": 1.1245, "margin_dpo/margin_mean": 395.6450500488281, "margin_dpo/margin_std": 574.4161376953125, "step": 582 }, { "KL/chosen_KL_mean": -788.7952880859375, "KL/mean": -915.5237426757812, "KL/rejected_KL_mean": -1042.252197265625, "KL/std": 566.1787109375, "epoch": 0.856093979441997, "fcm_dpo/beta": 0.0009400502895005047, "fcm_dpo/delta": -0.008407074958086014, "fcm_dpo/margin": 253.45693969726562, "fcm_dpo/q_t": 0.44672808051109314, "grad_norm": 57.46125411987305, "learning_rate": 3.159440233840763e-08, "logits/chosen": -0.9853731393814087, "logits/rejected": -0.978585422039032, "logps/chosen": -845.6581420898438, "logps/ref_chosen": -56.86286163330078, "logps/ref_rejected": -88.4039306640625, "logps/rejected": -1130.6561279296875, "loss": 1.2656, "margin_dpo/margin_mean": 253.45692443847656, "margin_dpo/margin_std": 682.3837890625, "step": 583 }, { "KL/chosen_KL_mean": -670.3692626953125, "KL/mean": -927.6574096679688, "KL/rejected_KL_mean": -1184.945556640625, "KL/std": 594.9502563476562, "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.000934588722884655, "fcm_dpo/delta": -0.0828862339258194, "fcm_dpo/margin": 514.5762939453125, "fcm_dpo/q_t": 0.39284205436706543, "grad_norm": 29.62038230895996, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -1.0637917518615723, "logits/rejected": -1.0736515522003174, "logps/chosen": -727.2699584960938, "logps/ref_chosen": -56.90068054199219, "logps/ref_rejected": -97.63606262207031, "logps/rejected": -1282.5816650390625, "loss": 1.0451, "margin_dpo/margin_mean": 514.5762939453125, "margin_dpo/margin_std": 644.3516845703125, "step": 584 }, { "KL/chosen_KL_mean": -772.5379638671875, "KL/mean": -1022.5894775390625, "KL/rejected_KL_mean": -1272.6409912109375, "KL/std": 647.20556640625, "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.0009266703855246305, "fcm_dpo/delta": -0.06506932526826859, "fcm_dpo/margin": 500.10296630859375, "fcm_dpo/q_t": 0.3975642919540405, "grad_norm": 34.582088470458984, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.089249849319458, "logits/rejected": -1.112525224685669, "logps/chosen": -833.511962890625, "logps/ref_chosen": -60.973968505859375, "logps/ref_rejected": -84.16952514648438, "logps/rejected": -1356.810546875, "loss": 1.0897, "margin_dpo/margin_mean": 500.10296630859375, "margin_dpo/margin_std": 768.308349609375, "step": 585 }, { "KL/chosen_KL_mean": -807.0784912109375, "KL/mean": -980.6554565429688, "KL/rejected_KL_mean": -1154.232421875, "KL/std": 573.5162963867188, "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.0009305774001404643, "fcm_dpo/delta": 0.07824563980102539, "fcm_dpo/margin": 347.15399169921875, "fcm_dpo/q_t": 0.42543265223503113, "grad_norm": 33.32202911376953, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.1124446392059326, "logits/rejected": -1.137439250946045, "logps/chosen": -863.93408203125, "logps/ref_chosen": -56.85559844970703, "logps/ref_rejected": -91.80261993408203, "logps/rejected": -1246.03515625, "loss": 1.1743, "margin_dpo/margin_mean": 347.15399169921875, "margin_dpo/margin_std": 641.89892578125, "step": 586 }, { "KL/chosen_KL_mean": -590.2689208984375, "KL/mean": -805.8556518554688, "KL/rejected_KL_mean": -1021.4425048828125, "KL/std": 601.960205078125, "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0009339781245216727, "fcm_dpo/delta": -0.002780054695904255, "fcm_dpo/margin": 431.173583984375, "fcm_dpo/q_t": 0.4075835347175598, "grad_norm": 36.960052490234375, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -1.1249654293060303, "logits/rejected": -1.1489168405532837, "logps/chosen": -634.96044921875, "logps/ref_chosen": -44.69159698486328, "logps/ref_rejected": -82.62385559082031, "logps/rejected": -1104.06640625, "loss": 1.0941, "margin_dpo/margin_mean": 431.1735534667969, "margin_dpo/margin_std": 591.1593017578125, "step": 587 }, { "KL/chosen_KL_mean": -746.4495849609375, "KL/mean": -930.6257934570312, "KL/rejected_KL_mean": -1114.802001953125, "KL/std": 514.0170288085938, "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.0009352130000479519, "fcm_dpo/delta": 0.05642148479819298, "fcm_dpo/margin": 368.352294921875, "fcm_dpo/q_t": 0.4200473427772522, "grad_norm": 30.264469146728516, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.071812391281128, "logits/rejected": -1.0980116128921509, "logps/chosen": -796.7445678710938, "logps/ref_chosen": -50.29494857788086, "logps/ref_rejected": -107.36988067626953, "logps/rejected": -1222.171875, "loss": 1.1337, "margin_dpo/margin_mean": 368.3523254394531, "margin_dpo/margin_std": 540.8074340820312, "step": 588 }, { "KL/chosen_KL_mean": -765.505615234375, "KL/mean": -1002.3455810546875, "KL/rejected_KL_mean": -1239.1854248046875, "KL/std": 586.9735717773438, "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.0009358683601021767, "fcm_dpo/delta": -0.04427178204059601, "fcm_dpo/margin": 473.67987060546875, "fcm_dpo/q_t": 0.3995531499385834, "grad_norm": 30.180952072143555, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.046541690826416, "logits/rejected": -1.0619137287139893, "logps/chosen": -825.435546875, "logps/ref_chosen": -59.929908752441406, "logps/ref_rejected": -111.65534973144531, "logps/rejected": -1350.8408203125, "loss": 1.071, "margin_dpo/margin_mean": 473.67987060546875, "margin_dpo/margin_std": 629.702392578125, "step": 589 }, { "KL/chosen_KL_mean": -648.0178833007812, "KL/mean": -879.0958862304688, "KL/rejected_KL_mean": -1110.173828125, "KL/std": 540.598388671875, "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.0009314791532233357, "fcm_dpo/delta": -0.031209638342261314, "fcm_dpo/margin": 462.15606689453125, "fcm_dpo/q_t": 0.40161752700805664, "grad_norm": 23.975587844848633, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.0303199291229248, "logits/rejected": -1.0523946285247803, "logps/chosen": -703.82763671875, "logps/ref_chosen": -55.80979537963867, "logps/ref_rejected": -106.06282043457031, "logps/rejected": -1216.23681640625, "loss": 1.0681, "margin_dpo/margin_mean": 462.1560363769531, "margin_dpo/margin_std": 576.578125, "step": 590 }, { "KL/chosen_KL_mean": -737.061767578125, "KL/mean": -956.1275024414062, "KL/rejected_KL_mean": -1175.1932373046875, "KL/std": 550.1436157226562, "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.0009308913140557706, "fcm_dpo/delta": -0.008016519248485565, "fcm_dpo/margin": 438.1313781738281, "fcm_dpo/q_t": 0.4057263135910034, "grad_norm": 30.565818786621094, "learning_rate": 2.678415274939408e-08, "logits/chosen": -1.1124975681304932, "logits/rejected": -1.0994548797607422, "logps/chosen": -793.3024291992188, "logps/ref_chosen": -56.24061965942383, "logps/ref_rejected": -83.78629302978516, "logps/rejected": -1258.9794921875, "loss": 1.0962, "margin_dpo/margin_mean": 438.13140869140625, "margin_dpo/margin_std": 615.4335327148438, "step": 591 }, { "KL/chosen_KL_mean": -743.1383666992188, "KL/mean": -941.9227905273438, "KL/rejected_KL_mean": -1140.707275390625, "KL/std": 541.6781616210938, "epoch": 0.869309838472834, "fcm_dpo/beta": 0.0009322206024080515, "fcm_dpo/delta": 0.029935326427221298, "fcm_dpo/margin": 397.5688171386719, "fcm_dpo/q_t": 0.416223406791687, "grad_norm": 30.50063705444336, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.1092453002929688, "logits/rejected": -1.1124801635742188, "logps/chosen": -791.07861328125, "logps/ref_chosen": -47.94025421142578, "logps/ref_rejected": -75.73287963867188, "logps/rejected": -1216.440185546875, "loss": 1.1635, "margin_dpo/margin_mean": 397.56884765625, "margin_dpo/margin_std": 738.2605590820312, "step": 592 }, { "KL/chosen_KL_mean": -737.1370849609375, "KL/mean": -894.179931640625, "KL/rejected_KL_mean": -1051.2227783203125, "KL/std": 630.5286865234375, "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.0009341684053651989, "fcm_dpo/delta": 0.006865399423986673, "fcm_dpo/margin": 314.085693359375, "fcm_dpo/q_t": 0.4349074959754944, "grad_norm": 45.077449798583984, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.925495982170105, "logits/rejected": -0.9140543937683105, "logps/chosen": -785.827880859375, "logps/ref_chosen": -48.690757751464844, "logps/ref_rejected": -60.90800094604492, "logps/rejected": -1112.130859375, "loss": 1.2123, "margin_dpo/margin_mean": 314.085693359375, "margin_dpo/margin_std": 687.1311645507812, "step": 593 }, { "KL/chosen_KL_mean": -696.455810546875, "KL/mean": -892.4623413085938, "KL/rejected_KL_mean": -1088.468994140625, "KL/std": 599.05810546875, "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0009395517408847809, "fcm_dpo/delta": 0.03171183168888092, "fcm_dpo/margin": 392.01312255859375, "fcm_dpo/q_t": 0.4168938100337982, "grad_norm": 32.01091003417969, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.0287201404571533, "logits/rejected": -1.0127062797546387, "logps/chosen": -751.3906860351562, "logps/ref_chosen": -54.93488693237305, "logps/ref_rejected": -86.09967803955078, "logps/rejected": -1174.568603515625, "loss": 1.1387, "margin_dpo/margin_mean": 392.01312255859375, "margin_dpo/margin_std": 633.80126953125, "step": 594 }, { "KL/chosen_KL_mean": -645.5115356445312, "KL/mean": -869.5858154296875, "KL/rejected_KL_mean": -1093.66015625, "KL/std": 564.3441772460938, "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.0009344806894659996, "fcm_dpo/delta": -0.01929367519915104, "fcm_dpo/margin": 448.14862060546875, "fcm_dpo/q_t": 0.40738070011138916, "grad_norm": 33.268638610839844, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.0355620384216309, "logits/rejected": -1.0576274394989014, "logps/chosen": -694.9319458007812, "logps/ref_chosen": -49.4204216003418, "logps/ref_rejected": -80.62731170654297, "logps/rejected": -1174.2874755859375, "loss": 1.0873, "margin_dpo/margin_mean": 448.14862060546875, "margin_dpo/margin_std": 625.08984375, "step": 595 }, { "KL/chosen_KL_mean": -755.9296875, "KL/mean": -896.432373046875, "KL/rejected_KL_mean": -1036.93505859375, "KL/std": 544.8298950195312, "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.000945114647038281, "fcm_dpo/delta": 0.1361963152885437, "fcm_dpo/margin": 281.0052795410156, "fcm_dpo/q_t": 0.4416961669921875, "grad_norm": 70.67866516113281, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.043994665145874, "logits/rejected": -1.005753517150879, "logps/chosen": -815.7213745117188, "logps/ref_chosen": -59.791683197021484, "logps/ref_rejected": -80.09111785888672, "logps/rejected": -1117.026123046875, "loss": 1.2272, "margin_dpo/margin_mean": 281.00531005859375, "margin_dpo/margin_std": 656.0474853515625, "step": 596 }, { "KL/chosen_KL_mean": -699.6539306640625, "KL/mean": -960.3472290039062, "KL/rejected_KL_mean": -1221.04052734375, "KL/std": 631.4920043945312, "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.0009366714511997998, "fcm_dpo/delta": -0.09220761060714722, "fcm_dpo/margin": 521.3866577148438, "fcm_dpo/q_t": 0.39339327812194824, "grad_norm": 30.50818634033203, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -0.9620223045349121, "logits/rejected": -1.0084235668182373, "logps/chosen": -756.9146728515625, "logps/ref_chosen": -57.26078796386719, "logps/ref_rejected": -100.6937255859375, "logps/rejected": -1321.7342529296875, "loss": 1.0648, "margin_dpo/margin_mean": 521.3866577148438, "margin_dpo/margin_std": 711.2468872070312, "step": 597 }, { "KL/chosen_KL_mean": -694.2019653320312, "KL/mean": -900.156494140625, "KL/rejected_KL_mean": -1106.111083984375, "KL/std": 543.1419677734375, "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.0009368107421323657, "fcm_dpo/delta": 0.013848438858985901, "fcm_dpo/margin": 411.9090881347656, "fcm_dpo/q_t": 0.41072171926498413, "grad_norm": 36.10586166381836, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -1.0843559503555298, "logits/rejected": -1.0926883220672607, "logps/chosen": -746.720458984375, "logps/ref_chosen": -52.51850509643555, "logps/ref_rejected": -89.44385528564453, "logps/rejected": -1195.554931640625, "loss": 1.1177, "margin_dpo/margin_mean": 411.9090881347656, "margin_dpo/margin_std": 603.3158569335938, "step": 598 }, { "KL/chosen_KL_mean": -711.0474853515625, "KL/mean": -883.8363037109375, "KL/rejected_KL_mean": -1056.625244140625, "KL/std": 532.8595581054688, "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.0009362338460050523, "fcm_dpo/delta": -0.048169832676649094, "fcm_dpo/margin": 345.5777587890625, "fcm_dpo/q_t": 0.4225807189941406, "grad_norm": 32.4377326965332, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -0.9888404011726379, "logits/rejected": -0.994670569896698, "logps/chosen": -760.85009765625, "logps/ref_chosen": -49.802677154541016, "logps/ref_rejected": -82.978515625, "logps/rejected": -1139.603759765625, "loss": 1.1487, "margin_dpo/margin_mean": 345.5777587890625, "margin_dpo/margin_std": 510.5614318847656, "step": 599 }, { "KL/chosen_KL_mean": -777.1749267578125, "KL/mean": -945.5758056640625, "KL/rejected_KL_mean": -1113.9766845703125, "KL/std": 559.109619140625, "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.000941394129768014, "fcm_dpo/delta": 0.0843147411942482, "fcm_dpo/margin": 336.8017272949219, "fcm_dpo/q_t": 0.42964982986450195, "grad_norm": 36.367210388183594, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -1.1235439777374268, "logits/rejected": -1.1162173748016357, "logps/chosen": -843.6098022460938, "logps/ref_chosen": -66.43487548828125, "logps/ref_rejected": -85.45649719238281, "logps/rejected": -1199.43310546875, "loss": 1.1856, "margin_dpo/margin_mean": 336.8017272949219, "margin_dpo/margin_std": 669.5869140625, "step": 600 }, { "KL/chosen_KL_mean": -782.239501953125, "KL/mean": -993.9614868164062, "KL/rejected_KL_mean": -1205.683349609375, "KL/std": 575.0701904296875, "epoch": 0.882525697503671, "fcm_dpo/beta": 0.0009437098633497953, "fcm_dpo/delta": 0.00038953591138124466, "fcm_dpo/margin": 423.4439697265625, "fcm_dpo/q_t": 0.4073420763015747, "grad_norm": 30.614376068115234, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.0543596744537354, "logits/rejected": -1.0704107284545898, "logps/chosen": -841.373046875, "logps/ref_chosen": -59.13361358642578, "logps/ref_rejected": -94.69093322753906, "logps/rejected": -1300.3743896484375, "loss": 1.0983, "margin_dpo/margin_mean": 423.4439697265625, "margin_dpo/margin_std": 586.927001953125, "step": 601 }, { "KL/chosen_KL_mean": -553.6896362304688, "KL/mean": -824.164794921875, "KL/rejected_KL_mean": -1094.64013671875, "KL/std": 556.9711303710938, "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.0009391128551214933, "fcm_dpo/delta": -0.11084318161010742, "fcm_dpo/margin": 540.950439453125, "fcm_dpo/q_t": 0.3849470615386963, "grad_norm": 61.494728088378906, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -1.071298360824585, "logits/rejected": -1.1071069240570068, "logps/chosen": -602.2831420898438, "logps/ref_chosen": -48.59352111816406, "logps/ref_rejected": -87.6685562133789, "logps/rejected": -1182.30859375, "loss": 1.014, "margin_dpo/margin_mean": 540.950439453125, "margin_dpo/margin_std": 597.431396484375, "step": 602 }, { "KL/chosen_KL_mean": -677.567138671875, "KL/mean": -920.722900390625, "KL/rejected_KL_mean": -1163.8787841796875, "KL/std": 601.8591918945312, "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.0009299052762798965, "fcm_dpo/delta": -0.05344250053167343, "fcm_dpo/margin": 486.3115234375, "fcm_dpo/q_t": 0.40060174465179443, "grad_norm": 30.66020393371582, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.1096835136413574, "logits/rejected": -1.0989437103271484, "logps/chosen": -747.9818115234375, "logps/ref_chosen": -70.41461944580078, "logps/ref_rejected": -100.32559967041016, "logps/rejected": -1264.204345703125, "loss": 1.0749, "margin_dpo/margin_mean": 486.3115234375, "margin_dpo/margin_std": 690.6925048828125, "step": 603 }, { "KL/chosen_KL_mean": -667.291015625, "KL/mean": -937.2022705078125, "KL/rejected_KL_mean": -1207.113525390625, "KL/std": 592.4906616210938, "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.0009219404309988022, "fcm_dpo/delta": -0.1001720055937767, "fcm_dpo/margin": 539.822509765625, "fcm_dpo/q_t": 0.3912475109100342, "grad_norm": 30.202251434326172, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.0218074321746826, "logits/rejected": -1.0635687112808228, "logps/chosen": -713.7491455078125, "logps/ref_chosen": -46.45808029174805, "logps/ref_rejected": -91.8544921875, "logps/rejected": -1298.968017578125, "loss": 1.0509, "margin_dpo/margin_mean": 539.822509765625, "margin_dpo/margin_std": 732.5137939453125, "step": 604 }, { "KL/chosen_KL_mean": -696.3240966796875, "KL/mean": -882.376708984375, "KL/rejected_KL_mean": -1068.42919921875, "KL/std": 532.2725219726562, "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.0009225456742569804, "fcm_dpo/delta": 0.05772203952074051, "fcm_dpo/margin": 372.1051025390625, "fcm_dpo/q_t": 0.4235140085220337, "grad_norm": 36.44785690307617, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.0042846202850342, "logits/rejected": -1.0125752687454224, "logps/chosen": -762.573486328125, "logps/ref_chosen": -66.24933624267578, "logps/ref_rejected": -102.30496978759766, "logps/rejected": -1170.734130859375, "loss": 1.148, "margin_dpo/margin_mean": 372.1051025390625, "margin_dpo/margin_std": 632.8191528320312, "step": 605 }, { "KL/chosen_KL_mean": -676.556396484375, "KL/mean": -889.0665283203125, "KL/rejected_KL_mean": -1101.5765380859375, "KL/std": 537.904296875, "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.0009270627633668482, "fcm_dpo/delta": 0.005940597504377365, "fcm_dpo/margin": 425.02008056640625, "fcm_dpo/q_t": 0.4101853370666504, "grad_norm": 31.26183319091797, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.0010316371917725, "logits/rejected": -1.0146284103393555, "logps/chosen": -731.3755493164062, "logps/ref_chosen": -54.819122314453125, "logps/ref_rejected": -98.37146759033203, "logps/rejected": -1199.947998046875, "loss": 1.0951, "margin_dpo/margin_mean": 425.02008056640625, "margin_dpo/margin_std": 566.8992919921875, "step": 606 }, { "KL/chosen_KL_mean": -745.302978515625, "KL/mean": -913.755615234375, "KL/rejected_KL_mean": -1082.208251953125, "KL/std": 564.3204345703125, "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0009324230486527085, "fcm_dpo/delta": 0.08710216730833054, "fcm_dpo/margin": 336.9051818847656, "fcm_dpo/q_t": 0.4282468557357788, "grad_norm": 29.58710479736328, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -0.9958839416503906, "logits/rejected": -0.9777606129646301, "logps/chosen": -803.3870239257812, "logps/ref_chosen": -58.08403778076172, "logps/ref_rejected": -79.777099609375, "logps/rejected": -1161.9853515625, "loss": 1.1734, "margin_dpo/margin_mean": 336.9051513671875, "margin_dpo/margin_std": 613.3471069335938, "step": 607 }, { "KL/chosen_KL_mean": -618.9171142578125, "KL/mean": -831.6253662109375, "KL/rejected_KL_mean": -1044.3336181640625, "KL/std": 493.6427001953125, "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.000934786512516439, "fcm_dpo/delta": 0.002314794808626175, "fcm_dpo/margin": 425.41650390625, "fcm_dpo/q_t": 0.4072082042694092, "grad_norm": 28.977453231811523, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.0519757270812988, "logits/rejected": -1.0653681755065918, "logps/chosen": -676.367919921875, "logps/ref_chosen": -57.450836181640625, "logps/ref_rejected": -94.77339172363281, "logps/rejected": -1139.10693359375, "loss": 1.0788, "margin_dpo/margin_mean": 425.41650390625, "margin_dpo/margin_std": 498.175048828125, "step": 608 }, { "KL/chosen_KL_mean": -661.7661743164062, "KL/mean": -925.5433349609375, "KL/rejected_KL_mean": -1189.320556640625, "KL/std": 666.0048828125, "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.0009277096251025796, "fcm_dpo/delta": -0.09163244068622589, "fcm_dpo/margin": 527.5543823242188, "fcm_dpo/q_t": 0.3955400884151459, "grad_norm": 30.367385864257812, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -1.1196882724761963, "logits/rejected": -1.1408710479736328, "logps/chosen": -720.571533203125, "logps/ref_chosen": -58.805355072021484, "logps/ref_rejected": -88.81600952148438, "logps/rejected": -1278.136474609375, "loss": 1.0618, "margin_dpo/margin_mean": 527.5543823242188, "margin_dpo/margin_std": 781.7075805664062, "step": 609 }, { "KL/chosen_KL_mean": -647.3359375, "KL/mean": -821.218505859375, "KL/rejected_KL_mean": -995.10107421875, "KL/std": 509.16314697265625, "epoch": 0.895741556534508, "fcm_dpo/beta": 0.0009309325832873583, "fcm_dpo/delta": 0.07747611403465271, "fcm_dpo/margin": 347.76507568359375, "fcm_dpo/q_t": 0.42488405108451843, "grad_norm": 41.63359451293945, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.0002460479736328, "logits/rejected": -0.9726261496543884, "logps/chosen": -713.031005859375, "logps/ref_chosen": -65.69503784179688, "logps/ref_rejected": -83.40538787841797, "logps/rejected": -1078.5064697265625, "loss": 1.1582, "margin_dpo/margin_mean": 347.76507568359375, "margin_dpo/margin_std": 584.4713745117188, "step": 610 }, { "KL/chosen_KL_mean": -680.7236328125, "KL/mean": -954.4203491210938, "KL/rejected_KL_mean": -1228.1170654296875, "KL/std": 674.031494140625, "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.0009249848080798984, "fcm_dpo/delta": -0.1090783104300499, "fcm_dpo/margin": 547.3934326171875, "fcm_dpo/q_t": 0.3915501832962036, "grad_norm": 40.06733703613281, "learning_rate": 1.6421423736208e-08, "logits/chosen": -1.0733530521392822, "logits/rejected": -1.1131242513656616, "logps/chosen": -733.3231201171875, "logps/ref_chosen": -52.59946823120117, "logps/ref_rejected": -86.33099365234375, "logps/rejected": -1314.447998046875, "loss": 1.057, "margin_dpo/margin_mean": 547.3934326171875, "margin_dpo/margin_std": 783.890869140625, "step": 611 }, { "KL/chosen_KL_mean": -730.0038452148438, "KL/mean": -944.6229248046875, "KL/rejected_KL_mean": -1159.2420654296875, "KL/std": 537.7095336914062, "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.0009237851481884718, "fcm_dpo/delta": 0.003468703478574753, "fcm_dpo/margin": 429.2381896972656, "fcm_dpo/q_t": 0.40873080492019653, "grad_norm": 30.787771224975586, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -1.054150104522705, "logits/rejected": -1.0440592765808105, "logps/chosen": -789.3275756835938, "logps/ref_chosen": -59.32372283935547, "logps/ref_rejected": -88.31239318847656, "logps/rejected": -1247.554443359375, "loss": 1.0915, "margin_dpo/margin_mean": 429.2381591796875, "margin_dpo/margin_std": 562.5599975585938, "step": 612 }, { "KL/chosen_KL_mean": -648.699951171875, "KL/mean": -902.3504028320312, "KL/rejected_KL_mean": -1156.0009765625, "KL/std": 597.5882568359375, "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.0009168373653665185, "fcm_dpo/delta": -0.066818006336689, "fcm_dpo/margin": 507.301025390625, "fcm_dpo/q_t": 0.39431166648864746, "grad_norm": 34.06965255737305, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.0495436191558838, "logits/rejected": -1.0977256298065186, "logps/chosen": -708.4298706054688, "logps/ref_chosen": -59.72996520996094, "logps/ref_rejected": -105.10752868652344, "logps/rejected": -1261.1083984375, "loss": 1.048, "margin_dpo/margin_mean": 507.301025390625, "margin_dpo/margin_std": 619.2185668945312, "step": 613 }, { "KL/chosen_KL_mean": -726.5540771484375, "KL/mean": -973.306640625, "KL/rejected_KL_mean": -1220.0592041015625, "KL/std": 595.4200439453125, "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0009136309381574392, "fcm_dpo/delta": -0.05203912407159805, "fcm_dpo/margin": 493.5052185058594, "fcm_dpo/q_t": 0.3989720642566681, "grad_norm": 43.53131866455078, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.0377488136291504, "logits/rejected": -1.1064826250076294, "logps/chosen": -779.4930419921875, "logps/ref_chosen": -52.93898010253906, "logps/ref_rejected": -104.67938232421875, "logps/rejected": -1324.738525390625, "loss": 1.0802, "margin_dpo/margin_mean": 493.50518798828125, "margin_dpo/margin_std": 707.6263427734375, "step": 614 }, { "KL/chosen_KL_mean": -656.21240234375, "KL/mean": -860.7072143554688, "KL/rejected_KL_mean": -1065.2021484375, "KL/std": 597.1307373046875, "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0009152768179774284, "fcm_dpo/delta": 0.025946544483304024, "fcm_dpo/margin": 408.9896545410156, "fcm_dpo/q_t": 0.4139822721481323, "grad_norm": 33.06850814819336, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -0.9963966608047485, "logits/rejected": -1.0074797868728638, "logps/chosen": -722.0296630859375, "logps/ref_chosen": -65.81727600097656, "logps/ref_rejected": -95.17749786376953, "logps/rejected": -1160.379638671875, "loss": 1.1254, "margin_dpo/margin_mean": 408.9896545410156, "margin_dpo/margin_std": 637.387451171875, "step": 615 }, { "KL/chosen_KL_mean": -777.4338989257812, "KL/mean": -951.5509033203125, "KL/rejected_KL_mean": -1125.66796875, "KL/std": 516.37451171875, "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.0009200784843415022, "fcm_dpo/delta": 0.08084507286548615, "fcm_dpo/margin": 348.23406982421875, "fcm_dpo/q_t": 0.4268506169319153, "grad_norm": 27.89613151550293, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.0936863422393799, "logits/rejected": -1.075373649597168, "logps/chosen": -842.5667724609375, "logps/ref_chosen": -65.13285827636719, "logps/ref_rejected": -74.70050048828125, "logps/rejected": -1200.3685302734375, "loss": 1.1618, "margin_dpo/margin_mean": 348.2341003417969, "margin_dpo/margin_std": 590.9950561523438, "step": 616 }, { "KL/chosen_KL_mean": -742.44384765625, "KL/mean": -881.6417846679688, "KL/rejected_KL_mean": -1020.8397216796875, "KL/std": 495.73095703125, "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.0009322729893028736, "fcm_dpo/delta": 0.1421297788619995, "fcm_dpo/margin": 278.3958740234375, "fcm_dpo/q_t": 0.4408647119998932, "grad_norm": 45.14849853515625, "learning_rate": 1.378797888467345e-08, "logits/chosen": -0.9931391477584839, "logits/rejected": -0.9588996171951294, "logps/chosen": -805.4493408203125, "logps/ref_chosen": -63.005550384521484, "logps/ref_rejected": -64.234130859375, "logps/rejected": -1085.0738525390625, "loss": 1.2146, "margin_dpo/margin_mean": 278.3958740234375, "margin_dpo/margin_std": 587.977294921875, "step": 617 }, { "KL/chosen_KL_mean": -761.5391845703125, "KL/mean": -1049.184814453125, "KL/rejected_KL_mean": -1336.83056640625, "KL/std": 683.9238891601562, "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.0009282865794375539, "fcm_dpo/delta": -0.1377088725566864, "fcm_dpo/margin": 575.2913818359375, "fcm_dpo/q_t": 0.3874184489250183, "grad_norm": 37.7491455078125, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.0628851652145386, "logits/rejected": -1.1012842655181885, "logps/chosen": -828.6405639648438, "logps/ref_chosen": -67.10134887695312, "logps/ref_rejected": -92.15340423583984, "logps/rejected": -1428.98388671875, "loss": 1.0682, "margin_dpo/margin_mean": 575.2913818359375, "margin_dpo/margin_std": 880.9173583984375, "step": 618 }, { "KL/chosen_KL_mean": -778.9534301757812, "KL/mean": -977.70556640625, "KL/rejected_KL_mean": -1176.457763671875, "KL/std": 610.49609375, "epoch": 0.908957415565345, "fcm_dpo/beta": 0.0009244297398254275, "fcm_dpo/delta": 0.03313371166586876, "fcm_dpo/margin": 397.50433349609375, "fcm_dpo/q_t": 0.4229835271835327, "grad_norm": 41.30840301513672, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.0145196914672852, "logits/rejected": -1.0216684341430664, "logps/chosen": -834.931640625, "logps/ref_chosen": -55.978233337402344, "logps/ref_rejected": -93.1854019165039, "logps/rejected": -1269.64306640625, "loss": 1.1829, "margin_dpo/margin_mean": 397.5043029785156, "margin_dpo/margin_std": 825.1723022460938, "step": 619 }, { "KL/chosen_KL_mean": -700.730224609375, "KL/mean": -897.155517578125, "KL/rejected_KL_mean": -1093.5806884765625, "KL/std": 533.931640625, "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.0009257015772163868, "fcm_dpo/delta": 0.03692508488893509, "fcm_dpo/margin": 392.8504638671875, "fcm_dpo/q_t": 0.4179452955722809, "grad_norm": 41.25801467895508, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -1.0765249729156494, "logits/rejected": -1.0782887935638428, "logps/chosen": -760.5277099609375, "logps/ref_chosen": -59.79750061035156, "logps/ref_rejected": -78.41075134277344, "logps/rejected": -1171.991455078125, "loss": 1.1321, "margin_dpo/margin_mean": 392.8504943847656, "margin_dpo/margin_std": 619.072998046875, "step": 620 }, { "KL/chosen_KL_mean": -700.159912109375, "KL/mean": -985.0526123046875, "KL/rejected_KL_mean": -1269.9453125, "KL/std": 644.7921142578125, "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.000923096900805831, "fcm_dpo/delta": -0.12948347628116608, "fcm_dpo/margin": 569.7854614257812, "fcm_dpo/q_t": 0.3818792700767517, "grad_norm": 36.94768142700195, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.092390775680542, "logits/rejected": -1.13364839553833, "logps/chosen": -754.0936279296875, "logps/ref_chosen": -53.93375778198242, "logps/ref_rejected": -88.36951446533203, "logps/rejected": -1358.31494140625, "loss": 1.027, "margin_dpo/margin_mean": 569.7854614257812, "margin_dpo/margin_std": 705.4525146484375, "step": 621 }, { "KL/chosen_KL_mean": -697.857421875, "KL/mean": -898.1604614257812, "KL/rejected_KL_mean": -1098.463623046875, "KL/std": 515.55908203125, "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0009166207164525986, "fcm_dpo/delta": 0.0332704596221447, "fcm_dpo/margin": 400.6061096191406, "fcm_dpo/q_t": 0.4171668291091919, "grad_norm": 37.14033889770508, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -0.9720423221588135, "logits/rejected": -0.9517272710800171, "logps/chosen": -758.1432495117188, "logps/ref_chosen": -60.28582000732422, "logps/ref_rejected": -85.51873779296875, "logps/rejected": -1183.9822998046875, "loss": 1.1303, "margin_dpo/margin_mean": 400.6060791015625, "margin_dpo/margin_std": 636.8636474609375, "step": 622 }, { "KL/chosen_KL_mean": -729.516845703125, "KL/mean": -890.4384765625, "KL/rejected_KL_mean": -1051.3599853515625, "KL/std": 516.193359375, "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.0009268617141060531, "fcm_dpo/delta": 0.10320872068405151, "fcm_dpo/margin": 321.8431701660156, "fcm_dpo/q_t": 0.43382978439331055, "grad_norm": 35.06966018676758, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -1.096697211265564, "logits/rejected": -1.092256784439087, "logps/chosen": -793.673828125, "logps/ref_chosen": -64.1569595336914, "logps/ref_rejected": -85.08304595947266, "logps/rejected": -1136.443115234375, "loss": 1.1852, "margin_dpo/margin_mean": 321.8431396484375, "margin_dpo/margin_std": 620.9134521484375, "step": 623 }, { "KL/chosen_KL_mean": -732.138916015625, "KL/mean": -979.6350708007812, "KL/rejected_KL_mean": -1227.131103515625, "KL/std": 541.4876708984375, "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.0009254544274881482, "fcm_dpo/delta": -0.05943404883146286, "fcm_dpo/margin": 494.9923400878906, "fcm_dpo/q_t": 0.39315974712371826, "grad_norm": 33.594234466552734, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -1.132512092590332, "logits/rejected": -1.1252646446228027, "logps/chosen": -804.0575561523438, "logps/ref_chosen": -71.91862487792969, "logps/ref_rejected": -97.13203430175781, "logps/rejected": -1324.26318359375, "loss": 1.0492, "margin_dpo/margin_mean": 494.9923400878906, "margin_dpo/margin_std": 585.3521118164062, "step": 624 }, { "KL/chosen_KL_mean": -680.2445068359375, "KL/mean": -972.3754272460938, "KL/rejected_KL_mean": -1264.50634765625, "KL/std": 626.7379150390625, "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.0009159696055576205, "fcm_dpo/delta": -0.13877010345458984, "fcm_dpo/margin": 584.261962890625, "fcm_dpo/q_t": 0.3786303400993347, "grad_norm": 48.92192459106445, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.0401864051818848, "logits/rejected": -1.0676482915878296, "logps/chosen": -738.5865478515625, "logps/ref_chosen": -58.342071533203125, "logps/ref_rejected": -86.09038543701172, "logps/rejected": -1350.5968017578125, "loss": 0.9865, "margin_dpo/margin_mean": 584.261962890625, "margin_dpo/margin_std": 589.0737915039062, "step": 625 }, { "KL/chosen_KL_mean": -864.771484375, "KL/mean": -1021.89208984375, "KL/rejected_KL_mean": -1179.0126953125, "KL/std": 683.1890869140625, "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.0009178274776786566, "fcm_dpo/delta": 0.1132790595293045, "fcm_dpo/margin": 314.2411193847656, "fcm_dpo/q_t": 0.4319329857826233, "grad_norm": 38.202056884765625, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.0161819458007812, "logits/rejected": -1.007856011390686, "logps/chosen": -939.8840942382812, "logps/ref_chosen": -75.11260986328125, "logps/ref_rejected": -99.188720703125, "logps/rejected": -1278.201416015625, "loss": 1.2125, "margin_dpo/margin_mean": 314.2411193847656, "margin_dpo/margin_std": 675.1224365234375, "step": 626 }, { "KL/chosen_KL_mean": -591.8564453125, "KL/mean": -881.7455444335938, "KL/rejected_KL_mean": -1171.6346435546875, "KL/std": 698.69775390625, "epoch": 0.920704845814978, "fcm_dpo/beta": 0.0009149962570518255, "fcm_dpo/delta": -0.13404129445552826, "fcm_dpo/margin": 579.7781372070312, "fcm_dpo/q_t": 0.3895081877708435, "grad_norm": 31.1433162689209, "learning_rate": 9.897955805412e-09, "logits/chosen": -0.9388647079467773, "logits/rejected": -1.0119527578353882, "logps/chosen": -639.599609375, "logps/ref_chosen": -47.74314880371094, "logps/ref_rejected": -106.75448608398438, "logps/rejected": -1278.38916015625, "loss": 1.0424, "margin_dpo/margin_mean": 579.7781372070312, "margin_dpo/margin_std": 799.517578125, "step": 627 }, { "KL/chosen_KL_mean": -759.9505615234375, "KL/mean": -988.3452758789062, "KL/rejected_KL_mean": -1216.739990234375, "KL/std": 587.220458984375, "epoch": 0.922173274596182, "fcm_dpo/beta": 0.0009073036490008235, "fcm_dpo/delta": -0.014781368896365166, "fcm_dpo/margin": 456.78936767578125, "fcm_dpo/q_t": 0.40726301074028015, "grad_norm": 34.24974060058594, "learning_rate": 9.543589206795238e-09, "logits/chosen": -1.096536636352539, "logits/rejected": -1.1101562976837158, "logps/chosen": -820.1334838867188, "logps/ref_chosen": -60.182945251464844, "logps/ref_rejected": -101.55467224121094, "logps/rejected": -1318.294677734375, "loss": 1.1024, "margin_dpo/margin_mean": 456.78936767578125, "margin_dpo/margin_std": 680.5447998046875, "step": 628 }, { "KL/chosen_KL_mean": -751.41064453125, "KL/mean": -967.8169555664062, "KL/rejected_KL_mean": -1184.2232666015625, "KL/std": 572.2176513671875, "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0009084055200219154, "fcm_dpo/delta": 0.006959038320928812, "fcm_dpo/margin": 432.81256103515625, "fcm_dpo/q_t": 0.40856897830963135, "grad_norm": 38.99527359008789, "learning_rate": 9.19555885822887e-09, "logits/chosen": -1.0798592567443848, "logits/rejected": -1.0886130332946777, "logps/chosen": -815.6241455078125, "logps/ref_chosen": -64.21354675292969, "logps/ref_rejected": -91.65367126464844, "logps/rejected": -1275.876953125, "loss": 1.0948, "margin_dpo/margin_mean": 432.81256103515625, "margin_dpo/margin_std": 571.304443359375, "step": 629 }, { "KL/chosen_KL_mean": -677.60888671875, "KL/mean": -813.6373291015625, "KL/rejected_KL_mean": -949.665771484375, "KL/std": 625.6536254882812, "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.0009094603592529893, "fcm_dpo/delta": 0.0273385439068079, "fcm_dpo/margin": 272.05682373046875, "fcm_dpo/q_t": 0.4511290192604065, "grad_norm": 44.24210739135742, "learning_rate": 8.85387393063622e-09, "logits/chosen": -1.0548908710479736, "logits/rejected": -1.0271886587142944, "logps/chosen": -736.89990234375, "logps/ref_chosen": -59.29100036621094, "logps/ref_rejected": -83.59829711914062, "logps/rejected": -1033.2640380859375, "loss": 1.2546, "margin_dpo/margin_mean": 272.0568542480469, "margin_dpo/margin_std": 752.2532958984375, "step": 630 }, { "KL/chosen_KL_mean": -818.7833862304688, "KL/mean": -1021.9134521484375, "KL/rejected_KL_mean": -1225.04345703125, "KL/std": 593.9693603515625, "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.0009135643485933542, "fcm_dpo/delta": 0.029337339103221893, "fcm_dpo/margin": 406.260009765625, "fcm_dpo/q_t": 0.4151837229728699, "grad_norm": 36.602779388427734, "learning_rate": 8.518543427732949e-09, "logits/chosen": -1.141347885131836, "logits/rejected": -1.1473594903945923, "logps/chosen": -878.2369995117188, "logps/ref_chosen": -59.45360565185547, "logps/ref_rejected": -80.95156860351562, "logps/rejected": -1305.9949951171875, "loss": 1.14, "margin_dpo/margin_mean": 406.2600402832031, "margin_dpo/margin_std": 679.403564453125, "step": 631 }, { "KL/chosen_KL_mean": -730.484130859375, "KL/mean": -936.6009521484375, "KL/rejected_KL_mean": -1142.7177734375, "KL/std": 546.6138305664062, "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.0009125604410655797, "fcm_dpo/delta": 0.024038948118686676, "fcm_dpo/margin": 412.233642578125, "fcm_dpo/q_t": 0.4137340188026428, "grad_norm": 39.51885986328125, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.1271458864212036, "logits/rejected": -1.126634955406189, "logps/chosen": -791.835693359375, "logps/ref_chosen": -61.35155487060547, "logps/ref_rejected": -86.16017150878906, "logps/rejected": -1228.8779296875, "loss": 1.1307, "margin_dpo/margin_mean": 412.233642578125, "margin_dpo/margin_std": 653.526611328125, "step": 632 }, { "KL/chosen_KL_mean": -793.9225463867188, "KL/mean": -951.2642822265625, "KL/rejected_KL_mean": -1108.6060791015625, "KL/std": 532.9287109375, "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.0009164665825664997, "fcm_dpo/delta": 0.019403798505663872, "fcm_dpo/margin": 314.6834716796875, "fcm_dpo/q_t": 0.4337080717086792, "grad_norm": 44.8255615234375, "learning_rate": 7.866980873399015e-09, "logits/chosen": -1.1348984241485596, "logits/rejected": -1.147522211074829, "logps/chosen": -851.20068359375, "logps/ref_chosen": -57.27816390991211, "logps/ref_rejected": -91.58395385742188, "logps/rejected": -1200.1900634765625, "loss": 1.1933, "margin_dpo/margin_mean": 314.6835021972656, "margin_dpo/margin_std": 608.6378784179688, "step": 633 }, { "KL/chosen_KL_mean": -881.3724365234375, "KL/mean": -1016.420654296875, "KL/rejected_KL_mean": -1151.4688720703125, "KL/std": 623.2406005859375, "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.0009277792414650321, "fcm_dpo/delta": 0.15131084620952606, "fcm_dpo/margin": 270.09649658203125, "fcm_dpo/q_t": 0.4442192018032074, "grad_norm": 36.3792724609375, "learning_rate": 7.550765991247654e-09, "logits/chosen": -0.985339879989624, "logits/rejected": -0.9769416451454163, "logps/chosen": -947.991455078125, "logps/ref_chosen": -66.61896514892578, "logps/ref_rejected": -107.12564849853516, "logps/rejected": -1258.594482421875, "loss": 1.2225, "margin_dpo/margin_mean": 270.09649658203125, "margin_dpo/margin_std": 597.4962768554688, "step": 634 }, { "KL/chosen_KL_mean": -775.334716796875, "KL/mean": -957.4782104492188, "KL/rejected_KL_mean": -1139.62158203125, "KL/std": 646.2401733398438, "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.0009354540961794555, "fcm_dpo/delta": 0.060258638113737106, "fcm_dpo/margin": 364.28692626953125, "fcm_dpo/q_t": 0.4247502088546753, "grad_norm": 43.95642852783203, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.0711841583251953, "logits/rejected": -1.0444178581237793, "logps/chosen": -849.290283203125, "logps/ref_chosen": -73.95551300048828, "logps/ref_rejected": -82.50045776367188, "logps/rejected": -1222.1220703125, "loss": 1.163, "margin_dpo/margin_mean": 364.28692626953125, "margin_dpo/margin_std": 672.04248046875, "step": 635 }, { "KL/chosen_KL_mean": -732.2950439453125, "KL/mean": -943.854248046875, "KL/rejected_KL_mean": -1155.413330078125, "KL/std": 642.1853637695312, "epoch": 0.933920704845815, "fcm_dpo/beta": 0.0009403472067788243, "fcm_dpo/delta": 0.0016737580299377441, "fcm_dpo/margin": 423.1182861328125, "fcm_dpo/q_t": 0.41292804479599, "grad_norm": 25.051197052001953, "learning_rate": 6.937510679537628e-09, "logits/chosen": -0.9883960485458374, "logits/rejected": -0.9851752519607544, "logps/chosen": -791.9239501953125, "logps/ref_chosen": -59.628910064697266, "logps/ref_rejected": -81.97883605957031, "logps/rejected": -1237.3922119140625, "loss": 1.1096, "margin_dpo/margin_mean": 423.1182861328125, "margin_dpo/margin_std": 657.2255859375, "step": 636 }, { "KL/chosen_KL_mean": -731.9287719726562, "KL/mean": -977.4234619140625, "KL/rejected_KL_mean": -1222.9180908203125, "KL/std": 657.0423583984375, "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0009306943975389004, "fcm_dpo/delta": -0.05869361758232117, "fcm_dpo/margin": 490.98931884765625, "fcm_dpo/q_t": 0.39948800206184387, "grad_norm": 32.110877990722656, "learning_rate": 6.640486409826785e-09, "logits/chosen": -1.1167852878570557, "logits/rejected": -1.1629787683486938, "logps/chosen": -781.5814208984375, "logps/ref_chosen": -49.652687072753906, "logps/ref_rejected": -98.40513610839844, "logps/rejected": -1321.3232421875, "loss": 1.0728, "margin_dpo/margin_mean": 490.98931884765625, "margin_dpo/margin_std": 690.1351318359375, "step": 637 }, { "KL/chosen_KL_mean": -686.4628295898438, "KL/mean": -884.8165283203125, "KL/rejected_KL_mean": -1083.1702880859375, "KL/std": 587.4907836914062, "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.0009277926292270422, "fcm_dpo/delta": -0.08301420509815216, "fcm_dpo/margin": 396.7074279785156, "fcm_dpo/q_t": 0.4109494686126709, "grad_norm": 36.82447052001953, "learning_rate": 6.349874889624962e-09, "logits/chosen": -0.9534268379211426, "logits/rejected": -0.9300310611724854, "logps/chosen": -744.6195068359375, "logps/ref_chosen": -58.156639099121094, "logps/ref_rejected": -79.3014907836914, "logps/rejected": -1162.4716796875, "loss": 1.1481, "margin_dpo/margin_mean": 396.7074279785156, "margin_dpo/margin_std": 668.0907592773438, "step": 638 }, { "KL/chosen_KL_mean": -987.62060546875, "KL/mean": -1060.539306640625, "KL/rejected_KL_mean": -1133.458251953125, "KL/std": 582.9657592773438, "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.0009239416103810072, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 145.8376007080078, "fcm_dpo/q_t": 0.46903330087661743, "grad_norm": 139.1773681640625, "learning_rate": 6.065683776815933e-09, "logits/chosen": -0.9747638702392578, "logits/rejected": -0.9038654565811157, "logps/chosen": -1059.94384765625, "logps/ref_chosen": -72.32319641113281, "logps/ref_rejected": -74.2749252319336, "logps/rejected": -1207.733154296875, "loss": 1.3733, "margin_dpo/margin_mean": 145.83761596679688, "margin_dpo/margin_std": 759.8252563476562, "step": 639 }, { "KL/chosen_KL_mean": -737.2076416015625, "KL/mean": -1026.8175048828125, "KL/rejected_KL_mean": -1316.42724609375, "KL/std": 669.41357421875, "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.00091610784875229, "fcm_dpo/delta": -0.13407810032367706, "fcm_dpo/margin": 579.2196044921875, "fcm_dpo/q_t": 0.3861461579799652, "grad_norm": 52.59541320800781, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -0.9671276807785034, "logits/rejected": -0.9929375648498535, "logps/chosen": -793.342041015625, "logps/ref_chosen": -56.13436508178711, "logps/ref_rejected": -108.60014343261719, "logps/rejected": -1425.02734375, "loss": 1.0344, "margin_dpo/margin_mean": 579.2196044921875, "margin_dpo/margin_std": 785.578369140625, "step": 640 }, { "KL/chosen_KL_mean": -859.6257934570312, "KL/mean": -1046.6640625, "KL/rejected_KL_mean": -1233.7021484375, "KL/std": 573.8028564453125, "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.0009146465454250574, "fcm_dpo/delta": 0.05887487903237343, "fcm_dpo/margin": 374.076416015625, "fcm_dpo/q_t": 0.42490917444229126, "grad_norm": 37.22019958496094, "learning_rate": 5.516592558795746e-09, "logits/chosen": -1.0567216873168945, "logits/rejected": -1.0576525926589966, "logps/chosen": -924.6226806640625, "logps/ref_chosen": -64.99689483642578, "logps/ref_rejected": -86.99232482910156, "logps/rejected": -1320.694580078125, "loss": 1.1778, "margin_dpo/margin_mean": 374.076416015625, "margin_dpo/margin_std": 746.7939453125, "step": 641 }, { "KL/chosen_KL_mean": -806.0206298828125, "KL/mean": -1029.8985595703125, "KL/rejected_KL_mean": -1253.7764892578125, "KL/std": 721.0260620117188, "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.0009186153765767813, "fcm_dpo/delta": -0.011798828840255737, "fcm_dpo/margin": 447.75579833984375, "fcm_dpo/q_t": 0.41545358300209045, "grad_norm": 39.59940719604492, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.001420021057129, "logits/rejected": -1.0281758308410645, "logps/chosen": -871.7098999023438, "logps/ref_chosen": -65.68924713134766, "logps/ref_rejected": -110.24205017089844, "logps/rejected": -1364.0185546875, "loss": 1.1498, "margin_dpo/margin_mean": 447.75579833984375, "margin_dpo/margin_std": 873.0943603515625, "step": 642 }, { "KL/chosen_KL_mean": -727.7049560546875, "KL/mean": -922.38037109375, "KL/rejected_KL_mean": -1117.055908203125, "KL/std": 554.1806640625, "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.0009108081576414406, "fcm_dpo/delta": -0.05616462975740433, "fcm_dpo/margin": 389.350830078125, "fcm_dpo/q_t": 0.41762280464172363, "grad_norm": 45.18101119995117, "learning_rate": 4.993270631642038e-09, "logits/chosen": -1.1256649494171143, "logits/rejected": -1.1269869804382324, "logps/chosen": -779.6549072265625, "logps/ref_chosen": -51.94999694824219, "logps/ref_rejected": -87.46833801269531, "logps/rejected": -1204.524169921875, "loss": 1.1243, "margin_dpo/margin_mean": 389.3507995605469, "margin_dpo/margin_std": 536.47412109375, "step": 643 }, { "KL/chosen_KL_mean": -711.8359375, "KL/mean": -895.3848876953125, "KL/rejected_KL_mean": -1078.9339599609375, "KL/std": 636.5994873046875, "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0009149383986368775, "fcm_dpo/delta": 0.06524142622947693, "fcm_dpo/margin": 367.0979919433594, "fcm_dpo/q_t": 0.42591458559036255, "grad_norm": 48.07936096191406, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.0160619020462036, "logits/rejected": -1.021895170211792, "logps/chosen": -770.8535766601562, "logps/ref_chosen": -59.017662048339844, "logps/ref_rejected": -87.13668823242188, "logps/rejected": -1166.070556640625, "loss": 1.1859, "margin_dpo/margin_mean": 367.0979919433594, "margin_dpo/margin_std": 747.0296020507812, "step": 644 }, { "KL/chosen_KL_mean": -738.4013061523438, "KL/mean": -835.6104125976562, "KL/rejected_KL_mean": -932.8195190429688, "KL/std": 500.1334228515625, "epoch": 0.947136563876652, "fcm_dpo/beta": 0.0009230460273101926, "fcm_dpo/delta": 0.068226657807827, "fcm_dpo/margin": 194.41815185546875, "fcm_dpo/q_t": 0.4604693055152893, "grad_norm": 92.3852310180664, "learning_rate": 4.495773155069299e-09, "logits/chosen": -0.9825325608253479, "logits/rejected": -0.969984769821167, "logps/chosen": -794.27734375, "logps/ref_chosen": -55.87602233886719, "logps/ref_rejected": -97.78080749511719, "logps/rejected": -1030.600341796875, "loss": 1.3114, "margin_dpo/margin_mean": 194.41818237304688, "margin_dpo/margin_std": 666.4583740234375, "step": 645 }, { "KL/chosen_KL_mean": -734.58056640625, "KL/mean": -900.8929443359375, "KL/rejected_KL_mean": -1067.205322265625, "KL/std": 515.5413818359375, "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.0009276814525946975, "fcm_dpo/delta": 0.09289085865020752, "fcm_dpo/margin": 332.624755859375, "fcm_dpo/q_t": 0.43122583627700806, "grad_norm": 54.124847412109375, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.0459076166152954, "logits/rejected": -1.022136926651001, "logps/chosen": -795.8563232421875, "logps/ref_chosen": -61.275787353515625, "logps/ref_rejected": -77.50580596923828, "logps/rejected": -1144.711181640625, "loss": 1.184, "margin_dpo/margin_mean": 332.624755859375, "margin_dpo/margin_std": 628.484130859375, "step": 646 }, { "KL/chosen_KL_mean": -620.7327880859375, "KL/mean": -821.4715576171875, "KL/rejected_KL_mean": -1022.2103271484375, "KL/std": 562.453125, "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0009334119968116283, "fcm_dpo/delta": 0.025739368051290512, "fcm_dpo/margin": 401.47760009765625, "fcm_dpo/q_t": 0.4135919511318207, "grad_norm": 33.10821533203125, "learning_rate": 4.024152566816791e-09, "logits/chosen": -0.9425860047340393, "logits/rejected": -0.9679901599884033, "logps/chosen": -675.585205078125, "logps/ref_chosen": -54.8524169921875, "logps/ref_rejected": -93.5194091796875, "logps/rejected": -1115.729736328125, "loss": 1.1112, "margin_dpo/margin_mean": 401.47760009765625, "margin_dpo/margin_std": 560.4625244140625, "step": 647 }, { "KL/chosen_KL_mean": -693.855224609375, "KL/mean": -988.9381103515625, "KL/rejected_KL_mean": -1284.02099609375, "KL/std": 669.7720947265625, "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.000923770247027278, "fcm_dpo/delta": -0.14923778176307678, "fcm_dpo/margin": 590.1658935546875, "fcm_dpo/q_t": 0.3826466500759125, "grad_norm": 28.995315551757812, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.0780737400054932, "logits/rejected": -1.1365818977355957, "logps/chosen": -748.026611328125, "logps/ref_chosen": -54.17146682739258, "logps/ref_rejected": -98.7127914428711, "logps/rejected": -1382.73388671875, "loss": 1.024, "margin_dpo/margin_mean": 590.1658935546875, "margin_dpo/margin_std": 785.5125122070312, "step": 648 }, { "KL/chosen_KL_mean": -715.6250610351562, "KL/mean": -850.495361328125, "KL/rejected_KL_mean": -985.3656005859375, "KL/std": 527.7951049804688, "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.0009227098198607564, "fcm_dpo/delta": 0.05385739728808403, "fcm_dpo/margin": 269.74053955078125, "fcm_dpo/q_t": 0.44609135389328003, "grad_norm": 33.012882232666016, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -1.1264615058898926, "logits/rejected": -1.1146023273468018, "logps/chosen": -778.1054077148438, "logps/ref_chosen": -62.480350494384766, "logps/ref_rejected": -80.07717895507812, "logps/rejected": -1065.44287109375, "loss": 1.2419, "margin_dpo/margin_mean": 269.7405700683594, "margin_dpo/margin_std": 652.0167236328125, "step": 649 }, { "KL/chosen_KL_mean": -752.7981567382812, "KL/mean": -973.55517578125, "KL/rejected_KL_mean": -1194.312255859375, "KL/std": 645.759765625, "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.0009236993500962853, "fcm_dpo/delta": -0.008039167150855064, "fcm_dpo/margin": 441.51409912109375, "fcm_dpo/q_t": 0.4098489284515381, "grad_norm": 28.0744571685791, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.0201561450958252, "logits/rejected": -1.0400092601776123, "logps/chosen": -808.8909912109375, "logps/ref_chosen": -56.09281921386719, "logps/ref_rejected": -98.26483917236328, "logps/rejected": -1292.5771484375, "loss": 1.1285, "margin_dpo/margin_mean": 441.51409912109375, "margin_dpo/margin_std": 749.44873046875, "step": 650 }, { "KL/chosen_KL_mean": -520.7056884765625, "KL/mean": -799.834228515625, "KL/rejected_KL_mean": -1078.9627685546875, "KL/std": 597.411865234375, "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.0009163864306174219, "fcm_dpo/delta": -0.11453430354595184, "fcm_dpo/margin": 558.2569580078125, "fcm_dpo/q_t": 0.3837953507900238, "grad_norm": 45.06791687011719, "learning_rate": 3.158738163478475e-09, "logits/chosen": -1.0805871486663818, "logits/rejected": -1.1315158605575562, "logps/chosen": -564.1311645507812, "logps/ref_chosen": -43.42544937133789, "logps/ref_rejected": -99.95791625976562, "logps/rejected": -1178.920654296875, "loss": 1.0074, "margin_dpo/margin_mean": 558.2569580078125, "margin_dpo/margin_std": 605.560791015625, "step": 651 }, { "KL/chosen_KL_mean": -649.9320678710938, "KL/mean": -872.9180908203125, "KL/rejected_KL_mean": -1095.904052734375, "KL/std": 631.821044921875, "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.000913501949980855, "fcm_dpo/delta": -0.007549758069217205, "fcm_dpo/margin": 445.9720153808594, "fcm_dpo/q_t": 0.4097236096858978, "grad_norm": 41.35665512084961, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -1.05128812789917, "logits/rejected": -1.0744919776916504, "logps/chosen": -712.5089111328125, "logps/ref_chosen": -62.57680892944336, "logps/ref_rejected": -111.76779174804688, "logps/rejected": -1207.671875, "loss": 1.1206, "margin_dpo/margin_mean": 445.9720153808594, "margin_dpo/margin_std": 735.88134765625, "step": 652 }, { "KL/chosen_KL_mean": -779.2682495117188, "KL/mean": -977.2479858398438, "KL/rejected_KL_mean": -1175.2276611328125, "KL/std": 638.5052490234375, "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.0009161117486655712, "fcm_dpo/delta": 0.0379294790327549, "fcm_dpo/margin": 395.95941162109375, "fcm_dpo/q_t": 0.41951340436935425, "grad_norm": 39.457916259765625, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -1.0988178253173828, "logits/rejected": -1.118626594543457, "logps/chosen": -840.3812255859375, "logps/ref_chosen": -61.11295700073242, "logps/ref_rejected": -103.24960327148438, "logps/rejected": -1278.477294921875, "loss": 1.1425, "margin_dpo/margin_mean": 395.95941162109375, "margin_dpo/margin_std": 670.7147827148438, "step": 653 }, { "KL/chosen_KL_mean": -704.796875, "KL/mean": -902.2587890625, "KL/rejected_KL_mean": -1099.720703125, "KL/std": 529.0486450195312, "epoch": 0.960352422907489, "fcm_dpo/beta": 0.000919223006349057, "fcm_dpo/delta": 0.03766302019357681, "fcm_dpo/margin": 394.923828125, "fcm_dpo/q_t": 0.41876959800720215, "grad_norm": 32.23603820800781, "learning_rate": 2.577954022936174e-09, "logits/chosen": -1.0824410915374756, "logits/rejected": -1.1002042293548584, "logps/chosen": -766.5250244140625, "logps/ref_chosen": -61.7281379699707, "logps/ref_rejected": -98.7738037109375, "logps/rejected": -1198.4945068359375, "loss": 1.1299, "margin_dpo/margin_mean": 394.923828125, "margin_dpo/margin_std": 613.5523681640625, "step": 654 }, { "KL/chosen_KL_mean": -666.8828125, "KL/mean": -870.272705078125, "KL/rejected_KL_mean": -1073.66259765625, "KL/std": 549.2578735351562, "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.0009220571955665946, "fcm_dpo/delta": 0.025397051125764847, "fcm_dpo/margin": 406.7799072265625, "fcm_dpo/q_t": 0.41584494709968567, "grad_norm": 34.27344512939453, "learning_rate": 2.397392281198729e-09, "logits/chosen": -1.0440378189086914, "logits/rejected": -1.0873972177505493, "logps/chosen": -716.4595947265625, "logps/ref_chosen": -49.576812744140625, "logps/ref_rejected": -98.29183197021484, "logps/rejected": -1171.9544677734375, "loss": 1.1277, "margin_dpo/margin_mean": 406.7799072265625, "margin_dpo/margin_std": 646.5740966796875, "step": 655 }, { "KL/chosen_KL_mean": -702.065673828125, "KL/mean": -1036.923828125, "KL/rejected_KL_mean": -1371.781982421875, "KL/std": 697.924072265625, "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.0009094408596865833, "fcm_dpo/delta": -0.2153792828321457, "fcm_dpo/margin": 669.7164306640625, "fcm_dpo/q_t": 0.3670673072338104, "grad_norm": 81.25033569335938, "learning_rate": 2.223355098446622e-09, "logits/chosen": -0.9668943881988525, "logits/rejected": -1.0353293418884277, "logps/chosen": -754.6151123046875, "logps/ref_chosen": -52.54943084716797, "logps/ref_rejected": -113.67464447021484, "logps/rejected": -1485.456787109375, "loss": 0.9595, "margin_dpo/margin_mean": 669.7164306640625, "margin_dpo/margin_std": 723.3271484375, "step": 656 }, { "KL/chosen_KL_mean": -684.0799560546875, "KL/mean": -962.5208129882812, "KL/rejected_KL_mean": -1240.961669921875, "KL/std": 694.053466796875, "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.0008950576884672046, "fcm_dpo/delta": -0.10112009942531586, "fcm_dpo/margin": 556.8817138671875, "fcm_dpo/q_t": 0.38936832547187805, "grad_norm": 43.923797607421875, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.0988727807998657, "logits/rejected": -1.1430981159210205, "logps/chosen": -730.780517578125, "logps/ref_chosen": -46.700538635253906, "logps/ref_rejected": -97.91487121582031, "logps/rejected": -1338.87646484375, "loss": 1.0394, "margin_dpo/margin_mean": 556.8817138671875, "margin_dpo/margin_std": 717.0012817382812, "step": 657 }, { "KL/chosen_KL_mean": -721.3847045898438, "KL/mean": -944.3277587890625, "KL/rejected_KL_mean": -1167.27099609375, "KL/std": 536.6946411132812, "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.0008913222118280828, "fcm_dpo/delta": 0.002288389950990677, "fcm_dpo/margin": 445.8861389160156, "fcm_dpo/q_t": 0.4090738594532013, "grad_norm": 33.75960159301758, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -1.0495469570159912, "logits/rejected": -1.0703763961791992, "logps/chosen": -782.3428955078125, "logps/ref_chosen": -60.95820999145508, "logps/ref_rejected": -95.93949127197266, "logps/rejected": -1263.21044921875, "loss": 1.1046, "margin_dpo/margin_mean": 445.8861083984375, "margin_dpo/margin_std": 629.4903564453125, "step": 658 }, { "KL/chosen_KL_mean": -631.7986450195312, "KL/mean": -848.4490966796875, "KL/rejected_KL_mean": -1065.099609375, "KL/std": 545.1578369140625, "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.0008934010984376073, "fcm_dpo/delta": 0.013058962300419807, "fcm_dpo/margin": 433.3009033203125, "fcm_dpo/q_t": 0.41191983222961426, "grad_norm": 32.50177001953125, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -0.9998725056648254, "logits/rejected": -0.9671860933303833, "logps/chosen": -708.5416259765625, "logps/ref_chosen": -76.74298095703125, "logps/ref_rejected": -87.4709701538086, "logps/rejected": -1152.570556640625, "loss": 1.1012, "margin_dpo/margin_mean": 433.3009338378906, "margin_dpo/margin_std": 592.5736083984375, "step": 659 }, { "KL/chosen_KL_mean": -696.3098754882812, "KL/mean": -962.7720947265625, "KL/rejected_KL_mean": -1229.234375, "KL/std": 621.4789428710938, "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.0008908716263249516, "fcm_dpo/delta": -0.07654912024736404, "fcm_dpo/margin": 532.9244995117188, "fcm_dpo/q_t": 0.3918866515159607, "grad_norm": 45.116607666015625, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.0789703130722046, "logits/rejected": -1.089571237564087, "logps/chosen": -755.3577880859375, "logps/ref_chosen": -59.04788589477539, "logps/ref_rejected": -75.96005249023438, "logps/rejected": -1305.1944580078125, "loss": 1.0467, "margin_dpo/margin_mean": 532.9244995117188, "margin_dpo/margin_std": 662.8816528320312, "step": 660 }, { "KL/chosen_KL_mean": -641.9539794921875, "KL/mean": -880.4549560546875, "KL/rejected_KL_mean": -1118.9559326171875, "KL/std": 687.37255859375, "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.000888321315869689, "fcm_dpo/delta": -0.02433537133038044, "fcm_dpo/margin": 477.001953125, "fcm_dpo/q_t": 0.40677112340927124, "grad_norm": 43.87881851196289, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.1158676147460938, "logits/rejected": -1.1237622499465942, "logps/chosen": -692.6279296875, "logps/ref_chosen": -50.673973083496094, "logps/ref_rejected": -86.00569152832031, "logps/rejected": -1204.961669921875, "loss": 1.0893, "margin_dpo/margin_mean": 477.001953125, "margin_dpo/margin_std": 703.0227661132812, "step": 661 }, { "KL/chosen_KL_mean": -701.1622314453125, "KL/mean": -907.7703857421875, "KL/rejected_KL_mean": -1114.378662109375, "KL/std": 595.3218994140625, "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.0008879530942067504, "fcm_dpo/delta": 0.03370830416679382, "fcm_dpo/margin": 413.2164306640625, "fcm_dpo/q_t": 0.4180566072463989, "grad_norm": 28.001462936401367, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.0258715152740479, "logits/rejected": -1.023505449295044, "logps/chosen": -770.4232788085938, "logps/ref_chosen": -69.26106262207031, "logps/ref_rejected": -89.05593872070312, "logps/rejected": -1203.4345703125, "loss": 1.1412, "margin_dpo/margin_mean": 413.2164306640625, "margin_dpo/margin_std": 703.9146728515625, "step": 662 }, { "KL/chosen_KL_mean": -683.2754516601562, "KL/mean": -903.65966796875, "KL/rejected_KL_mean": -1124.0439453125, "KL/std": 640.8971557617188, "epoch": 0.973568281938326, "fcm_dpo/beta": 0.0008887812728062272, "fcm_dpo/delta": 0.008383674547076225, "fcm_dpo/margin": 440.7684020996094, "fcm_dpo/q_t": 0.41332346200942993, "grad_norm": 30.94547462463379, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -1.0513726472854614, "logits/rejected": -1.0720903873443604, "logps/chosen": -748.1544189453125, "logps/ref_chosen": -64.87890625, "logps/ref_rejected": -113.92536926269531, "logps/rejected": -1237.96923828125, "loss": 1.1309, "margin_dpo/margin_mean": 440.7684020996094, "margin_dpo/margin_std": 747.7581787109375, "step": 663 }, { "KL/chosen_KL_mean": -748.7786254882812, "KL/mean": -1007.6143798828125, "KL/rejected_KL_mean": -1266.4501953125, "KL/std": 679.8283081054688, "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.0008842787938192487, "fcm_dpo/delta": -0.05937542766332626, "fcm_dpo/margin": 517.6715698242188, "fcm_dpo/q_t": 0.39915400743484497, "grad_norm": 30.055925369262695, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.0716259479522705, "logits/rejected": -1.1003488302230835, "logps/chosen": -809.6671142578125, "logps/ref_chosen": -60.88847351074219, "logps/ref_rejected": -105.521728515625, "logps/rejected": -1371.971923828125, "loss": 1.0709, "margin_dpo/margin_mean": 517.6715698242188, "margin_dpo/margin_std": 723.2990112304688, "step": 664 }, { "KL/chosen_KL_mean": -657.037109375, "KL/mean": -874.8280029296875, "KL/rejected_KL_mean": -1092.618896484375, "KL/std": 550.8089599609375, "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.0008855801424942911, "fcm_dpo/delta": 0.014537844806909561, "fcm_dpo/margin": 435.581787109375, "fcm_dpo/q_t": 0.41070112586021423, "grad_norm": 56.73260498046875, "learning_rate": 9.513254770636137e-10, "logits/chosen": -1.1636441946029663, "logits/rejected": -1.1826890707015991, "logps/chosen": -717.6011962890625, "logps/ref_chosen": -60.56413269042969, "logps/ref_rejected": -84.80882263183594, "logps/rejected": -1177.427734375, "loss": 1.0941, "margin_dpo/margin_mean": 435.581787109375, "margin_dpo/margin_std": 550.626708984375, "step": 665 }, { "KL/chosen_KL_mean": -698.910400390625, "KL/mean": -909.08203125, "KL/rejected_KL_mean": -1119.2535400390625, "KL/std": 539.8720092773438, "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.0008871962781995535, "fcm_dpo/delta": 0.027589332312345505, "fcm_dpo/margin": 420.3431396484375, "fcm_dpo/q_t": 0.415420264005661, "grad_norm": 30.555706024169922, "learning_rate": 8.427576920763956e-10, "logits/chosen": -0.9608547687530518, "logits/rejected": -0.9632211923599243, "logps/chosen": -763.3303833007812, "logps/ref_chosen": -64.41996002197266, "logps/ref_rejected": -95.8916244506836, "logps/rejected": -1215.145263671875, "loss": 1.1199, "margin_dpo/margin_mean": 420.3431396484375, "margin_dpo/margin_std": 612.9646606445312, "step": 666 }, { "KL/chosen_KL_mean": -784.1095581054688, "KL/mean": -1053.423828125, "KL/rejected_KL_mean": -1322.738037109375, "KL/std": 630.7745361328125, "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.0008815985638648272, "fcm_dpo/delta": -0.07696354389190674, "fcm_dpo/margin": 538.6284790039062, "fcm_dpo/q_t": 0.3935006856918335, "grad_norm": 44.02839660644531, "learning_rate": 7.407554321417764e-10, "logits/chosen": -0.9979052543640137, "logits/rejected": -0.9953716993331909, "logps/chosen": -853.3865966796875, "logps/ref_chosen": -69.27702331542969, "logps/ref_rejected": -87.83549499511719, "logps/rejected": -1410.573486328125, "loss": 1.0497, "margin_dpo/margin_mean": 538.6284790039062, "margin_dpo/margin_std": 688.3106689453125, "step": 667 }, { "KL/chosen_KL_mean": -829.9423217773438, "KL/mean": -1009.3814697265625, "KL/rejected_KL_mean": -1188.820556640625, "KL/std": 652.5074462890625, "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.0008891418110579252, "fcm_dpo/delta": 0.0817713588476181, "fcm_dpo/margin": 358.8783264160156, "fcm_dpo/q_t": 0.4309133291244507, "grad_norm": 50.39697265625, "learning_rate": 6.453213851142225e-10, "logits/chosen": -1.052807092666626, "logits/rejected": -1.0501337051391602, "logps/chosen": -902.5463256835938, "logps/ref_chosen": -72.60400390625, "logps/ref_rejected": -103.73905944824219, "logps/rejected": -1292.5596923828125, "loss": 1.2006, "margin_dpo/margin_mean": 358.8783264160156, "margin_dpo/margin_std": 768.2645263671875, "step": 668 }, { "KL/chosen_KL_mean": -619.5028076171875, "KL/mean": -885.7294311523438, "KL/rejected_KL_mean": -1151.9560546875, "KL/std": 607.2215576171875, "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.0008850685553625226, "fcm_dpo/delta": -0.07294955104589462, "fcm_dpo/margin": 532.453369140625, "fcm_dpo/q_t": 0.39315131306648254, "grad_norm": 24.650182723999023, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.0345218181610107, "logits/rejected": -1.041704773902893, "logps/chosen": -665.6192016601562, "logps/ref_chosen": -46.116416931152344, "logps/ref_rejected": -77.92434692382812, "logps/rejected": -1229.8804931640625, "loss": 1.054, "margin_dpo/margin_mean": 532.453369140625, "margin_dpo/margin_std": 684.6708984375, "step": 669 }, { "KL/chosen_KL_mean": -620.4657592773438, "KL/mean": -882.550048828125, "KL/rejected_KL_mean": -1144.6343994140625, "KL/std": 581.8966674804688, "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.0008802937809377909, "fcm_dpo/delta": -0.06295409053564072, "fcm_dpo/margin": 524.1685791015625, "fcm_dpo/q_t": 0.3964523673057556, "grad_norm": 27.264894485473633, "learning_rate": 4.741678157389739e-10, "logits/chosen": -0.9793117642402649, "logits/rejected": -0.9908572435379028, "logps/chosen": -682.8115234375, "logps/ref_chosen": -62.34575271606445, "logps/ref_rejected": -96.9405517578125, "logps/rejected": -1241.574951171875, "loss": 1.0678, "margin_dpo/margin_mean": 524.1685791015625, "margin_dpo/margin_std": 703.9255981445312, "step": 670 }, { "KL/chosen_KL_mean": -780.59716796875, "KL/mean": -994.5520629882812, "KL/rejected_KL_mean": -1208.5069580078125, "KL/std": 590.3243408203125, "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.000880036735907197, "fcm_dpo/delta": 0.02367694117128849, "fcm_dpo/margin": 427.90972900390625, "fcm_dpo/q_t": 0.413568913936615, "grad_norm": 35.296512603759766, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.0900222063064575, "logits/rejected": -1.1154989004135132, "logps/chosen": -828.5972900390625, "logps/ref_chosen": -48.00010681152344, "logps/ref_rejected": -83.81932067871094, "logps/rejected": -1292.3262939453125, "loss": 1.1384, "margin_dpo/margin_mean": 427.90972900390625, "margin_dpo/margin_std": 720.406005859375, "step": 671 }, { "KL/chosen_KL_mean": -845.4471435546875, "KL/mean": -1062.668701171875, "KL/rejected_KL_mean": -1279.8902587890625, "KL/std": 706.8720703125, "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0008793792221695185, "fcm_dpo/delta": 0.01829908788204193, "fcm_dpo/margin": 434.44317626953125, "fcm_dpo/q_t": 0.4161950945854187, "grad_norm": 38.76650619506836, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.1482672691345215, "logits/rejected": -1.1567609310150146, "logps/chosen": -904.0303955078125, "logps/ref_chosen": -58.58328628540039, "logps/ref_rejected": -93.14015197753906, "logps/rejected": -1373.030517578125, "loss": 1.1456, "margin_dpo/margin_mean": 434.4431457519531, "margin_dpo/margin_std": 770.93408203125, "step": 672 }, { "KL/chosen_KL_mean": -725.902587890625, "KL/mean": -928.3306274414062, "KL/rejected_KL_mean": -1130.758544921875, "KL/std": 572.4488525390625, "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.0008829334983602166, "fcm_dpo/delta": 0.04332459345459938, "fcm_dpo/margin": 404.85595703125, "fcm_dpo/q_t": 0.41951626539230347, "grad_norm": 26.817668914794922, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -1.091918706893921, "logits/rejected": -1.0944968461990356, "logps/chosen": -772.6258544921875, "logps/ref_chosen": -46.72320556640625, "logps/ref_rejected": -85.29623413085938, "logps/rejected": -1216.0548095703125, "loss": 1.1323, "margin_dpo/margin_mean": 404.85595703125, "margin_dpo/margin_std": 645.776611328125, "step": 673 }, { "KL/chosen_KL_mean": -647.2359008789062, "KL/mean": -898.0692138671875, "KL/rejected_KL_mean": -1148.902587890625, "KL/std": 574.1219482421875, "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.0008830685401335359, "fcm_dpo/delta": -0.044042401015758514, "fcm_dpo/margin": 501.666748046875, "fcm_dpo/q_t": 0.4016549587249756, "grad_norm": 40.91920852661133, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -0.9943381547927856, "logits/rejected": -1.0083208084106445, "logps/chosen": -692.681396484375, "logps/ref_chosen": -45.445526123046875, "logps/ref_rejected": -70.04593658447266, "logps/rejected": -1218.948486328125, "loss": 1.075, "margin_dpo/margin_mean": 501.666748046875, "margin_dpo/margin_std": 687.1994018554688, "step": 674 }, { "KL/chosen_KL_mean": -708.86376953125, "KL/mean": -968.4271240234375, "KL/rejected_KL_mean": -1227.990478515625, "KL/std": 640.6890258789062, "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.0008735453011468053, "fcm_dpo/delta": -0.05547531321644783, "fcm_dpo/margin": 519.1265869140625, "fcm_dpo/q_t": 0.3998996317386627, "grad_norm": 34.21153259277344, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.0351924896240234, "logits/rejected": -1.0452386140823364, "logps/chosen": -753.0400390625, "logps/ref_chosen": -44.17628479003906, "logps/ref_rejected": -74.09197998046875, "logps/rejected": -1302.0823974609375, "loss": 1.07, "margin_dpo/margin_mean": 519.1265869140625, "margin_dpo/margin_std": 684.087646484375, "step": 675 }, { "KL/chosen_KL_mean": -776.6356201171875, "KL/mean": -1018.5604858398438, "KL/rejected_KL_mean": -1260.4853515625, "KL/std": 617.2457885742188, "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.0008749394328333437, "fcm_dpo/delta": -0.023866407573223114, "fcm_dpo/margin": 483.8497619628906, "fcm_dpo/q_t": 0.40422454476356506, "grad_norm": 28.231639862060547, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.0005314350128174, "logits/rejected": -1.0063731670379639, "logps/chosen": -848.0341186523438, "logps/ref_chosen": -71.39852905273438, "logps/ref_rejected": -88.3587646484375, "logps/rejected": -1348.8441162109375, "loss": 1.0782, "margin_dpo/margin_mean": 483.8497314453125, "margin_dpo/margin_std": 629.48583984375, "step": 676 }, { "KL/chosen_KL_mean": -752.8453979492188, "KL/mean": -1003.5128173828125, "KL/rejected_KL_mean": -1254.1802978515625, "KL/std": 636.5200805664062, "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.000870502321049571, "fcm_dpo/delta": -0.037262238562107086, "fcm_dpo/margin": 501.3348693847656, "fcm_dpo/q_t": 0.4067116677761078, "grad_norm": 33.513336181640625, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.0794928073883057, "logits/rejected": -1.0712941884994507, "logps/chosen": -809.372802734375, "logps/ref_chosen": -56.527435302734375, "logps/ref_rejected": -78.22654724121094, "logps/rejected": -1332.40673828125, "loss": 1.0906, "margin_dpo/margin_mean": 501.3348693847656, "margin_dpo/margin_std": 767.4678344726562, "step": 677 }, { "KL/chosen_KL_mean": -626.833251953125, "KL/mean": -919.983642578125, "KL/rejected_KL_mean": -1213.134033203125, "KL/std": 699.7562255859375, "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.0008654722478240728, "fcm_dpo/delta": -0.1102285087108612, "fcm_dpo/margin": 586.30078125, "fcm_dpo/q_t": 0.3892865777015686, "grad_norm": 35.17583465576172, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.0084434747695923, "logits/rejected": -1.0442242622375488, "logps/chosen": -672.9677734375, "logps/ref_chosen": -46.13447570800781, "logps/ref_rejected": -80.60462951660156, "logps/rejected": -1293.73876953125, "loss": 1.0515, "margin_dpo/margin_mean": 586.30078125, "margin_dpo/margin_std": 820.827392578125, "step": 678 }, { "KL/chosen_KL_mean": -740.3648681640625, "KL/mean": -935.5812377929688, "KL/rejected_KL_mean": -1130.797607421875, "KL/std": 556.906005859375, "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.000864343426655978, "fcm_dpo/delta": 0.06362688541412354, "fcm_dpo/margin": 390.4327087402344, "fcm_dpo/q_t": 0.4237492084503174, "grad_norm": 33.905616760253906, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -1.0304598808288574, "logits/rejected": -1.0171773433685303, "logps/chosen": -790.6597900390625, "logps/ref_chosen": -50.294921875, "logps/ref_rejected": -76.59813690185547, "logps/rejected": -1207.395751953125, "loss": 1.1458, "margin_dpo/margin_mean": 390.4327087402344, "margin_dpo/margin_std": 631.3106079101562, "step": 679 }, { "KL/chosen_KL_mean": -745.6943359375, "KL/mean": -1011.821533203125, "KL/rejected_KL_mean": -1277.94873046875, "KL/std": 722.32958984375, "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.0008604623144492507, "fcm_dpo/delta": -0.059656865894794464, "fcm_dpo/margin": 532.2544555664062, "fcm_dpo/q_t": 0.39625048637390137, "grad_norm": 34.24609375, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.0160003900527954, "logits/rejected": -1.0428314208984375, "logps/chosen": -822.6099853515625, "logps/ref_chosen": -76.91569519042969, "logps/ref_rejected": -112.384765625, "logps/rejected": -1390.33349609375, "loss": 1.0828, "margin_dpo/margin_mean": 532.2544555664062, "margin_dpo/margin_std": 783.269287109375, "step": 680 }, { "KL/chosen_KL_mean": -750.117919921875, "KL/mean": -952.0081787109375, "KL/rejected_KL_mean": -1153.8985595703125, "KL/std": 590.4039306640625, "epoch": 1.0, "fcm_dpo/beta": 0.0008671648683957756, "fcm_dpo/delta": 0.05011419206857681, "fcm_dpo/margin": 403.78057861328125, "fcm_dpo/q_t": 0.4205058813095093, "grad_norm": 33.01895523071289, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -1.0952690839767456, "logits/rejected": -1.1073505878448486, "logps/chosen": -811.0751953125, "logps/ref_chosen": -60.957279205322266, "logps/ref_rejected": -88.55797576904297, "logps/rejected": -1242.45654296875, "loss": 1.1452, "margin_dpo/margin_mean": 403.7805480957031, "margin_dpo/margin_std": 639.1034545898438, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0870012780118095, "train_runtime": 1735.4876, "train_samples_per_second": 25.121, "train_steps_per_second": 0.392 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }