{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 681, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0014684287812041115, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06123492121696472, "fcm_dpo/q_t": 0.4984707534313202, "grad_norm": 38.81780242919922, "learning_rate": 0.0, "logits/chosen": 0.2677389979362488, "logits/rejected": 0.6191319227218628, "logps/chosen": -85.37664031982422, "logps/ref_chosen": -85.43083190917969, "logps/ref_rejected": -79.90458679199219, "logps/rejected": -79.91163635253906, "loss": 1.3807, "margin_dpo/margin_mean": 0.061235010623931885, "margin_dpo/margin_std": 0.4424324035644531, "step": 1 }, { "epoch": 0.002936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.12442219257354736, "fcm_dpo/q_t": 0.5031089186668396, "grad_norm": 29.75464630126953, "learning_rate": 7.246376811594203e-09, "logits/chosen": 0.47771453857421875, "logits/rejected": 0.5908545851707458, "logps/chosen": -82.11383056640625, "logps/ref_chosen": -82.06893157958984, "logps/ref_rejected": -81.65457916259766, "logps/rejected": -81.57505798339844, "loss": 1.3992, "margin_dpo/margin_mean": -0.12442246079444885, "margin_dpo/margin_std": 0.41358208656311035, "step": 2 }, { "epoch": 0.004405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.014965459704399109, "fcm_dpo/q_t": 0.4996257424354553, "grad_norm": 26.07487678527832, "learning_rate": 1.4492753623188406e-08, "logits/chosen": 0.4268788695335388, "logits/rejected": 0.6501424312591553, "logps/chosen": -93.79121398925781, "logps/ref_chosen": -93.81098937988281, "logps/ref_rejected": -74.22950744628906, "logps/rejected": -74.22470092773438, "loss": 1.3853, "margin_dpo/margin_mean": 0.014965400099754333, "margin_dpo/margin_std": 0.45240455865859985, "step": 3 }, { "epoch": 0.005873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02085861563682556, "fcm_dpo/q_t": 0.4994773864746094, "grad_norm": 32.71585464477539, "learning_rate": 2.1739130434782606e-08, "logits/chosen": 0.6463205814361572, "logits/rejected": 0.7779839038848877, "logps/chosen": -87.30850219726562, "logps/ref_chosen": -87.29246520996094, "logps/ref_rejected": -93.82425689697266, "logps/rejected": -93.86114501953125, "loss": 1.3846, "margin_dpo/margin_mean": 0.020858705043792725, "margin_dpo/margin_std": 0.39117318391799927, "step": 4 }, { "epoch": 0.007342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.050249382853507996, "fcm_dpo/q_t": 0.4987432062625885, "grad_norm": 35.20225143432617, "learning_rate": 2.898550724637681e-08, "logits/chosen": 0.4468464255332947, "logits/rejected": 0.6582976579666138, "logps/chosen": -89.32503509521484, "logps/ref_chosen": -89.33676147460938, "logps/ref_rejected": -88.74783325195312, "logps/rejected": -88.78636169433594, "loss": 1.3818, "margin_dpo/margin_mean": 0.050250083208084106, "margin_dpo/margin_std": 0.44784289598464966, "step": 5 }, { "epoch": 0.00881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01285526156425476, "fcm_dpo/q_t": 0.5003200173377991, "grad_norm": 36.15748596191406, "learning_rate": 3.6231884057971014e-08, "logits/chosen": 0.48705005645751953, "logits/rejected": 0.803049623966217, "logps/chosen": -97.32769775390625, "logps/ref_chosen": -97.32147216796875, "logps/ref_rejected": -97.88345336914062, "logps/rejected": -97.87681579589844, "loss": 1.3881, "margin_dpo/margin_mean": -0.012855678796768188, "margin_dpo/margin_std": 0.42675212025642395, "step": 6 }, { "epoch": 0.010279001468428781, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06436635553836823, "fcm_dpo/q_t": 0.49839529395103455, "grad_norm": 37.96323776245117, "learning_rate": 4.347826086956521e-08, "logits/chosen": 0.49750369787216187, "logits/rejected": 0.5453470945358276, "logps/chosen": -86.62813568115234, "logps/ref_chosen": -86.64852905273438, "logps/ref_rejected": -109.61618041992188, "logps/rejected": -109.66015625, "loss": 1.3806, "margin_dpo/margin_mean": 0.06436613202095032, "margin_dpo/margin_std": 0.5262112021446228, "step": 7 }, { "epoch": 0.011747430249632892, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.1121322363615036, "fcm_dpo/q_t": 0.5028019547462463, "grad_norm": 32.900169372558594, "learning_rate": 5.0724637681159424e-08, "logits/chosen": 0.436749666929245, "logits/rejected": 0.7514491677284241, "logps/chosen": -90.00808715820312, "logps/ref_chosen": -89.9236831665039, "logps/ref_rejected": -86.22804260253906, "logps/rejected": -86.20030975341797, "loss": 1.398, "margin_dpo/margin_mean": -0.11213211715221405, "margin_dpo/margin_std": 0.4109366536140442, "step": 8 }, { "epoch": 0.013215859030837005, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02080167829990387, "fcm_dpo/q_t": 0.499479740858078, "grad_norm": 36.028873443603516, "learning_rate": 5.797101449275362e-08, "logits/chosen": 0.661954939365387, "logits/rejected": 0.8412786722183228, "logps/chosen": -103.87676239013672, "logps/ref_chosen": -103.85713195800781, "logps/ref_rejected": -104.31932067871094, "logps/rejected": -104.35975646972656, "loss": 1.3848, "margin_dpo/margin_mean": 0.020801380276679993, "margin_dpo/margin_std": 0.4883061647415161, "step": 9 }, { "epoch": 0.014684287812041116, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07037302851676941, "fcm_dpo/q_t": 0.5017586350440979, "grad_norm": 33.41664505004883, "learning_rate": 6.521739130434782e-08, "logits/chosen": 0.6513626575469971, "logits/rejected": 0.870223879814148, "logps/chosen": -76.17118072509766, "logps/ref_chosen": -76.20436096191406, "logps/ref_rejected": -87.152099609375, "logps/rejected": -87.04855346679688, "loss": 1.3939, "margin_dpo/margin_mean": -0.07037210464477539, "margin_dpo/margin_std": 0.4575299322605133, "step": 10 }, { "epoch": 0.016152716593245228, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.053068071603775024, "fcm_dpo/q_t": 0.49867451190948486, "grad_norm": 36.69655990600586, "learning_rate": 7.246376811594203e-08, "logits/chosen": 0.277323842048645, "logits/rejected": 0.4923889636993408, "logps/chosen": -82.32759094238281, "logps/ref_chosen": -82.36650085449219, "logps/ref_rejected": -94.26461791992188, "logps/rejected": -94.2787857055664, "loss": 1.3814, "margin_dpo/margin_mean": 0.053068608045578, "margin_dpo/margin_std": 0.41766154766082764, "step": 11 }, { "epoch": 0.01762114537444934, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01172947883605957, "fcm_dpo/q_t": 0.4997062683105469, "grad_norm": 40.26313781738281, "learning_rate": 7.971014492753623e-08, "logits/chosen": 0.40038958191871643, "logits/rejected": 0.5286105871200562, "logps/chosen": -99.0933837890625, "logps/ref_chosen": -99.10549926757812, "logps/ref_rejected": -110.27140808105469, "logps/rejected": -110.27102661132812, "loss": 1.3858, "margin_dpo/margin_mean": 0.011729925870895386, "margin_dpo/margin_std": 0.5183511972427368, "step": 12 }, { "epoch": 0.01908957415565345, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.059926122426986694, "fcm_dpo/q_t": 0.501498281955719, "grad_norm": 41.93380355834961, "learning_rate": 8.695652173913042e-08, "logits/chosen": 0.5050075054168701, "logits/rejected": 0.7639053463935852, "logps/chosen": -90.6314697265625, "logps/ref_chosen": -90.55972290039062, "logps/ref_rejected": -93.69110107421875, "logps/rejected": -93.70291137695312, "loss": 1.3928, "margin_dpo/margin_mean": -0.05992648005485535, "margin_dpo/margin_std": 0.4534745216369629, "step": 13 }, { "epoch": 0.020558002936857563, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.019990622997283936, "fcm_dpo/q_t": 0.500498354434967, "grad_norm": 35.67725372314453, "learning_rate": 9.420289855072464e-08, "logits/chosen": 1.0646815299987793, "logits/rejected": 1.1590429544448853, "logps/chosen": -99.85965728759766, "logps/ref_chosen": -99.82717895507812, "logps/ref_rejected": -108.94200134277344, "logps/rejected": -108.95449829101562, "loss": 1.3888, "margin_dpo/margin_mean": -0.01999080181121826, "margin_dpo/margin_std": 0.4524303078651428, "step": 14 }, { "epoch": 0.022026431718061675, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.06978121399879456, "fcm_dpo/q_t": 0.49825772643089294, "grad_norm": 31.65287971496582, "learning_rate": 1.0144927536231885e-07, "logits/chosen": 0.4620896875858307, "logits/rejected": 0.6329752802848816, "logps/chosen": -78.88731384277344, "logps/ref_chosen": -78.90997314453125, "logps/ref_rejected": -90.06234741210938, "logps/rejected": -90.10946655273438, "loss": 1.3799, "margin_dpo/margin_mean": 0.06978106498718262, "margin_dpo/margin_std": 0.4777568280696869, "step": 15 }, { "epoch": 0.023494860499265784, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01587851345539093, "fcm_dpo/q_t": 0.49960121512413025, "grad_norm": 34.593482971191406, "learning_rate": 1.0869565217391303e-07, "logits/chosen": 0.7190598249435425, "logits/rejected": 0.9916927814483643, "logps/chosen": -97.41981506347656, "logps/ref_chosen": -97.42327880859375, "logps/ref_rejected": -90.59945678710938, "logps/rejected": -90.61186981201172, "loss": 1.3855, "margin_dpo/margin_mean": 0.015878498554229736, "margin_dpo/margin_std": 0.5510396361351013, "step": 16 }, { "epoch": 0.024963289280469897, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.08168734610080719, "fcm_dpo/q_t": 0.49796023964881897, "grad_norm": 35.512939453125, "learning_rate": 1.1594202898550725e-07, "logits/chosen": 0.4151402413845062, "logits/rejected": 0.6361797451972961, "logps/chosen": -104.31832122802734, "logps/ref_chosen": -104.36430358886719, "logps/ref_rejected": -90.46772766113281, "logps/rejected": -90.50341796875, "loss": 1.3787, "margin_dpo/margin_mean": 0.08168637752532959, "margin_dpo/margin_std": 0.47371718287467957, "step": 17 }, { "epoch": 0.02643171806167401, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03281433880329132, "fcm_dpo/q_t": 0.5008206367492676, "grad_norm": 42.083641052246094, "learning_rate": 1.2318840579710146e-07, "logits/chosen": 0.47482335567474365, "logits/rejected": 0.8290694355964661, "logps/chosen": -87.07789611816406, "logps/ref_chosen": -87.09195709228516, "logps/ref_rejected": -81.85072326660156, "logps/rejected": -81.80384826660156, "loss": 1.3903, "margin_dpo/margin_mean": -0.03281354904174805, "margin_dpo/margin_std": 0.543687105178833, "step": 18 }, { "epoch": 0.027900146842878122, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.07938456535339355, "fcm_dpo/q_t": 0.501983642578125, "grad_norm": 31.655494689941406, "learning_rate": 1.3043478260869563e-07, "logits/chosen": 0.5405235886573792, "logits/rejected": 0.8114136457443237, "logps/chosen": -105.96595764160156, "logps/ref_chosen": -105.87354278564453, "logps/ref_rejected": -96.93023681640625, "logps/rejected": -96.94326782226562, "loss": 1.3946, "margin_dpo/margin_mean": -0.07938426733016968, "margin_dpo/margin_std": 0.3811599910259247, "step": 19 }, { "epoch": 0.02936857562408223, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.025069117546081543, "fcm_dpo/q_t": 0.4993741512298584, "grad_norm": 34.66242599487305, "learning_rate": 1.3768115942028986e-07, "logits/chosen": 0.23079314827919006, "logits/rejected": 0.5058917999267578, "logps/chosen": -90.76351928710938, "logps/ref_chosen": -90.75811767578125, "logps/ref_rejected": -85.91232299804688, "logps/rejected": -85.94279479980469, "loss": 1.3844, "margin_dpo/margin_mean": 0.025068938732147217, "margin_dpo/margin_std": 0.4899085462093353, "step": 20 }, { "epoch": 0.030837004405286344, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.091633141040802, "fcm_dpo/q_t": 0.4977106750011444, "grad_norm": 31.416053771972656, "learning_rate": 1.4492753623188405e-07, "logits/chosen": 0.5428293347358704, "logits/rejected": 0.7209779620170593, "logps/chosen": -80.27669525146484, "logps/ref_chosen": -80.33345794677734, "logps/ref_rejected": -83.93373107910156, "logps/rejected": -83.9686050415039, "loss": 1.3776, "margin_dpo/margin_mean": 0.09163343906402588, "margin_dpo/margin_std": 0.40345489978790283, "step": 21 }, { "epoch": 0.032305433186490456, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0237371027469635, "fcm_dpo/q_t": 0.4994067847728729, "grad_norm": 42.76321029663086, "learning_rate": 1.5217391304347825e-07, "logits/chosen": 0.4888147711753845, "logits/rejected": 0.6019254326820374, "logps/chosen": -95.3747329711914, "logps/ref_chosen": -95.39530181884766, "logps/ref_rejected": -103.47351837158203, "logps/rejected": -103.47669219970703, "loss": 1.3843, "margin_dpo/margin_mean": 0.02373659610748291, "margin_dpo/margin_std": 0.41101372241973877, "step": 22 }, { "epoch": 0.033773861967694566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.054152026772499084, "fcm_dpo/q_t": 0.5013521909713745, "grad_norm": 31.53017807006836, "learning_rate": 1.5942028985507245e-07, "logits/chosen": 0.7403199076652527, "logits/rejected": 0.9192092418670654, "logps/chosen": -90.61726379394531, "logps/ref_chosen": -90.63751220703125, "logps/ref_rejected": -86.59425354003906, "logps/rejected": -86.51985168457031, "loss": 1.3921, "margin_dpo/margin_mean": -0.05415254831314087, "margin_dpo/margin_std": 0.4045397639274597, "step": 23 }, { "epoch": 0.03524229074889868, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.038981035351753235, "fcm_dpo/q_t": 0.4990260601043701, "grad_norm": 44.209590911865234, "learning_rate": 1.6666666666666665e-07, "logits/chosen": 0.6239376068115234, "logits/rejected": 0.6544588804244995, "logps/chosen": -69.91582489013672, "logps/ref_chosen": -69.91728973388672, "logps/ref_rejected": -106.60990142822266, "logps/rejected": -106.64741516113281, "loss": 1.383, "margin_dpo/margin_mean": 0.03898155689239502, "margin_dpo/margin_std": 0.4552841782569885, "step": 24 }, { "epoch": 0.03671071953010279, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.050283268094062805, "fcm_dpo/q_t": 0.4987441301345825, "grad_norm": 36.85416030883789, "learning_rate": 1.7391304347826085e-07, "logits/chosen": 0.5321314334869385, "logits/rejected": 0.7012242078781128, "logps/chosen": -80.85140991210938, "logps/ref_chosen": -80.82548522949219, "logps/ref_rejected": -95.95710754394531, "logps/rejected": -96.0333251953125, "loss": 1.3816, "margin_dpo/margin_mean": 0.05028343200683594, "margin_dpo/margin_std": 0.37170085310935974, "step": 25 }, { "epoch": 0.0381791483113069, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07252025604248047, "fcm_dpo/q_t": 0.4981876611709595, "grad_norm": 40.67619705200195, "learning_rate": 1.8115942028985507e-07, "logits/chosen": 0.36407941579818726, "logits/rejected": 0.5750868320465088, "logps/chosen": -88.88731384277344, "logps/ref_chosen": -88.90116882324219, "logps/ref_rejected": -109.82819366455078, "logps/rejected": -109.88685607910156, "loss": 1.3795, "margin_dpo/margin_mean": 0.072520911693573, "margin_dpo/margin_std": 0.4113643765449524, "step": 26 }, { "epoch": 0.039647577092511016, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0897430032491684, "fcm_dpo/q_t": 0.4977561831474304, "grad_norm": 44.181434631347656, "learning_rate": 1.8840579710144927e-07, "logits/chosen": 0.2009836733341217, "logits/rejected": 0.38905927538871765, "logps/chosen": -77.58397674560547, "logps/ref_chosen": -77.59600830078125, "logps/ref_rejected": -103.93850708007812, "logps/rejected": -104.0162124633789, "loss": 1.3779, "margin_dpo/margin_mean": 0.08974292874336243, "margin_dpo/margin_std": 0.4670272469520569, "step": 27 }, { "epoch": 0.041116005873715125, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12937551736831665, "fcm_dpo/q_t": 0.49676674604415894, "grad_norm": 36.718238830566406, "learning_rate": 1.9565217391304347e-07, "logits/chosen": 0.7008620500564575, "logits/rejected": 0.9329044818878174, "logps/chosen": -102.15823364257812, "logps/ref_chosen": -102.22856903076172, "logps/ref_rejected": -96.9594955444336, "logps/rejected": -97.01853942871094, "loss": 1.3738, "margin_dpo/margin_mean": 0.12937626242637634, "margin_dpo/margin_std": 0.42088428139686584, "step": 28 }, { "epoch": 0.042584434654919234, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11628834903240204, "fcm_dpo/q_t": 0.49709412455558777, "grad_norm": 42.49556350708008, "learning_rate": 2.028985507246377e-07, "logits/chosen": 0.37622809410095215, "logits/rejected": 0.638656497001648, "logps/chosen": -88.61930084228516, "logps/ref_chosen": -88.64704895019531, "logps/ref_rejected": -102.96011352539062, "logps/rejected": -103.04864501953125, "loss": 1.3753, "margin_dpo/margin_mean": 0.11628936231136322, "margin_dpo/margin_std": 0.48608508706092834, "step": 29 }, { "epoch": 0.04405286343612335, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11767734587192535, "fcm_dpo/q_t": 0.49706411361694336, "grad_norm": 39.036170959472656, "learning_rate": 2.1014492753623187e-07, "logits/chosen": 0.8277263045310974, "logits/rejected": 0.9531229734420776, "logps/chosen": -88.38198852539062, "logps/ref_chosen": -88.38838195800781, "logps/ref_rejected": -102.31889343261719, "logps/rejected": -102.43018341064453, "loss": 1.3752, "margin_dpo/margin_mean": 0.11767733097076416, "margin_dpo/margin_std": 0.5070103406906128, "step": 30 }, { "epoch": 0.04552129221732746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05853770673274994, "fcm_dpo/q_t": 0.49853795766830444, "grad_norm": 31.66895866394043, "learning_rate": 2.1739130434782607e-07, "logits/chosen": 0.718494713306427, "logits/rejected": 0.9628149271011353, "logps/chosen": -101.16627502441406, "logps/ref_chosen": -101.12565612792969, "logps/ref_rejected": -79.8584213256836, "logps/rejected": -79.95758056640625, "loss": 1.381, "margin_dpo/margin_mean": 0.05853772163391113, "margin_dpo/margin_std": 0.4458237588405609, "step": 31 }, { "epoch": 0.04698972099853157, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.14585043489933014, "fcm_dpo/q_t": 0.4963546395301819, "grad_norm": 35.37748718261719, "learning_rate": 2.2463768115942027e-07, "logits/chosen": 0.3660997748374939, "logits/rejected": 0.5037564635276794, "logps/chosen": -96.66069030761719, "logps/ref_chosen": -96.61702728271484, "logps/ref_rejected": -96.224365234375, "logps/rejected": -96.41387176513672, "loss": 1.3723, "margin_dpo/margin_mean": 0.14585140347480774, "margin_dpo/margin_std": 0.44825515151023865, "step": 32 }, { "epoch": 0.048458149779735685, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16191767156124115, "fcm_dpo/q_t": 0.4959545135498047, "grad_norm": 33.53567123413086, "learning_rate": 2.318840579710145e-07, "logits/chosen": 0.5710635185241699, "logits/rejected": 0.7510501742362976, "logps/chosen": -81.49163055419922, "logps/ref_chosen": -81.52101135253906, "logps/ref_rejected": -93.80595397949219, "logps/rejected": -93.93849182128906, "loss": 1.3709, "margin_dpo/margin_mean": 0.1619182527065277, "margin_dpo/margin_std": 0.5155723094940186, "step": 33 }, { "epoch": 0.049926578560939794, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19678400456905365, "fcm_dpo/q_t": 0.4950863718986511, "grad_norm": 40.77206802368164, "learning_rate": 2.391304347826087e-07, "logits/chosen": 0.5938290357589722, "logits/rejected": 0.7664215564727783, "logps/chosen": -77.15574645996094, "logps/ref_chosen": -77.20204162597656, "logps/ref_rejected": -106.71876525878906, "logps/rejected": -106.86925506591797, "loss": 1.3673, "margin_dpo/margin_mean": 0.19678422808647156, "margin_dpo/margin_std": 0.4836421608924866, "step": 34 }, { "epoch": 0.0513950073421439, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1951250433921814, "fcm_dpo/q_t": 0.4951246976852417, "grad_norm": 42.55626678466797, "learning_rate": 2.463768115942029e-07, "logits/chosen": 0.1299372911453247, "logits/rejected": 0.2869094908237457, "logps/chosen": -77.56372833251953, "logps/ref_chosen": -77.57035064697266, "logps/ref_rejected": -112.18855285644531, "logps/rejected": -112.3770523071289, "loss": 1.3673, "margin_dpo/margin_mean": 0.1951252818107605, "margin_dpo/margin_std": 0.41524142026901245, "step": 35 }, { "epoch": 0.05286343612334802, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12170365452766418, "fcm_dpo/q_t": 0.4969629645347595, "grad_norm": 32.65164566040039, "learning_rate": 2.536231884057971e-07, "logits/chosen": 0.40253281593322754, "logits/rejected": 0.5915440320968628, "logps/chosen": -83.32685089111328, "logps/ref_chosen": -83.28823852539062, "logps/ref_rejected": -91.75741577148438, "logps/rejected": -91.91773223876953, "loss": 1.375, "margin_dpo/margin_mean": 0.12170398235321045, "margin_dpo/margin_std": 0.55162113904953, "step": 36 }, { "epoch": 0.05433186490455213, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3155759572982788, "fcm_dpo/q_t": 0.4921198785305023, "grad_norm": 36.134918212890625, "learning_rate": 2.6086956521739126e-07, "logits/chosen": 0.5907583236694336, "logits/rejected": 0.7441189885139465, "logps/chosen": -94.78392028808594, "logps/ref_chosen": -94.77108764648438, "logps/ref_rejected": -85.71726989746094, "logps/rejected": -86.04566955566406, "loss": 1.3558, "margin_dpo/margin_mean": 0.3155764639377594, "margin_dpo/margin_std": 0.5700342655181885, "step": 37 }, { "epoch": 0.055800293685756244, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19760102033615112, "fcm_dpo/q_t": 0.4950709939002991, "grad_norm": 37.12024688720703, "learning_rate": 2.681159420289855e-07, "logits/chosen": 0.5169904232025146, "logits/rejected": 0.6563962697982788, "logps/chosen": -75.88797760009766, "logps/ref_chosen": -75.97850036621094, "logps/ref_rejected": -104.10401916503906, "logps/rejected": -104.21109008789062, "loss": 1.3675, "margin_dpo/margin_mean": 0.197601780295372, "margin_dpo/margin_std": 0.5769727230072021, "step": 38 }, { "epoch": 0.05726872246696035, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3096907138824463, "fcm_dpo/q_t": 0.492266446352005, "grad_norm": 36.1661376953125, "learning_rate": 2.753623188405797e-07, "logits/chosen": 0.6212589740753174, "logits/rejected": 0.818722128868103, "logps/chosen": -81.21092987060547, "logps/ref_chosen": -81.18577575683594, "logps/ref_rejected": -84.1959228515625, "logps/rejected": -84.53076171875, "loss": 1.3563, "margin_dpo/margin_mean": 0.30969101190567017, "margin_dpo/margin_std": 0.5184357762336731, "step": 39 }, { "epoch": 0.05873715124816446, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24696290493011475, "fcm_dpo/q_t": 0.49383074045181274, "grad_norm": 29.007984161376953, "learning_rate": 2.8260869565217386e-07, "logits/chosen": 0.6268089413642883, "logits/rejected": 0.8235390186309814, "logps/chosen": -83.29570007324219, "logps/ref_chosen": -83.33257293701172, "logps/ref_rejected": -80.25591278076172, "logps/rejected": -80.46599578857422, "loss": 1.3624, "margin_dpo/margin_mean": 0.24696248769760132, "margin_dpo/margin_std": 0.489422082901001, "step": 40 }, { "epoch": 0.06020558002936858, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3290652930736542, "fcm_dpo/q_t": 0.4917981028556824, "grad_norm": 33.2258415222168, "learning_rate": 2.898550724637681e-07, "logits/chosen": 0.6201198697090149, "logits/rejected": 0.8052632808685303, "logps/chosen": -93.12638854980469, "logps/ref_chosen": -93.14866638183594, "logps/ref_rejected": -102.07920837402344, "logps/rejected": -102.38600158691406, "loss": 1.3551, "margin_dpo/margin_mean": 0.32906582951545715, "margin_dpo/margin_std": 0.7185872197151184, "step": 41 }, { "epoch": 0.06167400881057269, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4246216416358948, "fcm_dpo/q_t": 0.4893936514854431, "grad_norm": 40.833499908447266, "learning_rate": 2.971014492753623e-07, "logits/chosen": 0.3932719826698303, "logits/rejected": 0.5345179438591003, "logps/chosen": -90.62908172607422, "logps/ref_chosen": -90.70162200927734, "logps/ref_rejected": -113.8646469116211, "logps/rejected": -114.21672821044922, "loss": 1.3449, "margin_dpo/margin_mean": 0.4246218204498291, "margin_dpo/margin_std": 0.5078005790710449, "step": 42 }, { "epoch": 0.0631424375917768, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4403374493122101, "fcm_dpo/q_t": 0.4890177845954895, "grad_norm": 37.972869873046875, "learning_rate": 3.043478260869565e-07, "logits/chosen": 0.5793946981430054, "logits/rejected": 0.7695983052253723, "logps/chosen": -89.6876220703125, "logps/ref_chosen": -89.64402770996094, "logps/ref_rejected": -103.95185852050781, "logps/rejected": -104.435791015625, "loss": 1.3441, "margin_dpo/margin_mean": 0.440337598323822, "margin_dpo/margin_std": 0.7237739562988281, "step": 43 }, { "epoch": 0.06461086637298091, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3891373872756958, "fcm_dpo/q_t": 0.49028751254081726, "grad_norm": 34.31340026855469, "learning_rate": 3.115942028985507e-07, "logits/chosen": 0.8794394135475159, "logits/rejected": 1.000215768814087, "logps/chosen": -81.93470764160156, "logps/ref_chosen": -81.8783187866211, "logps/ref_rejected": -113.421630859375, "logps/rejected": -113.86715698242188, "loss": 1.3487, "margin_dpo/margin_mean": 0.3891381621360779, "margin_dpo/margin_std": 0.6131525039672852, "step": 44 }, { "epoch": 0.06607929515418502, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.1822262555360794, "fcm_dpo/q_t": 0.49545037746429443, "grad_norm": 24.454357147216797, "learning_rate": 3.188405797101449e-07, "logits/chosen": 0.6332857608795166, "logits/rejected": 0.7449449300765991, "logps/chosen": -77.44635009765625, "logps/ref_chosen": -77.34459686279297, "logps/ref_rejected": -84.24774169921875, "logps/rejected": -84.53172302246094, "loss": 1.3691, "margin_dpo/margin_mean": 0.18222709000110626, "margin_dpo/margin_std": 0.5951623916625977, "step": 45 }, { "epoch": 0.06754772393538913, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.48641544580459595, "fcm_dpo/q_t": 0.4878547787666321, "grad_norm": 32.93342208862305, "learning_rate": 3.260869565217391e-07, "logits/chosen": 0.5085247755050659, "logits/rejected": 0.7518465518951416, "logps/chosen": -90.32666015625, "logps/ref_chosen": -90.3408203125, "logps/ref_rejected": -100.676513671875, "logps/rejected": -101.14876556396484, "loss": 1.339, "margin_dpo/margin_mean": 0.4864157736301422, "margin_dpo/margin_std": 0.5446327328681946, "step": 46 }, { "epoch": 0.06901615271659324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3937411308288574, "fcm_dpo/q_t": 0.49017661809921265, "grad_norm": 34.04849624633789, "learning_rate": 3.333333333333333e-07, "logits/chosen": 0.45205074548721313, "logits/rejected": 0.6340922117233276, "logps/chosen": -104.42767333984375, "logps/ref_chosen": -104.41130065917969, "logps/ref_rejected": -101.25489807128906, "logps/rejected": -101.6650161743164, "loss": 1.3487, "margin_dpo/margin_mean": 0.39373981952667236, "margin_dpo/margin_std": 0.7378332614898682, "step": 47 }, { "epoch": 0.07048458149779736, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6404982805252075, "fcm_dpo/q_t": 0.4840244948863983, "grad_norm": 38.5645751953125, "learning_rate": 3.4057971014492755e-07, "logits/chosen": 0.29659873247146606, "logits/rejected": 0.5025741457939148, "logps/chosen": -91.0909423828125, "logps/ref_chosen": -91.10027313232422, "logps/ref_rejected": -95.08058166503906, "logps/rejected": -95.71174621582031, "loss": 1.3251, "margin_dpo/margin_mean": 0.6404985189437866, "margin_dpo/margin_std": 0.856590986251831, "step": 48 }, { "epoch": 0.07195301027900147, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7378369569778442, "fcm_dpo/q_t": 0.4816128611564636, "grad_norm": 39.67837905883789, "learning_rate": 3.478260869565217e-07, "logits/chosen": 0.10201732814311981, "logits/rejected": 0.5736548900604248, "logps/chosen": -93.02400207519531, "logps/ref_chosen": -93.00367736816406, "logps/ref_rejected": -91.74899291992188, "logps/rejected": -92.50715637207031, "loss": 1.316, "margin_dpo/margin_mean": 0.7378373742103577, "margin_dpo/margin_std": 0.9256041646003723, "step": 49 }, { "epoch": 0.07342143906020558, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7001360654830933, "fcm_dpo/q_t": 0.4825558662414551, "grad_norm": 33.148658752441406, "learning_rate": 3.5507246376811595e-07, "logits/chosen": 0.5446861982345581, "logits/rejected": 0.7148386240005493, "logps/chosen": -94.72885131835938, "logps/ref_chosen": -94.62681579589844, "logps/ref_rejected": -103.57435607910156, "logps/rejected": -104.37653350830078, "loss": 1.3197, "margin_dpo/margin_mean": 0.7001360654830933, "margin_dpo/margin_std": 0.9148711562156677, "step": 50 }, { "epoch": 0.07488986784140969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6637753248214722, "fcm_dpo/q_t": 0.48347365856170654, "grad_norm": 30.014331817626953, "learning_rate": 3.6231884057971015e-07, "logits/chosen": 0.7485964298248291, "logits/rejected": 0.8959745168685913, "logps/chosen": -87.53605651855469, "logps/ref_chosen": -87.50727844238281, "logps/ref_rejected": -83.47235107421875, "logps/rejected": -84.16490936279297, "loss": 1.3236, "margin_dpo/margin_mean": 0.6637749671936035, "margin_dpo/margin_std": 1.0224616527557373, "step": 51 }, { "epoch": 0.0763582966226138, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.873137354850769, "fcm_dpo/q_t": 0.47824588418006897, "grad_norm": 32.00587844848633, "learning_rate": 3.695652173913043e-07, "logits/chosen": 0.49871087074279785, "logits/rejected": 0.7171226739883423, "logps/chosen": -90.64942169189453, "logps/ref_chosen": -90.63026428222656, "logps/ref_rejected": -87.0390625, "logps/rejected": -87.93136596679688, "loss": 1.3029, "margin_dpo/margin_mean": 0.8731374740600586, "margin_dpo/margin_std": 0.9018590450286865, "step": 52 }, { "epoch": 0.07782672540381791, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1096992492675781, "fcm_dpo/q_t": 0.47249969840049744, "grad_norm": 39.590824127197266, "learning_rate": 3.7681159420289855e-07, "logits/chosen": 0.07211381196975708, "logits/rejected": 0.1840796172618866, "logps/chosen": -81.551025390625, "logps/ref_chosen": -81.58307647705078, "logps/ref_rejected": -95.66152954101562, "logps/rejected": -96.73918151855469, "loss": 1.2831, "margin_dpo/margin_mean": 1.1096991300582886, "margin_dpo/margin_std": 1.3239152431488037, "step": 53 }, { "epoch": 0.07929515418502203, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1452147960662842, "fcm_dpo/q_t": 0.4715806841850281, "grad_norm": 39.977752685546875, "learning_rate": 3.8405797101449274e-07, "logits/chosen": 0.47239190340042114, "logits/rejected": 0.6545497179031372, "logps/chosen": -88.91471099853516, "logps/ref_chosen": -88.91017150878906, "logps/ref_rejected": -99.1175537109375, "logps/rejected": -100.2673110961914, "loss": 1.2792, "margin_dpo/margin_mean": 1.1452147960662842, "margin_dpo/margin_std": 1.2928842306137085, "step": 54 }, { "epoch": 0.08076358296622614, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.180344820022583, "fcm_dpo/q_t": 0.4708316922187805, "grad_norm": 34.956790924072266, "learning_rate": 3.9130434782608694e-07, "logits/chosen": 0.10857650637626648, "logits/rejected": 0.4178081750869751, "logps/chosen": -92.31873321533203, "logps/ref_chosen": -92.45591735839844, "logps/ref_rejected": -92.97093200683594, "logps/rejected": -94.01409149169922, "loss": 1.2787, "margin_dpo/margin_mean": 1.180344581604004, "margin_dpo/margin_std": 1.6435117721557617, "step": 55 }, { "epoch": 0.08223201174743025, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.060053825378418, "fcm_dpo/q_t": 0.4738434851169586, "grad_norm": 33.11833572387695, "learning_rate": 3.9855072463768114e-07, "logits/chosen": 0.2020319402217865, "logits/rejected": 0.5419068336486816, "logps/chosen": -87.31375122070312, "logps/ref_chosen": -87.23665618896484, "logps/ref_rejected": -100.64553833007812, "logps/rejected": -101.78268432617188, "loss": 1.2901, "margin_dpo/margin_mean": 1.0600528717041016, "margin_dpo/margin_std": 1.6760348081588745, "step": 56 }, { "epoch": 0.08370044052863436, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1467325687408447, "fcm_dpo/q_t": 0.4716096520423889, "grad_norm": 33.74794387817383, "learning_rate": 4.057971014492754e-07, "logits/chosen": 0.31785082817077637, "logits/rejected": 0.5342439413070679, "logps/chosen": -98.2012939453125, "logps/ref_chosen": -98.15074157714844, "logps/ref_rejected": -101.05284118652344, "logps/rejected": -102.25011444091797, "loss": 1.2808, "margin_dpo/margin_mean": 1.146733283996582, "margin_dpo/margin_std": 1.5416290760040283, "step": 57 }, { "epoch": 0.08516886930983847, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5004881620407104, "fcm_dpo/q_t": 0.46311402320861816, "grad_norm": 37.92626190185547, "learning_rate": 4.1304347826086954e-07, "logits/chosen": 0.29364755749702454, "logits/rejected": 0.6253389120101929, "logps/chosen": -99.32469177246094, "logps/ref_chosen": -99.58097076416016, "logps/ref_rejected": -91.22227478027344, "logps/rejected": -92.46648406982422, "loss": 1.2512, "margin_dpo/margin_mean": 1.50048828125, "margin_dpo/margin_std": 1.8517991304397583, "step": 58 }, { "epoch": 0.08663729809104258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2302589416503906, "fcm_dpo/q_t": 0.469511479139328, "grad_norm": 34.269100189208984, "learning_rate": 4.2028985507246374e-07, "logits/chosen": 0.13759316504001617, "logits/rejected": 0.3323509693145752, "logps/chosen": -89.81268310546875, "logps/ref_chosen": -89.80232238769531, "logps/ref_rejected": -94.27667236328125, "logps/rejected": -95.51730346679688, "loss": 1.2723, "margin_dpo/margin_mean": 1.230258584022522, "margin_dpo/margin_std": 1.4574449062347412, "step": 59 }, { "epoch": 0.0881057268722467, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1427745819091797, "fcm_dpo/q_t": 0.4716527462005615, "grad_norm": 28.718473434448242, "learning_rate": 4.2753623188405794e-07, "logits/chosen": 0.3863717317581177, "logits/rejected": 0.6563723087310791, "logps/chosen": -95.13877868652344, "logps/ref_chosen": -95.15571594238281, "logps/ref_rejected": -91.3724365234375, "logps/rejected": -92.4982681274414, "loss": 1.2804, "margin_dpo/margin_mean": 1.1427747011184692, "margin_dpo/margin_std": 1.4296643733978271, "step": 60 }, { "epoch": 0.08957415565345081, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1766782999038696, "fcm_dpo/q_t": 0.47107481956481934, "grad_norm": 28.268402099609375, "learning_rate": 4.3478260869565214e-07, "logits/chosen": 0.2091524302959442, "logits/rejected": 0.34982138872146606, "logps/chosen": -85.9404296875, "logps/ref_chosen": -85.73230743408203, "logps/ref_rejected": -97.96575927734375, "logps/rejected": -99.3505630493164, "loss": 1.2807, "margin_dpo/margin_mean": 1.1766784191131592, "margin_dpo/margin_std": 1.8282430171966553, "step": 61 }, { "epoch": 0.09104258443465492, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2114932537078857, "fcm_dpo/q_t": 0.47012367844581604, "grad_norm": 28.394248962402344, "learning_rate": 4.420289855072464e-07, "logits/chosen": 0.222091406583786, "logits/rejected": 0.4194578528404236, "logps/chosen": -81.77252197265625, "logps/ref_chosen": -81.63538360595703, "logps/ref_rejected": -84.03831481933594, "logps/rejected": -85.38694763183594, "loss": 1.2787, "margin_dpo/margin_mean": 1.2114924192428589, "margin_dpo/margin_std": 1.857269048690796, "step": 62 }, { "epoch": 0.09251101321585903, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4869378805160522, "fcm_dpo/q_t": 0.46316325664520264, "grad_norm": 35.04859924316406, "learning_rate": 4.4927536231884053e-07, "logits/chosen": 0.33780792355537415, "logits/rejected": 0.5467942357063293, "logps/chosen": -103.6246337890625, "logps/ref_chosen": -103.62405395507812, "logps/ref_rejected": -103.40303039550781, "logps/rejected": -104.89054870605469, "loss": 1.2495, "margin_dpo/margin_mean": 1.4869377613067627, "margin_dpo/margin_std": 1.5714600086212158, "step": 63 }, { "epoch": 0.09397944199706314, "fcm_dpo/beta": 0.10389302670955658, "fcm_dpo/delta": 0.19095781445503235, "fcm_dpo/margin": 1.7406915426254272, "fcm_dpo/q_t": 0.45685333013534546, "grad_norm": 35.974021911621094, "learning_rate": 4.5652173913043473e-07, "logits/chosen": 0.3727140724658966, "logits/rejected": 0.584496259689331, "logps/chosen": -87.18478393554688, "logps/ref_chosen": -87.0015869140625, "logps/ref_rejected": -100.5854721069336, "logps/rejected": -102.50936126708984, "loss": 1.2273, "margin_dpo/margin_mean": 1.7406916618347168, "margin_dpo/margin_std": 2.1689281463623047, "step": 64 }, { "epoch": 0.09544787077826726, "fcm_dpo/beta": 0.10389302670955658, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.7801563739776611, "fcm_dpo/q_t": 0.4546605348587036, "grad_norm": 39.58053970336914, "learning_rate": 4.63768115942029e-07, "logits/chosen": 0.08340189605951309, "logits/rejected": 0.22737503051757812, "logps/chosen": -91.27607727050781, "logps/ref_chosen": -91.22191619873047, "logps/ref_rejected": -115.33553314208984, "logps/rejected": -117.16984558105469, "loss": 1.2216, "margin_dpo/margin_mean": 1.780156135559082, "margin_dpo/margin_std": 2.092562437057495, "step": 65 }, { "epoch": 0.09691629955947137, "fcm_dpo/beta": 0.10389302670955658, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2560508251190186, "fcm_dpo/q_t": 0.46791407465934753, "grad_norm": 26.43943977355957, "learning_rate": 4.7101449275362313e-07, "logits/chosen": 0.25161832571029663, "logits/rejected": 0.460490345954895, "logps/chosen": -84.09396362304688, "logps/ref_chosen": -83.78422546386719, "logps/ref_rejected": -82.7520980834961, "logps/rejected": -84.31788635253906, "loss": 1.2702, "margin_dpo/margin_mean": 1.2560513019561768, "margin_dpo/margin_std": 1.885561466217041, "step": 66 }, { "epoch": 0.09838472834067548, "fcm_dpo/beta": 0.10389302670955658, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1451356410980225, "fcm_dpo/q_t": 0.47134000062942505, "grad_norm": 25.080429077148438, "learning_rate": 4.782608695652174e-07, "logits/chosen": 0.3801344335079193, "logits/rejected": 0.6603550910949707, "logps/chosen": -88.52891540527344, "logps/ref_chosen": -87.67295837402344, "logps/ref_rejected": -79.08674621582031, "logps/rejected": -81.08783721923828, "loss": 1.2902, "margin_dpo/margin_mean": 1.1451354026794434, "margin_dpo/margin_std": 2.6883256435394287, "step": 67 }, { "epoch": 0.09985315712187959, "fcm_dpo/beta": 0.10986920446157455, "fcm_dpo/delta": 0.3722814917564392, "fcm_dpo/margin": 2.152301788330078, "fcm_dpo/q_t": 0.44556739926338196, "grad_norm": 29.909442901611328, "learning_rate": 4.855072463768116e-07, "logits/chosen": -0.003267938271164894, "logits/rejected": 0.2983437776565552, "logps/chosen": -97.34490966796875, "logps/ref_chosen": -97.07884216308594, "logps/ref_rejected": -84.11872863769531, "logps/rejected": -86.53709411621094, "loss": 1.1887, "margin_dpo/margin_mean": 2.15230131149292, "margin_dpo/margin_std": 2.906233072280884, "step": 68 }, { "epoch": 0.1013215859030837, "fcm_dpo/beta": 0.1179037019610405, "fcm_dpo/delta": 0.3364790081977844, "fcm_dpo/margin": 2.307511806488037, "fcm_dpo/q_t": 0.4372074604034424, "grad_norm": 33.9486083984375, "learning_rate": 4.927536231884058e-07, "logits/chosen": 0.2297275960445404, "logits/rejected": 0.3395928740501404, "logps/chosen": -86.02600860595703, "logps/ref_chosen": -85.71971130371094, "logps/ref_rejected": -109.48020935058594, "logps/rejected": -112.0940170288086, "loss": 1.1623, "margin_dpo/margin_mean": 2.307512044906616, "margin_dpo/margin_std": 3.0080556869506836, "step": 69 }, { "epoch": 0.1027900146842878, "fcm_dpo/beta": 0.12486476451158524, "fcm_dpo/delta": 0.2788800001144409, "fcm_dpo/margin": 2.644542694091797, "fcm_dpo/q_t": 0.4251058101654053, "grad_norm": 33.3270378112793, "learning_rate": 5e-07, "logits/chosen": 0.3216286301612854, "logits/rejected": 0.5021553635597229, "logps/chosen": -95.56564331054688, "logps/ref_chosen": -95.00994873046875, "logps/ref_rejected": -96.21272277832031, "logps/rejected": -99.4129638671875, "loss": 1.1246, "margin_dpo/margin_mean": 2.644542932510376, "margin_dpo/margin_std": 3.2647337913513184, "step": 70 }, { "epoch": 0.10425844346549193, "fcm_dpo/beta": 0.12874360382556915, "fcm_dpo/delta": 0.13646240532398224, "fcm_dpo/margin": 3.6519854068756104, "fcm_dpo/q_t": 0.3953820765018463, "grad_norm": 39.30204772949219, "learning_rate": 4.999967061337492e-07, "logits/chosen": -0.15218183398246765, "logits/rejected": 0.21293151378631592, "logps/chosen": -91.0079345703125, "logps/ref_chosen": -90.97735595703125, "logps/ref_rejected": -102.59103393554688, "logps/rejected": -106.27359771728516, "loss": 1.0304, "margin_dpo/margin_mean": 3.6519851684570312, "margin_dpo/margin_std": 4.069307804107666, "step": 71 }, { "epoch": 0.10572687224669604, "fcm_dpo/beta": 0.13415689766407013, "fcm_dpo/delta": 0.2152521312236786, "fcm_dpo/margin": 2.9328012466430664, "fcm_dpo/q_t": 0.4104134142398834, "grad_norm": 35.07579803466797, "learning_rate": 4.999868246217933e-07, "logits/chosen": -0.1298740804195404, "logits/rejected": 0.19889742136001587, "logps/chosen": -98.41287231445312, "logps/ref_chosen": -97.89152526855469, "logps/ref_rejected": -100.19171142578125, "logps/rejected": -103.64585876464844, "loss": 1.0781, "margin_dpo/margin_mean": 2.932800769805908, "margin_dpo/margin_std": 3.356008529663086, "step": 72 }, { "epoch": 0.10719530102790015, "fcm_dpo/beta": 0.1394435167312622, "fcm_dpo/delta": 0.17690719664096832, "fcm_dpo/margin": 3.0887374877929688, "fcm_dpo/q_t": 0.4099673926830292, "grad_norm": 33.995853424072266, "learning_rate": 4.999703557245192e-07, "logits/chosen": -0.2099972814321518, "logits/rejected": 0.10542194545269012, "logps/chosen": -96.5327377319336, "logps/ref_chosen": -95.7690200805664, "logps/ref_rejected": -95.93243408203125, "logps/rejected": -99.78489685058594, "loss": 1.1024, "margin_dpo/margin_mean": 3.0887370109558105, "margin_dpo/margin_std": 4.92803430557251, "step": 73 }, { "epoch": 0.10866372980910426, "fcm_dpo/beta": 0.14254900813102722, "fcm_dpo/delta": 0.1032918393611908, "fcm_dpo/margin": 3.521709442138672, "fcm_dpo/q_t": 0.3906998634338379, "grad_norm": 36.36503219604492, "learning_rate": 4.999472998758977e-07, "logits/chosen": -0.10034316778182983, "logits/rejected": 0.0426066517829895, "logps/chosen": -79.76939392089844, "logps/ref_chosen": -78.80839538574219, "logps/ref_rejected": -101.64677429199219, "logps/rejected": -106.12947082519531, "loss": 1.0323, "margin_dpo/margin_mean": 3.5217092037200928, "margin_dpo/margin_std": 4.384693145751953, "step": 74 }, { "epoch": 0.11013215859030837, "fcm_dpo/beta": 0.14341631531715393, "fcm_dpo/delta": -0.055621419101953506, "fcm_dpo/margin": 4.54258918762207, "fcm_dpo/q_t": 0.3650940954685211, "grad_norm": 36.794151306152344, "learning_rate": 4.999176576834721e-07, "logits/chosen": -0.20140711963176727, "logits/rejected": -0.000715896487236023, "logps/chosen": -79.31031799316406, "logps/ref_chosen": -78.28185272216797, "logps/ref_rejected": -115.40311431884766, "logps/rejected": -120.97416687011719, "loss": 0.959, "margin_dpo/margin_mean": 4.542590141296387, "margin_dpo/margin_std": 5.413492679595947, "step": 75 }, { "epoch": 0.11160058737151249, "fcm_dpo/beta": 0.14552651345729828, "fcm_dpo/delta": 0.16189850866794586, "fcm_dpo/margin": 3.059915542602539, "fcm_dpo/q_t": 0.40516766905784607, "grad_norm": 30.741098403930664, "learning_rate": 4.998814299283415e-07, "logits/chosen": -0.614639401435852, "logits/rejected": -0.23801718652248383, "logps/chosen": -89.27581787109375, "logps/ref_chosen": -87.87713623046875, "logps/ref_rejected": -85.71968078613281, "logps/rejected": -90.17826843261719, "loss": 1.0865, "margin_dpo/margin_mean": 3.059915542602539, "margin_dpo/margin_std": 4.615878105163574, "step": 76 }, { "epoch": 0.1130690161527166, "fcm_dpo/beta": 0.14861461520195007, "fcm_dpo/delta": 0.10367655009031296, "fcm_dpo/margin": 3.3754987716674805, "fcm_dpo/q_t": 0.3928101658821106, "grad_norm": 37.48255920410156, "learning_rate": 4.998386175651409e-07, "logits/chosen": -0.18063946068286896, "logits/rejected": 0.18999367952346802, "logps/chosen": -101.2906723022461, "logps/ref_chosen": -99.70034790039062, "logps/ref_rejected": -98.20576477050781, "logps/rejected": -103.17158508300781, "loss": 1.0692, "margin_dpo/margin_mean": 3.3754987716674805, "margin_dpo/margin_std": 5.01793098449707, "step": 77 }, { "epoch": 0.1145374449339207, "fcm_dpo/beta": 0.15083855390548706, "fcm_dpo/delta": 0.049115367233753204, "fcm_dpo/margin": 3.670267105102539, "fcm_dpo/q_t": 0.3794415593147278, "grad_norm": 31.02741241455078, "learning_rate": 4.997892217220159e-07, "logits/chosen": 0.14838413894176483, "logits/rejected": 0.38533467054367065, "logps/chosen": -90.91835021972656, "logps/ref_chosen": -90.29670715332031, "logps/ref_rejected": -91.13772583007812, "logps/rejected": -95.42964172363281, "loss": 0.9953, "margin_dpo/margin_mean": 3.670267105102539, "margin_dpo/margin_std": 4.296913146972656, "step": 78 }, { "epoch": 0.11600587371512482, "fcm_dpo/beta": 0.14908111095428467, "fcm_dpo/delta": -0.10986006259918213, "fcm_dpo/margin": 4.710598945617676, "fcm_dpo/q_t": 0.355399489402771, "grad_norm": 32.22175598144531, "learning_rate": 4.997332437005931e-07, "logits/chosen": -0.6625052690505981, "logits/rejected": -0.28548330068588257, "logps/chosen": -87.22914123535156, "logps/ref_chosen": -86.37832641601562, "logps/ref_rejected": -94.10777282714844, "logps/rejected": -99.66917419433594, "loss": 0.9479, "margin_dpo/margin_mean": 4.710598945617676, "margin_dpo/margin_std": 5.8075103759765625, "step": 79 }, { "epoch": 0.11747430249632893, "fcm_dpo/beta": 0.1489923596382141, "fcm_dpo/delta": 0.04605354368686676, "fcm_dpo/margin": 3.7351717948913574, "fcm_dpo/q_t": 0.3830299377441406, "grad_norm": 33.21147537231445, "learning_rate": 4.996706849759452e-07, "logits/chosen": -0.6652204990386963, "logits/rejected": -0.30884838104248047, "logps/chosen": -95.46414947509766, "logps/ref_chosen": -93.97032165527344, "logps/ref_rejected": -92.57441711425781, "logps/rejected": -97.80342102050781, "loss": 1.0157, "margin_dpo/margin_mean": 3.7351717948913574, "margin_dpo/margin_std": 4.898221015930176, "step": 80 }, { "epoch": 0.11894273127753303, "fcm_dpo/beta": 0.14419203996658325, "fcm_dpo/delta": -0.18252533674240112, "fcm_dpo/margin": 5.311590194702148, "fcm_dpo/q_t": 0.3500846028327942, "grad_norm": 32.420406341552734, "learning_rate": 4.996015471965529e-07, "logits/chosen": -0.641904890537262, "logits/rejected": -0.22355474531650543, "logps/chosen": -100.79706573486328, "logps/ref_chosen": -99.83012390136719, "logps/ref_rejected": -133.67247009277344, "logps/rejected": -139.95098876953125, "loss": 0.9325, "margin_dpo/margin_mean": 5.311590194702148, "margin_dpo/margin_std": 6.772265434265137, "step": 81 }, { "epoch": 0.12041116005873716, "fcm_dpo/beta": 0.14681574702262878, "fcm_dpo/delta": 0.07446617633104324, "fcm_dpo/margin": 3.593234062194824, "fcm_dpo/q_t": 0.38746753334999084, "grad_norm": 33.34585952758789, "learning_rate": 4.995258321842611e-07, "logits/chosen": -0.10603200644254684, "logits/rejected": 0.11097083985805511, "logps/chosen": -84.93232727050781, "logps/ref_chosen": -83.04598236083984, "logps/ref_rejected": -94.52595520019531, "logps/rejected": -100.00553131103516, "loss": 1.0399, "margin_dpo/margin_mean": 3.593233823776245, "margin_dpo/margin_std": 4.823699951171875, "step": 82 }, { "epoch": 0.12187958883994127, "fcm_dpo/beta": 0.14609777927398682, "fcm_dpo/delta": -0.0029450468719005585, "fcm_dpo/margin": 4.125781536102295, "fcm_dpo/q_t": 0.38040691614151, "grad_norm": 39.993614196777344, "learning_rate": 4.994435419342304e-07, "logits/chosen": -0.4488382339477539, "logits/rejected": -0.025571677833795547, "logps/chosen": -94.34185028076172, "logps/ref_chosen": -92.17621612548828, "logps/ref_rejected": -107.74464416503906, "logps/rejected": -114.03605651855469, "loss": 1.0479, "margin_dpo/margin_mean": 4.125782012939453, "margin_dpo/margin_std": 6.356573104858398, "step": 83 }, { "epoch": 0.12334801762114538, "fcm_dpo/beta": 0.1460931897163391, "fcm_dpo/delta": 0.0006008504424244165, "fcm_dpo/margin": 2.527191638946533, "fcm_dpo/q_t": 0.422789990901947, "grad_norm": 44.41518020629883, "learning_rate": 4.993546786148857e-07, "logits/chosen": -0.4026920795440674, "logits/rejected": -0.021030515432357788, "logps/chosen": -104.27578735351562, "logps/ref_chosen": -101.5264892578125, "logps/ref_rejected": -92.42608642578125, "logps/rejected": -97.70257568359375, "loss": 1.1681, "margin_dpo/margin_mean": 2.5271923542022705, "margin_dpo/margin_std": 4.771922588348389, "step": 84 }, { "epoch": 0.12481644640234948, "fcm_dpo/beta": 0.1492038369178772, "fcm_dpo/delta": 0.09993791580200195, "fcm_dpo/margin": 3.376296043395996, "fcm_dpo/q_t": 0.3976307511329651, "grad_norm": 42.52838134765625, "learning_rate": 4.992592445678582e-07, "logits/chosen": -0.22517216205596924, "logits/rejected": 0.10873828083276749, "logps/chosen": -98.53033447265625, "logps/ref_chosen": -96.12738037109375, "logps/ref_rejected": -85.05519104003906, "logps/rejected": -90.83444213867188, "loss": 1.1171, "margin_dpo/margin_mean": 3.3762965202331543, "margin_dpo/margin_std": 5.977930545806885, "step": 85 }, { "epoch": 0.1262848751835536, "fcm_dpo/beta": 0.14783477783203125, "fcm_dpo/delta": -0.044383928179740906, "fcm_dpo/margin": 4.338801383972168, "fcm_dpo/q_t": 0.37967485189437866, "grad_norm": 30.416973114013672, "learning_rate": 4.991572423079235e-07, "logits/chosen": -0.4074563682079315, "logits/rejected": -0.16365355253219604, "logps/chosen": -83.54771423339844, "logps/ref_chosen": -81.70426177978516, "logps/ref_rejected": -93.65550231933594, "logps/rejected": -99.8377456665039, "loss": 1.0355, "margin_dpo/margin_mean": 4.338800430297852, "margin_dpo/margin_std": 6.941473960876465, "step": 86 }, { "epoch": 0.1277533039647577, "fcm_dpo/beta": 0.1455276608467102, "fcm_dpo/delta": -0.07910901308059692, "fcm_dpo/margin": 4.626860618591309, "fcm_dpo/q_t": 0.36312583088874817, "grad_norm": 38.379615783691406, "learning_rate": 4.990486745229364e-07, "logits/chosen": -0.8469661474227905, "logits/rejected": -0.48443603515625, "logps/chosen": -94.81890869140625, "logps/ref_chosen": -92.68596649169922, "logps/ref_rejected": -102.91818237304688, "logps/rejected": -109.67797088623047, "loss": 1.0375, "margin_dpo/margin_mean": 4.626861572265625, "margin_dpo/margin_std": 7.198031425476074, "step": 87 }, { "epoch": 0.12922173274596183, "fcm_dpo/beta": 0.14544734358787537, "fcm_dpo/delta": 0.05892753228545189, "fcm_dpo/margin": 3.7355780601501465, "fcm_dpo/q_t": 0.39093679189682007, "grad_norm": 35.78076171875, "learning_rate": 4.989335440737586e-07, "logits/chosen": -0.09812933206558228, "logits/rejected": 0.08861222863197327, "logps/chosen": -103.42122650146484, "logps/ref_chosen": -100.76298522949219, "logps/ref_rejected": -113.15037536621094, "logps/rejected": -119.54420471191406, "loss": 1.087, "margin_dpo/margin_mean": 3.7355780601501465, "margin_dpo/margin_std": 6.172787666320801, "step": 88 }, { "epoch": 0.13069016152716592, "fcm_dpo/beta": 0.147188201546669, "fcm_dpo/delta": 0.028476351872086525, "fcm_dpo/margin": 3.892774820327759, "fcm_dpo/q_t": 0.3894566297531128, "grad_norm": 30.757919311523438, "learning_rate": 4.988118539941847e-07, "logits/chosen": -0.6192222833633423, "logits/rejected": -0.28894031047821045, "logps/chosen": -91.76663208007812, "logps/ref_chosen": -89.69108581542969, "logps/ref_rejected": -88.56832885742188, "logps/rejected": -94.53666687011719, "loss": 1.0588, "margin_dpo/margin_mean": 3.892774820327759, "margin_dpo/margin_std": 6.543310165405273, "step": 89 }, { "epoch": 0.13215859030837004, "fcm_dpo/beta": 0.14558328688144684, "fcm_dpo/delta": -0.14587749540805817, "fcm_dpo/margin": 5.050919532775879, "fcm_dpo/q_t": 0.35632702708244324, "grad_norm": 32.816978454589844, "learning_rate": 4.986836074908615e-07, "logits/chosen": -0.28241512179374695, "logits/rejected": -0.16540709137916565, "logps/chosen": -83.23228454589844, "logps/ref_chosen": -81.38255310058594, "logps/ref_rejected": -117.77714538574219, "logps/rejected": -124.67778015136719, "loss": 0.9688, "margin_dpo/margin_mean": 5.050920486450195, "margin_dpo/margin_std": 6.877577781677246, "step": 90 }, { "epoch": 0.13362701908957417, "fcm_dpo/beta": 0.14302682876586914, "fcm_dpo/delta": -0.03883608058094978, "fcm_dpo/margin": 4.4493865966796875, "fcm_dpo/q_t": 0.37537193298339844, "grad_norm": 35.73554611206055, "learning_rate": 4.985488079432037e-07, "logits/chosen": -0.2560047507286072, "logits/rejected": 0.15471355617046356, "logps/chosen": -99.27584838867188, "logps/ref_chosen": -97.22187805175781, "logps/ref_rejected": -92.97674560546875, "logps/rejected": -99.4801025390625, "loss": 1.0129, "margin_dpo/margin_mean": 4.449386119842529, "margin_dpo/margin_std": 6.499499320983887, "step": 91 }, { "epoch": 0.13509544787077826, "fcm_dpo/beta": 0.14428263902664185, "fcm_dpo/delta": 0.06443801522254944, "fcm_dpo/margin": 3.7344517707824707, "fcm_dpo/q_t": 0.39002060890197754, "grad_norm": 30.80695343017578, "learning_rate": 4.984074589033043e-07, "logits/chosen": -0.8915022611618042, "logits/rejected": -0.5445601344108582, "logps/chosen": -86.91651153564453, "logps/ref_chosen": -84.5302734375, "logps/ref_rejected": -84.5013198852539, "logps/rejected": -90.62200927734375, "loss": 1.0648, "margin_dpo/margin_mean": 3.7344512939453125, "margin_dpo/margin_std": 5.879533290863037, "step": 92 }, { "epoch": 0.13656387665198239, "fcm_dpo/beta": 0.14631253480911255, "fcm_dpo/delta": 0.07103022187948227, "fcm_dpo/margin": 3.6395092010498047, "fcm_dpo/q_t": 0.39281517267227173, "grad_norm": 39.31964111328125, "learning_rate": 4.982595640958425e-07, "logits/chosen": -0.8906035423278809, "logits/rejected": -0.34005117416381836, "logps/chosen": -92.74388122558594, "logps/ref_chosen": -90.25043487548828, "logps/ref_rejected": -84.0942153930664, "logps/rejected": -90.2271728515625, "loss": 1.1067, "margin_dpo/margin_mean": 3.6395092010498047, "margin_dpo/margin_std": 6.321407318115234, "step": 93 }, { "epoch": 0.13803230543318648, "fcm_dpo/beta": 0.14079803228378296, "fcm_dpo/delta": -0.1881101429462433, "fcm_dpo/margin": 5.460287094116211, "fcm_dpo/q_t": 0.3449600040912628, "grad_norm": 34.704139709472656, "learning_rate": 4.98105127417984e-07, "logits/chosen": -0.45207756757736206, "logits/rejected": -0.15455085039138794, "logps/chosen": -94.94246673583984, "logps/ref_chosen": -92.4542236328125, "logps/ref_rejected": -105.2472915649414, "logps/rejected": -113.19581604003906, "loss": 0.954, "margin_dpo/margin_mean": 5.4602861404418945, "margin_dpo/margin_std": 6.94245719909668, "step": 94 }, { "epoch": 0.1395007342143906, "fcm_dpo/beta": 0.14194484055042267, "fcm_dpo/delta": -0.0035935528576374054, "fcm_dpo/margin": 4.245328903198242, "fcm_dpo/q_t": 0.38038820028305054, "grad_norm": 29.971302032470703, "learning_rate": 4.979441529392784e-07, "logits/chosen": -0.8304082155227661, "logits/rejected": -0.5099495053291321, "logps/chosen": -80.91264343261719, "logps/ref_chosen": -78.87370300292969, "logps/ref_rejected": -83.59121704101562, "logps/rejected": -89.87547302246094, "loss": 1.029, "margin_dpo/margin_mean": 4.245327949523926, "margin_dpo/margin_std": 6.151725769042969, "step": 95 }, { "epoch": 0.14096916299559473, "fcm_dpo/beta": 0.13753080368041992, "fcm_dpo/delta": -0.15204164385795593, "fcm_dpo/margin": 5.382932186126709, "fcm_dpo/q_t": 0.35132813453674316, "grad_norm": 29.244211196899414, "learning_rate": 4.977766449015534e-07, "logits/chosen": -0.7205407619476318, "logits/rejected": -0.30231958627700806, "logps/chosen": -108.13746643066406, "logps/ref_chosen": -106.5921630859375, "logps/ref_rejected": -101.76802825927734, "logps/rejected": -108.6962661743164, "loss": 0.9483, "margin_dpo/margin_mean": 5.382932662963867, "margin_dpo/margin_std": 6.747956275939941, "step": 96 }, { "epoch": 0.14243759177679882, "fcm_dpo/beta": 0.1367703527212143, "fcm_dpo/delta": -0.06670509278774261, "fcm_dpo/margin": 4.833524703979492, "fcm_dpo/q_t": 0.36010098457336426, "grad_norm": 32.554283142089844, "learning_rate": 4.976026077188012e-07, "logits/chosen": -0.327857106924057, "logits/rejected": 0.07270251214504242, "logps/chosen": -91.24253845214844, "logps/ref_chosen": -88.67988586425781, "logps/ref_rejected": -84.81229400634766, "logps/rejected": -92.20846557617188, "loss": 0.9517, "margin_dpo/margin_mean": 4.833524227142334, "margin_dpo/margin_std": 5.391106605529785, "step": 97 }, { "epoch": 0.14390602055800295, "fcm_dpo/beta": 0.1318550407886505, "fcm_dpo/delta": -0.11207875609397888, "fcm_dpo/margin": 5.328702926635742, "fcm_dpo/q_t": 0.35097742080688477, "grad_norm": 30.32093620300293, "learning_rate": 4.974220459770639e-07, "logits/chosen": -0.5091695785522461, "logits/rejected": -0.2531248927116394, "logps/chosen": -94.11245727539062, "logps/ref_chosen": -92.24249267578125, "logps/ref_rejected": -101.51948547363281, "logps/rejected": -108.7181396484375, "loss": 0.917, "margin_dpo/margin_mean": 5.328701972961426, "margin_dpo/margin_std": 5.660694122314453, "step": 98 }, { "epoch": 0.14537444933920704, "fcm_dpo/beta": 0.12901289761066437, "fcm_dpo/delta": -0.14893479645252228, "fcm_dpo/margin": 5.719096660614014, "fcm_dpo/q_t": 0.35661765933036804, "grad_norm": 27.99370002746582, "learning_rate": 4.972349644343108e-07, "logits/chosen": -0.3547890782356262, "logits/rejected": -0.14507462084293365, "logps/chosen": -73.60296630859375, "logps/ref_chosen": -72.18464660644531, "logps/ref_rejected": -91.88131713867188, "logps/rejected": -99.01873016357422, "loss": 0.9419, "margin_dpo/margin_mean": 5.719097137451172, "margin_dpo/margin_std": 7.474823474884033, "step": 99 }, { "epoch": 0.14684287812041116, "fcm_dpo/beta": 0.1281561702489853, "fcm_dpo/delta": 0.03391501307487488, "fcm_dpo/margin": 4.430019378662109, "fcm_dpo/q_t": 0.3762722611427307, "grad_norm": 26.248151779174805, "learning_rate": 4.970413680203148e-07, "logits/chosen": -0.6022966504096985, "logits/rejected": -0.14130988717079163, "logps/chosen": -91.0530014038086, "logps/ref_chosen": -89.51382446289062, "logps/ref_rejected": -81.21713256835938, "logps/rejected": -87.18632507324219, "loss": 0.9831, "margin_dpo/margin_mean": 4.430019378662109, "margin_dpo/margin_std": 4.9434709548950195, "step": 100 }, { "epoch": 0.14684287812041116, "eval_fcm_dpo/beta": 0.12897978723049164, "eval_logits/chosen": 0.03100498393177986, "eval_logits/rejected": 0.2742968201637268, "eval_logps/chosen": -103.41716766357422, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -100.0186767578125, "eval_loss": 0.5851320028305054, "eval_margin_dpo/margin_mean": 3.027308940887451, "eval_margin_dpo/margin_std": 6.074099063873291, "eval_runtime": 43.7521, "eval_samples_per_second": 53.46, "eval_steps_per_second": 1.691, "step": 100 }, { "epoch": 0.14831130690161526, "fcm_dpo/beta": 0.13015779852867126, "fcm_dpo/delta": 0.09972133487462997, "fcm_dpo/margin": 3.881521224975586, "fcm_dpo/q_t": 0.39363157749176025, "grad_norm": 33.8118896484375, "learning_rate": 4.968412618365215e-07, "logits/chosen": -0.4824643135070801, "logits/rejected": -0.21651335060596466, "logps/chosen": -97.09152221679688, "logps/ref_chosen": -94.4031982421875, "logps/ref_rejected": -89.91001892089844, "logps/rejected": -96.47987365722656, "loss": 1.0872, "margin_dpo/margin_mean": 3.881521224975586, "margin_dpo/margin_std": 6.2039475440979, "step": 101 }, { "epoch": 0.14977973568281938, "fcm_dpo/beta": 0.1360168159008026, "fcm_dpo/delta": 0.1453799605369568, "fcm_dpo/margin": 3.3603551387786865, "fcm_dpo/q_t": 0.4019954204559326, "grad_norm": 43.768089294433594, "learning_rate": 4.966346511559149e-07, "logits/chosen": -0.7954562902450562, "logits/rejected": -0.4590097665786743, "logps/chosen": -94.66258239746094, "logps/ref_chosen": -91.60942077636719, "logps/ref_rejected": -75.7689208984375, "logps/rejected": -82.18241882324219, "loss": 1.128, "margin_dpo/margin_mean": 3.3603553771972656, "margin_dpo/margin_std": 5.937575340270996, "step": 102 }, { "epoch": 0.1512481644640235, "fcm_dpo/beta": 0.13238456845283508, "fcm_dpo/delta": -0.1370110958814621, "fcm_dpo/margin": 5.488382339477539, "fcm_dpo/q_t": 0.34756556153297424, "grad_norm": 32.67784118652344, "learning_rate": 4.964215414228785e-07, "logits/chosen": -0.6738690733909607, "logits/rejected": -0.30804526805877686, "logps/chosen": -99.65579223632812, "logps/ref_chosen": -97.40238952636719, "logps/ref_rejected": -98.981689453125, "logps/rejected": -106.72347259521484, "loss": 0.9394, "margin_dpo/margin_mean": 5.4883832931518555, "margin_dpo/margin_std": 6.411886215209961, "step": 103 }, { "epoch": 0.1527165932452276, "fcm_dpo/beta": 0.12842795252799988, "fcm_dpo/delta": -0.16619333624839783, "fcm_dpo/margin": 5.866588115692139, "fcm_dpo/q_t": 0.34973686933517456, "grad_norm": 25.817495346069336, "learning_rate": 4.96201938253052e-07, "logits/chosen": -0.8139798045158386, "logits/rejected": -0.4907284379005432, "logps/chosen": -96.42156982421875, "logps/ref_chosen": -94.85636138916016, "logps/ref_rejected": -97.02669525146484, "logps/rejected": -104.45848846435547, "loss": 0.9237, "margin_dpo/margin_mean": 5.866589069366455, "margin_dpo/margin_std": 7.296744346618652, "step": 104 }, { "epoch": 0.15418502202643172, "fcm_dpo/beta": 0.12592965364456177, "fcm_dpo/delta": -0.12789560854434967, "fcm_dpo/margin": 5.70680046081543, "fcm_dpo/q_t": 0.3505769968032837, "grad_norm": 30.186168670654297, "learning_rate": 4.959758474331832e-07, "logits/chosen": -0.8461470603942871, "logits/rejected": -0.40368765592575073, "logps/chosen": -98.9314956665039, "logps/ref_chosen": -96.75790405273438, "logps/ref_rejected": -101.5774917602539, "logps/rejected": -109.4578857421875, "loss": 0.9491, "margin_dpo/margin_mean": 5.70680046081543, "margin_dpo/margin_std": 6.950338840484619, "step": 105 }, { "epoch": 0.15565345080763582, "fcm_dpo/beta": 0.12308678776025772, "fcm_dpo/delta": -0.1014535203576088, "fcm_dpo/margin": 5.641180515289307, "fcm_dpo/q_t": 0.3579603433609009, "grad_norm": 26.967708587646484, "learning_rate": 4.957432749209755e-07, "logits/chosen": -0.3590039014816284, "logits/rejected": 0.06661949306726456, "logps/chosen": -94.51701354980469, "logps/ref_chosen": -92.43791198730469, "logps/ref_rejected": -91.00970458984375, "logps/rejected": -98.72999572753906, "loss": 0.9482, "margin_dpo/margin_mean": 5.641180038452148, "margin_dpo/margin_std": 6.846166133880615, "step": 106 }, { "epoch": 0.15712187958883994, "fcm_dpo/beta": 0.11860156059265137, "fcm_dpo/delta": -0.1794869601726532, "fcm_dpo/margin": 6.456604480743408, "fcm_dpo/q_t": 0.3544139862060547, "grad_norm": 27.01357650756836, "learning_rate": 4.955042268449307e-07, "logits/chosen": -0.43201225996017456, "logits/rejected": 0.07057125866413116, "logps/chosen": -106.36869812011719, "logps/ref_chosen": -105.12500762939453, "logps/ref_rejected": -101.1824951171875, "logps/rejected": -108.8827896118164, "loss": 0.9456, "margin_dpo/margin_mean": 6.456604480743408, "margin_dpo/margin_std": 9.23145580291748, "step": 107 }, { "epoch": 0.15859030837004406, "fcm_dpo/beta": 0.11540443450212479, "fcm_dpo/delta": -0.08156667649745941, "fcm_dpo/margin": 5.848906993865967, "fcm_dpo/q_t": 0.3599376976490021, "grad_norm": 25.21524429321289, "learning_rate": 4.952587095041881e-07, "logits/chosen": -0.8443419933319092, "logits/rejected": -0.35467660427093506, "logps/chosen": -91.43235778808594, "logps/ref_chosen": -89.20936584472656, "logps/ref_rejected": -101.83784484863281, "logps/rejected": -109.90974426269531, "loss": 0.9559, "margin_dpo/margin_mean": 5.848907470703125, "margin_dpo/margin_std": 6.974748134613037, "step": 108 }, { "epoch": 0.16005873715124816, "fcm_dpo/beta": 0.11124076694250107, "fcm_dpo/delta": -0.23742878437042236, "fcm_dpo/margin": 7.345666408538818, "fcm_dpo/q_t": 0.33463311195373535, "grad_norm": 25.55326271057129, "learning_rate": 4.95006729368358e-07, "logits/chosen": -0.5116323232650757, "logits/rejected": -0.30790773034095764, "logps/chosen": -101.7799072265625, "logps/ref_chosen": -100.18708801269531, "logps/ref_rejected": -103.53739929199219, "logps/rejected": -112.47588348388672, "loss": 0.895, "margin_dpo/margin_mean": 7.34566593170166, "margin_dpo/margin_std": 8.431303024291992, "step": 109 }, { "epoch": 0.16152716593245228, "fcm_dpo/beta": 0.11126747727394104, "fcm_dpo/delta": 0.09853193908929825, "fcm_dpo/margin": 4.551311492919922, "fcm_dpo/q_t": 0.3907678723335266, "grad_norm": 26.554285049438477, "learning_rate": 4.947482930773511e-07, "logits/chosen": -0.20916850864887238, "logits/rejected": 0.15151086449623108, "logps/chosen": -109.06941223144531, "logps/ref_chosen": -106.89572143554688, "logps/ref_rejected": -85.81120300292969, "logps/rejected": -92.53620910644531, "loss": 1.0485, "margin_dpo/margin_mean": 4.551311492919922, "margin_dpo/margin_std": 6.169561386108398, "step": 110 }, { "epoch": 0.16299559471365638, "fcm_dpo/beta": 0.10644402354955673, "fcm_dpo/delta": -0.31627288460731506, "fcm_dpo/margin": 8.312642097473145, "fcm_dpo/q_t": 0.3186783790588379, "grad_norm": 23.229040145874023, "learning_rate": 4.944834074412042e-07, "logits/chosen": -0.9531800746917725, "logits/rejected": -0.6935971975326538, "logps/chosen": -95.22097778320312, "logps/ref_chosen": -93.3293685913086, "logps/ref_rejected": -104.177490234375, "logps/rejected": -114.38174438476562, "loss": 0.8389, "margin_dpo/margin_mean": 8.312643051147461, "margin_dpo/margin_std": 8.214632034301758, "step": 111 }, { "epoch": 0.1644640234948605, "fcm_dpo/beta": 0.10744242370128632, "fcm_dpo/delta": 0.12501248717308044, "fcm_dpo/margin": 4.4774169921875, "fcm_dpo/q_t": 0.39411237835884094, "grad_norm": 24.20155906677246, "learning_rate": 4.942120794399002e-07, "logits/chosen": -0.9408313632011414, "logits/rejected": -0.488766074180603, "logps/chosen": -84.89042663574219, "logps/ref_chosen": -82.74703979492188, "logps/ref_rejected": -71.5584487915039, "logps/rejected": -78.17925262451172, "loss": 1.0479, "margin_dpo/margin_mean": 4.4774169921875, "margin_dpo/margin_std": 5.751065254211426, "step": 112 }, { "epoch": 0.16593245227606462, "fcm_dpo/beta": 0.10971181094646454, "fcm_dpo/delta": 0.08851583302021027, "fcm_dpo/margin": 4.70300817489624, "fcm_dpo/q_t": 0.38598090410232544, "grad_norm": 28.11321449279785, "learning_rate": 4.939343162231841e-07, "logits/chosen": -0.6415950059890747, "logits/rejected": -0.14356252551078796, "logps/chosen": -100.6945571899414, "logps/ref_chosen": -97.89949035644531, "logps/ref_rejected": -84.41960144042969, "logps/rejected": -91.91766357421875, "loss": 1.0218, "margin_dpo/margin_mean": 4.703007698059082, "margin_dpo/margin_std": 5.460232257843018, "step": 113 }, { "epoch": 0.16740088105726872, "fcm_dpo/beta": 0.10798073559999466, "fcm_dpo/delta": -0.10012990236282349, "fcm_dpo/margin": 6.41129732131958, "fcm_dpo/q_t": 0.36218881607055664, "grad_norm": 28.432401657104492, "learning_rate": 4.936501251103751e-07, "logits/chosen": -1.0188771486282349, "logits/rejected": -0.4950348734855652, "logps/chosen": -99.10498046875, "logps/ref_chosen": -96.4951171875, "logps/ref_rejected": -92.01536560058594, "logps/rejected": -101.0365219116211, "loss": 0.9806, "margin_dpo/margin_mean": 6.411297798156738, "margin_dpo/margin_std": 8.566350936889648, "step": 114 }, { "epoch": 0.16886930983847284, "fcm_dpo/beta": 0.10620786994695663, "fcm_dpo/delta": -0.11520249396562576, "fcm_dpo/margin": 6.658008098602295, "fcm_dpo/q_t": 0.3513981103897095, "grad_norm": 23.633058547973633, "learning_rate": 4.933595135901732e-07, "logits/chosen": -0.5462427139282227, "logits/rejected": -0.1587030589580536, "logps/chosen": -105.50727844238281, "logps/ref_chosen": -103.18013000488281, "logps/ref_rejected": -105.71595764160156, "logps/rejected": -114.70111846923828, "loss": 0.9284, "margin_dpo/margin_mean": 6.658008575439453, "margin_dpo/margin_std": 7.443235874176025, "step": 115 }, { "epoch": 0.17033773861967694, "fcm_dpo/beta": 0.1062956303358078, "fcm_dpo/delta": 0.024708664044737816, "fcm_dpo/margin": 5.4217939376831055, "fcm_dpo/q_t": 0.37565532326698303, "grad_norm": 22.831968307495117, "learning_rate": 4.930624893204624e-07, "logits/chosen": -0.5053553581237793, "logits/rejected": -0.07657349109649658, "logps/chosen": -83.38106536865234, "logps/ref_chosen": -81.15837097167969, "logps/ref_rejected": -86.30586242675781, "logps/rejected": -93.95034790039062, "loss": 0.9908, "margin_dpo/margin_mean": 5.421794414520264, "margin_dpo/margin_std": 6.337811470031738, "step": 116 }, { "epoch": 0.17180616740088106, "fcm_dpo/beta": 0.1065780371427536, "fcm_dpo/delta": 0.09205998480319977, "fcm_dpo/margin": 4.804106712341309, "fcm_dpo/q_t": 0.38822945952415466, "grad_norm": 31.471729278564453, "learning_rate": 4.927590601281083e-07, "logits/chosen": -0.6579692363739014, "logits/rejected": -0.30281364917755127, "logps/chosen": -99.8343505859375, "logps/ref_chosen": -97.0809555053711, "logps/ref_rejected": -75.1399154663086, "logps/rejected": -82.69741821289062, "loss": 1.0457, "margin_dpo/margin_mean": 4.804106712341309, "margin_dpo/margin_std": 6.34288215637207, "step": 117 }, { "epoch": 0.17327459618208516, "fcm_dpo/beta": 0.10656137019395828, "fcm_dpo/delta": -0.0756685882806778, "fcm_dpo/margin": 6.291032791137695, "fcm_dpo/q_t": 0.35226526856422424, "grad_norm": 22.64267349243164, "learning_rate": 4.924492340087524e-07, "logits/chosen": -1.0219924449920654, "logits/rejected": -0.7220288515090942, "logps/chosen": -88.30572509765625, "logps/ref_chosen": -86.53623962402344, "logps/ref_rejected": -82.93563842773438, "logps/rejected": -90.99617004394531, "loss": 0.9216, "margin_dpo/margin_mean": 6.291032791137695, "margin_dpo/margin_std": 6.30666446685791, "step": 118 }, { "epoch": 0.17474302496328928, "fcm_dpo/beta": 0.10483279079198837, "fcm_dpo/delta": -0.08183208853006363, "fcm_dpo/margin": 6.448603630065918, "fcm_dpo/q_t": 0.3583568036556244, "grad_norm": 26.867589950561523, "learning_rate": 4.92133019126601e-07, "logits/chosen": -0.9083903431892395, "logits/rejected": -0.6197102665901184, "logps/chosen": -107.41993713378906, "logps/ref_chosen": -104.60908508300781, "logps/ref_rejected": -109.15780639648438, "logps/rejected": -118.41727447509766, "loss": 0.9898, "margin_dpo/margin_mean": 6.448603630065918, "margin_dpo/margin_std": 8.662832260131836, "step": 119 }, { "epoch": 0.1762114537444934, "fcm_dpo/beta": 0.10017947107553482, "fcm_dpo/delta": -0.2905012369155884, "fcm_dpo/margin": 8.636421203613281, "fcm_dpo/q_t": 0.33309102058410645, "grad_norm": 30.011741638183594, "learning_rate": 4.918104238142103e-07, "logits/chosen": -1.0279607772827148, "logits/rejected": -0.5229378938674927, "logps/chosen": -116.57168579101562, "logps/ref_chosen": -114.68742370605469, "logps/ref_rejected": -111.18586730957031, "logps/rejected": -121.70654296875, "loss": 0.915, "margin_dpo/margin_mean": 8.636421203613281, "margin_dpo/margin_std": 10.763944625854492, "step": 120 }, { "epoch": 0.1776798825256975, "fcm_dpo/beta": 0.0964190736413002, "fcm_dpo/delta": -0.12846185266971588, "fcm_dpo/margin": 7.4496049880981445, "fcm_dpo/q_t": 0.3460380434989929, "grad_norm": 25.585208892822266, "learning_rate": 4.91481456572267e-07, "logits/chosen": -0.7847539186477661, "logits/rejected": -0.4723082184791565, "logps/chosen": -93.91836547851562, "logps/ref_chosen": -92.11666107177734, "logps/ref_rejected": -114.972900390625, "logps/rejected": -124.22420501708984, "loss": 0.9003, "margin_dpo/margin_mean": 7.4496049880981445, "margin_dpo/margin_std": 7.533651351928711, "step": 121 }, { "epoch": 0.17914831130690162, "fcm_dpo/beta": 0.09395934641361237, "fcm_dpo/delta": -0.18832112848758698, "fcm_dpo/margin": 8.238999366760254, "fcm_dpo/q_t": 0.3329077959060669, "grad_norm": 26.141273498535156, "learning_rate": 4.911461260693638e-07, "logits/chosen": -0.47960373759269714, "logits/rejected": -0.35823744535446167, "logps/chosen": -87.03057861328125, "logps/ref_chosen": -84.87757110595703, "logps/ref_rejected": -111.0829849243164, "logps/rejected": -121.47499084472656, "loss": 0.8771, "margin_dpo/margin_mean": 8.238998413085938, "margin_dpo/margin_std": 8.030372619628906, "step": 122 }, { "epoch": 0.18061674008810572, "fcm_dpo/beta": 0.09212273359298706, "fcm_dpo/delta": 0.013646617531776428, "fcm_dpo/margin": 6.361550807952881, "fcm_dpo/q_t": 0.3727787137031555, "grad_norm": 22.423507690429688, "learning_rate": 4.908044411417711e-07, "logits/chosen": -0.9911929368972778, "logits/rejected": -0.7056489586830139, "logps/chosen": -100.84278869628906, "logps/ref_chosen": -98.47874450683594, "logps/ref_rejected": -93.85417175292969, "logps/rejected": -102.57976531982422, "loss": 0.9885, "margin_dpo/margin_mean": 6.361550331115723, "margin_dpo/margin_std": 7.391707420349121, "step": 123 }, { "epoch": 0.18208516886930984, "fcm_dpo/beta": 0.09153375774621964, "fcm_dpo/delta": -0.10555803775787354, "fcm_dpo/margin": 7.628256320953369, "fcm_dpo/q_t": 0.36510169506073, "grad_norm": 22.723674774169922, "learning_rate": 4.904564107932048e-07, "logits/chosen": -0.6390783786773682, "logits/rejected": -0.385051965713501, "logps/chosen": -104.82085418701172, "logps/ref_chosen": -102.76290130615234, "logps/ref_rejected": -123.005615234375, "logps/rejected": -132.69183349609375, "loss": 0.9851, "margin_dpo/margin_mean": 7.628256320953369, "margin_dpo/margin_std": 10.736438751220703, "step": 124 }, { "epoch": 0.18355359765051396, "fcm_dpo/beta": 0.09196379780769348, "fcm_dpo/delta": 0.07185746729373932, "fcm_dpo/margin": 5.784401893615723, "fcm_dpo/q_t": 0.38678333163261414, "grad_norm": 24.231779098510742, "learning_rate": 4.90102044194588e-07, "logits/chosen": -0.6529036164283752, "logits/rejected": -0.4633956253528595, "logps/chosen": -93.97822570800781, "logps/ref_chosen": -90.42627716064453, "logps/ref_rejected": -89.24901580810547, "logps/rejected": -98.58536529541016, "loss": 1.0442, "margin_dpo/margin_mean": 5.784401893615723, "margin_dpo/margin_std": 8.095062255859375, "step": 125 }, { "epoch": 0.18502202643171806, "fcm_dpo/beta": 0.09028153121471405, "fcm_dpo/delta": -0.15866564214229584, "fcm_dpo/margin": 8.273946762084961, "fcm_dpo/q_t": 0.3347667157649994, "grad_norm": 22.696521759033203, "learning_rate": 4.897413506838102e-07, "logits/chosen": -0.7492173910140991, "logits/rejected": -0.49526458978652954, "logps/chosen": -96.82389831542969, "logps/ref_chosen": -94.42288208007812, "logps/ref_rejected": -103.88156127929688, "logps/rejected": -114.55652618408203, "loss": 0.8628, "margin_dpo/margin_mean": 8.273946762084961, "margin_dpo/margin_std": 7.15079402923584, "step": 126 }, { "epoch": 0.18649045521292218, "fcm_dpo/beta": 0.09020580351352692, "fcm_dpo/delta": 0.022558212280273438, "fcm_dpo/margin": 6.407971382141113, "fcm_dpo/q_t": 0.377363920211792, "grad_norm": 23.82652473449707, "learning_rate": 4.89374339765481e-07, "logits/chosen": -0.6647278070449829, "logits/rejected": -0.35185229778289795, "logps/chosen": -87.22705841064453, "logps/ref_chosen": -84.75105285644531, "logps/ref_rejected": -83.54523468017578, "logps/rejected": -92.42921447753906, "loss": 1.0158, "margin_dpo/margin_mean": 6.407971382141113, "margin_dpo/margin_std": 8.475217819213867, "step": 127 }, { "epoch": 0.18795888399412627, "fcm_dpo/beta": 0.09033428132534027, "fcm_dpo/delta": 0.012706171721220016, "fcm_dpo/margin": 6.505819320678711, "fcm_dpo/q_t": 0.37230080366134644, "grad_norm": 25.748743057250977, "learning_rate": 4.890010211106795e-07, "logits/chosen": -0.5167370438575745, "logits/rejected": -0.09917855262756348, "logps/chosen": -93.13040161132812, "logps/ref_chosen": -90.61964416503906, "logps/ref_rejected": -83.10347747802734, "logps/rejected": -92.12004852294922, "loss": 1.0246, "margin_dpo/margin_mean": 6.505819320678711, "margin_dpo/margin_std": 9.225418090820312, "step": 128 }, { "epoch": 0.1894273127753304, "fcm_dpo/beta": 0.09042790532112122, "fcm_dpo/delta": -0.0049272216856479645, "fcm_dpo/margin": 6.681199550628662, "fcm_dpo/q_t": 0.37444210052490234, "grad_norm": 21.542911529541016, "learning_rate": 4.88621404556699e-07, "logits/chosen": -0.42197108268737793, "logits/rejected": -0.1338576376438141, "logps/chosen": -101.74089050292969, "logps/ref_chosen": -98.91870880126953, "logps/ref_rejected": -103.28419494628906, "logps/rejected": -112.78758239746094, "loss": 0.9973, "margin_dpo/margin_mean": 6.681199073791504, "margin_dpo/margin_std": 8.355775833129883, "step": 129 }, { "epoch": 0.19089574155653452, "fcm_dpo/beta": 0.08857293426990509, "fcm_dpo/delta": -0.17602676153182983, "fcm_dpo/margin": 8.605558395385742, "fcm_dpo/q_t": 0.3452799320220947, "grad_norm": 20.744892120361328, "learning_rate": 4.882355001067891e-07, "logits/chosen": -0.8332382440567017, "logits/rejected": -0.5825516581535339, "logps/chosen": -75.53247833251953, "logps/ref_chosen": -73.18368530273438, "logps/ref_rejected": -88.02154541015625, "logps/rejected": -98.97589111328125, "loss": 0.9116, "margin_dpo/margin_mean": 8.605558395385742, "margin_dpo/margin_std": 9.510639190673828, "step": 130 }, { "epoch": 0.19236417033773862, "fcm_dpo/beta": 0.08698062598705292, "fcm_dpo/delta": -0.04303303733468056, "fcm_dpo/margin": 7.350835800170898, "fcm_dpo/q_t": 0.3662865161895752, "grad_norm": 21.23581886291504, "learning_rate": 4.878433179298909e-07, "logits/chosen": -0.792221188545227, "logits/rejected": -0.576511800289154, "logps/chosen": -75.73124694824219, "logps/ref_chosen": -73.9564437866211, "logps/ref_rejected": -94.45582580566406, "logps/rejected": -103.58146667480469, "loss": 0.9788, "margin_dpo/margin_mean": 7.350835800170898, "margin_dpo/margin_std": 9.003894805908203, "step": 131 }, { "epoch": 0.19383259911894274, "fcm_dpo/beta": 0.08460888266563416, "fcm_dpo/delta": -0.08415042608976364, "fcm_dpo/margin": 8.008447647094727, "fcm_dpo/q_t": 0.3516160249710083, "grad_norm": 23.11528205871582, "learning_rate": 4.874448683603694e-07, "logits/chosen": -0.7575106620788574, "logits/rejected": -0.360861212015152, "logps/chosen": -95.101318359375, "logps/ref_chosen": -92.43948364257812, "logps/ref_rejected": -94.13252258300781, "logps/rejected": -104.80279541015625, "loss": 0.9311, "margin_dpo/margin_mean": 8.008447647094727, "margin_dpo/margin_std": 8.533775329589844, "step": 132 }, { "epoch": 0.19530102790014683, "fcm_dpo/beta": 0.08362139761447906, "fcm_dpo/delta": -0.1000952273607254, "fcm_dpo/margin": 8.29056167602539, "fcm_dpo/q_t": 0.35254591703414917, "grad_norm": 21.20782470703125, "learning_rate": 4.870401618977415e-07, "logits/chosen": -1.1061053276062012, "logits/rejected": -0.7356460690498352, "logps/chosen": -94.83768463134766, "logps/ref_chosen": -93.11666107177734, "logps/ref_rejected": -102.09490966796875, "logps/rejected": -112.10649108886719, "loss": 0.9185, "margin_dpo/margin_mean": 8.29056167602539, "margin_dpo/margin_std": 8.746664047241211, "step": 133 }, { "epoch": 0.19676945668135096, "fcm_dpo/beta": 0.08186972141265869, "fcm_dpo/delta": -0.14321613311767578, "fcm_dpo/margin": 8.948598861694336, "fcm_dpo/q_t": 0.3416699171066284, "grad_norm": 19.530412673950195, "learning_rate": 4.866292092063986e-07, "logits/chosen": -1.1816595792770386, "logits/rejected": -0.8657940626144409, "logps/chosen": -85.77832794189453, "logps/ref_chosen": -83.55012512207031, "logps/ref_rejected": -93.8815689086914, "logps/rejected": -105.05838012695312, "loss": 0.8911, "margin_dpo/margin_mean": 8.948599815368652, "margin_dpo/margin_std": 8.551406860351562, "step": 134 }, { "epoch": 0.19823788546255505, "fcm_dpo/beta": 0.0804293155670166, "fcm_dpo/delta": -0.045598309487104416, "fcm_dpo/margin": 7.988539695739746, "fcm_dpo/q_t": 0.3668595552444458, "grad_norm": 26.672277450561523, "learning_rate": 4.862120211153265e-07, "logits/chosen": -0.9140989780426025, "logits/rejected": -0.7617922425270081, "logps/chosen": -79.91949462890625, "logps/ref_chosen": -77.58262634277344, "logps/ref_rejected": -121.05648040771484, "logps/rejected": -131.38189697265625, "loss": 0.9718, "margin_dpo/margin_mean": 7.9885406494140625, "margin_dpo/margin_std": 9.902532577514648, "step": 135 }, { "epoch": 0.19970631424375918, "fcm_dpo/beta": 0.0786333680152893, "fcm_dpo/delta": -0.09831453859806061, "fcm_dpo/margin": 8.791913986206055, "fcm_dpo/q_t": 0.34778937697410583, "grad_norm": 19.27442169189453, "learning_rate": 4.857886086178193e-07, "logits/chosen": -1.0831172466278076, "logits/rejected": -0.6637728214263916, "logps/chosen": -96.29593658447266, "logps/ref_chosen": -94.1727294921875, "logps/ref_rejected": -103.26428985595703, "logps/rejected": -114.17940521240234, "loss": 0.8993, "margin_dpo/margin_mean": 8.791913986206055, "margin_dpo/margin_std": 8.21432113647461, "step": 136 }, { "epoch": 0.2011747430249633, "fcm_dpo/beta": 0.07321903109550476, "fcm_dpo/delta": -0.3953135013580322, "fcm_dpo/margin": 13.009666442871094, "fcm_dpo/q_t": 0.302865207195282, "grad_norm": 21.34975814819336, "learning_rate": 4.853589828711902e-07, "logits/chosen": -0.7808685302734375, "logits/rejected": -0.5483202934265137, "logps/chosen": -79.78280639648438, "logps/ref_chosen": -78.11874389648438, "logps/ref_rejected": -117.92237091064453, "logps/rejected": -132.59609985351562, "loss": 0.7814, "margin_dpo/margin_mean": 13.00966739654541, "margin_dpo/margin_std": 11.704439163208008, "step": 137 }, { "epoch": 0.2026431718061674, "fcm_dpo/beta": 0.07211971282958984, "fcm_dpo/delta": -0.01127841230481863, "fcm_dpo/margin": 8.464805603027344, "fcm_dpo/q_t": 0.36739087104797363, "grad_norm": 18.32305908203125, "learning_rate": 4.849231551964771e-07, "logits/chosen": -0.5027905702590942, "logits/rejected": -0.2673172950744629, "logps/chosen": -96.51560974121094, "logps/ref_chosen": -92.82994079589844, "logps/ref_rejected": -98.64840698242188, "logps/rejected": -110.79888153076172, "loss": 0.96, "margin_dpo/margin_mean": 8.464805603027344, "margin_dpo/margin_std": 9.157770156860352, "step": 138 }, { "epoch": 0.20411160058737152, "fcm_dpo/beta": 0.07099495828151703, "fcm_dpo/delta": -0.09988156706094742, "fcm_dpo/margin": 9.755861282348633, "fcm_dpo/q_t": 0.3473873734474182, "grad_norm": 18.42373275756836, "learning_rate": 4.844811370781446e-07, "logits/chosen": -0.8951112031936646, "logits/rejected": -0.577903687953949, "logps/chosen": -81.29713439941406, "logps/ref_chosen": -79.00301361083984, "logps/ref_rejected": -85.82733154296875, "logps/rejected": -97.87730407714844, "loss": 0.9312, "margin_dpo/margin_mean": 9.755861282348633, "margin_dpo/margin_std": 10.73771858215332, "step": 139 }, { "epoch": 0.2055800293685756, "fcm_dpo/beta": 0.07040712982416153, "fcm_dpo/delta": -0.03950156271457672, "fcm_dpo/margin": 9.047286987304688, "fcm_dpo/q_t": 0.36051124334335327, "grad_norm": 17.700237274169922, "learning_rate": 4.840329401637809e-07, "logits/chosen": -1.4827024936676025, "logits/rejected": -1.119741439819336, "logps/chosen": -90.76173400878906, "logps/ref_chosen": -88.35244750976562, "logps/ref_rejected": -90.57476806640625, "logps/rejected": -102.03134155273438, "loss": 0.9434, "margin_dpo/margin_mean": 9.047286033630371, "margin_dpo/margin_std": 9.427406311035156, "step": 140 }, { "epoch": 0.20704845814977973, "fcm_dpo/beta": 0.06885601580142975, "fcm_dpo/delta": -0.14049942791461945, "fcm_dpo/margin": 10.606624603271484, "fcm_dpo/q_t": 0.3436024785041809, "grad_norm": 19.141874313354492, "learning_rate": 4.83578576263792e-07, "logits/chosen": -1.1690764427185059, "logits/rejected": -0.8353248834609985, "logps/chosen": -110.29083251953125, "logps/ref_chosen": -107.22016143798828, "logps/ref_rejected": -104.71223449707031, "logps/rejected": -118.3895263671875, "loss": 0.8996, "margin_dpo/margin_mean": 10.606623649597168, "margin_dpo/margin_std": 10.912744522094727, "step": 141 }, { "epoch": 0.20851688693098386, "fcm_dpo/beta": 0.067325659096241, "fcm_dpo/delta": -0.07276973128318787, "fcm_dpo/margin": 9.913378715515137, "fcm_dpo/q_t": 0.3580227196216583, "grad_norm": 18.446134567260742, "learning_rate": 4.83118057351089e-07, "logits/chosen": -0.8606932759284973, "logits/rejected": -0.6145029067993164, "logps/chosen": -92.71931457519531, "logps/ref_chosen": -88.72894287109375, "logps/ref_rejected": -99.73038482666016, "logps/rejected": -113.63412475585938, "loss": 0.9319, "margin_dpo/margin_mean": 9.91337776184082, "margin_dpo/margin_std": 10.864763259887695, "step": 142 }, { "epoch": 0.20998531571218795, "fcm_dpo/beta": 0.06714846193790436, "fcm_dpo/delta": 0.021465405821800232, "fcm_dpo/margin": 8.62674331665039, "fcm_dpo/q_t": 0.37440505623817444, "grad_norm": 17.625202178955078, "learning_rate": 4.826513955607734e-07, "logits/chosen": -1.2188315391540527, "logits/rejected": -0.7806566953659058, "logps/chosen": -87.72565460205078, "logps/ref_chosen": -85.50457763671875, "logps/ref_rejected": -85.8056640625, "logps/rejected": -96.65348815917969, "loss": 0.9974, "margin_dpo/margin_mean": 8.62674331665039, "margin_dpo/margin_std": 10.493571281433105, "step": 143 }, { "epoch": 0.21145374449339208, "fcm_dpo/beta": 0.06795937567949295, "fcm_dpo/delta": -0.022204145789146423, "fcm_dpo/margin": 9.117193222045898, "fcm_dpo/q_t": 0.37021785974502563, "grad_norm": 22.30422019958496, "learning_rate": 4.821786031898176e-07, "logits/chosen": -1.0173625946044922, "logits/rejected": -0.5171209573745728, "logps/chosen": -107.23023223876953, "logps/ref_chosen": -104.68708801269531, "logps/ref_rejected": -87.71908569335938, "logps/rejected": -99.37940979003906, "loss": 1.0019, "margin_dpo/margin_mean": 9.117193222045898, "margin_dpo/margin_std": 12.013388633728027, "step": 144 }, { "epoch": 0.21292217327459617, "fcm_dpo/beta": 0.0664219856262207, "fcm_dpo/delta": -0.08367905020713806, "fcm_dpo/margin": 10.206480026245117, "fcm_dpo/q_t": 0.35269540548324585, "grad_norm": 18.23365592956543, "learning_rate": 4.816996926967401e-07, "logits/chosen": -0.9210905432701111, "logits/rejected": -0.36352017521858215, "logps/chosen": -90.94596862792969, "logps/ref_chosen": -88.51988983154297, "logps/ref_rejected": -83.76513671875, "logps/rejected": -96.397705078125, "loss": 0.9285, "margin_dpo/margin_mean": 10.206480979919434, "margin_dpo/margin_std": 10.862861633300781, "step": 145 }, { "epoch": 0.2143906020558003, "fcm_dpo/beta": 0.06422297656536102, "fcm_dpo/delta": -0.15030890703201294, "fcm_dpo/margin": 11.474971771240234, "fcm_dpo/q_t": 0.3387315571308136, "grad_norm": 17.679115295410156, "learning_rate": 4.812146767012779e-07, "logits/chosen": -1.0683330297470093, "logits/rejected": -0.4716656804084778, "logps/chosen": -105.42335510253906, "logps/ref_chosen": -102.90586853027344, "logps/ref_rejected": -88.9493637084961, "logps/rejected": -102.94181823730469, "loss": 0.8824, "margin_dpo/margin_mean": 11.474971771240234, "margin_dpo/margin_std": 10.659229278564453, "step": 146 }, { "epoch": 0.21585903083700442, "fcm_dpo/beta": 0.06264235824346542, "fcm_dpo/delta": -0.1676965206861496, "fcm_dpo/margin": 12.052690505981445, "fcm_dpo/q_t": 0.334587037563324, "grad_norm": 16.055294036865234, "learning_rate": 4.807235679840536e-07, "logits/chosen": -1.0700526237487793, "logits/rejected": -0.5048679113388062, "logps/chosen": -91.99482727050781, "logps/ref_chosen": -89.26219177246094, "logps/ref_rejected": -76.50011444091797, "logps/rejected": -91.28544616699219, "loss": 0.8674, "margin_dpo/margin_mean": 12.052690505981445, "margin_dpo/margin_std": 10.964826583862305, "step": 147 }, { "epoch": 0.2173274596182085, "fcm_dpo/beta": 0.06236677244305611, "fcm_dpo/delta": -0.03803453594446182, "fcm_dpo/margin": 10.167755126953125, "fcm_dpo/q_t": 0.3664420247077942, "grad_norm": 18.700401306152344, "learning_rate": 4.802263794862384e-07, "logits/chosen": -1.050333023071289, "logits/rejected": -0.6528257131576538, "logps/chosen": -93.0657958984375, "logps/ref_chosen": -91.76287841796875, "logps/ref_rejected": -110.38552856445312, "logps/rejected": -121.856201171875, "loss": 0.966, "margin_dpo/margin_mean": 10.167754173278809, "margin_dpo/margin_std": 11.42441177368164, "step": 148 }, { "epoch": 0.21879588839941264, "fcm_dpo/beta": 0.0613025538623333, "fcm_dpo/delta": -0.013467121869325638, "fcm_dpo/margin": 9.98930835723877, "fcm_dpo/q_t": 0.37364453077316284, "grad_norm": 17.756973266601562, "learning_rate": 4.797231243092118e-07, "logits/chosen": -0.9625898003578186, "logits/rejected": -0.6630960702896118, "logps/chosen": -97.20555114746094, "logps/ref_chosen": -93.48401641845703, "logps/ref_rejected": -105.6436996459961, "logps/rejected": -119.35454559326172, "loss": 0.991, "margin_dpo/margin_mean": 9.989309310913086, "margin_dpo/margin_std": 12.843564987182617, "step": 149 }, { "epoch": 0.22026431718061673, "fcm_dpo/beta": 0.060672082006931305, "fcm_dpo/delta": -0.07046893239021301, "fcm_dpo/margin": 10.968360900878906, "fcm_dpo/q_t": 0.35937947034835815, "grad_norm": 15.75436782836914, "learning_rate": 4.792138157142157e-07, "logits/chosen": -0.8642300367355347, "logits/rejected": -0.5881214141845703, "logps/chosen": -78.63943481445312, "logps/ref_chosen": -75.7786865234375, "logps/ref_rejected": -90.02676391601562, "logps/rejected": -103.85586547851562, "loss": 0.9473, "margin_dpo/margin_mean": 10.968360900878906, "margin_dpo/margin_std": 12.808120727539062, "step": 150 }, { "epoch": 0.22173274596182085, "fcm_dpo/beta": 0.06035682559013367, "fcm_dpo/delta": -0.029848817735910416, "fcm_dpo/margin": 10.404455184936523, "fcm_dpo/q_t": 0.3672201931476593, "grad_norm": 19.685895919799805, "learning_rate": 4.786984671220053e-07, "logits/chosen": -1.7692604064941406, "logits/rejected": -1.2120516300201416, "logps/chosen": -106.98040771484375, "logps/ref_chosen": -102.73422241210938, "logps/ref_rejected": -106.03781127929688, "logps/rejected": -120.68846130371094, "loss": 0.9769, "margin_dpo/margin_mean": 10.404455184936523, "margin_dpo/margin_std": 12.831315994262695, "step": 151 }, { "epoch": 0.22320117474302498, "fcm_dpo/beta": 0.05888533219695091, "fcm_dpo/delta": -0.14253246784210205, "fcm_dpo/margin": 12.427541732788086, "fcm_dpo/q_t": 0.34149640798568726, "grad_norm": 17.039073944091797, "learning_rate": 4.78177092112495e-07, "logits/chosen": -1.1869471073150635, "logits/rejected": -0.8218509554862976, "logps/chosen": -99.31748962402344, "logps/ref_chosen": -97.59074401855469, "logps/ref_rejected": -110.49913024902344, "logps/rejected": -124.6534194946289, "loss": 0.8913, "margin_dpo/margin_mean": 12.427543640136719, "margin_dpo/margin_std": 12.471309661865234, "step": 152 }, { "epoch": 0.22466960352422907, "fcm_dpo/beta": 0.05911749601364136, "fcm_dpo/delta": 0.06782151758670807, "fcm_dpo/margin": 9.064018249511719, "fcm_dpo/q_t": 0.3846895396709442, "grad_norm": 17.052255630493164, "learning_rate": 4.776497044244016e-07, "logits/chosen": -1.4898900985717773, "logits/rejected": -1.1749351024627686, "logps/chosen": -91.19923400878906, "logps/ref_chosen": -87.02560424804688, "logps/ref_rejected": -91.06854248046875, "logps/rejected": -104.30618286132812, "loss": 1.0342, "margin_dpo/margin_mean": 9.064018249511719, "margin_dpo/margin_std": 12.064422607421875, "step": 153 }, { "epoch": 0.2261380323054332, "fcm_dpo/beta": 0.05774786323308945, "fcm_dpo/delta": -0.2577056586742401, "fcm_dpo/margin": 14.486139297485352, "fcm_dpo/q_t": 0.319768488407135, "grad_norm": 16.834199905395508, "learning_rate": 4.771163179548808e-07, "logits/chosen": -1.1176913976669312, "logits/rejected": -0.7909313440322876, "logps/chosen": -93.70020294189453, "logps/ref_chosen": -91.10566711425781, "logps/ref_rejected": -111.37238311767578, "logps/rejected": -128.45306396484375, "loss": 0.8249, "margin_dpo/margin_mean": 14.486139297485352, "margin_dpo/margin_std": 12.080253601074219, "step": 154 }, { "epoch": 0.2276064610866373, "fcm_dpo/beta": 0.05554405599832535, "fcm_dpo/delta": -0.05215900391340256, "fcm_dpo/margin": 11.656644821166992, "fcm_dpo/q_t": 0.35945552587509155, "grad_norm": 20.919530868530273, "learning_rate": 4.7657694675916247e-07, "logits/chosen": -1.0245808362960815, "logits/rejected": -0.6745032072067261, "logps/chosen": -98.85678100585938, "logps/ref_chosen": -94.56097412109375, "logps/ref_rejected": -84.77359008789062, "logps/rejected": -100.72604370117188, "loss": 0.9947, "margin_dpo/margin_mean": 11.656644821166992, "margin_dpo/margin_std": 15.20389175415039, "step": 155 }, { "epoch": 0.2290748898678414, "fcm_dpo/beta": 0.055864423513412476, "fcm_dpo/delta": 0.03552495688199997, "fcm_dpo/margin": 10.137336730957031, "fcm_dpo/q_t": 0.37930774688720703, "grad_norm": 18.78835105895996, "learning_rate": 4.7603160505017893e-07, "logits/chosen": -1.3272318840026855, "logits/rejected": -0.8460725545883179, "logps/chosen": -100.33680725097656, "logps/ref_chosen": -94.65538787841797, "logps/ref_rejected": -84.41934204101562, "logps/rejected": -100.23809051513672, "loss": 1.0171, "margin_dpo/margin_mean": 10.137334823608398, "margin_dpo/margin_std": 13.136527061462402, "step": 156 }, { "epoch": 0.2305433186490455, "fcm_dpo/beta": 0.05504686385393143, "fcm_dpo/delta": -0.15157991647720337, "fcm_dpo/margin": 13.453115463256836, "fcm_dpo/q_t": 0.33803731203079224, "grad_norm": 17.70085334777832, "learning_rate": 4.7548030719819154e-07, "logits/chosen": -1.6043157577514648, "logits/rejected": -1.1462740898132324, "logps/chosen": -102.32630920410156, "logps/ref_chosen": -97.01239013671875, "logps/ref_rejected": -111.70133972167969, "logps/rejected": -130.4683837890625, "loss": 0.8766, "margin_dpo/margin_mean": 13.453115463256836, "margin_dpo/margin_std": 12.435213088989258, "step": 157 }, { "epoch": 0.23201174743024963, "fcm_dpo/beta": 0.053367845714092255, "fcm_dpo/delta": -0.1598518192768097, "fcm_dpo/margin": 14.018548011779785, "fcm_dpo/q_t": 0.3366357088088989, "grad_norm": 17.31310272216797, "learning_rate": 4.7492306773041136e-07, "logits/chosen": -1.1085999011993408, "logits/rejected": -0.8174431920051575, "logps/chosen": -99.63768005371094, "logps/ref_chosen": -94.71296691894531, "logps/ref_rejected": -120.93733978271484, "logps/rejected": -139.880615234375, "loss": 0.8714, "margin_dpo/margin_mean": 14.018546104431152, "margin_dpo/margin_std": 12.86544418334961, "step": 158 }, { "epoch": 0.23348017621145375, "fcm_dpo/beta": 0.05200641602277756, "fcm_dpo/delta": -0.09326906502246857, "fcm_dpo/margin": 13.204719543457031, "fcm_dpo/q_t": 0.35249608755111694, "grad_norm": 14.883583068847656, "learning_rate": 4.743599013306165e-07, "logits/chosen": -1.7073493003845215, "logits/rejected": -1.2342015504837036, "logps/chosen": -112.4176025390625, "logps/ref_chosen": -108.2182388305664, "logps/ref_rejected": -94.83901977539062, "logps/rejected": -112.24309539794922, "loss": 0.9328, "margin_dpo/margin_mean": 13.204719543457031, "margin_dpo/margin_std": 14.517641067504883, "step": 159 }, { "epoch": 0.23494860499265785, "fcm_dpo/beta": 0.05130424350500107, "fcm_dpo/delta": -0.05173896625638008, "fcm_dpo/margin": 12.633888244628906, "fcm_dpo/q_t": 0.3558708131313324, "grad_norm": 15.668502807617188, "learning_rate": 4.737908228387656e-07, "logits/chosen": -1.4224272966384888, "logits/rejected": -0.8817993402481079, "logps/chosen": -101.59320068359375, "logps/ref_chosen": -98.16340637207031, "logps/ref_rejected": -102.28097534179688, "logps/rejected": -118.34466552734375, "loss": 0.9332, "margin_dpo/margin_mean": 12.633889198303223, "margin_dpo/margin_std": 12.74842643737793, "step": 160 }, { "epoch": 0.23641703377386197, "fcm_dpo/beta": 0.05100458487868309, "fcm_dpo/delta": -0.0010594986379146576, "fcm_dpo/margin": 11.772229194641113, "fcm_dpo/q_t": 0.3696848750114441, "grad_norm": 16.675426483154297, "learning_rate": 4.7321584725060594e-07, "logits/chosen": -1.661262035369873, "logits/rejected": -1.1878881454467773, "logps/chosen": -92.0523681640625, "logps/ref_chosen": -86.59187316894531, "logps/ref_rejected": -89.01643371582031, "logps/rejected": -106.24916076660156, "loss": 0.9897, "margin_dpo/margin_mean": 11.772229194641113, "margin_dpo/margin_std": 14.080129623413086, "step": 161 }, { "epoch": 0.23788546255506607, "fcm_dpo/beta": 0.050782568752765656, "fcm_dpo/delta": -0.046759188175201416, "fcm_dpo/margin": 12.671504974365234, "fcm_dpo/q_t": 0.357089638710022, "grad_norm": 14.808147430419922, "learning_rate": 4.7263498971727905e-07, "logits/chosen": -0.9641022682189941, "logits/rejected": -0.5625172257423401, "logps/chosen": -102.07867431640625, "logps/ref_chosen": -97.40288543701172, "logps/ref_rejected": -95.86923217773438, "logps/rejected": -113.2165298461914, "loss": 0.9463, "margin_dpo/margin_mean": 12.671504974365234, "margin_dpo/margin_std": 13.506097793579102, "step": 162 }, { "epoch": 0.2393538913362702, "fcm_dpo/beta": 0.04923873394727707, "fcm_dpo/delta": -0.19184833765029907, "fcm_dpo/margin": 15.766294479370117, "fcm_dpo/q_t": 0.3363766372203827, "grad_norm": 13.101168632507324, "learning_rate": 4.720482655449212e-07, "logits/chosen": -1.5308029651641846, "logits/rejected": -1.145806908607483, "logps/chosen": -86.65454864501953, "logps/ref_chosen": -82.1753921508789, "logps/ref_rejected": -82.86473846435547, "logps/rejected": -103.11018371582031, "loss": 0.8757, "margin_dpo/margin_mean": 15.766292572021484, "margin_dpo/margin_std": 16.105995178222656, "step": 163 }, { "epoch": 0.24082232011747431, "fcm_dpo/beta": 0.04908021539449692, "fcm_dpo/delta": 0.08986516296863556, "fcm_dpo/margin": 10.468819618225098, "fcm_dpo/q_t": 0.38547319173812866, "grad_norm": 15.708305358886719, "learning_rate": 4.714556901942599e-07, "logits/chosen": -1.2432619333267212, "logits/rejected": -0.7992422580718994, "logps/chosen": -98.1744613647461, "logps/ref_chosen": -90.47427368164062, "logps/ref_rejected": -87.02809143066406, "logps/rejected": -105.19710540771484, "loss": 1.031, "margin_dpo/margin_mean": 10.468820571899414, "margin_dpo/margin_std": 12.44760799407959, "step": 164 }, { "epoch": 0.2422907488986784, "fcm_dpo/beta": 0.04876025393605232, "fcm_dpo/delta": -0.1661471575498581, "fcm_dpo/margin": 15.461935997009277, "fcm_dpo/q_t": 0.3421006202697754, "grad_norm": 13.452424049377441, "learning_rate": 4.708572792802069e-07, "logits/chosen": -1.3514103889465332, "logits/rejected": -0.941323459148407, "logps/chosen": -92.18637084960938, "logps/ref_chosen": -87.313232421875, "logps/ref_rejected": -81.902587890625, "logps/rejected": -102.23765563964844, "loss": 0.8884, "margin_dpo/margin_mean": 15.461935997009277, "margin_dpo/margin_std": 16.554237365722656, "step": 165 }, { "epoch": 0.24375917767988253, "fcm_dpo/beta": 0.04737546294927597, "fcm_dpo/delta": -0.10841958224773407, "fcm_dpo/margin": 14.792806625366211, "fcm_dpo/q_t": 0.35638824105262756, "grad_norm": 14.667271614074707, "learning_rate": 4.702530485714461e-07, "logits/chosen": -1.409278154373169, "logits/rejected": -1.1956782341003418, "logps/chosen": -90.57990264892578, "logps/ref_chosen": -81.58606719970703, "logps/ref_rejected": -103.16517639160156, "logps/rejected": -126.95182800292969, "loss": 0.9348, "margin_dpo/margin_mean": 14.792806625366211, "margin_dpo/margin_std": 17.73269271850586, "step": 166 }, { "epoch": 0.24522760646108663, "fcm_dpo/beta": 0.0457189604640007, "fcm_dpo/delta": -0.15542298555374146, "fcm_dpo/margin": 16.228336334228516, "fcm_dpo/q_t": 0.33740636706352234, "grad_norm": 14.068784713745117, "learning_rate": 4.6964301399001877e-07, "logits/chosen": -0.9149848222732544, "logits/rejected": -0.7234656810760498, "logps/chosen": -91.24684143066406, "logps/ref_chosen": -83.56167602539062, "logps/ref_rejected": -99.05305480957031, "logps/rejected": -122.966552734375, "loss": 0.8664, "margin_dpo/margin_mean": 16.228334426879883, "margin_dpo/margin_std": 14.279183387756348, "step": 167 }, { "epoch": 0.24669603524229075, "fcm_dpo/beta": 0.04559946060180664, "fcm_dpo/delta": -0.028263315558433533, "fcm_dpo/margin": 13.73760986328125, "fcm_dpo/q_t": 0.3622729182243347, "grad_norm": 15.556920051574707, "learning_rate": 4.690271916109034e-07, "logits/chosen": -1.2671973705291748, "logits/rejected": -0.9255896210670471, "logps/chosen": -87.52298736572266, "logps/ref_chosen": -79.93123626708984, "logps/ref_rejected": -80.10216522216797, "logps/rejected": -101.43153381347656, "loss": 0.9486, "margin_dpo/margin_mean": 13.73760986328125, "margin_dpo/margin_std": 14.369284629821777, "step": 168 }, { "epoch": 0.24816446402349487, "fcm_dpo/beta": 0.0447533056139946, "fcm_dpo/delta": -0.10509349405765533, "fcm_dpo/margin": 15.592907905578613, "fcm_dpo/q_t": 0.3495541214942932, "grad_norm": 14.77795124053955, "learning_rate": 4.6840559766159235e-07, "logits/chosen": -1.474048137664795, "logits/rejected": -1.2569692134857178, "logps/chosen": -89.0520248413086, "logps/ref_chosen": -82.90370178222656, "logps/ref_rejected": -90.12191009521484, "logps/rejected": -111.8631362915039, "loss": 0.9092, "margin_dpo/margin_mean": 15.592906951904297, "margin_dpo/margin_std": 15.848993301391602, "step": 169 }, { "epoch": 0.24963289280469897, "fcm_dpo/beta": 0.044094495475292206, "fcm_dpo/delta": -0.01727929711341858, "fcm_dpo/margin": 13.957527160644531, "fcm_dpo/q_t": 0.3613913655281067, "grad_norm": 15.097552299499512, "learning_rate": 4.6777824852166437e-07, "logits/chosen": -1.403516173362732, "logits/rejected": -1.1804269552230835, "logps/chosen": -82.96324157714844, "logps/ref_chosen": -74.98197174072266, "logps/ref_rejected": -82.16560363769531, "logps/rejected": -104.10440063476562, "loss": 0.9393, "margin_dpo/margin_mean": 13.957527160644531, "margin_dpo/margin_std": 13.041166305541992, "step": 170 }, { "epoch": 0.2511013215859031, "fcm_dpo/beta": 0.043724410235881805, "fcm_dpo/delta": -0.0946701318025589, "fcm_dpo/margin": 15.739529609680176, "fcm_dpo/q_t": 0.36089038848876953, "grad_norm": 16.572017669677734, "learning_rate": 4.6714516072235273e-07, "logits/chosen": -1.3717052936553955, "logits/rejected": -0.8343698978424072, "logps/chosen": -128.6714630126953, "logps/ref_chosen": -118.97853088378906, "logps/ref_rejected": -115.76930236816406, "logps/rejected": -141.2017822265625, "loss": 0.9651, "margin_dpo/margin_mean": 15.73952865600586, "margin_dpo/margin_std": 20.676483154296875, "step": 171 }, { "epoch": 0.2525697503671072, "fcm_dpo/beta": 0.043144889175891876, "fcm_dpo/delta": -0.026401542127132416, "fcm_dpo/margin": 14.471285820007324, "fcm_dpo/q_t": 0.35602670907974243, "grad_norm": 12.957140922546387, "learning_rate": 4.6650635094610966e-07, "logits/chosen": -1.9741286039352417, "logits/rejected": -1.2486335039138794, "logps/chosen": -111.53092956542969, "logps/ref_chosen": -101.15379333496094, "logps/ref_rejected": -91.00568389892578, "logps/rejected": -115.85411071777344, "loss": 0.9095, "margin_dpo/margin_mean": 14.471284866333008, "margin_dpo/margin_std": 10.961095809936523, "step": 172 }, { "epoch": 0.2540381791483113, "fcm_dpo/beta": 0.04372722655534744, "fcm_dpo/delta": 0.06630315631628036, "fcm_dpo/margin": 12.284669876098633, "fcm_dpo/q_t": 0.37949445843696594, "grad_norm": 13.340476989746094, "learning_rate": 4.6586183602616687e-07, "logits/chosen": -1.3271675109863281, "logits/rejected": -0.8779016733169556, "logps/chosen": -108.86444091796875, "logps/ref_chosen": -99.05668640136719, "logps/ref_rejected": -83.8507308959961, "logps/rejected": -105.94315338134766, "loss": 0.9963, "margin_dpo/margin_mean": 12.284669876098633, "margin_dpo/margin_std": 13.309188842773438, "step": 173 }, { "epoch": 0.2555066079295154, "fcm_dpo/beta": 0.04317288100719452, "fcm_dpo/delta": -0.11161762475967407, "fcm_dpo/margin": 16.303455352783203, "fcm_dpo/q_t": 0.34579095244407654, "grad_norm": 15.587017059326172, "learning_rate": 4.652116329460919e-07, "logits/chosen": -1.7453947067260742, "logits/rejected": -1.6798264980316162, "logps/chosen": -89.8681869506836, "logps/ref_chosen": -80.40890502929688, "logps/ref_rejected": -108.20171356201172, "logps/rejected": -133.96444702148438, "loss": 0.904, "margin_dpo/margin_mean": 16.303455352783203, "margin_dpo/margin_std": 16.183778762817383, "step": 174 }, { "epoch": 0.25697503671071953, "fcm_dpo/beta": 0.04247898980975151, "fcm_dpo/delta": -0.09163874387741089, "fcm_dpo/margin": 16.133861541748047, "fcm_dpo/q_t": 0.3471967577934265, "grad_norm": 15.230539321899414, "learning_rate": 4.645557588393406e-07, "logits/chosen": -0.9117776155471802, "logits/rejected": -0.6414705514907837, "logps/chosen": -95.48237609863281, "logps/ref_chosen": -82.71649169921875, "logps/ref_rejected": -96.66712951660156, "logps/rejected": -125.56687927246094, "loss": 0.8972, "margin_dpo/margin_mean": 16.133861541748047, "margin_dpo/margin_std": 14.272943496704102, "step": 175 }, { "epoch": 0.25844346549192365, "fcm_dpo/beta": 0.041571564972400665, "fcm_dpo/delta": -0.08201269805431366, "fcm_dpo/margin": 16.272031784057617, "fcm_dpo/q_t": 0.3564577102661133, "grad_norm": 13.990058898925781, "learning_rate": 4.638942309888058e-07, "logits/chosen": -1.4704281091690063, "logits/rejected": -1.2904351949691772, "logps/chosen": -90.04341888427734, "logps/ref_chosen": -79.62930297851562, "logps/ref_rejected": -102.6885986328125, "logps/rejected": -129.374755859375, "loss": 0.9461, "margin_dpo/margin_mean": 16.272031784057617, "margin_dpo/margin_std": 18.543813705444336, "step": 176 }, { "epoch": 0.2599118942731278, "fcm_dpo/beta": 0.04052150249481201, "fcm_dpo/delta": -0.1876201629638672, "fcm_dpo/margin": 19.08730125427246, "fcm_dpo/q_t": 0.33940553665161133, "grad_norm": 12.637894630432129, "learning_rate": 4.6322706682636137e-07, "logits/chosen": -1.425307273864746, "logits/rejected": -1.0769648551940918, "logps/chosen": -103.93244171142578, "logps/ref_chosen": -92.56109619140625, "logps/ref_rejected": -103.33839416503906, "logps/rejected": -133.79702758789062, "loss": 0.876, "margin_dpo/margin_mean": 19.08730125427246, "margin_dpo/margin_std": 19.590530395507812, "step": 177 }, { "epoch": 0.26138032305433184, "fcm_dpo/beta": 0.03819538652896881, "fcm_dpo/delta": -0.28372639417648315, "fcm_dpo/margin": 22.48978614807129, "fcm_dpo/q_t": 0.31603994965553284, "grad_norm": 14.502700805664062, "learning_rate": 4.6255428393240354e-07, "logits/chosen": -1.4522943496704102, "logits/rejected": -1.1993482112884521, "logps/chosen": -94.813720703125, "logps/ref_chosen": -84.26579284667969, "logps/ref_rejected": -110.79730987548828, "logps/rejected": -143.83502197265625, "loss": 0.8098, "margin_dpo/margin_mean": 22.489788055419922, "margin_dpo/margin_std": 19.596847534179688, "step": 178 }, { "epoch": 0.26284875183553597, "fcm_dpo/beta": 0.03770379722118378, "fcm_dpo/delta": 0.03393785282969475, "fcm_dpo/margin": 15.058501243591309, "fcm_dpo/q_t": 0.3792954683303833, "grad_norm": 14.292525291442871, "learning_rate": 4.6187590003538724e-07, "logits/chosen": -1.4518247842788696, "logits/rejected": -1.0252907276153564, "logps/chosen": -107.04389953613281, "logps/ref_chosen": -93.4479751586914, "logps/ref_rejected": -94.25541687011719, "logps/rejected": -122.90983581542969, "loss": 1.0249, "margin_dpo/margin_mean": 15.058500289916992, "margin_dpo/margin_std": 20.123884201049805, "step": 179 }, { "epoch": 0.2643171806167401, "fcm_dpo/beta": 0.037198685109615326, "fcm_dpo/delta": -0.1041245386004448, "fcm_dpo/margin": 18.714359283447266, "fcm_dpo/q_t": 0.34685850143432617, "grad_norm": 16.52766227722168, "learning_rate": 4.611919330113591e-07, "logits/chosen": -1.3025885820388794, "logits/rejected": -1.0561097860336304, "logps/chosen": -100.08863830566406, "logps/ref_chosen": -88.83554077148438, "logps/ref_rejected": -103.90696716308594, "logps/rejected": -133.87442016601562, "loss": 0.894, "margin_dpo/margin_mean": 18.714359283447266, "margin_dpo/margin_std": 17.619686126708984, "step": 180 }, { "epoch": 0.2657856093979442, "fcm_dpo/beta": 0.037504084408283234, "fcm_dpo/delta": 0.05481020361185074, "fcm_dpo/margin": 14.614221572875977, "fcm_dpo/q_t": 0.3806135952472687, "grad_norm": 13.32461166381836, "learning_rate": 4.605024008834863e-07, "logits/chosen": -1.3358888626098633, "logits/rejected": -0.8776803612709045, "logps/chosen": -90.83370208740234, "logps/ref_chosen": -79.23396301269531, "logps/ref_rejected": -69.17442321777344, "logps/rejected": -95.38838195800781, "loss": 1.0004, "margin_dpo/margin_mean": 14.614219665527344, "margin_dpo/margin_std": 17.222267150878906, "step": 181 }, { "epoch": 0.26725403817914833, "fcm_dpo/beta": 0.036797843873500824, "fcm_dpo/delta": -0.10142774879932404, "fcm_dpo/margin": 18.840681076049805, "fcm_dpo/q_t": 0.3518170118331909, "grad_norm": 13.206196784973145, "learning_rate": 4.598073218215817e-07, "logits/chosen": -0.9532027244567871, "logits/rejected": -0.7038797736167908, "logps/chosen": -90.80711364746094, "logps/ref_chosen": -76.08218383789062, "logps/ref_rejected": -94.9686050415039, "logps/rejected": -128.53421020507812, "loss": 0.9321, "margin_dpo/margin_mean": 18.840679168701172, "margin_dpo/margin_std": 20.858131408691406, "step": 182 }, { "epoch": 0.2687224669603524, "fcm_dpo/beta": 0.03612806648015976, "fcm_dpo/delta": -0.09574753046035767, "fcm_dpo/margin": 19.054080963134766, "fcm_dpo/q_t": 0.3482775390148163, "grad_norm": 13.609108924865723, "learning_rate": 4.5910671414162484e-07, "logits/chosen": -1.2038804292678833, "logits/rejected": -0.5809817314147949, "logps/chosen": -108.66018676757812, "logps/ref_chosen": -93.90790557861328, "logps/ref_rejected": -83.56061553955078, "logps/rejected": -117.36697387695312, "loss": 0.9029, "margin_dpo/margin_mean": 19.054080963134766, "margin_dpo/margin_std": 17.8356990814209, "step": 183 }, { "epoch": 0.2701908957415565, "fcm_dpo/beta": 0.03689391911029816, "fcm_dpo/delta": 0.14321771264076233, "fcm_dpo/margin": 12.56014633178711, "fcm_dpo/q_t": 0.3973623514175415, "grad_norm": 13.919024467468262, "learning_rate": 4.5840059630527985e-07, "logits/chosen": -1.4166245460510254, "logits/rejected": -1.154975175857544, "logps/chosen": -100.36515045166016, "logps/ref_chosen": -86.86790466308594, "logps/ref_rejected": -84.28456115722656, "logps/rejected": -110.34194946289062, "loss": 1.0914, "margin_dpo/margin_mean": 12.56014633178711, "margin_dpo/margin_std": 19.446250915527344, "step": 184 }, { "epoch": 0.27165932452276065, "fcm_dpo/beta": 0.03748057410120964, "fcm_dpo/delta": 0.027648886665701866, "fcm_dpo/margin": 15.299049377441406, "fcm_dpo/q_t": 0.3777276873588562, "grad_norm": 15.781747817993164, "learning_rate": 4.5768898691940836e-07, "logits/chosen": -1.9094171524047852, "logits/rejected": -1.3005274534225464, "logps/chosen": -108.41390991210938, "logps/ref_chosen": -94.02142333984375, "logps/ref_rejected": -81.45668029785156, "logps/rejected": -111.1482162475586, "loss": 1.0174, "margin_dpo/margin_mean": 15.299049377441406, "margin_dpo/margin_std": 20.224950790405273, "step": 185 }, { "epoch": 0.27312775330396477, "fcm_dpo/beta": 0.03701728954911232, "fcm_dpo/delta": -0.03318379819393158, "fcm_dpo/margin": 17.034767150878906, "fcm_dpo/q_t": 0.3616868853569031, "grad_norm": 15.530001640319824, "learning_rate": 4.5697190473557947e-07, "logits/chosen": -2.2073938846588135, "logits/rejected": -1.3490548133850098, "logps/chosen": -131.6390380859375, "logps/ref_chosen": -114.19168090820312, "logps/ref_rejected": -93.33912658691406, "logps/rejected": -127.82122802734375, "loss": 0.9536, "margin_dpo/margin_mean": 17.03476905822754, "margin_dpo/margin_std": 18.32429313659668, "step": 186 }, { "epoch": 0.2745961820851689, "fcm_dpo/beta": 0.03734853118658066, "fcm_dpo/delta": 0.018614832311868668, "fcm_dpo/margin": 15.587629318237305, "fcm_dpo/q_t": 0.37575846910476685, "grad_norm": 13.691910743713379, "learning_rate": 4.5624936864957555e-07, "logits/chosen": -1.4853415489196777, "logits/rejected": -1.2423460483551025, "logps/chosen": -92.11011505126953, "logps/ref_chosen": -76.86955261230469, "logps/ref_rejected": -88.16494750976562, "logps/rejected": -118.99314880371094, "loss": 1.013, "margin_dpo/margin_mean": 15.587630271911621, "margin_dpo/margin_std": 20.420238494873047, "step": 187 }, { "epoch": 0.27606461086637296, "fcm_dpo/beta": 0.0366520918905735, "fcm_dpo/delta": -0.07102610170841217, "fcm_dpo/margin": 18.150962829589844, "fcm_dpo/q_t": 0.35110145807266235, "grad_norm": 13.728299140930176, "learning_rate": 4.5552139770089454e-07, "logits/chosen": -1.6838798522949219, "logits/rejected": -1.4296061992645264, "logps/chosen": -90.44649505615234, "logps/ref_chosen": -79.1948013305664, "logps/ref_rejected": -96.82889556884766, "logps/rejected": -126.23155212402344, "loss": 0.9272, "margin_dpo/margin_mean": 18.150962829589844, "margin_dpo/margin_std": 17.76366424560547, "step": 188 }, { "epoch": 0.2775330396475771, "fcm_dpo/beta": 0.0365501269698143, "fcm_dpo/delta": -0.09013816714286804, "fcm_dpo/margin": 18.695587158203125, "fcm_dpo/q_t": 0.3573761284351349, "grad_norm": 12.32153606414795, "learning_rate": 4.5478801107224794e-07, "logits/chosen": -1.5201849937438965, "logits/rejected": -0.8749784231185913, "logps/chosen": -99.00448608398438, "logps/ref_chosen": -83.8927993774414, "logps/ref_rejected": -79.46589660644531, "logps/rejected": -113.2731704711914, "loss": 0.9442, "margin_dpo/margin_mean": 18.695587158203125, "margin_dpo/margin_std": 21.5902042388916, "step": 189 }, { "epoch": 0.2790014684287812, "fcm_dpo/beta": 0.03506568446755409, "fcm_dpo/delta": -0.180100679397583, "fcm_dpo/margin": 21.843788146972656, "fcm_dpo/q_t": 0.331352174282074, "grad_norm": 13.670723915100098, "learning_rate": 4.5404922808905543e-07, "logits/chosen": -1.5921630859375, "logits/rejected": -0.9873641133308411, "logps/chosen": -115.09228515625, "logps/ref_chosen": -103.44146728515625, "logps/ref_rejected": -109.21502685546875, "logps/rejected": -142.70962524414062, "loss": 0.8555, "margin_dpo/margin_mean": 21.843786239624023, "margin_dpo/margin_std": 19.022666931152344, "step": 190 }, { "epoch": 0.28046989720998533, "fcm_dpo/beta": 0.03367793187499046, "fcm_dpo/delta": -0.23534753918647766, "fcm_dpo/margin": 24.244056701660156, "fcm_dpo/q_t": 0.3260498046875, "grad_norm": 14.045138359069824, "learning_rate": 4.5330506821893565e-07, "logits/chosen": -1.439011812210083, "logits/rejected": -1.1144874095916748, "logps/chosen": -110.62017059326172, "logps/ref_chosen": -99.78551483154297, "logps/ref_rejected": -116.00028228759766, "logps/rejected": -151.0789794921875, "loss": 0.8445, "margin_dpo/margin_mean": 24.244056701660156, "margin_dpo/margin_std": 22.547290802001953, "step": 191 }, { "epoch": 0.28193832599118945, "fcm_dpo/beta": 0.03333042562007904, "fcm_dpo/delta": 0.03673925995826721, "fcm_dpo/margin": 16.96189308166504, "fcm_dpo/q_t": 0.37764817476272583, "grad_norm": 15.383365631103516, "learning_rate": 4.5255555107119336e-07, "logits/chosen": -1.293067455291748, "logits/rejected": -1.030430793762207, "logps/chosen": -120.11058807373047, "logps/ref_chosen": -100.86032104492188, "logps/ref_rejected": -109.44500732421875, "logps/rejected": -145.6571502685547, "loss": 1.0054, "margin_dpo/margin_mean": 16.961894989013672, "margin_dpo/margin_std": 21.036739349365234, "step": 192 }, { "epoch": 0.2834067547723935, "fcm_dpo/beta": 0.0341544970870018, "fcm_dpo/delta": 0.1115766242146492, "fcm_dpo/margin": 14.43603515625, "fcm_dpo/q_t": 0.3931388258934021, "grad_norm": 14.872236251831055, "learning_rate": 4.5180069639630236e-07, "logits/chosen": -1.807457685470581, "logits/rejected": -1.221724271774292, "logps/chosen": -108.34063720703125, "logps/ref_chosen": -92.93831634521484, "logps/ref_rejected": -83.10917663574219, "logps/rejected": -112.94754028320312, "loss": 1.0871, "margin_dpo/margin_mean": 14.43603515625, "margin_dpo/margin_std": 22.31793975830078, "step": 193 }, { "epoch": 0.28487518355359764, "fcm_dpo/beta": 0.0344829261302948, "fcm_dpo/delta": 0.020925231277942657, "fcm_dpo/margin": 16.809911727905273, "fcm_dpo/q_t": 0.3748980760574341, "grad_norm": 12.2451810836792, "learning_rate": 4.510405240853854e-07, "logits/chosen": -2.212796211242676, "logits/rejected": -1.8392068147659302, "logps/chosen": -79.75379180908203, "logps/ref_chosen": -63.79157257080078, "logps/ref_rejected": -68.72665405273438, "logps/rejected": -101.498779296875, "loss": 1.0019, "margin_dpo/margin_mean": 16.809911727905273, "margin_dpo/margin_std": 20.774627685546875, "step": 194 }, { "epoch": 0.28634361233480177, "fcm_dpo/beta": 0.03373473137617111, "fcm_dpo/delta": -0.14405208826065063, "fcm_dpo/margin": 21.749523162841797, "fcm_dpo/q_t": 0.3414178490638733, "grad_norm": 12.763561248779297, "learning_rate": 4.5027505416968985e-07, "logits/chosen": -1.0410107374191284, "logits/rejected": -0.9035718441009521, "logps/chosen": -99.03205871582031, "logps/ref_chosen": -83.88545989990234, "logps/ref_rejected": -104.00125122070312, "logps/rejected": -140.89736938476562, "loss": 0.9105, "margin_dpo/margin_mean": 21.749523162841797, "margin_dpo/margin_std": 22.94123649597168, "step": 195 }, { "epoch": 0.2878120411160059, "fcm_dpo/beta": 0.032888732850551605, "fcm_dpo/delta": -0.09442660957574844, "fcm_dpo/margin": 20.915809631347656, "fcm_dpo/q_t": 0.34680312871932983, "grad_norm": 11.398536682128906, "learning_rate": 4.495043068200599e-07, "logits/chosen": -1.0971794128417969, "logits/rejected": -0.6393330693244934, "logps/chosen": -106.94159698486328, "logps/ref_chosen": -89.4013671875, "logps/ref_rejected": -83.34735107421875, "logps/rejected": -121.80339050292969, "loss": 0.9011, "margin_dpo/margin_mean": 20.915809631347656, "margin_dpo/margin_std": 19.22466278076172, "step": 196 }, { "epoch": 0.28928046989721, "fcm_dpo/beta": 0.03359599411487579, "fcm_dpo/delta": 0.17620685696601868, "fcm_dpo/margin": 12.843008041381836, "fcm_dpo/q_t": 0.40246933698654175, "grad_norm": 13.048999786376953, "learning_rate": 4.4872830234640493e-07, "logits/chosen": -1.3644063472747803, "logits/rejected": -1.0448070764541626, "logps/chosen": -105.15078735351562, "logps/ref_chosen": -86.84070587158203, "logps/ref_rejected": -82.79289245605469, "logps/rejected": -113.94599914550781, "loss": 1.0851, "margin_dpo/margin_mean": 12.843008995056152, "margin_dpo/margin_std": 18.110435485839844, "step": 197 }, { "epoch": 0.2907488986784141, "fcm_dpo/beta": 0.03354540839791298, "fcm_dpo/delta": -0.08072011172771454, "fcm_dpo/margin": 20.13166046142578, "fcm_dpo/q_t": 0.3533306121826172, "grad_norm": 15.39021110534668, "learning_rate": 4.479470611971645e-07, "logits/chosen": -1.7479376792907715, "logits/rejected": -1.5593535900115967, "logps/chosen": -106.19425964355469, "logps/ref_chosen": -85.04824829101562, "logps/ref_rejected": -104.10365295410156, "logps/rejected": -145.38133239746094, "loss": 0.9229, "margin_dpo/margin_mean": 20.13166046142578, "margin_dpo/margin_std": 20.80800437927246, "step": 198 }, { "epoch": 0.2922173274596182, "fcm_dpo/beta": 0.033529043197631836, "fcm_dpo/delta": 0.020058371126651764, "fcm_dpo/margin": 17.328767776489258, "fcm_dpo/q_t": 0.373092383146286, "grad_norm": 13.264050483703613, "learning_rate": 4.471606039587695e-07, "logits/chosen": -1.4798874855041504, "logits/rejected": -1.2172993421554565, "logps/chosen": -102.33380889892578, "logps/ref_chosen": -81.78669738769531, "logps/ref_rejected": -90.790771484375, "logps/rejected": -128.66665649414062, "loss": 0.992, "margin_dpo/margin_mean": 17.328765869140625, "margin_dpo/margin_std": 20.446475982666016, "step": 199 }, { "epoch": 0.2936857562408223, "fcm_dpo/beta": 0.03291964530944824, "fcm_dpo/delta": -0.0771205723285675, "fcm_dpo/margin": 20.37543296813965, "fcm_dpo/q_t": 0.35671910643577576, "grad_norm": 12.404348373413086, "learning_rate": 4.4636895135509966e-07, "logits/chosen": -1.604175090789795, "logits/rejected": -1.086242437362671, "logps/chosen": -104.21473693847656, "logps/ref_chosen": -84.26809692382812, "logps/ref_rejected": -86.96786499023438, "logps/rejected": -127.28994750976562, "loss": 0.9603, "margin_dpo/margin_mean": 20.37543296813965, "margin_dpo/margin_std": 23.788284301757812, "step": 200 }, { "epoch": 0.2936857562408223, "eval_fcm_dpo/beta": 0.0329931266605854, "eval_logits/chosen": -1.439759373664856, "eval_logits/rejected": -1.1262362003326416, "eval_logps/chosen": -123.58830261230469, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -132.22947692871094, "eval_loss": 0.534487247467041, "eval_margin_dpo/margin_mean": 15.06699275970459, "eval_margin_dpo/margin_std": 21.761512756347656, "eval_runtime": 43.741, "eval_samples_per_second": 53.474, "eval_steps_per_second": 1.692, "step": 200 }, { "epoch": 0.29515418502202645, "fcm_dpo/beta": 0.03285796195268631, "fcm_dpo/delta": -0.0432654544711113, "fcm_dpo/margin": 19.492534637451172, "fcm_dpo/q_t": 0.36308830976486206, "grad_norm": 13.604676246643066, "learning_rate": 4.455721242469372e-07, "logits/chosen": -1.3040488958358765, "logits/rejected": -0.9086974263191223, "logps/chosen": -129.45172119140625, "logps/ref_chosen": -107.77249145507812, "logps/ref_rejected": -119.79248046875, "logps/rejected": -160.9642333984375, "loss": 0.9856, "margin_dpo/margin_mean": 19.492534637451172, "margin_dpo/margin_std": 24.657806396484375, "step": 201 }, { "epoch": 0.2966226138032305, "fcm_dpo/beta": 0.03282172232866287, "fcm_dpo/delta": 0.012272818945348263, "fcm_dpo/margin": 17.926525115966797, "fcm_dpo/q_t": 0.3727894723415375, "grad_norm": 13.259603500366211, "learning_rate": 4.4477014363141755e-07, "logits/chosen": -1.6802592277526855, "logits/rejected": -1.547353744506836, "logps/chosen": -96.56066131591797, "logps/ref_chosen": -75.97245788574219, "logps/ref_rejected": -94.4599838256836, "logps/rejected": -132.9747314453125, "loss": 0.9903, "margin_dpo/margin_mean": 17.926525115966797, "margin_dpo/margin_std": 21.052555084228516, "step": 202 }, { "epoch": 0.29809104258443464, "fcm_dpo/beta": 0.032457903027534485, "fcm_dpo/delta": -0.07600473612546921, "fcm_dpo/margin": 20.671689987182617, "fcm_dpo/q_t": 0.3520776033401489, "grad_norm": 11.885881423950195, "learning_rate": 4.439630306414758e-07, "logits/chosen": -1.69942307472229, "logits/rejected": -1.2441532611846924, "logps/chosen": -112.8557357788086, "logps/ref_chosen": -94.96715545654297, "logps/ref_rejected": -92.8876724243164, "logps/rejected": -131.44793701171875, "loss": 0.9224, "margin_dpo/margin_mean": 20.671688079833984, "margin_dpo/margin_std": 20.679279327392578, "step": 203 }, { "epoch": 0.29955947136563876, "fcm_dpo/beta": 0.032535798847675323, "fcm_dpo/delta": 0.02650514431297779, "fcm_dpo/margin": 17.66338348388672, "fcm_dpo/q_t": 0.3760603666305542, "grad_norm": 14.426183700561523, "learning_rate": 4.431508065452897e-07, "logits/chosen": -1.7683607339859009, "logits/rejected": -1.129054069519043, "logps/chosen": -139.35574340820312, "logps/ref_chosen": -116.35719299316406, "logps/ref_rejected": -93.39759063720703, "logps/rejected": -134.05953979492188, "loss": 1.0127, "margin_dpo/margin_mean": 17.66338348388672, "margin_dpo/margin_std": 22.486337661743164, "step": 204 }, { "epoch": 0.3010279001468429, "fcm_dpo/beta": 0.031746186316013336, "fcm_dpo/delta": -0.1341342329978943, "fcm_dpo/margin": 22.80514907836914, "fcm_dpo/q_t": 0.3369899392127991, "grad_norm": 11.1696195602417, "learning_rate": 4.4233349274571974e-07, "logits/chosen": -2.104504108428955, "logits/rejected": -1.677614688873291, "logps/chosen": -109.97764587402344, "logps/ref_chosen": -88.85934448242188, "logps/ref_rejected": -91.8544921875, "logps/rejected": -135.77793884277344, "loss": 0.8669, "margin_dpo/margin_mean": 22.80514907836914, "margin_dpo/margin_std": 18.854307174682617, "step": 205 }, { "epoch": 0.302496328928047, "fcm_dpo/beta": 0.031177151948213577, "fcm_dpo/delta": -0.0601668581366539, "fcm_dpo/margin": 21.0255069732666, "fcm_dpo/q_t": 0.3517179489135742, "grad_norm": 14.36100959777832, "learning_rate": 4.415111107797445e-07, "logits/chosen": -1.566483497619629, "logits/rejected": -1.3959765434265137, "logps/chosen": -98.07730102539062, "logps/ref_chosen": -76.54634857177734, "logps/ref_rejected": -102.95314025878906, "logps/rejected": -145.50961303710938, "loss": 0.9067, "margin_dpo/margin_mean": 21.0255069732666, "margin_dpo/margin_std": 18.027606964111328, "step": 206 }, { "epoch": 0.3039647577092511, "fcm_dpo/beta": 0.030731763690710068, "fcm_dpo/delta": -0.11052694916725159, "fcm_dpo/margin": 22.86851692199707, "fcm_dpo/q_t": 0.35563924908638, "grad_norm": 14.994292259216309, "learning_rate": 4.4068368231789365e-07, "logits/chosen": -1.886859655380249, "logits/rejected": -1.45271635055542, "logps/chosen": -109.18792724609375, "logps/ref_chosen": -86.23164367675781, "logps/ref_rejected": -90.65512084960938, "logps/rejected": -136.47991943359375, "loss": 0.9606, "margin_dpo/margin_mean": 22.86851692199707, "margin_dpo/margin_std": 28.827228546142578, "step": 207 }, { "epoch": 0.3054331864904552, "fcm_dpo/beta": 0.03015347383916378, "fcm_dpo/delta": -0.12842844426631927, "fcm_dpo/margin": 23.85171890258789, "fcm_dpo/q_t": 0.3396170735359192, "grad_norm": 13.180238723754883, "learning_rate": 4.398512291636768e-07, "logits/chosen": -1.829242467880249, "logits/rejected": -1.4700978994369507, "logps/chosen": -117.53231811523438, "logps/ref_chosen": -94.1595458984375, "logps/ref_rejected": -100.96233367919922, "logps/rejected": -148.18682861328125, "loss": 0.8877, "margin_dpo/margin_mean": 23.85171890258789, "margin_dpo/margin_std": 21.651004791259766, "step": 208 }, { "epoch": 0.3069016152716593, "fcm_dpo/beta": 0.029522810131311417, "fcm_dpo/delta": -0.018623225390911102, "fcm_dpo/margin": 20.891002655029297, "fcm_dpo/q_t": 0.36556458473205566, "grad_norm": 11.94274616241455, "learning_rate": 4.3901377325300857e-07, "logits/chosen": -1.8460993766784668, "logits/rejected": -1.3643391132354736, "logps/chosen": -104.04109954833984, "logps/ref_chosen": -84.17056274414062, "logps/ref_rejected": -87.61955261230469, "logps/rejected": -128.38108825683594, "loss": 0.9693, "margin_dpo/margin_mean": 20.891002655029297, "margin_dpo/margin_std": 23.366682052612305, "step": 209 }, { "epoch": 0.30837004405286345, "fcm_dpo/beta": 0.029583394527435303, "fcm_dpo/delta": 0.005795907229185104, "fcm_dpo/margin": 20.090076446533203, "fcm_dpo/q_t": 0.37067821621894836, "grad_norm": 11.102954864501953, "learning_rate": 4.381713366536311e-07, "logits/chosen": -1.9762842655181885, "logits/rejected": -1.5929546356201172, "logps/chosen": -103.7457504272461, "logps/ref_chosen": -81.17117309570312, "logps/ref_rejected": -84.17478942871094, "logps/rejected": -126.83943939208984, "loss": 0.9765, "margin_dpo/margin_mean": 20.090076446533203, "margin_dpo/margin_std": 22.627273559570312, "step": 210 }, { "epoch": 0.30983847283406757, "fcm_dpo/beta": 0.029787559062242508, "fcm_dpo/delta": 0.0069667138159275055, "fcm_dpo/margin": 19.919384002685547, "fcm_dpo/q_t": 0.37133318185806274, "grad_norm": 13.3599214553833, "learning_rate": 4.373239415645323e-07, "logits/chosen": -1.7666494846343994, "logits/rejected": -1.1323686838150024, "logps/chosen": -133.63140869140625, "logps/ref_chosen": -108.71270751953125, "logps/ref_rejected": -93.55564880371094, "logps/rejected": -138.39373779296875, "loss": 0.9858, "margin_dpo/margin_mean": 19.919384002685547, "margin_dpo/margin_std": 23.80620574951172, "step": 211 }, { "epoch": 0.31130690161527164, "fcm_dpo/beta": 0.028646890074014664, "fcm_dpo/delta": -0.22884619235992432, "fcm_dpo/margin": 28.252504348754883, "fcm_dpo/q_t": 0.32535678148269653, "grad_norm": 11.055304527282715, "learning_rate": 4.3647161031536086e-07, "logits/chosen": -1.8360583782196045, "logits/rejected": -1.422875165939331, "logps/chosen": -114.99956512451172, "logps/ref_chosen": -98.36195373535156, "logps/ref_rejected": -109.88999938964844, "logps/rejected": -154.78012084960938, "loss": 0.8428, "margin_dpo/margin_mean": 28.252504348754883, "margin_dpo/margin_std": 25.65581703186035, "step": 212 }, { "epoch": 0.31277533039647576, "fcm_dpo/beta": 0.028224751353263855, "fcm_dpo/delta": -0.024367500096559525, "fcm_dpo/margin": 22.059900283813477, "fcm_dpo/q_t": 0.3628522753715515, "grad_norm": 14.968526840209961, "learning_rate": 4.3561436536583774e-07, "logits/chosen": -1.8746771812438965, "logits/rejected": -1.1175596714019775, "logps/chosen": -128.48854064941406, "logps/ref_chosen": -108.05531311035156, "logps/ref_rejected": -100.14414978027344, "logps/rejected": -142.63729858398438, "loss": 0.9689, "margin_dpo/margin_mean": 22.059898376464844, "margin_dpo/margin_std": 25.182662963867188, "step": 213 }, { "epoch": 0.3142437591776799, "fcm_dpo/beta": 0.02846229076385498, "fcm_dpo/delta": 0.03237991780042648, "fcm_dpo/margin": 20.002635955810547, "fcm_dpo/q_t": 0.38030585646629333, "grad_norm": 11.99617862701416, "learning_rate": 4.3475222930516473e-07, "logits/chosen": -1.8247478008270264, "logits/rejected": -1.70417320728302, "logps/chosen": -99.7642593383789, "logps/ref_chosen": -77.80473327636719, "logps/ref_rejected": -84.27578735351562, "logps/rejected": -126.2379379272461, "loss": 1.0348, "margin_dpo/margin_mean": 20.002635955810547, "margin_dpo/margin_std": 28.244205474853516, "step": 214 }, { "epoch": 0.315712187958884, "fcm_dpo/beta": 0.027820151299238205, "fcm_dpo/delta": -0.1480301022529602, "fcm_dpo/margin": 26.492956161499023, "fcm_dpo/q_t": 0.3393728733062744, "grad_norm": 11.24252986907959, "learning_rate": 4.3388522485142885e-07, "logits/chosen": -1.7213172912597656, "logits/rejected": -1.2412421703338623, "logps/chosen": -103.63861083984375, "logps/ref_chosen": -85.1138916015625, "logps/ref_rejected": -96.86151123046875, "logps/rejected": -141.87918090820312, "loss": 0.8754, "margin_dpo/margin_mean": 26.492958068847656, "margin_dpo/margin_std": 24.533878326416016, "step": 215 }, { "epoch": 0.31718061674008813, "fcm_dpo/beta": 0.027603790163993835, "fcm_dpo/delta": -0.03746962174773216, "fcm_dpo/margin": 22.996862411499023, "fcm_dpo/q_t": 0.3604411780834198, "grad_norm": 10.716496467590332, "learning_rate": 4.330133748510036e-07, "logits/chosen": -1.9421966075897217, "logits/rejected": -1.5539422035217285, "logps/chosen": -101.21246337890625, "logps/ref_chosen": -80.59230041503906, "logps/ref_rejected": -81.41983795166016, "logps/rejected": -125.036865234375, "loss": 0.9377, "margin_dpo/margin_mean": 22.996862411499023, "margin_dpo/margin_std": 23.105133056640625, "step": 216 }, { "epoch": 0.3186490455212922, "fcm_dpo/beta": 0.027701357379555702, "fcm_dpo/delta": 0.04847228527069092, "fcm_dpo/margin": 19.994403839111328, "fcm_dpo/q_t": 0.38077569007873535, "grad_norm": 13.545554161071777, "learning_rate": 4.3213670227794757e-07, "logits/chosen": -1.772913932800293, "logits/rejected": -1.4578328132629395, "logps/chosen": -117.97203063964844, "logps/ref_chosen": -93.47257995605469, "logps/ref_rejected": -103.488525390625, "logps/rejected": -147.9823760986328, "loss": 1.0186, "margin_dpo/margin_mean": 19.994403839111328, "margin_dpo/margin_std": 25.860273361206055, "step": 217 }, { "epoch": 0.3201174743024963, "fcm_dpo/beta": 0.02740955352783203, "fcm_dpo/delta": -0.046444494277238846, "fcm_dpo/margin": 23.46440887451172, "fcm_dpo/q_t": 0.36053454875946045, "grad_norm": 11.716761589050293, "learning_rate": 4.3125523023339815e-07, "logits/chosen": -1.467943787574768, "logits/rejected": -1.0809111595153809, "logps/chosen": -112.5135498046875, "logps/ref_chosen": -89.05883026123047, "logps/ref_rejected": -94.30680847167969, "logps/rejected": -141.22592163085938, "loss": 0.9696, "margin_dpo/margin_mean": 23.46440887451172, "margin_dpo/margin_std": 27.73025894165039, "step": 218 }, { "epoch": 0.32158590308370044, "fcm_dpo/beta": 0.0276676956564188, "fcm_dpo/delta": 0.009598739445209503, "fcm_dpo/margin": 21.319698333740234, "fcm_dpo/q_t": 0.37368935346603394, "grad_norm": 14.510382652282715, "learning_rate": 4.303689819449636e-07, "logits/chosen": -1.6932134628295898, "logits/rejected": -1.259705662727356, "logps/chosen": -122.24336242675781, "logps/ref_chosen": -101.00733947753906, "logps/ref_rejected": -92.46794128417969, "logps/rejected": -135.02365112304688, "loss": 1.0173, "margin_dpo/margin_mean": 21.319698333740234, "margin_dpo/margin_std": 28.504629135131836, "step": 219 }, { "epoch": 0.32305433186490456, "fcm_dpo/beta": 0.027722153812646866, "fcm_dpo/delta": 0.056987300515174866, "fcm_dpo/margin": 19.699235916137695, "fcm_dpo/q_t": 0.3782733082771301, "grad_norm": 12.883475303649902, "learning_rate": 4.2947798076611047e-07, "logits/chosen": -1.2188053131103516, "logits/rejected": -0.8566290140151978, "logps/chosen": -117.09075164794922, "logps/ref_chosen": -95.53721618652344, "logps/ref_rejected": -94.30703735351562, "logps/rejected": -135.559814453125, "loss": 0.9964, "margin_dpo/margin_mean": 19.699234008789062, "margin_dpo/margin_std": 21.820940017700195, "step": 220 }, { "epoch": 0.3245227606461087, "fcm_dpo/beta": 0.02716447226703167, "fcm_dpo/delta": -0.2148474156856537, "fcm_dpo/margin": 29.37576675415039, "fcm_dpo/q_t": 0.3273373246192932, "grad_norm": 12.420080184936523, "learning_rate": 4.285822501755485e-07, "logits/chosen": -1.346145749092102, "logits/rejected": -1.196437120437622, "logps/chosen": -103.59893035888672, "logps/ref_chosen": -82.84486389160156, "logps/ref_rejected": -110.81179809570312, "logps/rejected": -160.94163513183594, "loss": 0.8462, "margin_dpo/margin_mean": 29.375770568847656, "margin_dpo/margin_std": 25.051647186279297, "step": 221 }, { "epoch": 0.32599118942731276, "fcm_dpo/beta": 0.026609212160110474, "fcm_dpo/delta": -0.045089878141880035, "fcm_dpo/margin": 24.11734962463379, "fcm_dpo/q_t": 0.36285531520843506, "grad_norm": 14.774637222290039, "learning_rate": 4.276818137766118e-07, "logits/chosen": -1.9378188848495483, "logits/rejected": -1.510035514831543, "logps/chosen": -121.65461730957031, "logps/ref_chosen": -95.14198303222656, "logps/ref_rejected": -106.80441284179688, "logps/rejected": -157.43441772460938, "loss": 0.9999, "margin_dpo/margin_mean": 24.117347717285156, "margin_dpo/margin_std": 31.577239990234375, "step": 222 }, { "epoch": 0.3274596182085169, "fcm_dpo/beta": 0.026801597326993942, "fcm_dpo/delta": 0.11458225548267365, "fcm_dpo/margin": 18.32587432861328, "fcm_dpo/q_t": 0.3927801251411438, "grad_norm": 13.29928970336914, "learning_rate": 4.2677669529663686e-07, "logits/chosen": -2.038062572479248, "logits/rejected": -1.6366686820983887, "logps/chosen": -110.8859634399414, "logps/ref_chosen": -85.57511138916016, "logps/ref_rejected": -86.45237731933594, "logps/rejected": -130.089111328125, "loss": 1.0622, "margin_dpo/margin_mean": 18.32587432861328, "margin_dpo/margin_std": 25.809633255004883, "step": 223 }, { "epoch": 0.328928046989721, "fcm_dpo/beta": 0.026815548539161682, "fcm_dpo/delta": -0.049758054316043854, "fcm_dpo/margin": 24.111448287963867, "fcm_dpo/q_t": 0.35970795154571533, "grad_norm": 12.745670318603516, "learning_rate": 4.2586691858633747e-07, "logits/chosen": -1.7115750312805176, "logits/rejected": -1.2611249685287476, "logps/chosen": -106.57003784179688, "logps/ref_chosen": -82.72380065917969, "logps/ref_rejected": -82.59538269042969, "logps/rejected": -130.55307006835938, "loss": 0.9542, "margin_dpo/margin_mean": 24.111448287963867, "margin_dpo/margin_std": 27.108388900756836, "step": 224 }, { "epoch": 0.3303964757709251, "fcm_dpo/beta": 0.026444047689437866, "fcm_dpo/delta": -0.10222186148166656, "fcm_dpo/margin": 26.288196563720703, "fcm_dpo/q_t": 0.35237205028533936, "grad_norm": 13.087396621704102, "learning_rate": 4.249525076191759e-07, "logits/chosen": -1.6562142372131348, "logits/rejected": -1.1619675159454346, "logps/chosen": -122.09429931640625, "logps/ref_chosen": -95.67768859863281, "logps/ref_rejected": -105.09687805175781, "logps/rejected": -157.80169677734375, "loss": 0.9433, "margin_dpo/margin_mean": 26.288192749023438, "margin_dpo/margin_std": 30.676586151123047, "step": 225 }, { "epoch": 0.33186490455212925, "fcm_dpo/beta": 0.02590518817305565, "fcm_dpo/delta": -0.03390205651521683, "fcm_dpo/margin": 24.35424041748047, "fcm_dpo/q_t": 0.3604390621185303, "grad_norm": 10.917281150817871, "learning_rate": 4.2403348649073167e-07, "logits/chosen": -1.681097388267517, "logits/rejected": -1.1455821990966797, "logps/chosen": -115.38078308105469, "logps/ref_chosen": -93.46092224121094, "logps/ref_rejected": -86.7017593383789, "logps/rejected": -132.97586059570312, "loss": 0.9398, "margin_dpo/margin_mean": 24.35424041748047, "margin_dpo/margin_std": 24.179988861083984, "step": 226 }, { "epoch": 0.3333333333333333, "fcm_dpo/beta": 0.02585548348724842, "fcm_dpo/delta": -0.045401234179735184, "fcm_dpo/margin": 24.849811553955078, "fcm_dpo/q_t": 0.362407386302948, "grad_norm": 12.172209739685059, "learning_rate": 4.2310987941806615e-07, "logits/chosen": -1.2314541339874268, "logits/rejected": -0.8789317607879639, "logps/chosen": -123.27052307128906, "logps/ref_chosen": -92.81427001953125, "logps/ref_rejected": -104.73693084716797, "logps/rejected": -160.04299926757812, "loss": 0.9737, "margin_dpo/margin_mean": 24.849811553955078, "margin_dpo/margin_std": 29.725021362304688, "step": 227 }, { "epoch": 0.33480176211453744, "fcm_dpo/beta": 0.026136575266718864, "fcm_dpo/delta": 0.09344188123941422, "fcm_dpo/margin": 19.564260482788086, "fcm_dpo/q_t": 0.3862323760986328, "grad_norm": 12.771748542785645, "learning_rate": 4.2218171073908463e-07, "logits/chosen": -1.6004002094268799, "logits/rejected": -1.1517033576965332, "logps/chosen": -122.49995422363281, "logps/ref_chosen": -94.03712463378906, "logps/ref_rejected": -96.02151489257812, "logps/rejected": -144.04861450195312, "loss": 1.0429, "margin_dpo/margin_mean": 19.56426239013672, "margin_dpo/margin_std": 25.722578048706055, "step": 228 }, { "epoch": 0.33627019089574156, "fcm_dpo/beta": 0.025405660271644592, "fcm_dpo/delta": -0.2284693717956543, "fcm_dpo/margin": 31.885055541992188, "fcm_dpo/q_t": 0.3289991319179535, "grad_norm": 11.33955192565918, "learning_rate": 4.212490049118951e-07, "logits/chosen": -1.9561712741851807, "logits/rejected": -1.348691701889038, "logps/chosen": -120.61212921142578, "logps/ref_chosen": -95.57766723632812, "logps/ref_rejected": -89.17379760742188, "logps/rejected": -146.0933074951172, "loss": 0.8603, "margin_dpo/margin_mean": 31.885055541992188, "margin_dpo/margin_std": 31.453580856323242, "step": 229 }, { "epoch": 0.3377386196769457, "fcm_dpo/beta": 0.02440287545323372, "fcm_dpo/delta": -0.1861468255519867, "fcm_dpo/margin": 31.624526977539062, "fcm_dpo/q_t": 0.3298434615135193, "grad_norm": 12.601603507995605, "learning_rate": 4.203117865141635e-07, "logits/chosen": -1.638891339302063, "logits/rejected": -1.555408239364624, "logps/chosen": -90.70736694335938, "logps/ref_chosen": -63.713626861572266, "logps/ref_rejected": -91.90870666503906, "logps/rejected": -150.5269775390625, "loss": 0.8553, "margin_dpo/margin_mean": 31.624528884887695, "margin_dpo/margin_std": 27.797245025634766, "step": 230 }, { "epoch": 0.3392070484581498, "fcm_dpo/beta": 0.023886041715741158, "fcm_dpo/delta": -0.04298706725239754, "fcm_dpo/margin": 26.76896858215332, "fcm_dpo/q_t": 0.3564389944076538, "grad_norm": 10.671087265014648, "learning_rate": 4.1937008024246625e-07, "logits/chosen": -1.6697638034820557, "logits/rejected": -1.1297423839569092, "logps/chosen": -125.79000091552734, "logps/ref_chosen": -95.45567321777344, "logps/ref_rejected": -80.95568084716797, "logps/rejected": -138.05897521972656, "loss": 0.9207, "margin_dpo/margin_mean": 26.76896858215332, "margin_dpo/margin_std": 24.022785186767578, "step": 231 }, { "epoch": 0.3406754772393539, "fcm_dpo/beta": 0.02407444454729557, "fcm_dpo/delta": 0.04270622134208679, "fcm_dpo/margin": 23.248355865478516, "fcm_dpo/q_t": 0.37612390518188477, "grad_norm": 11.53981876373291, "learning_rate": 4.1842391091163933e-07, "logits/chosen": -1.4995675086975098, "logits/rejected": -1.0280941724777222, "logps/chosen": -128.28302001953125, "logps/ref_chosen": -96.89726257324219, "logps/ref_rejected": -89.76461791992188, "logps/rejected": -144.39874267578125, "loss": 0.9973, "margin_dpo/margin_mean": 23.248355865478516, "margin_dpo/margin_std": 26.718120574951172, "step": 232 }, { "epoch": 0.342143906020558, "fcm_dpo/beta": 0.02357470616698265, "fcm_dpo/delta": -0.1532088667154312, "fcm_dpo/margin": 31.44921112060547, "fcm_dpo/q_t": 0.3387308120727539, "grad_norm": 11.674040794372559, "learning_rate": 4.174733034541245e-07, "logits/chosen": -1.721343994140625, "logits/rejected": -1.4938092231750488, "logps/chosen": -122.76866149902344, "logps/ref_chosen": -89.05032348632812, "logps/ref_rejected": -112.75917053222656, "logps/rejected": -177.92672729492188, "loss": 0.8881, "margin_dpo/margin_mean": 31.44921112060547, "margin_dpo/margin_std": 30.564104080200195, "step": 233 }, { "epoch": 0.3436123348017621, "fcm_dpo/beta": 0.022600866854190826, "fcm_dpo/delta": -0.17494013905525208, "fcm_dpo/margin": 33.52787780761719, "fcm_dpo/q_t": 0.3380278944969177, "grad_norm": 11.083850860595703, "learning_rate": 4.165182829193126e-07, "logits/chosen": -1.2067891359329224, "logits/rejected": -1.1254383325576782, "logps/chosen": -106.61786651611328, "logps/ref_chosen": -74.318359375, "logps/ref_rejected": -106.38758850097656, "logps/rejected": -172.2149658203125, "loss": 0.8916, "margin_dpo/margin_mean": 33.52787780761719, "margin_dpo/margin_std": 33.51675796508789, "step": 234 }, { "epoch": 0.34508076358296624, "fcm_dpo/beta": 0.022734005004167557, "fcm_dpo/delta": 0.01751571148633957, "fcm_dpo/margin": 25.663055419921875, "fcm_dpo/q_t": 0.36923179030418396, "grad_norm": 10.767046928405762, "learning_rate": 4.1555887447288255e-07, "logits/chosen": -1.3730831146240234, "logits/rejected": -0.8100583553314209, "logps/chosen": -136.84141540527344, "logps/ref_chosen": -98.217041015625, "logps/ref_rejected": -97.24677276611328, "logps/rejected": -161.53419494628906, "loss": 0.9624, "margin_dpo/margin_mean": 25.663055419921875, "margin_dpo/margin_std": 25.689247131347656, "step": 235 }, { "epoch": 0.3465491923641703, "fcm_dpo/beta": 0.022359684109687805, "fcm_dpo/delta": -0.12736767530441284, "fcm_dpo/margin": 32.12920379638672, "fcm_dpo/q_t": 0.35286009311676025, "grad_norm": 11.318601608276367, "learning_rate": 4.1459510339613946e-07, "logits/chosen": -1.2687420845031738, "logits/rejected": -1.2513408660888672, "logps/chosen": -111.73262023925781, "logps/ref_chosen": -78.83773040771484, "logps/ref_rejected": -109.06343078613281, "logps/rejected": -174.0875244140625, "loss": 0.924, "margin_dpo/margin_mean": 32.12920379638672, "margin_dpo/margin_std": 37.443138122558594, "step": 236 }, { "epoch": 0.34801762114537443, "fcm_dpo/beta": 0.02228570729494095, "fcm_dpo/delta": 0.0468742772936821, "fcm_dpo/margin": 24.937557220458984, "fcm_dpo/q_t": 0.3862718939781189, "grad_norm": 11.445574760437012, "learning_rate": 4.136269950853473e-07, "logits/chosen": -1.4098989963531494, "logits/rejected": -1.0293853282928467, "logps/chosen": -123.49139404296875, "logps/ref_chosen": -85.21128845214844, "logps/ref_rejected": -99.90999603271484, "logps/rejected": -163.12767028808594, "loss": 1.065, "margin_dpo/margin_mean": 24.937557220458984, "margin_dpo/margin_std": 39.00885009765625, "step": 237 }, { "epoch": 0.34948604992657856, "fcm_dpo/beta": 0.022245164960622787, "fcm_dpo/delta": -0.023596640676259995, "fcm_dpo/margin": 27.963947296142578, "fcm_dpo/q_t": 0.36633193492889404, "grad_norm": 10.700308799743652, "learning_rate": 4.126545750510605e-07, "logits/chosen": -1.0491883754730225, "logits/rejected": -0.8851519227027893, "logps/chosen": -120.75160217285156, "logps/ref_chosen": -78.73123168945312, "logps/ref_rejected": -95.41840362548828, "logps/rejected": -165.4027099609375, "loss": 0.9704, "margin_dpo/margin_mean": 27.963947296142578, "margin_dpo/margin_std": 32.5536003112793, "step": 238 }, { "epoch": 0.3509544787077827, "fcm_dpo/beta": 0.022143233567476273, "fcm_dpo/delta": 0.018351107835769653, "fcm_dpo/margin": 26.272315979003906, "fcm_dpo/q_t": 0.37351781129837036, "grad_norm": 11.168644905090332, "learning_rate": 4.116778689174514e-07, "logits/chosen": -1.7788722515106201, "logits/rejected": -1.3021533489227295, "logps/chosen": -131.14578247070312, "logps/ref_chosen": -92.60093688964844, "logps/ref_rejected": -100.51769256591797, "logps/rejected": -165.3348388671875, "loss": 0.9967, "margin_dpo/margin_mean": 26.272315979003906, "margin_dpo/margin_std": 30.892166137695312, "step": 239 }, { "epoch": 0.3524229074889868, "fcm_dpo/beta": 0.022500187158584595, "fcm_dpo/delta": 0.07234937697649002, "fcm_dpo/margin": 23.62274932861328, "fcm_dpo/q_t": 0.3824573755264282, "grad_norm": 12.867546081542969, "learning_rate": 4.106969024216348e-07, "logits/chosen": -1.9507150650024414, "logits/rejected": -1.6044831275939941, "logps/chosen": -125.81748962402344, "logps/ref_chosen": -86.15977478027344, "logps/ref_rejected": -80.45567321777344, "logps/rejected": -143.7361297607422, "loss": 1.0213, "margin_dpo/margin_mean": 23.62274932861328, "margin_dpo/margin_std": 28.90222930908203, "step": 240 }, { "epoch": 0.35389133627019087, "fcm_dpo/beta": 0.0217452310025692, "fcm_dpo/delta": -0.3266553282737732, "fcm_dpo/margin": 41.317291259765625, "fcm_dpo/q_t": 0.30971965193748474, "grad_norm": 10.297433853149414, "learning_rate": 4.097117014129903e-07, "logits/chosen": -1.5102795362472534, "logits/rejected": -0.8343143463134766, "logps/chosen": -136.01959228515625, "logps/ref_chosen": -101.04594421386719, "logps/ref_rejected": -94.04934692382812, "logps/rejected": -170.34030151367188, "loss": 0.8042, "margin_dpo/margin_mean": 41.317291259765625, "margin_dpo/margin_std": 36.07255554199219, "step": 241 }, { "epoch": 0.355359765051395, "fcm_dpo/beta": 0.021258074790239334, "fcm_dpo/delta": -0.021016836166381836, "fcm_dpo/margin": 29.1357479095459, "fcm_dpo/q_t": 0.36231163144111633, "grad_norm": 11.23208999633789, "learning_rate": 4.087222918524807e-07, "logits/chosen": -1.8308205604553223, "logits/rejected": -1.1828442811965942, "logps/chosen": -138.07565307617188, "logps/ref_chosen": -95.67266845703125, "logps/ref_rejected": -90.65454864501953, "logps/rejected": -162.19329833984375, "loss": 0.9561, "margin_dpo/margin_mean": 29.135746002197266, "margin_dpo/margin_std": 30.477272033691406, "step": 242 }, { "epoch": 0.3568281938325991, "fcm_dpo/beta": 0.021303560584783554, "fcm_dpo/delta": -0.015803933143615723, "fcm_dpo/margin": 28.77163314819336, "fcm_dpo/q_t": 0.36690443754196167, "grad_norm": 11.050640106201172, "learning_rate": 4.07728699811968e-07, "logits/chosen": -1.8111510276794434, "logits/rejected": -1.0000836849212646, "logps/chosen": -139.14649963378906, "logps/ref_chosen": -98.03140258789062, "logps/ref_rejected": -83.18806457519531, "logps/rejected": -153.07479858398438, "loss": 0.9904, "margin_dpo/margin_mean": 28.77163314819336, "margin_dpo/margin_std": 34.02631378173828, "step": 243 }, { "epoch": 0.35829662261380324, "fcm_dpo/beta": 0.020892877131700516, "fcm_dpo/delta": -0.03988625854253769, "fcm_dpo/margin": 30.495805740356445, "fcm_dpo/q_t": 0.36164575815200806, "grad_norm": 11.020362854003906, "learning_rate": 4.067309514735267e-07, "logits/chosen": -2.001028060913086, "logits/rejected": -1.656126856803894, "logps/chosen": -131.77944946289062, "logps/ref_chosen": -88.8939208984375, "logps/ref_rejected": -102.57278442382812, "logps/rejected": -175.95413208007812, "loss": 0.9556, "margin_dpo/margin_mean": 30.495803833007812, "margin_dpo/margin_std": 33.84397888183594, "step": 244 }, { "epoch": 0.35976505139500736, "fcm_dpo/beta": 0.02063831314444542, "fcm_dpo/delta": -0.0902470201253891, "fcm_dpo/margin": 33.148372650146484, "fcm_dpo/q_t": 0.34877562522888184, "grad_norm": 10.457184791564941, "learning_rate": 4.057290731287531e-07, "logits/chosen": -2.292057514190674, "logits/rejected": -1.4950196743011475, "logps/chosen": -140.51174926757812, "logps/ref_chosen": -104.19400024414062, "logps/ref_rejected": -92.65645599365234, "logps/rejected": -162.12255859375, "loss": 0.9075, "margin_dpo/margin_mean": 33.14836883544922, "margin_dpo/margin_std": 31.503116607666016, "step": 245 }, { "epoch": 0.36123348017621143, "fcm_dpo/beta": 0.020679624751210213, "fcm_dpo/delta": 0.07523977756500244, "fcm_dpo/margin": 25.567752838134766, "fcm_dpo/q_t": 0.3833276629447937, "grad_norm": 11.005114555358887, "learning_rate": 4.047230911780736e-07, "logits/chosen": -1.9735589027404785, "logits/rejected": -1.4642459154129028, "logps/chosen": -149.89358520507812, "logps/ref_chosen": -103.21904754638672, "logps/ref_rejected": -90.99221801757812, "logps/rejected": -163.2344970703125, "loss": 1.021, "margin_dpo/margin_mean": 25.567752838134766, "margin_dpo/margin_std": 31.323787689208984, "step": 246 }, { "epoch": 0.36270190895741555, "fcm_dpo/beta": 0.020045291632413864, "fcm_dpo/delta": -0.22309784591197968, "fcm_dpo/margin": 40.07710647583008, "fcm_dpo/q_t": 0.328274130821228, "grad_norm": 10.08717155456543, "learning_rate": 4.0371303213004814e-07, "logits/chosen": -2.199232578277588, "logits/rejected": -1.9928696155548096, "logps/chosen": -129.20089721679688, "logps/ref_chosen": -86.99437713623047, "logps/ref_rejected": -111.33802795410156, "logps/rejected": -193.62164306640625, "loss": 0.8532, "margin_dpo/margin_mean": 40.07710647583008, "margin_dpo/margin_std": 37.57669448852539, "step": 247 }, { "epoch": 0.3641703377386197, "fcm_dpo/beta": 0.020135609433054924, "fcm_dpo/delta": 0.0492338202893734, "fcm_dpo/margin": 27.472003936767578, "fcm_dpo/q_t": 0.3776289224624634, "grad_norm": 11.691912651062012, "learning_rate": 4.0269892260067197e-07, "logits/chosen": -1.5299792289733887, "logits/rejected": -1.482997179031372, "logps/chosen": -123.07820129394531, "logps/ref_chosen": -74.7855224609375, "logps/ref_rejected": -98.27689361572266, "logps/rejected": -174.0415802001953, "loss": 0.9963, "margin_dpo/margin_mean": 27.472003936767578, "margin_dpo/margin_std": 31.208698272705078, "step": 248 }, { "epoch": 0.3656387665198238, "fcm_dpo/beta": 0.02031414583325386, "fcm_dpo/delta": 0.08933912217617035, "fcm_dpo/margin": 25.36626434326172, "fcm_dpo/q_t": 0.38820499181747437, "grad_norm": 12.994771957397461, "learning_rate": 4.0168078931267426e-07, "logits/chosen": -2.2492823600769043, "logits/rejected": -1.739271640777588, "logps/chosen": -148.12538146972656, "logps/ref_chosen": -95.70379638671875, "logps/ref_rejected": -87.14646911621094, "logps/rejected": -164.934326171875, "loss": 1.0546, "margin_dpo/margin_mean": 25.36626434326172, "margin_dpo/margin_std": 35.47053527832031, "step": 249 }, { "epoch": 0.3671071953010279, "fcm_dpo/beta": 0.020489081740379333, "fcm_dpo/delta": 0.04790526628494263, "fcm_dpo/margin": 27.043155670166016, "fcm_dpo/q_t": 0.37720349431037903, "grad_norm": 13.011922836303711, "learning_rate": 4.006586590948141e-07, "logits/chosen": -1.7810206413269043, "logits/rejected": -0.8627009391784668, "logps/chosen": -161.8379364013672, "logps/ref_chosen": -114.05221557617188, "logps/ref_rejected": -81.08767700195312, "logps/rejected": -155.91656494140625, "loss": 1.0209, "margin_dpo/margin_mean": 27.043155670166016, "margin_dpo/margin_std": 33.19232940673828, "step": 250 }, { "epoch": 0.368575624082232, "fcm_dpo/beta": 0.020762145519256592, "fcm_dpo/delta": 0.07583161443471909, "fcm_dpo/margin": 25.410945892333984, "fcm_dpo/q_t": 0.3874654173851013, "grad_norm": 12.704300880432129, "learning_rate": 3.9963255888117325e-07, "logits/chosen": -1.9245502948760986, "logits/rejected": -1.244246006011963, "logps/chosen": -144.52914428710938, "logps/ref_chosen": -97.71128845214844, "logps/ref_rejected": -83.52742004394531, "logps/rejected": -155.7562255859375, "loss": 1.0538, "margin_dpo/margin_mean": 25.410945892333984, "margin_dpo/margin_std": 35.388427734375, "step": 251 }, { "epoch": 0.3700440528634361, "fcm_dpo/beta": 0.02117466926574707, "fcm_dpo/delta": 0.05208258330821991, "fcm_dpo/margin": 26.01020050048828, "fcm_dpo/q_t": 0.37829625606536865, "grad_norm": 13.06750774383545, "learning_rate": 3.9860251571044666e-07, "logits/chosen": -1.8967103958129883, "logits/rejected": -1.3722825050354004, "logps/chosen": -155.25262451171875, "logps/ref_chosen": -108.9861068725586, "logps/ref_rejected": -91.56423950195312, "logps/rejected": -163.84097290039062, "loss": 1.0169, "margin_dpo/margin_mean": 26.01020050048828, "margin_dpo/margin_std": 32.340423583984375, "step": 252 }, { "epoch": 0.37151248164464024, "fcm_dpo/beta": 0.021657634526491165, "fcm_dpo/delta": 0.07324262708425522, "fcm_dpo/margin": 24.389892578125, "fcm_dpo/q_t": 0.3817564845085144, "grad_norm": 12.639901161193848, "learning_rate": 3.9756855672522986e-07, "logits/chosen": -2.0871338844299316, "logits/rejected": -1.880314588546753, "logps/chosen": -150.90634155273438, "logps/ref_chosen": -100.21630096435547, "logps/ref_rejected": -105.67670440673828, "logps/rejected": -180.7566375732422, "loss": 1.0268, "margin_dpo/margin_mean": 24.389892578125, "margin_dpo/margin_std": 29.48230743408203, "step": 253 }, { "epoch": 0.37298091042584436, "fcm_dpo/beta": 0.021658308804035187, "fcm_dpo/delta": 0.0799470990896225, "fcm_dpo/margin": 24.19228744506836, "fcm_dpo/q_t": 0.38415348529815674, "grad_norm": 12.239904403686523, "learning_rate": 3.965307091713037e-07, "logits/chosen": -1.8840199708938599, "logits/rejected": -1.340732455253601, "logps/chosen": -148.34524536132812, "logps/ref_chosen": -98.73518371582031, "logps/ref_rejected": -93.73825073242188, "logps/rejected": -167.5406036376953, "loss": 1.0444, "margin_dpo/margin_mean": 24.19228744506836, "margin_dpo/margin_std": 32.14631652832031, "step": 254 }, { "epoch": 0.3744493392070485, "fcm_dpo/beta": 0.021404996514320374, "fcm_dpo/delta": -0.153507798910141, "fcm_dpo/margin": 34.67428207397461, "fcm_dpo/q_t": 0.33902448415756226, "grad_norm": 11.967893600463867, "learning_rate": 3.954890003969163e-07, "logits/chosen": -1.910501480102539, "logits/rejected": -1.5700030326843262, "logps/chosen": -141.0276641845703, "logps/ref_chosen": -90.382568359375, "logps/ref_rejected": -97.07624816894531, "logps/rejected": -182.3956298828125, "loss": 0.8959, "margin_dpo/margin_mean": 34.674278259277344, "margin_dpo/margin_std": 34.8538818359375, "step": 255 }, { "epoch": 0.37591776798825255, "fcm_dpo/beta": 0.02097788080573082, "fcm_dpo/delta": -0.10094420611858368, "fcm_dpo/margin": 33.08401107788086, "fcm_dpo/q_t": 0.3532879948616028, "grad_norm": 12.47603702545166, "learning_rate": 3.944434578520628e-07, "logits/chosen": -1.874246597290039, "logits/rejected": -1.5234124660491943, "logps/chosen": -136.12049865722656, "logps/ref_chosen": -88.7528076171875, "logps/ref_rejected": -98.49383544921875, "logps/rejected": -178.94552612304688, "loss": 0.9704, "margin_dpo/margin_mean": 33.08401107788086, "margin_dpo/margin_std": 41.36225891113281, "step": 256 }, { "epoch": 0.37738619676945667, "fcm_dpo/beta": 0.020358510315418243, "fcm_dpo/delta": -0.06614407151937485, "fcm_dpo/margin": 32.34702682495117, "fcm_dpo/q_t": 0.35797828435897827, "grad_norm": 11.72060775756836, "learning_rate": 3.933941090877615e-07, "logits/chosen": -1.9517452716827393, "logits/rejected": -1.572911262512207, "logps/chosen": -132.3583984375, "logps/ref_chosen": -82.80352020263672, "logps/ref_rejected": -85.8677978515625, "logps/rejected": -167.7696990966797, "loss": 0.9599, "margin_dpo/margin_mean": 32.34702682495117, "margin_dpo/margin_std": 36.041168212890625, "step": 257 }, { "epoch": 0.3788546255506608, "fcm_dpo/beta": 0.02057628706097603, "fcm_dpo/delta": -0.028431270271539688, "fcm_dpo/margin": 30.42020034790039, "fcm_dpo/q_t": 0.36126869916915894, "grad_norm": 12.226306915283203, "learning_rate": 3.923409817553284e-07, "logits/chosen": -2.099985122680664, "logits/rejected": -1.7437987327575684, "logps/chosen": -140.68870544433594, "logps/ref_chosen": -90.187744140625, "logps/ref_rejected": -103.47068786621094, "logps/rejected": -184.39186096191406, "loss": 0.9628, "margin_dpo/margin_mean": 30.42020034790039, "margin_dpo/margin_std": 32.89296340942383, "step": 258 }, { "epoch": 0.3803230543318649, "fcm_dpo/beta": 0.020515816286206245, "fcm_dpo/delta": 0.08078932762145996, "fcm_dpo/margin": 25.49618148803711, "fcm_dpo/q_t": 0.3858429789543152, "grad_norm": 12.687917709350586, "learning_rate": 3.9128410360564793e-07, "logits/chosen": -1.4180266857147217, "logits/rejected": -1.1736819744110107, "logps/chosen": -146.1452178955078, "logps/ref_chosen": -90.77254486083984, "logps/ref_rejected": -94.58816528320312, "logps/rejected": -175.45701599121094, "loss": 1.0682, "margin_dpo/margin_mean": 25.49618148803711, "margin_dpo/margin_std": 38.16563415527344, "step": 259 }, { "epoch": 0.38179148311306904, "fcm_dpo/beta": 0.021066930145025253, "fcm_dpo/delta": 0.11459525674581528, "fcm_dpo/margin": 23.313533782958984, "fcm_dpo/q_t": 0.39374488592147827, "grad_norm": 14.11849594116211, "learning_rate": 3.9022350248844246e-07, "logits/chosen": -1.6266734600067139, "logits/rejected": -1.6325747966766357, "logps/chosen": -131.62429809570312, "logps/ref_chosen": -75.59269714355469, "logps/ref_rejected": -100.84554290771484, "logps/rejected": -180.190673828125, "loss": 1.113, "margin_dpo/margin_mean": 23.313533782958984, "margin_dpo/margin_std": 39.60096740722656, "step": 260 }, { "epoch": 0.3832599118942731, "fcm_dpo/beta": 0.02059752494096756, "fcm_dpo/delta": -0.1640065312385559, "fcm_dpo/margin": 36.42674255371094, "fcm_dpo/q_t": 0.33720719814300537, "grad_norm": 11.14237117767334, "learning_rate": 3.891592063515376e-07, "logits/chosen": -1.8282723426818848, "logits/rejected": -1.453067421913147, "logps/chosen": -138.38714599609375, "logps/ref_chosen": -94.25491333007812, "logps/ref_rejected": -93.65699005126953, "logps/rejected": -174.21595764160156, "loss": 0.8778, "margin_dpo/margin_mean": 36.42674255371094, "margin_dpo/margin_std": 34.84246826171875, "step": 261 }, { "epoch": 0.38472834067547723, "fcm_dpo/beta": 0.020204119384288788, "fcm_dpo/delta": -0.09102602303028107, "fcm_dpo/margin": 33.87533950805664, "fcm_dpo/q_t": 0.3513264060020447, "grad_norm": 11.49547290802002, "learning_rate": 3.880912432401264e-07, "logits/chosen": -1.7930582761764526, "logits/rejected": -1.2549785375595093, "logps/chosen": -133.25894165039062, "logps/ref_chosen": -85.26730346679688, "logps/ref_rejected": -90.82609558105469, "logps/rejected": -172.69308471679688, "loss": 0.9266, "margin_dpo/margin_mean": 33.87533950805664, "margin_dpo/margin_std": 35.84063720703125, "step": 262 }, { "epoch": 0.38619676945668135, "fcm_dpo/beta": 0.019306186586618423, "fcm_dpo/delta": -0.29907822608947754, "fcm_dpo/margin": 45.224769592285156, "fcm_dpo/q_t": 0.3136481046676636, "grad_norm": 10.701422691345215, "learning_rate": 3.870196412960302e-07, "logits/chosen": -1.9696038961410522, "logits/rejected": -1.4115229845046997, "logps/chosen": -134.94964599609375, "logps/ref_chosen": -95.75790405273438, "logps/ref_rejected": -101.83377075195312, "logps/rejected": -186.2502899169922, "loss": 0.83, "margin_dpo/margin_mean": 45.224769592285156, "margin_dpo/margin_std": 42.148963928222656, "step": 263 }, { "epoch": 0.3876651982378855, "fcm_dpo/beta": 0.01862514019012451, "fcm_dpo/delta": -0.11325480043888092, "fcm_dpo/margin": 37.84281921386719, "fcm_dpo/q_t": 0.34645789861679077, "grad_norm": 10.080230712890625, "learning_rate": 3.8594442875695665e-07, "logits/chosen": -2.0475120544433594, "logits/rejected": -1.7664682865142822, "logps/chosen": -134.43109130859375, "logps/ref_chosen": -90.6226577758789, "logps/ref_rejected": -100.32554626464844, "logps/rejected": -181.976806640625, "loss": 0.9163, "margin_dpo/margin_mean": 37.84281921386719, "margin_dpo/margin_std": 38.84701919555664, "step": 264 }, { "epoch": 0.3891336270190896, "fcm_dpo/beta": 0.018290959298610687, "fcm_dpo/delta": -0.06356701254844666, "fcm_dpo/margin": 35.99824523925781, "fcm_dpo/q_t": 0.3593207895755768, "grad_norm": 10.072009086608887, "learning_rate": 3.848656339557562e-07, "logits/chosen": -1.925950050354004, "logits/rejected": -1.5249955654144287, "logps/chosen": -138.44578552246094, "logps/ref_chosen": -92.37232971191406, "logps/ref_rejected": -94.62757873535156, "logps/rejected": -176.69927978515625, "loss": 0.9603, "margin_dpo/margin_mean": 35.99824905395508, "margin_dpo/margin_std": 41.63231658935547, "step": 265 }, { "epoch": 0.39060205580029367, "fcm_dpo/beta": 0.018374420702457428, "fcm_dpo/delta": 0.031407758593559265, "fcm_dpo/margin": 31.039928436279297, "fcm_dpo/q_t": 0.3749583959579468, "grad_norm": 12.884138107299805, "learning_rate": 3.8378328531967507e-07, "logits/chosen": -2.2668302059173584, "logits/rejected": -1.4866418838500977, "logps/chosen": -155.75039672851562, "logps/ref_chosen": -102.20002746582031, "logps/ref_rejected": -74.36642456054688, "logps/rejected": -158.9567108154297, "loss": 1.002, "margin_dpo/margin_mean": 31.039928436279297, "margin_dpo/margin_std": 37.078948974609375, "step": 266 }, { "epoch": 0.3920704845814978, "fcm_dpo/beta": 0.018142802640795708, "fcm_dpo/delta": -0.07580061256885529, "fcm_dpo/margin": 36.915672302246094, "fcm_dpo/q_t": 0.35151273012161255, "grad_norm": 10.919096946716309, "learning_rate": 3.8269741136960646e-07, "logits/chosen": -1.7907602787017822, "logits/rejected": -1.1931796073913574, "logps/chosen": -150.03753662109375, "logps/ref_chosen": -104.28599548339844, "logps/ref_rejected": -95.98719024658203, "logps/rejected": -178.6544189453125, "loss": 0.9123, "margin_dpo/margin_mean": 36.915672302246094, "margin_dpo/margin_std": 34.50056076049805, "step": 267 }, { "epoch": 0.3935389133627019, "fcm_dpo/beta": 0.017767982557415962, "fcm_dpo/delta": -0.12636685371398926, "fcm_dpo/margin": 40.33136749267578, "fcm_dpo/q_t": 0.34127211570739746, "grad_norm": 10.990910530090332, "learning_rate": 3.8160804071933894e-07, "logits/chosen": -1.6720902919769287, "logits/rejected": -1.495853304862976, "logps/chosen": -132.07077026367188, "logps/ref_chosen": -86.69622039794922, "logps/ref_rejected": -109.19184112548828, "logps/rejected": -194.89776611328125, "loss": 0.8877, "margin_dpo/margin_mean": 40.33136749267578, "margin_dpo/margin_std": 36.7901611328125, "step": 268 }, { "epoch": 0.39500734214390604, "fcm_dpo/beta": 0.017438966780900955, "fcm_dpo/delta": -0.1256072223186493, "fcm_dpo/margin": 41.10211944580078, "fcm_dpo/q_t": 0.3434303104877472, "grad_norm": 10.632892608642578, "learning_rate": 3.8051520207480204e-07, "logits/chosen": -1.8700554370880127, "logits/rejected": -1.408653974533081, "logps/chosen": -152.26226806640625, "logps/ref_chosen": -104.97181701660156, "logps/ref_rejected": -112.47647094726562, "logps/rejected": -200.86904907226562, "loss": 0.8995, "margin_dpo/margin_mean": 41.10211944580078, "margin_dpo/margin_std": 40.0078125, "step": 269 }, { "epoch": 0.3964757709251101, "fcm_dpo/beta": 0.017290018498897552, "fcm_dpo/delta": -0.01898367889225483, "fcm_dpo/margin": 35.722171783447266, "fcm_dpo/q_t": 0.3673768937587738, "grad_norm": 10.380319595336914, "learning_rate": 3.794189242333106e-07, "logits/chosen": -2.5234313011169434, "logits/rejected": -2.1158437728881836, "logps/chosen": -153.2642822265625, "logps/ref_chosen": -101.07383728027344, "logps/ref_rejected": -117.75289154052734, "logps/rejected": -205.66552734375, "loss": 1.0017, "margin_dpo/margin_mean": 35.722171783447266, "margin_dpo/margin_std": 45.860267639160156, "step": 270 }, { "epoch": 0.39794419970631423, "fcm_dpo/beta": 0.0168063472956419, "fcm_dpo/delta": -0.1483364999294281, "fcm_dpo/margin": 43.856292724609375, "fcm_dpo/q_t": 0.3364192843437195, "grad_norm": 10.36974811553955, "learning_rate": 3.7831923608280514e-07, "logits/chosen": -1.4026761054992676, "logits/rejected": -0.9475973844528198, "logps/chosen": -140.18299865722656, "logps/ref_chosen": -96.72459411621094, "logps/ref_rejected": -98.5244140625, "logps/rejected": -185.839111328125, "loss": 0.9012, "margin_dpo/margin_mean": 43.856292724609375, "margin_dpo/margin_std": 43.59748840332031, "step": 271 }, { "epoch": 0.39941262848751835, "fcm_dpo/beta": 0.016488002613186836, "fcm_dpo/delta": -0.08753229677677155, "fcm_dpo/margin": 41.335079193115234, "fcm_dpo/q_t": 0.35309892892837524, "grad_norm": 10.801867485046387, "learning_rate": 3.772161666010912e-07, "logits/chosen": -1.7714588642120361, "logits/rejected": -1.6390788555145264, "logps/chosen": -117.77945709228516, "logps/ref_chosen": -80.97721862792969, "logps/ref_rejected": -108.55535888671875, "logps/rejected": -186.69268798828125, "loss": 0.9354, "margin_dpo/margin_mean": 41.3350830078125, "margin_dpo/margin_std": 44.84138107299805, "step": 272 }, { "epoch": 0.4008810572687225, "fcm_dpo/beta": 0.016001848503947258, "fcm_dpo/delta": -0.10035118460655212, "fcm_dpo/margin": 43.12675857543945, "fcm_dpo/q_t": 0.350558340549469, "grad_norm": 9.880205154418945, "learning_rate": 3.761097448550755e-07, "logits/chosen": -1.484649419784546, "logits/rejected": -1.0760293006896973, "logps/chosen": -142.65541076660156, "logps/ref_chosen": -92.22460174560547, "logps/ref_rejected": -97.3630599975586, "logps/rejected": -190.9206085205078, "loss": 0.9532, "margin_dpo/margin_mean": 43.12675857543945, "margin_dpo/margin_std": 49.11865234375, "step": 273 }, { "epoch": 0.4023494860499266, "fcm_dpo/beta": 0.016166478395462036, "fcm_dpo/delta": 0.05010783299803734, "fcm_dpo/margin": 34.18390655517578, "fcm_dpo/q_t": 0.37679600715637207, "grad_norm": 12.417606353759766, "learning_rate": 3.75e-07, "logits/chosen": -1.9811569452285767, "logits/rejected": -1.5822125673294067, "logps/chosen": -133.99008178710938, "logps/ref_chosen": -82.26089477539062, "logps/ref_rejected": -83.87699890136719, "logps/rejected": -169.79010009765625, "loss": 0.9924, "margin_dpo/margin_mean": 34.18390655517578, "margin_dpo/margin_std": 37.70836639404297, "step": 274 }, { "epoch": 0.40381791483113066, "fcm_dpo/beta": 0.016330569982528687, "fcm_dpo/delta": -0.013110160827636719, "fcm_dpo/margin": 37.44886016845703, "fcm_dpo/q_t": 0.3667539358139038, "grad_norm": 11.84593391418457, "learning_rate": 3.738869612786737e-07, "logits/chosen": -2.191549301147461, "logits/rejected": -1.8682620525360107, "logps/chosen": -123.36338806152344, "logps/ref_chosen": -79.68696594238281, "logps/ref_rejected": -98.7509765625, "logps/rejected": -179.87625122070312, "loss": 1.0023, "margin_dpo/margin_mean": 37.44886016845703, "margin_dpo/margin_std": 47.065589904785156, "step": 275 }, { "epoch": 0.4052863436123348, "fcm_dpo/beta": 0.016055840998888016, "fcm_dpo/delta": -0.008165836334228516, "fcm_dpo/margin": 37.754920959472656, "fcm_dpo/q_t": 0.3685857951641083, "grad_norm": 12.060040473937988, "learning_rate": 3.7277065802070204e-07, "logits/chosen": -2.0615105628967285, "logits/rejected": -1.5026049613952637, "logps/chosen": -140.79737854003906, "logps/ref_chosen": -86.53970336914062, "logps/ref_rejected": -77.85394287109375, "logps/rejected": -169.86654663085938, "loss": 0.9909, "margin_dpo/margin_mean": 37.754920959472656, "margin_dpo/margin_std": 45.30341339111328, "step": 276 }, { "epoch": 0.4067547723935389, "fcm_dpo/beta": 0.016346724703907967, "fcm_dpo/delta": 0.08950374275445938, "fcm_dpo/margin": 31.514963150024414, "fcm_dpo/q_t": 0.38510024547576904, "grad_norm": 13.390870094299316, "learning_rate": 3.71651119641714e-07, "logits/chosen": -2.061767101287842, "logits/rejected": -1.646317481994629, "logps/chosen": -141.05987548828125, "logps/ref_chosen": -84.24411010742188, "logps/ref_rejected": -98.83421325683594, "logps/rejected": -187.1649169921875, "loss": 1.0305, "margin_dpo/margin_mean": 31.514963150024414, "margin_dpo/margin_std": 38.913230895996094, "step": 277 }, { "epoch": 0.40822320117474303, "fcm_dpo/beta": 0.0163891538977623, "fcm_dpo/delta": -0.006686456501483917, "fcm_dpo/margin": 36.96751403808594, "fcm_dpo/q_t": 0.36568570137023926, "grad_norm": 12.524548530578613, "learning_rate": 3.705283756425872e-07, "logits/chosen": -2.1564512252807617, "logits/rejected": -1.948979139328003, "logps/chosen": -136.97509765625, "logps/ref_chosen": -82.431884765625, "logps/ref_rejected": -97.85691833496094, "logps/rejected": -189.3676300048828, "loss": 0.9865, "margin_dpo/margin_mean": 36.9675178527832, "margin_dpo/margin_std": 43.24678039550781, "step": 278 }, { "epoch": 0.40969162995594716, "fcm_dpo/beta": 0.01617228053510189, "fcm_dpo/delta": -0.08998230844736099, "fcm_dpo/margin": 42.24311447143555, "fcm_dpo/q_t": 0.35100412368774414, "grad_norm": 11.406697273254395, "learning_rate": 3.6940245560867e-07, "logits/chosen": -1.8248711824417114, "logits/rejected": -1.607770562171936, "logps/chosen": -143.13755798339844, "logps/ref_chosen": -85.16799926757812, "logps/ref_rejected": -94.12664794921875, "logps/rejected": -194.33932495117188, "loss": 0.9281, "margin_dpo/margin_mean": 42.24311828613281, "margin_dpo/margin_std": 44.3424072265625, "step": 279 }, { "epoch": 0.4111600587371512, "fcm_dpo/beta": 0.01589279994368553, "fcm_dpo/delta": -0.12768404185771942, "fcm_dpo/margin": 45.2197265625, "fcm_dpo/q_t": 0.34247255325317383, "grad_norm": 11.279327392578125, "learning_rate": 3.6827338920900253e-07, "logits/chosen": -1.7638534307479858, "logits/rejected": -1.555558204650879, "logps/chosen": -143.5164337158203, "logps/ref_chosen": -85.85641479492188, "logps/ref_rejected": -104.11859130859375, "logps/rejected": -206.9983367919922, "loss": 0.8992, "margin_dpo/margin_mean": 45.2197265625, "margin_dpo/margin_std": 43.852996826171875, "step": 280 }, { "epoch": 0.41262848751835535, "fcm_dpo/beta": 0.015449896454811096, "fcm_dpo/delta": -0.1223934069275856, "fcm_dpo/margin": 46.17839050292969, "fcm_dpo/q_t": 0.3489765524864197, "grad_norm": 9.723531723022461, "learning_rate": 3.6714120619553435e-07, "logits/chosen": -2.1463782787323, "logits/rejected": -1.422343373298645, "logps/chosen": -147.69259643554688, "logps/ref_chosen": -86.55082702636719, "logps/ref_rejected": -88.62866973876953, "logps/rejected": -195.94882202148438, "loss": 0.9317, "margin_dpo/margin_mean": 46.17839050292969, "margin_dpo/margin_std": 52.58583068847656, "step": 281 }, { "epoch": 0.41409691629955947, "fcm_dpo/beta": 0.014763940125703812, "fcm_dpo/delta": -0.25571346282958984, "fcm_dpo/margin": 56.46161651611328, "fcm_dpo/q_t": 0.3185679316520691, "grad_norm": 9.334823608398438, "learning_rate": 3.660059364023408e-07, "logits/chosen": -1.9072599411010742, "logits/rejected": -1.542052984237671, "logps/chosen": -164.61256408691406, "logps/ref_chosen": -105.10511016845703, "logps/ref_rejected": -102.85336303710938, "logps/rejected": -218.82244873046875, "loss": 0.8212, "margin_dpo/margin_mean": 56.46161651611328, "margin_dpo/margin_std": 48.17137908935547, "step": 282 }, { "epoch": 0.4155653450807636, "fcm_dpo/beta": 0.014660017564892769, "fcm_dpo/delta": 0.026379965245723724, "fcm_dpo/margin": 39.232139587402344, "fcm_dpo/q_t": 0.3760044276714325, "grad_norm": 12.626619338989258, "learning_rate": 3.6486760974483685e-07, "logits/chosen": -2.1855897903442383, "logits/rejected": -1.8464176654815674, "logps/chosen": -164.91322326660156, "logps/ref_chosen": -95.05259704589844, "logps/ref_rejected": -103.54454803466797, "logps/rejected": -212.63731384277344, "loss": 1.0276, "margin_dpo/margin_mean": 39.232139587402344, "margin_dpo/margin_std": 53.09544372558594, "step": 283 }, { "epoch": 0.4170337738619677, "fcm_dpo/beta": 0.01422587689012289, "fcm_dpo/delta": -0.17442496120929718, "fcm_dpo/margin": 53.390506744384766, "fcm_dpo/q_t": 0.33068281412124634, "grad_norm": 9.979324340820312, "learning_rate": 3.6372625621898863e-07, "logits/chosen": -1.7961182594299316, "logits/rejected": -1.4766089916229248, "logps/chosen": -148.95257568359375, "logps/ref_chosen": -87.66647338867188, "logps/ref_rejected": -98.75103759765625, "logps/rejected": -213.42764282226562, "loss": 0.8488, "margin_dpo/margin_mean": 53.3905029296875, "margin_dpo/margin_std": 44.12660217285156, "step": 284 }, { "epoch": 0.4185022026431718, "fcm_dpo/beta": 0.01436292938888073, "fcm_dpo/delta": 0.06020905822515488, "fcm_dpo/margin": 37.7760009765625, "fcm_dpo/q_t": 0.3767179250717163, "grad_norm": 14.075101852416992, "learning_rate": 3.625819059005228e-07, "logits/chosen": -2.3716185092926025, "logits/rejected": -2.0925536155700684, "logps/chosen": -164.59915161132812, "logps/ref_chosen": -94.43303680419922, "logps/ref_rejected": -104.07194519042969, "logps/rejected": -212.01405334472656, "loss": 0.9954, "margin_dpo/margin_mean": 37.7760009765625, "margin_dpo/margin_std": 40.709510803222656, "step": 285 }, { "epoch": 0.4199706314243759, "fcm_dpo/beta": 0.014110833406448364, "fcm_dpo/delta": -0.10124547779560089, "fcm_dpo/margin": 49.17796325683594, "fcm_dpo/q_t": 0.3466648757457733, "grad_norm": 10.761466026306152, "learning_rate": 3.614345889441346e-07, "logits/chosen": -1.7981414794921875, "logits/rejected": -1.32814621925354, "logps/chosen": -178.29949951171875, "logps/ref_chosen": -103.72039794921875, "logps/ref_rejected": -96.25775909423828, "logps/rejected": -220.01480102539062, "loss": 0.9137, "margin_dpo/margin_mean": 49.17796325683594, "margin_dpo/margin_std": 48.17053985595703, "step": 286 }, { "epoch": 0.42143906020558003, "fcm_dpo/beta": 0.014512348920106888, "fcm_dpo/delta": 0.16535604000091553, "fcm_dpo/margin": 30.320154190063477, "fcm_dpo/q_t": 0.4061054587364197, "grad_norm": 11.057209968566895, "learning_rate": 3.6028433558269275e-07, "logits/chosen": -2.0745434761047363, "logits/rejected": -1.4789698123931885, "logps/chosen": -181.6774444580078, "logps/ref_chosen": -93.88988494873047, "logps/ref_rejected": -83.33365631103516, "logps/rejected": -201.44137573242188, "loss": 1.1162, "margin_dpo/margin_mean": 30.320154190063477, "margin_dpo/margin_std": 49.62172317504883, "step": 287 }, { "epoch": 0.42290748898678415, "fcm_dpo/beta": 0.014212418347597122, "fcm_dpo/delta": -0.13853885233402252, "fcm_dpo/margin": 51.22802734375, "fcm_dpo/q_t": 0.34565305709838867, "grad_norm": 10.420333862304688, "learning_rate": 3.5913117612644327e-07, "logits/chosen": -1.952336311340332, "logits/rejected": -1.555095911026001, "logps/chosen": -172.57025146484375, "logps/ref_chosen": -88.15602111816406, "logps/ref_rejected": -93.28195190429688, "logps/rejected": -228.92422485351562, "loss": 0.9128, "margin_dpo/margin_mean": 51.22802734375, "margin_dpo/margin_std": 54.86450958251953, "step": 288 }, { "epoch": 0.4243759177679883, "fcm_dpo/beta": 0.013875823467969894, "fcm_dpo/delta": -0.14725634455680847, "fcm_dpo/margin": 53.087974548339844, "fcm_dpo/q_t": 0.34943529963493347, "grad_norm": 9.893397331237793, "learning_rate": 3.5797514096221024e-07, "logits/chosen": -1.9582253694534302, "logits/rejected": -1.757028341293335, "logps/chosen": -158.78115844726562, "logps/ref_chosen": -75.39292907714844, "logps/ref_rejected": -93.15428161621094, "logps/rejected": -229.6304931640625, "loss": 0.9302, "margin_dpo/margin_mean": 53.087974548339844, "margin_dpo/margin_std": 61.871124267578125, "step": 289 }, { "epoch": 0.42584434654919234, "fcm_dpo/beta": 0.013547876849770546, "fcm_dpo/delta": -0.14979226887226105, "fcm_dpo/margin": 54.51033020019531, "fcm_dpo/q_t": 0.3426121473312378, "grad_norm": 10.341540336608887, "learning_rate": 3.568162605525952e-07, "logits/chosen": -1.157566785812378, "logits/rejected": -1.1871843338012695, "logps/chosen": -179.93722534179688, "logps/ref_chosen": -88.04197692871094, "logps/ref_rejected": -123.212158203125, "logps/rejected": -269.61773681640625, "loss": 0.9071, "margin_dpo/margin_mean": 54.51033020019531, "margin_dpo/margin_std": 57.139404296875, "step": 290 }, { "epoch": 0.42731277533039647, "fcm_dpo/beta": 0.013345437124371529, "fcm_dpo/delta": -0.0021981801837682724, "fcm_dpo/margin": 45.109642028808594, "fcm_dpo/q_t": 0.3656271994113922, "grad_norm": 10.851905822753906, "learning_rate": 3.5565456543517485e-07, "logits/chosen": -2.3467676639556885, "logits/rejected": -1.8936057090759277, "logps/chosen": -181.8697967529297, "logps/ref_chosen": -94.09524536132812, "logps/ref_rejected": -96.05006408691406, "logps/rejected": -228.93426513671875, "loss": 0.9784, "margin_dpo/margin_mean": 45.109642028808594, "margin_dpo/margin_std": 51.156219482421875, "step": 291 }, { "epoch": 0.4287812041116006, "fcm_dpo/beta": 0.013189633376896381, "fcm_dpo/delta": -0.062153078615665436, "fcm_dpo/margin": 49.88909912109375, "fcm_dpo/q_t": 0.3522568643093109, "grad_norm": 10.579051971435547, "learning_rate": 3.5449008622169583e-07, "logits/chosen": -2.1592330932617188, "logits/rejected": -1.730700969696045, "logps/chosen": -178.15887451171875, "logps/ref_chosen": -88.25041198730469, "logps/ref_rejected": -96.41764068603516, "logps/rejected": -236.21517944335938, "loss": 0.9094, "margin_dpo/margin_mean": 49.889095306396484, "margin_dpo/margin_std": 44.906211853027344, "step": 292 }, { "epoch": 0.4302496328928047, "fcm_dpo/beta": 0.013408501632511616, "fcm_dpo/delta": 0.0868854746222496, "fcm_dpo/margin": 38.51031494140625, "fcm_dpo/q_t": 0.3900435268878937, "grad_norm": 9.791333198547363, "learning_rate": 3.5332285359726846e-07, "logits/chosen": -2.625321388244629, "logits/rejected": -2.0894412994384766, "logps/chosen": -180.33511352539062, "logps/ref_chosen": -87.37654113769531, "logps/ref_rejected": -85.75579833984375, "logps/rejected": -217.22467041015625, "loss": 1.0576, "margin_dpo/margin_mean": 38.51031494140625, "margin_dpo/margin_std": 54.91911315917969, "step": 293 }, { "epoch": 0.43171806167400884, "fcm_dpo/beta": 0.013757754117250443, "fcm_dpo/delta": 0.16101771593093872, "fcm_dpo/margin": 32.374855041503906, "fcm_dpo/q_t": 0.4055211544036865, "grad_norm": 11.367213249206543, "learning_rate": 3.5215289831955786e-07, "logits/chosen": -1.6977550983428955, "logits/rejected": -1.5189887285232544, "logps/chosen": -170.7028045654297, "logps/ref_chosen": -73.50794982910156, "logps/ref_rejected": -88.08877563476562, "logps/rejected": -217.65847778320312, "loss": 1.1326, "margin_dpo/margin_mean": 32.374855041503906, "margin_dpo/margin_std": 57.856483459472656, "step": 294 }, { "epoch": 0.4331864904552129, "fcm_dpo/beta": 0.013606134802103043, "fcm_dpo/delta": -0.06477653235197067, "fcm_dpo/margin": 48.46016311645508, "fcm_dpo/q_t": 0.3579823672771454, "grad_norm": 10.51223373413086, "learning_rate": 3.509802512179737e-07, "logits/chosen": -1.6513359546661377, "logits/rejected": -1.5038331747055054, "logps/chosen": -172.31985473632812, "logps/ref_chosen": -77.76548767089844, "logps/ref_rejected": -94.24726867675781, "logps/rejected": -237.2617950439453, "loss": 0.9532, "margin_dpo/margin_mean": 48.46016311645508, "margin_dpo/margin_std": 54.77019500732422, "step": 295 }, { "epoch": 0.434654919236417, "fcm_dpo/beta": 0.01337181031703949, "fcm_dpo/delta": -0.10661374777555466, "fcm_dpo/margin": 52.22368240356445, "fcm_dpo/q_t": 0.34752362966537476, "grad_norm": 11.95097827911377, "learning_rate": 3.498049431928577e-07, "logits/chosen": -2.5504400730133057, "logits/rejected": -2.0862655639648438, "logps/chosen": -198.88778686523438, "logps/ref_chosen": -97.85641479492188, "logps/ref_rejected": -100.81631469726562, "logps/rejected": -254.07135009765625, "loss": 0.9174, "margin_dpo/margin_mean": 52.22368240356445, "margin_dpo/margin_std": 53.6197509765625, "step": 296 }, { "epoch": 0.43612334801762115, "fcm_dpo/beta": 0.013420840725302696, "fcm_dpo/delta": 0.018867414444684982, "fcm_dpo/margin": 43.375144958496094, "fcm_dpo/q_t": 0.37541282176971436, "grad_norm": 13.033909797668457, "learning_rate": 3.486270052146694e-07, "logits/chosen": -1.2705745697021484, "logits/rejected": -0.9568313360214233, "logps/chosen": -192.59295654296875, "logps/ref_chosen": -88.56583404541016, "logps/ref_rejected": -101.55656433105469, "logps/rejected": -248.9588165283203, "loss": 1.0032, "margin_dpo/margin_mean": 43.375144958496094, "margin_dpo/margin_std": 54.68861389160156, "step": 297 }, { "epoch": 0.43759177679882527, "fcm_dpo/beta": 0.013230125419795513, "fcm_dpo/delta": -0.11564385145902634, "fcm_dpo/margin": 53.48369216918945, "fcm_dpo/q_t": 0.35127192735671997, "grad_norm": 16.36045265197754, "learning_rate": 3.474464683231698e-07, "logits/chosen": -2.012399911880493, "logits/rejected": -1.9581105709075928, "logps/chosen": -186.6326904296875, "logps/ref_chosen": -94.88043975830078, "logps/ref_rejected": -122.31101989746094, "logps/rejected": -267.54693603515625, "loss": 0.9249, "margin_dpo/margin_mean": 53.48369216918945, "margin_dpo/margin_std": 59.249088287353516, "step": 298 }, { "epoch": 0.4390602055800294, "fcm_dpo/beta": 0.012806476093828678, "fcm_dpo/delta": -0.13323679566383362, "fcm_dpo/margin": 56.45828628540039, "fcm_dpo/q_t": 0.34347003698349, "grad_norm": 10.563586235046387, "learning_rate": 3.462633636266041e-07, "logits/chosen": -0.9475471377372742, "logits/rejected": -0.7138333320617676, "logps/chosen": -175.18414306640625, "logps/ref_chosen": -80.40835571289062, "logps/ref_rejected": -89.53717041015625, "logps/rejected": -240.771240234375, "loss": 0.9033, "margin_dpo/margin_mean": 56.458290100097656, "margin_dpo/margin_std": 57.19697570800781, "step": 299 }, { "epoch": 0.44052863436123346, "fcm_dpo/beta": 0.012716519646346569, "fcm_dpo/delta": 0.005831874907016754, "fcm_dpo/margin": 46.72724151611328, "fcm_dpo/q_t": 0.37154459953308105, "grad_norm": 11.477516174316406, "learning_rate": 3.4507772230088147e-07, "logits/chosen": -1.3495773077011108, "logits/rejected": -1.1283777952194214, "logps/chosen": -187.920654296875, "logps/ref_chosen": -88.15890502929688, "logps/ref_rejected": -100.93919372558594, "logps/rejected": -247.4281768798828, "loss": 1.0079, "margin_dpo/margin_mean": 46.727237701416016, "margin_dpo/margin_std": 59.988731384277344, "step": 300 }, { "epoch": 0.44052863436123346, "eval_fcm_dpo/beta": 0.01277273241430521, "eval_logits/chosen": -1.7159112691879272, "eval_logits/rejected": -1.3590905666351318, "eval_logps/chosen": -199.09510803222656, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -235.35121154785156, "eval_loss": 0.5174583792686462, "eval_margin_dpo/margin_mean": 42.681915283203125, "eval_margin_dpo/margin_std": 56.65989685058594, "eval_runtime": 43.743, "eval_samples_per_second": 53.471, "eval_steps_per_second": 1.692, "step": 300 }, { "epoch": 0.4419970631424376, "fcm_dpo/beta": 0.012361442670226097, "fcm_dpo/delta": -0.16100239753723145, "fcm_dpo/margin": 60.36054992675781, "fcm_dpo/q_t": 0.3411176800727844, "grad_norm": 10.854103088378906, "learning_rate": 3.4388957558875316e-07, "logits/chosen": -1.7507340908050537, "logits/rejected": -1.315748691558838, "logps/chosen": -178.2888946533203, "logps/ref_chosen": -85.50491333007812, "logps/ref_rejected": -98.91683959960938, "logps/rejected": -252.06137084960938, "loss": 0.9009, "margin_dpo/margin_mean": 60.36054992675781, "margin_dpo/margin_std": 62.385398864746094, "step": 301 }, { "epoch": 0.4434654919236417, "fcm_dpo/beta": 0.012308208271861076, "fcm_dpo/delta": -0.004877906292676926, "fcm_dpo/margin": 49.0919189453125, "fcm_dpo/q_t": 0.3680340647697449, "grad_norm": 11.78134536743164, "learning_rate": 3.426989547989902e-07, "logits/chosen": -1.391321063041687, "logits/rejected": -1.2118873596191406, "logps/chosen": -172.2723388671875, "logps/ref_chosen": -79.45040893554688, "logps/ref_rejected": -103.0591049194336, "logps/rejected": -244.97293090820312, "loss": 0.9875, "margin_dpo/margin_mean": 49.091915130615234, "margin_dpo/margin_std": 58.554237365722656, "step": 302 }, { "epoch": 0.44493392070484583, "fcm_dpo/beta": 0.01226123422384262, "fcm_dpo/delta": -0.08874759823083878, "fcm_dpo/margin": 55.66910934448242, "fcm_dpo/q_t": 0.3502175211906433, "grad_norm": 10.105868339538574, "learning_rate": 3.4150589130555773e-07, "logits/chosen": -1.8867647647857666, "logits/rejected": -1.4471575021743774, "logps/chosen": -179.00445556640625, "logps/ref_chosen": -95.59829711914062, "logps/ref_rejected": -92.0645751953125, "logps/rejected": -231.1398468017578, "loss": 0.9267, "margin_dpo/margin_mean": 55.669105529785156, "margin_dpo/margin_std": 56.76445007324219, "step": 303 }, { "epoch": 0.44640234948604995, "fcm_dpo/beta": 0.011957298964262009, "fcm_dpo/delta": -0.11238184571266174, "fcm_dpo/margin": 58.92472839355469, "fcm_dpo/q_t": 0.3453929126262665, "grad_norm": 11.284574508666992, "learning_rate": 3.403104165467883e-07, "logits/chosen": -2.2299141883850098, "logits/rejected": -1.8196072578430176, "logps/chosen": -183.23292541503906, "logps/ref_chosen": -99.4531478881836, "logps/ref_rejected": -98.08226013183594, "logps/rejected": -240.78677368164062, "loss": 0.9148, "margin_dpo/margin_mean": 58.92472839355469, "margin_dpo/margin_std": 60.10872268676758, "step": 304 }, { "epoch": 0.447870778267254, "fcm_dpo/beta": 0.011641704477369785, "fcm_dpo/delta": -0.06990103423595428, "fcm_dpo/margin": 56.961299896240234, "fcm_dpo/q_t": 0.35451453924179077, "grad_norm": 10.1016263961792, "learning_rate": 3.391125620245535e-07, "logits/chosen": -2.020512104034424, "logits/rejected": -1.5305230617523193, "logps/chosen": -190.84634399414062, "logps/ref_chosen": -103.26339721679688, "logps/ref_rejected": -97.82897186279297, "logps/rejected": -242.3732147216797, "loss": 0.9438, "margin_dpo/margin_mean": 56.96129608154297, "margin_dpo/margin_std": 59.46485137939453, "step": 305 }, { "epoch": 0.44933920704845814, "fcm_dpo/beta": 0.011314811185002327, "fcm_dpo/delta": -0.20089520514011383, "fcm_dpo/margin": 69.29483032226562, "fcm_dpo/q_t": 0.3232324719429016, "grad_norm": 10.051947593688965, "learning_rate": 3.3791235930343417e-07, "logits/chosen": -2.1980226039886475, "logits/rejected": -1.6485023498535156, "logps/chosen": -177.880615234375, "logps/ref_chosen": -100.3038330078125, "logps/ref_rejected": -92.23086547851562, "logps/rejected": -239.10247802734375, "loss": 0.8261, "margin_dpo/margin_mean": 69.29483032226562, "margin_dpo/margin_std": 51.61286926269531, "step": 306 }, { "epoch": 0.45080763582966227, "fcm_dpo/beta": 0.011268500238656998, "fcm_dpo/delta": 0.04018982872366905, "fcm_dpo/margin": 49.86820983886719, "fcm_dpo/q_t": 0.37846463918685913, "grad_norm": 10.908080101013184, "learning_rate": 3.367098400098881e-07, "logits/chosen": -2.024630308151245, "logits/rejected": -1.6950145959854126, "logps/chosen": -181.67286682128906, "logps/ref_chosen": -96.64976501464844, "logps/ref_rejected": -93.34860229492188, "logps/rejected": -228.23992919921875, "loss": 1.026, "margin_dpo/margin_mean": 49.86820602416992, "margin_dpo/margin_std": 66.2600326538086, "step": 307 }, { "epoch": 0.4522760646108664, "fcm_dpo/beta": 0.011317985132336617, "fcm_dpo/delta": 0.04552755504846573, "fcm_dpo/margin": 49.12928009033203, "fcm_dpo/q_t": 0.38196253776550293, "grad_norm": 11.59121322631836, "learning_rate": 3.355050358314172e-07, "logits/chosen": -1.7888622283935547, "logits/rejected": -1.5279297828674316, "logps/chosen": -173.8710479736328, "logps/ref_chosen": -85.72467041015625, "logps/ref_rejected": -86.42317962646484, "logps/rejected": -223.69882202148438, "loss": 1.0387, "margin_dpo/margin_mean": 49.12928009033203, "margin_dpo/margin_std": 66.89312744140625, "step": 308 }, { "epoch": 0.45374449339207046, "fcm_dpo/beta": 0.011532912030816078, "fcm_dpo/delta": 0.044411551207304, "fcm_dpo/margin": 48.37786102294922, "fcm_dpo/q_t": 0.3778493404388428, "grad_norm": 14.045127868652344, "learning_rate": 3.3429797851573183e-07, "logits/chosen": -1.8127508163452148, "logits/rejected": -1.389906883239746, "logps/chosen": -179.91854858398438, "logps/ref_chosen": -83.15145874023438, "logps/ref_rejected": -86.58602142333984, "logps/rejected": -231.73095703125, "loss": 1.0173, "margin_dpo/margin_mean": 48.37785720825195, "margin_dpo/margin_std": 60.719688415527344, "step": 309 }, { "epoch": 0.4552129221732746, "fcm_dpo/beta": 0.011562330648303032, "fcm_dpo/delta": 0.013482505455613136, "fcm_dpo/margin": 50.79509735107422, "fcm_dpo/q_t": 0.3710099160671234, "grad_norm": 10.965914726257324, "learning_rate": 3.3308869986991487e-07, "logits/chosen": -2.2912349700927734, "logits/rejected": -1.8876441717147827, "logps/chosen": -175.0748291015625, "logps/ref_chosen": -89.73799133300781, "logps/ref_rejected": -87.07083129882812, "logps/rejected": -223.2027587890625, "loss": 0.9925, "margin_dpo/margin_mean": 50.79509353637695, "margin_dpo/margin_std": 60.45803451538086, "step": 310 }, { "epoch": 0.4566813509544787, "fcm_dpo/beta": 0.011323259212076664, "fcm_dpo/delta": -0.14433330297470093, "fcm_dpo/margin": 64.79643249511719, "fcm_dpo/q_t": 0.33913880586624146, "grad_norm": 10.768723487854004, "learning_rate": 3.3187723175958346e-07, "logits/chosen": -1.8347514867782593, "logits/rejected": -1.1266942024230957, "logps/chosen": -192.47508239746094, "logps/ref_chosen": -88.52644348144531, "logps/ref_rejected": -82.26608276367188, "logps/rejected": -251.0111541748047, "loss": 0.875, "margin_dpo/margin_mean": 64.79643249511719, "margin_dpo/margin_std": 58.919742584228516, "step": 311 }, { "epoch": 0.4581497797356828, "fcm_dpo/beta": 0.011255646124482155, "fcm_dpo/delta": -0.024957779794931412, "fcm_dpo/margin": 55.35394287109375, "fcm_dpo/q_t": 0.36366888880729675, "grad_norm": 10.373575210571289, "learning_rate": 3.306636061080487e-07, "logits/chosen": -1.5337181091308594, "logits/rejected": -1.003326654434204, "logps/chosen": -179.2374267578125, "logps/ref_chosen": -80.40069580078125, "logps/ref_rejected": -82.4471435546875, "logps/rejected": -236.63783264160156, "loss": 0.9707, "margin_dpo/margin_mean": 55.35394287109375, "margin_dpo/margin_std": 63.06084442138672, "step": 312 }, { "epoch": 0.45961820851688695, "fcm_dpo/beta": 0.01121255662292242, "fcm_dpo/delta": 0.0006225090473890305, "fcm_dpo/margin": 53.44978332519531, "fcm_dpo/q_t": 0.3703840374946594, "grad_norm": 11.122648239135742, "learning_rate": 3.2944785489547537e-07, "logits/chosen": -2.4546828269958496, "logits/rejected": -2.0480782985687256, "logps/chosen": -176.66587829589844, "logps/ref_chosen": -84.14190673828125, "logps/ref_rejected": -93.42684173583984, "logps/rejected": -239.40060424804688, "loss": 1.008, "margin_dpo/margin_mean": 53.44977569580078, "margin_dpo/margin_std": 69.2515869140625, "step": 313 }, { "epoch": 0.461086637298091, "fcm_dpo/beta": 0.011162392795085907, "fcm_dpo/delta": 0.010066971182823181, "fcm_dpo/margin": 52.88270568847656, "fcm_dpo/q_t": 0.3719868063926697, "grad_norm": 11.244734764099121, "learning_rate": 3.2823001015803857e-07, "logits/chosen": -1.9083623886108398, "logits/rejected": -1.6891626119613647, "logps/chosen": -181.61160278320312, "logps/ref_chosen": -88.80972290039062, "logps/ref_rejected": -104.15892028808594, "logps/rejected": -249.843505859375, "loss": 0.9914, "margin_dpo/margin_mean": 52.88270568847656, "margin_dpo/margin_std": 63.093101501464844, "step": 314 }, { "epoch": 0.46255506607929514, "fcm_dpo/beta": 0.011188652366399765, "fcm_dpo/delta": -0.006126678083091974, "fcm_dpo/margin": 54.138145446777344, "fcm_dpo/q_t": 0.37136292457580566, "grad_norm": 12.068286895751953, "learning_rate": 3.270101039870797e-07, "logits/chosen": -1.4232127666473389, "logits/rejected": -1.2410612106323242, "logps/chosen": -171.7213134765625, "logps/ref_chosen": -80.793701171875, "logps/ref_rejected": -92.13711547851562, "logps/rejected": -237.202880859375, "loss": 1.0249, "margin_dpo/margin_mean": 54.138145446777344, "margin_dpo/margin_std": 74.93022155761719, "step": 315 }, { "epoch": 0.46402349486049926, "fcm_dpo/beta": 0.011153796687722206, "fcm_dpo/delta": -0.032783642411231995, "fcm_dpo/margin": 56.54811096191406, "fcm_dpo/q_t": 0.364329993724823, "grad_norm": 10.284096717834473, "learning_rate": 3.2578816852826086e-07, "logits/chosen": -1.860671043395996, "logits/rejected": -1.8614685535430908, "logps/chosen": -179.6904296875, "logps/ref_chosen": -84.18424987792969, "logps/ref_rejected": -105.6908187866211, "logps/rejected": -257.7451171875, "loss": 0.9706, "margin_dpo/margin_mean": 56.54811096191406, "margin_dpo/margin_std": 67.02822875976562, "step": 316 }, { "epoch": 0.4654919236417034, "fcm_dpo/beta": 0.010856937617063522, "fcm_dpo/delta": -0.19055171310901642, "fcm_dpo/margin": 71.48945617675781, "fcm_dpo/q_t": 0.3373391628265381, "grad_norm": 12.972799301147461, "learning_rate": 3.2456423598071783e-07, "logits/chosen": -1.9898185729980469, "logits/rejected": -1.512327790260315, "logps/chosen": -185.63662719726562, "logps/ref_chosen": -87.87348937988281, "logps/ref_rejected": -108.26519775390625, "logps/rejected": -277.5177917480469, "loss": 0.9042, "margin_dpo/margin_mean": 71.48945617675781, "margin_dpo/margin_std": 77.33946228027344, "step": 317 }, { "epoch": 0.4669603524229075, "fcm_dpo/beta": 0.010556013323366642, "fcm_dpo/delta": -0.08519938588142395, "fcm_dpo/margin": 64.34666442871094, "fcm_dpo/q_t": 0.351060688495636, "grad_norm": 10.77450180053711, "learning_rate": 3.233383385962115e-07, "logits/chosen": -2.1584458351135254, "logits/rejected": -1.4637277126312256, "logps/chosen": -193.25021362304688, "logps/ref_chosen": -98.20553588867188, "logps/ref_rejected": -88.13629150390625, "logps/rejected": -247.52764892578125, "loss": 0.925, "margin_dpo/margin_mean": 64.34667205810547, "margin_dpo/margin_std": 66.24368286132812, "step": 318 }, { "epoch": 0.4684287812041116, "fcm_dpo/beta": 0.010318400338292122, "fcm_dpo/delta": -0.1146102324128151, "fcm_dpo/margin": 68.42571258544922, "fcm_dpo/q_t": 0.3458014726638794, "grad_norm": 10.647491455078125, "learning_rate": 3.2211050867827805e-07, "logits/chosen": -1.9602619409561157, "logits/rejected": -1.8778321743011475, "logps/chosen": -164.64324951171875, "logps/ref_chosen": -82.50337219238281, "logps/ref_rejected": -118.65068054199219, "logps/rejected": -269.21624755859375, "loss": 0.9143, "margin_dpo/margin_mean": 68.42571258544922, "margin_dpo/margin_std": 70.17927551269531, "step": 319 }, { "epoch": 0.4698972099853157, "fcm_dpo/beta": 0.010274864733219147, "fcm_dpo/delta": 0.008025115355849266, "fcm_dpo/margin": 57.651084899902344, "fcm_dpo/q_t": 0.369371235370636, "grad_norm": 10.72866439819336, "learning_rate": 3.208807785813777e-07, "logits/chosen": -2.084186553955078, "logits/rejected": -1.8974592685699463, "logps/chosen": -172.93923950195312, "logps/ref_chosen": -87.13722229003906, "logps/ref_rejected": -103.22412109375, "logps/rejected": -246.67721557617188, "loss": 0.974, "margin_dpo/margin_mean": 57.651084899902344, "margin_dpo/margin_std": 64.01679992675781, "step": 320 }, { "epoch": 0.4713656387665198, "fcm_dpo/beta": 0.010156042873859406, "fcm_dpo/delta": -0.06489168107509613, "fcm_dpo/margin": 64.97313690185547, "fcm_dpo/q_t": 0.3531816005706787, "grad_norm": 10.770258903503418, "learning_rate": 3.1964918071004217e-07, "logits/chosen": -2.1117072105407715, "logits/rejected": -1.7369625568389893, "logps/chosen": -188.44654846191406, "logps/ref_chosen": -93.10466766357422, "logps/ref_rejected": -97.4681396484375, "logps/rejected": -257.78314208984375, "loss": 0.9341, "margin_dpo/margin_mean": 64.97313690185547, "margin_dpo/margin_std": 66.21844482421875, "step": 321 }, { "epoch": 0.47283406754772395, "fcm_dpo/beta": 0.009905429556965828, "fcm_dpo/delta": -0.12228944152593613, "fcm_dpo/margin": 71.8099594116211, "fcm_dpo/q_t": 0.3459140360355377, "grad_norm": 13.35485553741455, "learning_rate": 3.184157475180207e-07, "logits/chosen": -1.7865667343139648, "logits/rejected": -1.4356210231781006, "logps/chosen": -182.97547912597656, "logps/ref_chosen": -92.19291687011719, "logps/ref_rejected": -100.89321899414062, "logps/rejected": -263.4857177734375, "loss": 0.9237, "margin_dpo/margin_mean": 71.8099594116211, "margin_dpo/margin_std": 75.52076721191406, "step": 322 }, { "epoch": 0.47430249632892807, "fcm_dpo/beta": 0.009861964732408524, "fcm_dpo/delta": -0.021682463586330414, "fcm_dpo/margin": 62.875831604003906, "fcm_dpo/q_t": 0.3650767207145691, "grad_norm": 10.933591842651367, "learning_rate": 3.171805115074251e-07, "logits/chosen": -2.0441133975982666, "logits/rejected": -1.6695351600646973, "logps/chosen": -184.3953857421875, "logps/ref_chosen": -80.27210235595703, "logps/ref_rejected": -80.3033447265625, "logps/rejected": -247.302490234375, "loss": 0.9606, "margin_dpo/margin_mean": 62.875831604003906, "margin_dpo/margin_std": 68.70439147949219, "step": 323 }, { "epoch": 0.47577092511013214, "fcm_dpo/beta": 0.009814316406846046, "fcm_dpo/delta": -0.05098802596330643, "fcm_dpo/margin": 65.9963150024414, "fcm_dpo/q_t": 0.3542330861091614, "grad_norm": 10.099251747131348, "learning_rate": 3.1594350522787295e-07, "logits/chosen": -1.5629843473434448, "logits/rejected": -0.8973981142044067, "logps/chosen": -200.52267456054688, "logps/ref_chosen": -95.72935485839844, "logps/ref_rejected": -94.16924285888672, "logps/rejected": -264.9588623046875, "loss": 0.9235, "margin_dpo/margin_mean": 65.9963150024414, "margin_dpo/margin_std": 62.014060974121094, "step": 324 }, { "epoch": 0.47723935389133626, "fcm_dpo/beta": 0.009698811918497086, "fcm_dpo/delta": -0.06353026628494263, "fcm_dpo/margin": 67.9854965209961, "fcm_dpo/q_t": 0.3524365723133087, "grad_norm": 8.840824127197266, "learning_rate": 3.147047612756302e-07, "logits/chosen": -1.7049955129623413, "logits/rejected": -0.977316677570343, "logps/chosen": -196.92950439453125, "logps/ref_chosen": -98.70686340332031, "logps/ref_rejected": -76.40809631347656, "logps/rejected": -242.61624145507812, "loss": 0.9246, "margin_dpo/margin_mean": 67.9854965209961, "margin_dpo/margin_std": 65.83146667480469, "step": 325 }, { "epoch": 0.4787077826725404, "fcm_dpo/beta": 0.009657101705670357, "fcm_dpo/delta": 0.0018943261820822954, "fcm_dpo/margin": 61.94574737548828, "fcm_dpo/q_t": 0.36700719594955444, "grad_norm": 10.065813064575195, "learning_rate": 3.134643122927519e-07, "logits/chosen": -2.4169249534606934, "logits/rejected": -1.762148380279541, "logps/chosen": -214.70034790039062, "logps/ref_chosen": -104.66130828857422, "logps/ref_rejected": -86.82537841796875, "logps/rejected": -258.8101806640625, "loss": 0.9749, "margin_dpo/margin_mean": 61.94574737548828, "margin_dpo/margin_std": 68.8028793334961, "step": 326 }, { "epoch": 0.4801762114537445, "fcm_dpo/beta": 0.00965862162411213, "fcm_dpo/delta": 0.015111330896615982, "fcm_dpo/margin": 60.6379508972168, "fcm_dpo/q_t": 0.3714819848537445, "grad_norm": 10.074041366577148, "learning_rate": 3.1222219096622264e-07, "logits/chosen": -1.8287179470062256, "logits/rejected": -1.238955020904541, "logps/chosen": -213.91966247558594, "logps/ref_chosen": -107.44023895263672, "logps/ref_rejected": -117.05323028564453, "logps/rejected": -284.17059326171875, "loss": 0.9792, "margin_dpo/margin_mean": 60.6379508972168, "margin_dpo/margin_std": 67.89471435546875, "step": 327 }, { "epoch": 0.48164464023494863, "fcm_dpo/beta": 0.0094821248203516, "fcm_dpo/delta": -0.14909884333610535, "fcm_dpo/margin": 77.84373474121094, "fcm_dpo/q_t": 0.33735549449920654, "grad_norm": 12.075971603393555, "learning_rate": 3.1097843002709427e-07, "logits/chosen": -1.6724544763565063, "logits/rejected": -1.3806228637695312, "logps/chosen": -192.94517517089844, "logps/ref_chosen": -86.6106948852539, "logps/ref_rejected": -97.40188598632812, "logps/rejected": -281.5801086425781, "loss": 0.8778, "margin_dpo/margin_mean": 77.84373474121094, "margin_dpo/margin_std": 71.33271789550781, "step": 328 }, { "epoch": 0.4831130690161527, "fcm_dpo/beta": 0.0093916654586792, "fcm_dpo/delta": 0.019437361508607864, "fcm_dpo/margin": 61.90422058105469, "fcm_dpo/q_t": 0.371404230594635, "grad_norm": 11.608654975891113, "learning_rate": 3.0973306224962437e-07, "logits/chosen": -1.5943433046340942, "logits/rejected": -1.0977287292480469, "logps/chosen": -212.25148010253906, "logps/ref_chosen": -100.61308288574219, "logps/ref_rejected": -104.95370483398438, "logps/rejected": -278.496337890625, "loss": 0.9812, "margin_dpo/margin_mean": 61.90422058105469, "margin_dpo/margin_std": 68.27637481689453, "step": 329 }, { "epoch": 0.4845814977973568, "fcm_dpo/beta": 0.00939023308455944, "fcm_dpo/delta": -0.020257247611880302, "fcm_dpo/margin": 65.90046691894531, "fcm_dpo/q_t": 0.3579813539981842, "grad_norm": 9.650596618652344, "learning_rate": 3.084861204504122e-07, "logits/chosen": -2.022364377975464, "logits/rejected": -1.9466552734375, "logps/chosen": -175.4729766845703, "logps/ref_chosen": -76.95843505859375, "logps/ref_rejected": -92.81494140625, "logps/rejected": -257.22991943359375, "loss": 0.9311, "margin_dpo/margin_mean": 65.90046691894531, "margin_dpo/margin_std": 57.93687438964844, "step": 330 }, { "epoch": 0.48604992657856094, "fcm_dpo/beta": 0.009334873408079147, "fcm_dpo/delta": -0.08815296739339828, "fcm_dpo/margin": 73.05988311767578, "fcm_dpo/q_t": 0.34697699546813965, "grad_norm": 12.887969970703125, "learning_rate": 3.072376374875335e-07, "logits/chosen": -1.7896149158477783, "logits/rejected": -1.4816310405731201, "logps/chosen": -188.48745727539062, "logps/ref_chosen": -83.05941772460938, "logps/ref_rejected": -94.13113403320312, "logps/rejected": -272.6190490722656, "loss": 0.8953, "margin_dpo/margin_mean": 73.05988311767578, "margin_dpo/margin_std": 62.86804962158203, "step": 331 }, { "epoch": 0.48751835535976507, "fcm_dpo/beta": 0.00939233973622322, "fcm_dpo/delta": 0.1038806289434433, "fcm_dpo/margin": 53.36603927612305, "fcm_dpo/q_t": 0.3885546326637268, "grad_norm": 9.494643211364746, "learning_rate": 3.059876462596758e-07, "logits/chosen": -2.1970367431640625, "logits/rejected": -1.778074026107788, "logps/chosen": -191.7408447265625, "logps/ref_chosen": -81.93089294433594, "logps/ref_rejected": -82.84590148925781, "logps/rejected": -246.02191162109375, "loss": 1.0397, "margin_dpo/margin_mean": 53.36603546142578, "margin_dpo/margin_std": 68.24900817871094, "step": 332 }, { "epoch": 0.4889867841409692, "fcm_dpo/beta": 0.009271856397390366, "fcm_dpo/delta": -0.06427404284477234, "fcm_dpo/margin": 71.00698852539062, "fcm_dpo/q_t": 0.35842740535736084, "grad_norm": 10.024383544921875, "learning_rate": 3.0473617970527015e-07, "logits/chosen": -1.5461020469665527, "logits/rejected": -1.212287425994873, "logps/chosen": -216.07131958007812, "logps/ref_chosen": -88.53257751464844, "logps/ref_rejected": -102.46601867675781, "logps/rejected": -301.01171875, "loss": 0.968, "margin_dpo/margin_mean": 71.00698852539062, "margin_dpo/margin_std": 83.59139251708984, "step": 333 }, { "epoch": 0.49045521292217326, "fcm_dpo/beta": 0.009264590218663216, "fcm_dpo/delta": 0.004638519138097763, "fcm_dpo/margin": 64.21815490722656, "fcm_dpo/q_t": 0.3684922456741333, "grad_norm": 11.035482406616211, "learning_rate": 3.034832708016243e-07, "logits/chosen": -1.366419792175293, "logits/rejected": -1.0714776515960693, "logps/chosen": -213.7954864501953, "logps/ref_chosen": -97.75778198242188, "logps/ref_rejected": -100.0058364868164, "logps/rejected": -280.2616882324219, "loss": 1.0037, "margin_dpo/margin_mean": 64.21814727783203, "margin_dpo/margin_std": 79.95184326171875, "step": 334 }, { "epoch": 0.4919236417033774, "fcm_dpo/beta": 0.009400870651006699, "fcm_dpo/delta": 0.05613856390118599, "fcm_dpo/margin": 58.17888641357422, "fcm_dpo/q_t": 0.38059741258621216, "grad_norm": 11.443718910217285, "learning_rate": 3.022289525640531e-07, "logits/chosen": -2.1963934898376465, "logits/rejected": -1.7753444910049438, "logps/chosen": -212.1436767578125, "logps/ref_chosen": -96.37603759765625, "logps/ref_rejected": -94.92401123046875, "logps/rejected": -268.87054443359375, "loss": 1.0258, "margin_dpo/margin_mean": 58.17888641357422, "margin_dpo/margin_std": 75.49417114257812, "step": 335 }, { "epoch": 0.4933920704845815, "fcm_dpo/beta": 0.009332764893770218, "fcm_dpo/delta": -0.08782423287630081, "fcm_dpo/margin": 73.06648254394531, "fcm_dpo/q_t": 0.3505851626396179, "grad_norm": 11.681970596313477, "learning_rate": 3.009732580450086e-07, "logits/chosen": -1.6232037544250488, "logits/rejected": -1.4630122184753418, "logps/chosen": -203.955078125, "logps/ref_chosen": -88.52952575683594, "logps/ref_rejected": -109.5557861328125, "logps/rejected": -298.04779052734375, "loss": 0.9236, "margin_dpo/margin_mean": 73.06648254394531, "margin_dpo/margin_std": 76.55876159667969, "step": 336 }, { "epoch": 0.4948604992657856, "fcm_dpo/beta": 0.009262502193450928, "fcm_dpo/delta": -0.02163381688296795, "fcm_dpo/margin": 66.955322265625, "fcm_dpo/q_t": 0.36714088916778564, "grad_norm": 12.159340858459473, "learning_rate": 2.9971622033320914e-07, "logits/chosen": -2.4223062992095947, "logits/rejected": -2.1396877765655518, "logps/chosen": -213.5079345703125, "logps/ref_chosen": -97.3123779296875, "logps/ref_rejected": -110.13134765625, "logps/rejected": -293.2822265625, "loss": 0.9977, "margin_dpo/margin_mean": 66.955322265625, "margin_dpo/margin_std": 85.39122772216797, "step": 337 }, { "epoch": 0.49632892804698975, "fcm_dpo/beta": 0.00921336468309164, "fcm_dpo/delta": 0.004760187119245529, "fcm_dpo/margin": 64.63055419921875, "fcm_dpo/q_t": 0.36690112948417664, "grad_norm": 10.842123985290527, "learning_rate": 2.984578725527675e-07, "logits/chosen": -1.7986483573913574, "logits/rejected": -1.6366183757781982, "logps/chosen": -202.72119140625, "logps/ref_chosen": -83.63011169433594, "logps/ref_rejected": -93.69235229492188, "logps/rejected": -277.41400146484375, "loss": 0.9782, "margin_dpo/margin_mean": 64.63055419921875, "margin_dpo/margin_std": 72.1498794555664, "step": 338 }, { "epoch": 0.4977973568281938, "fcm_dpo/beta": 0.009260453283786774, "fcm_dpo/delta": -0.030668731778860092, "fcm_dpo/margin": 67.80081939697266, "fcm_dpo/q_t": 0.3631947636604309, "grad_norm": 10.40760612487793, "learning_rate": 2.9719824786231796e-07, "logits/chosen": -3.007730484008789, "logits/rejected": -2.590148448944092, "logps/chosen": -223.9995880126953, "logps/ref_chosen": -99.85182189941406, "logps/ref_rejected": -109.0743637084961, "logps/rejected": -301.02294921875, "loss": 0.9867, "margin_dpo/margin_mean": 67.80081939697266, "margin_dpo/margin_std": 81.80381774902344, "step": 339 }, { "epoch": 0.49926578560939794, "fcm_dpo/beta": 0.009243492037057877, "fcm_dpo/delta": 0.04011049121618271, "fcm_dpo/margin": 60.805809020996094, "fcm_dpo/q_t": 0.37543347477912903, "grad_norm": 12.52755355834961, "learning_rate": 2.959373794541426e-07, "logits/chosen": -1.4719023704528809, "logits/rejected": -0.9998151659965515, "logps/chosen": -218.23899841308594, "logps/ref_chosen": -84.97396850585938, "logps/ref_rejected": -78.80033874511719, "logps/rejected": -272.87115478515625, "loss": 1.0069, "margin_dpo/margin_mean": 60.805809020996094, "margin_dpo/margin_std": 75.30453491210938, "step": 340 }, { "epoch": 0.5007342143906021, "fcm_dpo/beta": 0.009315932169556618, "fcm_dpo/delta": 0.06518840789794922, "fcm_dpo/margin": 57.787986755371094, "fcm_dpo/q_t": 0.383889377117157, "grad_norm": 12.319915771484375, "learning_rate": 2.946753005532965e-07, "logits/chosen": -1.5798420906066895, "logits/rejected": -1.3175228834152222, "logps/chosen": -224.97544860839844, "logps/ref_chosen": -85.26399230957031, "logps/ref_rejected": -106.08595275878906, "logps/rejected": -303.58538818359375, "loss": 1.0502, "margin_dpo/margin_mean": 57.787986755371094, "margin_dpo/margin_std": 81.41255187988281, "step": 341 }, { "epoch": 0.5022026431718062, "fcm_dpo/beta": 0.009050893597304821, "fcm_dpo/delta": -0.1713675707578659, "fcm_dpo/margin": 83.44084167480469, "fcm_dpo/q_t": 0.33266481757164, "grad_norm": 10.313692092895508, "learning_rate": 2.934120444167326e-07, "logits/chosen": -1.3843135833740234, "logits/rejected": -0.633280336856842, "logps/chosen": -212.62014770507812, "logps/ref_chosen": -101.03860473632812, "logps/ref_rejected": -98.89851379394531, "logps/rejected": -293.9208984375, "loss": 0.8903, "margin_dpo/margin_mean": 83.44084167480469, "margin_dpo/margin_std": 78.70523834228516, "step": 342 }, { "epoch": 0.5036710719530103, "fcm_dpo/beta": 0.009080128744244576, "fcm_dpo/delta": -0.036977700889110565, "fcm_dpo/margin": 69.78305053710938, "fcm_dpo/q_t": 0.3612483739852905, "grad_norm": 12.353619575500488, "learning_rate": 2.9214764433242476e-07, "logits/chosen": -2.0943527221679688, "logits/rejected": -1.9702728986740112, "logps/chosen": -203.23928833007812, "logps/ref_chosen": -83.74736022949219, "logps/ref_rejected": -116.57908630371094, "logps/rejected": -305.85406494140625, "loss": 0.9632, "margin_dpo/margin_mean": 69.78305053710938, "margin_dpo/margin_std": 76.31233215332031, "step": 343 }, { "epoch": 0.5051395007342144, "fcm_dpo/beta": 0.008997892960906029, "fcm_dpo/delta": -0.030076302587985992, "fcm_dpo/margin": 69.76876068115234, "fcm_dpo/q_t": 0.3652355372905731, "grad_norm": 10.976330757141113, "learning_rate": 2.9088213361849126e-07, "logits/chosen": -1.680814504623413, "logits/rejected": -1.425823450088501, "logps/chosen": -202.2406768798828, "logps/ref_chosen": -78.25396728515625, "logps/ref_rejected": -95.42870330810547, "logps/rejected": -289.1842041015625, "loss": 0.9842, "margin_dpo/margin_mean": 69.76876068115234, "margin_dpo/margin_std": 85.37368774414062, "step": 344 }, { "epoch": 0.5066079295154186, "fcm_dpo/beta": 0.008805211633443832, "fcm_dpo/delta": -0.08041529357433319, "fcm_dpo/margin": 76.61846160888672, "fcm_dpo/q_t": 0.3534577190876007, "grad_norm": 11.980306625366211, "learning_rate": 2.896155456223163e-07, "logits/chosen": -1.5205774307250977, "logits/rejected": -1.2206867933273315, "logps/chosen": -221.85317993164062, "logps/ref_chosen": -98.0794906616211, "logps/ref_rejected": -104.03477478027344, "logps/rejected": -304.42694091796875, "loss": 0.9355, "margin_dpo/margin_mean": 76.61846923828125, "margin_dpo/margin_std": 82.15892028808594, "step": 345 }, { "epoch": 0.5080763582966226, "fcm_dpo/beta": 0.008871208876371384, "fcm_dpo/delta": 0.0582590326666832, "fcm_dpo/margin": 61.423545837402344, "fcm_dpo/q_t": 0.3825676441192627, "grad_norm": 10.243391036987305, "learning_rate": 2.883479137196714e-07, "logits/chosen": -1.9583630561828613, "logits/rejected": -1.651193618774414, "logps/chosen": -219.33963012695312, "logps/ref_chosen": -80.68348693847656, "logps/ref_rejected": -85.89260864257812, "logps/rejected": -285.9722900390625, "loss": 1.0424, "margin_dpo/margin_mean": 61.423545837402344, "margin_dpo/margin_std": 86.02749633789062, "step": 346 }, { "epoch": 0.5095447870778267, "fcm_dpo/beta": 0.009024476632475853, "fcm_dpo/delta": 0.0737469345331192, "fcm_dpo/margin": 58.69518280029297, "fcm_dpo/q_t": 0.385664701461792, "grad_norm": 11.432808876037598, "learning_rate": 2.8707927131383614e-07, "logits/chosen": -1.6128592491149902, "logits/rejected": -1.2691833972930908, "logps/chosen": -223.65142822265625, "logps/ref_chosen": -93.67938232421875, "logps/ref_rejected": -97.54251098632812, "logps/rejected": -286.209716796875, "loss": 1.047, "margin_dpo/margin_mean": 58.69518280029297, "margin_dpo/margin_std": 81.67510986328125, "step": 347 }, { "epoch": 0.5110132158590308, "fcm_dpo/beta": 0.008929546922445297, "fcm_dpo/delta": -0.0036687031388282776, "fcm_dpo/margin": 67.33739471435547, "fcm_dpo/q_t": 0.3671216368675232, "grad_norm": 9.5287446975708, "learning_rate": 2.858096518347179e-07, "logits/chosen": -1.8974089622497559, "logits/rejected": -1.6134772300720215, "logps/chosen": -202.92189025878906, "logps/ref_chosen": -90.25224304199219, "logps/ref_rejected": -95.77027893066406, "logps/rejected": -275.7773132324219, "loss": 0.9868, "margin_dpo/margin_mean": 67.33739471435547, "margin_dpo/margin_std": 76.04698181152344, "step": 348 }, { "epoch": 0.5124816446402349, "fcm_dpo/beta": 0.009099038317799568, "fcm_dpo/delta": 0.019386742264032364, "fcm_dpo/margin": 63.853363037109375, "fcm_dpo/q_t": 0.3738081157207489, "grad_norm": 11.725251197814941, "learning_rate": 2.845390887379706e-07, "logits/chosen": -1.996931552886963, "logits/rejected": -1.7708909511566162, "logps/chosen": -214.01986694335938, "logps/ref_chosen": -92.05809020996094, "logps/ref_rejected": -103.96527099609375, "logps/rejected": -289.7804260253906, "loss": 1.0031, "margin_dpo/margin_mean": 63.853363037109375, "margin_dpo/margin_std": 78.11573791503906, "step": 349 }, { "epoch": 0.5139500734214391, "fcm_dpo/beta": 0.009226538240909576, "fcm_dpo/delta": 0.1344294250011444, "fcm_dpo/margin": 51.163047790527344, "fcm_dpo/q_t": 0.39684417843818665, "grad_norm": 12.850371360778809, "learning_rate": 2.8326761550411346e-07, "logits/chosen": -1.785254955291748, "logits/rejected": -1.556738018989563, "logps/chosen": -219.18960571289062, "logps/ref_chosen": -87.05419921875, "logps/ref_rejected": -95.72193145751953, "logps/rejected": -279.0203857421875, "loss": 1.0736, "margin_dpo/margin_mean": 51.16304397583008, "margin_dpo/margin_std": 73.25152587890625, "step": 350 }, { "epoch": 0.5154185022026432, "fcm_dpo/beta": 0.009196523576974869, "fcm_dpo/delta": -0.1071363314986229, "fcm_dpo/margin": 76.08428192138672, "fcm_dpo/q_t": 0.34812554717063904, "grad_norm": 11.473617553710938, "learning_rate": 2.819952656376487e-07, "logits/chosen": -1.5400876998901367, "logits/rejected": -1.2160134315490723, "logps/chosen": -218.47886657714844, "logps/ref_chosen": -89.16903686523438, "logps/ref_rejected": -108.29801940917969, "logps/rejected": -313.692138671875, "loss": 0.9389, "margin_dpo/margin_mean": 76.08428192138672, "margin_dpo/margin_std": 84.34982299804688, "step": 351 }, { "epoch": 0.5168869309838473, "fcm_dpo/beta": 0.009029903449118137, "fcm_dpo/delta": -0.0580860860645771, "fcm_dpo/margin": 72.45895385742188, "fcm_dpo/q_t": 0.3607955574989319, "grad_norm": 11.246063232421875, "learning_rate": 2.8072207266617854e-07, "logits/chosen": -1.9249012470245361, "logits/rejected": -1.3872631788253784, "logps/chosen": -215.51181030273438, "logps/ref_chosen": -96.81147766113281, "logps/ref_rejected": -84.40483093261719, "logps/rejected": -275.5640869140625, "loss": 0.9591, "margin_dpo/margin_mean": 72.45895385742188, "margin_dpo/margin_std": 83.17478942871094, "step": 352 }, { "epoch": 0.5183553597650514, "fcm_dpo/beta": 0.009232399985194206, "fcm_dpo/delta": 0.1866103559732437, "fcm_dpo/margin": 45.66265869140625, "fcm_dpo/q_t": 0.40671753883361816, "grad_norm": 12.68930721282959, "learning_rate": 2.794480701395219e-07, "logits/chosen": -1.9192724227905273, "logits/rejected": -1.527640700340271, "logps/chosen": -221.19326782226562, "logps/ref_chosen": -85.36248779296875, "logps/ref_rejected": -87.36732482910156, "logps/rejected": -268.86077880859375, "loss": 1.1274, "margin_dpo/margin_mean": 45.66265869140625, "margin_dpo/margin_std": 76.734375, "step": 353 }, { "epoch": 0.5198237885462555, "fcm_dpo/beta": 0.009293053299188614, "fcm_dpo/delta": -0.0588521845638752, "fcm_dpo/margin": 70.47891235351562, "fcm_dpo/q_t": 0.35432109236717224, "grad_norm": 10.813716888427734, "learning_rate": 2.781732916288303e-07, "logits/chosen": -1.8877973556518555, "logits/rejected": -1.4194233417510986, "logps/chosen": -215.95494079589844, "logps/ref_chosen": -95.8011703491211, "logps/ref_rejected": -94.86614227294922, "logps/rejected": -285.49884033203125, "loss": 0.9172, "margin_dpo/margin_mean": 70.47891235351562, "margin_dpo/margin_std": 65.81661987304688, "step": 354 }, { "epoch": 0.5212922173274597, "fcm_dpo/beta": 0.009149078279733658, "fcm_dpo/delta": -0.0197719968855381, "fcm_dpo/margin": 67.51188659667969, "fcm_dpo/q_t": 0.3642021119594574, "grad_norm": 11.531266212463379, "learning_rate": 2.7689777072570284e-07, "logits/chosen": -2.4402689933776855, "logits/rejected": -1.9639763832092285, "logps/chosen": -205.29983520507812, "logps/ref_chosen": -86.185546875, "logps/ref_rejected": -88.4615707397461, "logps/rejected": -275.0877685546875, "loss": 0.9754, "margin_dpo/margin_mean": 67.51187896728516, "margin_dpo/margin_std": 75.14073181152344, "step": 355 }, { "epoch": 0.5227606461086637, "fcm_dpo/beta": 0.009418745525181293, "fcm_dpo/delta": 0.13494420051574707, "fcm_dpo/margin": 49.970237731933594, "fcm_dpo/q_t": 0.3972604274749756, "grad_norm": 13.176185607910156, "learning_rate": 2.7562154104130176e-07, "logits/chosen": -1.8549089431762695, "logits/rejected": -1.394954800605774, "logps/chosen": -231.3189697265625, "logps/ref_chosen": -90.818115234375, "logps/ref_rejected": -82.1273193359375, "logps/rejected": -272.5984191894531, "loss": 1.1045, "margin_dpo/margin_mean": 49.970237731933594, "margin_dpo/margin_std": 81.81539916992188, "step": 356 }, { "epoch": 0.5242290748898678, "fcm_dpo/beta": 0.009089802391827106, "fcm_dpo/delta": -0.18602949380874634, "fcm_dpo/margin": 84.42666625976562, "fcm_dpo/q_t": 0.3301994502544403, "grad_norm": 9.98563003540039, "learning_rate": 2.7434463620546594e-07, "logits/chosen": -2.099191188812256, "logits/rejected": -1.7402398586273193, "logps/chosen": -202.91748046875, "logps/ref_chosen": -87.5047607421875, "logps/ref_rejected": -92.24937438964844, "logps/rejected": -292.0887756347656, "loss": 0.861, "margin_dpo/margin_mean": 84.42666625976562, "margin_dpo/margin_std": 71.71357727050781, "step": 357 }, { "epoch": 0.5256975036710719, "fcm_dpo/beta": 0.009189213626086712, "fcm_dpo/delta": 0.059452179819345474, "fcm_dpo/margin": 59.17223358154297, "fcm_dpo/q_t": 0.3846597373485565, "grad_norm": 11.328146934509277, "learning_rate": 2.730670898658255e-07, "logits/chosen": -1.47278892993927, "logits/rejected": -0.9826061725616455, "logps/chosen": -225.228271484375, "logps/ref_chosen": -95.05546569824219, "logps/ref_rejected": -96.47611999511719, "logps/rejected": -285.8211669921875, "loss": 1.0297, "margin_dpo/margin_mean": 59.17223358154297, "margin_dpo/margin_std": 79.3198471069336, "step": 358 }, { "epoch": 0.527165932452276, "fcm_dpo/beta": 0.009268088266253471, "fcm_dpo/delta": 0.09260153770446777, "fcm_dpo/margin": 55.203125, "fcm_dpo/q_t": 0.39258408546447754, "grad_norm": 11.787094116210938, "learning_rate": 2.717889356869146e-07, "logits/chosen": -1.5292563438415527, "logits/rejected": -0.9679604768753052, "logps/chosen": -230.56655883789062, "logps/ref_chosen": -87.37332916259766, "logps/ref_rejected": -88.77188110351562, "logps/rejected": -287.1682434082031, "loss": 1.0731, "margin_dpo/margin_mean": 55.203128814697266, "margin_dpo/margin_std": 82.28927612304688, "step": 359 }, { "epoch": 0.5286343612334802, "fcm_dpo/beta": 0.009283397346735, "fcm_dpo/delta": -0.06985802948474884, "fcm_dpo/margin": 71.65193176269531, "fcm_dpo/q_t": 0.35196417570114136, "grad_norm": 14.178664207458496, "learning_rate": 2.7051020734928443e-07, "logits/chosen": -1.6124732494354248, "logits/rejected": -1.2661912441253662, "logps/chosen": -209.75942993164062, "logps/ref_chosen": -75.22344970703125, "logps/ref_rejected": -76.09400177001953, "logps/rejected": -282.28192138671875, "loss": 0.9315, "margin_dpo/margin_mean": 71.65193176269531, "margin_dpo/margin_std": 73.52704620361328, "step": 360 }, { "epoch": 0.5301027900146843, "fcm_dpo/beta": 0.009296550415456295, "fcm_dpo/delta": 0.019596338272094727, "fcm_dpo/margin": 62.54603958129883, "fcm_dpo/q_t": 0.37713533639907837, "grad_norm": 12.717004776000977, "learning_rate": 2.6923093854861593e-07, "logits/chosen": -1.715510606765747, "logits/rejected": -1.4632185697555542, "logps/chosen": -218.74740600585938, "logps/ref_chosen": -81.95333099365234, "logps/ref_rejected": -96.67919921875, "logps/rejected": -296.0193176269531, "loss": 1.0228, "margin_dpo/margin_mean": 62.54604721069336, "margin_dpo/margin_std": 84.71768188476562, "step": 361 }, { "epoch": 0.5315712187958884, "fcm_dpo/beta": 0.009247587993741035, "fcm_dpo/delta": -0.015775587409734726, "fcm_dpo/margin": 66.46771240234375, "fcm_dpo/q_t": 0.3636864423751831, "grad_norm": 11.942319869995117, "learning_rate": 2.679511629948319e-07, "logits/chosen": -1.6847763061523438, "logits/rejected": -1.467708706855774, "logps/chosen": -230.143310546875, "logps/ref_chosen": -89.33971405029297, "logps/ref_rejected": -110.50196838378906, "logps/rejected": -317.7733154296875, "loss": 0.9741, "margin_dpo/margin_mean": 66.46771240234375, "margin_dpo/margin_std": 74.62181854248047, "step": 362 }, { "epoch": 0.5330396475770925, "fcm_dpo/beta": 0.009058544412255287, "fcm_dpo/delta": -0.18251146376132965, "fcm_dpo/margin": 84.8752670288086, "fcm_dpo/q_t": 0.33618301153182983, "grad_norm": 12.65180778503418, "learning_rate": 2.6667091441120816e-07, "logits/chosen": -1.7909729480743408, "logits/rejected": -1.3521623611450195, "logps/chosen": -214.4105987548828, "logps/ref_chosen": -80.77499389648438, "logps/ref_rejected": -79.43267822265625, "logps/rejected": -297.94354248046875, "loss": 0.8884, "margin_dpo/margin_mean": 84.87525939941406, "margin_dpo/margin_std": 86.01889038085938, "step": 363 }, { "epoch": 0.5345080763582967, "fcm_dpo/beta": 0.008983662351965904, "fcm_dpo/delta": -0.018669523298740387, "fcm_dpo/margin": 68.58975219726562, "fcm_dpo/q_t": 0.36190158128738403, "grad_norm": 13.727071762084961, "learning_rate": 2.6539022653348575e-07, "logits/chosen": -2.0300331115722656, "logits/rejected": -1.877135992050171, "logps/chosen": -209.77499389648438, "logps/ref_chosen": -76.52960205078125, "logps/ref_rejected": -97.56173706054688, "logps/rejected": -299.39691162109375, "loss": 0.9511, "margin_dpo/margin_mean": 68.58975219726562, "margin_dpo/margin_std": 66.92192840576172, "step": 364 }, { "epoch": 0.5359765051395007, "fcm_dpo/beta": 0.008843779563903809, "fcm_dpo/delta": -0.06726966798305511, "fcm_dpo/margin": 74.93986511230469, "fcm_dpo/q_t": 0.35855019092559814, "grad_norm": 10.768917083740234, "learning_rate": 2.641091331089811e-07, "logits/chosen": -1.5261728763580322, "logits/rejected": -1.3805526494979858, "logps/chosen": -210.90951538085938, "logps/ref_chosen": -80.54489135742188, "logps/ref_rejected": -99.32316589355469, "logps/rejected": -304.62762451171875, "loss": 0.9621, "margin_dpo/margin_mean": 74.93986511230469, "margin_dpo/margin_std": 88.24283599853516, "step": 365 }, { "epoch": 0.5374449339207048, "fcm_dpo/beta": 0.008807620033621788, "fcm_dpo/delta": 0.024917863309383392, "fcm_dpo/margin": 65.45736694335938, "fcm_dpo/q_t": 0.37715762853622437, "grad_norm": 10.28955364227295, "learning_rate": 2.6282766789569736e-07, "logits/chosen": -1.706976294517517, "logits/rejected": -1.7277071475982666, "logps/chosen": -201.939697265625, "logps/ref_chosen": -72.23806762695312, "logps/ref_rejected": -91.67182922363281, "logps/rejected": -286.830810546875, "loss": 0.9911, "margin_dpo/margin_mean": 65.45736694335938, "margin_dpo/margin_std": 78.65293884277344, "step": 366 }, { "epoch": 0.5389133627019089, "fcm_dpo/beta": 0.008751322515308857, "fcm_dpo/delta": -0.05687132105231285, "fcm_dpo/margin": 74.63752746582031, "fcm_dpo/q_t": 0.3563315272331238, "grad_norm": 10.187382698059082, "learning_rate": 2.615458646614349e-07, "logits/chosen": -1.7047667503356934, "logits/rejected": -1.3686751127243042, "logps/chosen": -207.76351928710938, "logps/ref_chosen": -85.42201232910156, "logps/ref_rejected": -83.51779174804688, "logps/rejected": -280.496826171875, "loss": 0.9312, "margin_dpo/margin_mean": 74.63752746582031, "margin_dpo/margin_std": 76.12283325195312, "step": 367 }, { "epoch": 0.540381791483113, "fcm_dpo/beta": 0.008638489991426468, "fcm_dpo/delta": -0.10402532666921616, "fcm_dpo/margin": 80.64979553222656, "fcm_dpo/q_t": 0.34605395793914795, "grad_norm": 10.933233261108398, "learning_rate": 2.6026375718290083e-07, "logits/chosen": -1.6838431358337402, "logits/rejected": -1.6408352851867676, "logps/chosen": -189.81283569335938, "logps/ref_chosen": -73.84112548828125, "logps/ref_rejected": -104.69528198242188, "logps/rejected": -301.3167724609375, "loss": 0.9216, "margin_dpo/margin_mean": 80.64979553222656, "margin_dpo/margin_std": 81.0087661743164, "step": 368 }, { "epoch": 0.5418502202643172, "fcm_dpo/beta": 0.008491117507219315, "fcm_dpo/delta": -0.05438908189535141, "fcm_dpo/margin": 76.65316772460938, "fcm_dpo/q_t": 0.35426294803619385, "grad_norm": 11.064504623413086, "learning_rate": 2.589813792448196e-07, "logits/chosen": -1.6709437370300293, "logits/rejected": -1.30281400680542, "logps/chosen": -224.9449920654297, "logps/ref_chosen": -98.46195983886719, "logps/ref_rejected": -99.15672302246094, "logps/rejected": -302.29290771484375, "loss": 0.915, "margin_dpo/margin_mean": 76.65316772460938, "margin_dpo/margin_std": 69.63180541992188, "step": 369 }, { "epoch": 0.5433186490455213, "fcm_dpo/beta": 0.008652754127979279, "fcm_dpo/delta": 0.14852631092071533, "fcm_dpo/margin": 52.930091857910156, "fcm_dpo/q_t": 0.3984871506690979, "grad_norm": 11.095954895019531, "learning_rate": 2.5769876463904263e-07, "logits/chosen": -1.8884553909301758, "logits/rejected": -1.686102032661438, "logps/chosen": -236.5087432861328, "logps/ref_chosen": -96.77574157714844, "logps/ref_rejected": -102.74411010742188, "logps/rejected": -295.4071960449219, "loss": 1.0924, "margin_dpo/margin_mean": 52.930091857910156, "margin_dpo/margin_std": 80.57627868652344, "step": 370 }, { "epoch": 0.5447870778267254, "fcm_dpo/beta": 0.008863227441906929, "fcm_dpo/delta": 0.09689568728208542, "fcm_dpo/margin": 57.24983215332031, "fcm_dpo/q_t": 0.38820239901542664, "grad_norm": 12.403904914855957, "learning_rate": 2.5641594716365744e-07, "logits/chosen": -1.7923095226287842, "logits/rejected": -1.426107406616211, "logps/chosen": -235.0094451904297, "logps/ref_chosen": -102.99020385742188, "logps/ref_rejected": -105.35062408447266, "logps/rejected": -294.61968994140625, "loss": 1.0522, "margin_dpo/margin_mean": 57.24983215332031, "margin_dpo/margin_std": 77.89881134033203, "step": 371 }, { "epoch": 0.5462555066079295, "fcm_dpo/beta": 0.00865775067359209, "fcm_dpo/delta": -0.1739071160554886, "fcm_dpo/margin": 87.87632751464844, "fcm_dpo/q_t": 0.3345174789428711, "grad_norm": 9.950636863708496, "learning_rate": 2.551329606220976e-07, "logits/chosen": -1.8596062660217285, "logits/rejected": -1.2332671880722046, "logps/chosen": -204.2379608154297, "logps/ref_chosen": -91.02623748779297, "logps/ref_rejected": -85.6102294921875, "logps/rejected": -286.69830322265625, "loss": 0.8762, "margin_dpo/margin_mean": 87.87632751464844, "margin_dpo/margin_std": 83.01412963867188, "step": 372 }, { "epoch": 0.5477239353891337, "fcm_dpo/beta": 0.008587082847952843, "fcm_dpo/delta": 0.02046929858624935, "fcm_dpo/margin": 67.62651824951172, "fcm_dpo/q_t": 0.3730928897857666, "grad_norm": 13.208666801452637, "learning_rate": 2.538498388222517e-07, "logits/chosen": -2.059213161468506, "logits/rejected": -1.5383615493774414, "logps/chosen": -229.08303833007812, "logps/ref_chosen": -100.07307434082031, "logps/ref_rejected": -91.11634826660156, "logps/rejected": -287.7528076171875, "loss": 1.0004, "margin_dpo/margin_mean": 67.62651062011719, "margin_dpo/margin_std": 82.54972839355469, "step": 373 }, { "epoch": 0.5491923641703378, "fcm_dpo/beta": 0.008557072840631008, "fcm_dpo/delta": -0.01339799165725708, "fcm_dpo/margin": 71.54954528808594, "fcm_dpo/q_t": 0.36381733417510986, "grad_norm": 10.388188362121582, "learning_rate": 2.525666155755725e-07, "logits/chosen": -2.204723834991455, "logits/rejected": -1.814666986465454, "logps/chosen": -211.86734008789062, "logps/ref_chosen": -98.01234436035156, "logps/ref_rejected": -100.75224304199219, "logps/rejected": -286.1567687988281, "loss": 0.9477, "margin_dpo/margin_mean": 71.54954528808594, "margin_dpo/margin_std": 71.1581802368164, "step": 374 }, { "epoch": 0.5506607929515418, "fcm_dpo/beta": 0.00840807519853115, "fcm_dpo/delta": -0.07696233689785004, "fcm_dpo/margin": 79.625244140625, "fcm_dpo/q_t": 0.3517950177192688, "grad_norm": 10.864441871643066, "learning_rate": 2.512833246961859e-07, "logits/chosen": -1.7122812271118164, "logits/rejected": -1.4867812395095825, "logps/chosen": -202.69105529785156, "logps/ref_chosen": -84.39627838134766, "logps/ref_rejected": -95.7188491821289, "logps/rejected": -293.63885498046875, "loss": 0.9258, "margin_dpo/margin_mean": 79.62525177001953, "margin_dpo/margin_std": 77.73374938964844, "step": 375 }, { "epoch": 0.5521292217327459, "fcm_dpo/beta": 0.00827401876449585, "fcm_dpo/delta": -0.13464483618736267, "fcm_dpo/margin": 87.5845718383789, "fcm_dpo/q_t": 0.3414178490638733, "grad_norm": 11.585356712341309, "learning_rate": 2.5e-07, "logits/chosen": -1.7216506004333496, "logits/rejected": -1.4160125255584717, "logps/chosen": -206.31048583984375, "logps/ref_chosen": -95.73770141601562, "logps/ref_rejected": -111.97331237792969, "logps/rejected": -310.13067626953125, "loss": 0.8974, "margin_dpo/margin_mean": 87.5845718383789, "margin_dpo/margin_std": 86.00065612792969, "step": 376 }, { "epoch": 0.55359765051395, "fcm_dpo/beta": 0.008009790442883968, "fcm_dpo/delta": -0.14207610487937927, "fcm_dpo/margin": 91.17427062988281, "fcm_dpo/q_t": 0.339458703994751, "grad_norm": 10.219127655029297, "learning_rate": 2.487166753038141e-07, "logits/chosen": -1.5154132843017578, "logits/rejected": -1.391338586807251, "logps/chosen": -199.823486328125, "logps/ref_chosen": -78.92694854736328, "logps/ref_rejected": -105.89892578125, "logps/rejected": -317.9697265625, "loss": 0.8889, "margin_dpo/margin_mean": 91.17427062988281, "margin_dpo/margin_std": 85.8505859375, "step": 377 }, { "epoch": 0.5550660792951542, "fcm_dpo/beta": 0.008050184696912766, "fcm_dpo/delta": 0.0010249018669128418, "fcm_dpo/margin": 74.31245422363281, "fcm_dpo/q_t": 0.36496371030807495, "grad_norm": 10.996868133544922, "learning_rate": 2.4743338442442754e-07, "logits/chosen": -2.0298149585723877, "logits/rejected": -1.7492952346801758, "logps/chosen": -192.58639526367188, "logps/ref_chosen": -75.37240600585938, "logps/ref_rejected": -93.70409393310547, "logps/rejected": -285.2305603027344, "loss": 0.9746, "margin_dpo/margin_mean": 74.31245422363281, "margin_dpo/margin_std": 80.56228637695312, "step": 378 }, { "epoch": 0.5565345080763583, "fcm_dpo/beta": 0.008063051849603653, "fcm_dpo/delta": 0.05794180929660797, "fcm_dpo/margin": 67.62254333496094, "fcm_dpo/q_t": 0.38331836462020874, "grad_norm": 10.951027870178223, "learning_rate": 2.461501611777483e-07, "logits/chosen": -1.8049771785736084, "logits/rejected": -1.708174467086792, "logps/chosen": -221.08468627929688, "logps/ref_chosen": -88.85568237304688, "logps/ref_rejected": -121.38114166259766, "logps/rejected": -321.2326965332031, "loss": 1.0446, "margin_dpo/margin_mean": 67.62254333496094, "margin_dpo/margin_std": 94.80967712402344, "step": 379 }, { "epoch": 0.5580029368575624, "fcm_dpo/beta": 0.008038859814405441, "fcm_dpo/delta": -0.04042629525065422, "fcm_dpo/margin": 79.3418197631836, "fcm_dpo/q_t": 0.364249050617218, "grad_norm": 13.981470108032227, "learning_rate": 2.4486703937790243e-07, "logits/chosen": -1.3323712348937988, "logits/rejected": -1.299829125404358, "logps/chosen": -227.69309997558594, "logps/ref_chosen": -79.83321380615234, "logps/ref_rejected": -110.7447509765625, "logps/rejected": -337.94647216796875, "loss": 0.9705, "margin_dpo/margin_mean": 79.3418197631836, "margin_dpo/margin_std": 95.55670928955078, "step": 380 }, { "epoch": 0.5594713656387665, "fcm_dpo/beta": 0.00806824117898941, "fcm_dpo/delta": 0.03291797637939453, "fcm_dpo/margin": 70.51676940917969, "fcm_dpo/q_t": 0.37344565987586975, "grad_norm": 12.015775680541992, "learning_rate": 2.435840528363426e-07, "logits/chosen": -1.5715270042419434, "logits/rejected": -1.0036929845809937, "logps/chosen": -214.4466552734375, "logps/ref_chosen": -86.56683349609375, "logps/ref_rejected": -84.416748046875, "logps/rejected": -282.8133544921875, "loss": 0.9954, "margin_dpo/margin_mean": 70.51676940917969, "margin_dpo/margin_std": 82.17634582519531, "step": 381 }, { "epoch": 0.5609397944199707, "fcm_dpo/beta": 0.007982393726706505, "fcm_dpo/delta": -0.02618768811225891, "fcm_dpo/margin": 78.04951477050781, "fcm_dpo/q_t": 0.36266398429870605, "grad_norm": 10.413619041442871, "learning_rate": 2.4230123536095745e-07, "logits/chosen": -1.9479053020477295, "logits/rejected": -1.6788208484649658, "logps/chosen": -233.09474182128906, "logps/ref_chosen": -104.46180725097656, "logps/ref_rejected": -114.60223388671875, "logps/rejected": -321.28466796875, "loss": 0.9563, "margin_dpo/margin_mean": 78.04950714111328, "margin_dpo/margin_std": 81.13053894042969, "step": 382 }, { "epoch": 0.5624082232011748, "fcm_dpo/beta": 0.008084449917078018, "fcm_dpo/delta": 0.05731251835823059, "fcm_dpo/margin": 67.51455688476562, "fcm_dpo/q_t": 0.3798357844352722, "grad_norm": 11.326423645019531, "learning_rate": 2.4101862075518037e-07, "logits/chosen": -1.5943937301635742, "logits/rejected": -1.4752601385116577, "logps/chosen": -219.62017822265625, "logps/ref_chosen": -82.11607360839844, "logps/ref_rejected": -98.77716064453125, "logps/rejected": -303.79583740234375, "loss": 1.0111, "margin_dpo/margin_mean": 67.51455688476562, "margin_dpo/margin_std": 80.03602600097656, "step": 383 }, { "epoch": 0.5638766519823789, "fcm_dpo/beta": 0.008265078067779541, "fcm_dpo/delta": 0.04529654234647751, "fcm_dpo/margin": 67.14823913574219, "fcm_dpo/q_t": 0.37842482328414917, "grad_norm": 13.126636505126953, "learning_rate": 2.397362428170992e-07, "logits/chosen": -1.7413570880889893, "logits/rejected": -1.5120550394058228, "logps/chosen": -228.9114990234375, "logps/ref_chosen": -80.58525085449219, "logps/ref_rejected": -91.44789123535156, "logps/rejected": -306.92236328125, "loss": 1.0505, "margin_dpo/margin_mean": 67.14823913574219, "margin_dpo/margin_std": 92.4796371459961, "step": 384 }, { "epoch": 0.5653450807635829, "fcm_dpo/beta": 0.008029671385884285, "fcm_dpo/delta": -0.09009828418493271, "fcm_dpo/margin": 84.93191528320312, "fcm_dpo/q_t": 0.35304510593414307, "grad_norm": 12.234119415283203, "learning_rate": 2.3845413533856514e-07, "logits/chosen": -2.2260751724243164, "logits/rejected": -1.6197378635406494, "logps/chosen": -217.18515014648438, "logps/ref_chosen": -92.76016235351562, "logps/ref_rejected": -84.45997619628906, "logps/rejected": -293.81689453125, "loss": 0.9373, "margin_dpo/margin_mean": 84.93191528320312, "margin_dpo/margin_std": 90.8809814453125, "step": 385 }, { "epoch": 0.566813509544787, "fcm_dpo/beta": 0.007991664111614227, "fcm_dpo/delta": -0.05802081525325775, "fcm_dpo/margin": 81.86547088623047, "fcm_dpo/q_t": 0.35287386178970337, "grad_norm": 9.855354309082031, "learning_rate": 2.3717233210430254e-07, "logits/chosen": -1.597925066947937, "logits/rejected": -1.3119235038757324, "logps/chosen": -215.71453857421875, "logps/ref_chosen": -85.35896301269531, "logps/ref_rejected": -96.99065399169922, "logps/rejected": -309.211669921875, "loss": 0.9444, "margin_dpo/margin_mean": 81.86546325683594, "margin_dpo/margin_std": 86.34203338623047, "step": 386 }, { "epoch": 0.5682819383259912, "fcm_dpo/beta": 0.007988503202795982, "fcm_dpo/delta": 0.00018469803035259247, "fcm_dpo/margin": 75.06538391113281, "fcm_dpo/q_t": 0.36637941002845764, "grad_norm": 12.132233619689941, "learning_rate": 2.3589086689101889e-07, "logits/chosen": -2.5154049396514893, "logits/rejected": -1.8449430465698242, "logps/chosen": -253.63768005371094, "logps/ref_chosen": -102.68814086914062, "logps/ref_rejected": -99.42858123779297, "logps/rejected": -325.4435119628906, "loss": 0.9641, "margin_dpo/margin_mean": 75.06538391113281, "margin_dpo/margin_std": 79.062255859375, "step": 387 }, { "epoch": 0.5697503671071953, "fcm_dpo/beta": 0.00789298489689827, "fcm_dpo/delta": -0.05572465807199478, "fcm_dpo/margin": 82.60919952392578, "fcm_dpo/q_t": 0.35591650009155273, "grad_norm": 10.612791061401367, "learning_rate": 2.3460977346651428e-07, "logits/chosen": -1.7140839099884033, "logits/rejected": -1.5836071968078613, "logps/chosen": -234.08363342285156, "logps/ref_chosen": -83.58523559570312, "logps/ref_rejected": -110.10037231445312, "logps/rejected": -343.2079772949219, "loss": 0.9393, "margin_dpo/margin_mean": 82.60919189453125, "margin_dpo/margin_std": 85.55877685546875, "step": 388 }, { "epoch": 0.5712187958883994, "fcm_dpo/beta": 0.007903015241026878, "fcm_dpo/delta": 0.023557592183351517, "fcm_dpo/margin": 73.11224365234375, "fcm_dpo/q_t": 0.3752698302268982, "grad_norm": 12.45739459991455, "learning_rate": 2.3332908558879177e-07, "logits/chosen": -2.490940570831299, "logits/rejected": -2.0545895099639893, "logps/chosen": -244.65711975097656, "logps/ref_chosen": -89.822265625, "logps/ref_rejected": -84.85376739501953, "logps/rejected": -312.80084228515625, "loss": 1.0175, "margin_dpo/margin_mean": 73.11224365234375, "margin_dpo/margin_std": 96.38984680175781, "step": 389 }, { "epoch": 0.5726872246696035, "fcm_dpo/beta": 0.007932371459901333, "fcm_dpo/delta": 0.060371283441782, "fcm_dpo/margin": 68.37907409667969, "fcm_dpo/q_t": 0.3861868381500244, "grad_norm": 10.967287063598633, "learning_rate": 2.320488370051681e-07, "logits/chosen": -1.6026420593261719, "logits/rejected": -1.389652132987976, "logps/chosen": -234.77609252929688, "logps/ref_chosen": -77.53244018554688, "logps/ref_rejected": -92.58392333984375, "logps/rejected": -318.2066650390625, "loss": 1.0473, "margin_dpo/margin_mean": 68.37908172607422, "margin_dpo/margin_std": 97.29962158203125, "step": 390 }, { "epoch": 0.5741556534508077, "fcm_dpo/beta": 0.007975282147526741, "fcm_dpo/delta": -0.00302848219871521, "fcm_dpo/margin": 75.55020141601562, "fcm_dpo/q_t": 0.3686513304710388, "grad_norm": 11.11927604675293, "learning_rate": 2.3076906145138405e-07, "logits/chosen": -1.8575489521026611, "logits/rejected": -1.4008793830871582, "logps/chosen": -229.98764038085938, "logps/ref_chosen": -88.2329330444336, "logps/ref_rejected": -93.00756072998047, "logps/rejected": -310.3124694824219, "loss": 0.9856, "margin_dpo/margin_mean": 75.55020141601562, "margin_dpo/margin_std": 88.94224548339844, "step": 391 }, { "epoch": 0.5756240822320118, "fcm_dpo/beta": 0.007913609966635704, "fcm_dpo/delta": -0.104721799492836, "fcm_dpo/margin": 88.1395263671875, "fcm_dpo/q_t": 0.3488277792930603, "grad_norm": 13.28966999053955, "learning_rate": 2.294897926507156e-07, "logits/chosen": -1.2604981660842896, "logits/rejected": -1.1343798637390137, "logps/chosen": -231.86595153808594, "logps/ref_chosen": -87.74442291259766, "logps/ref_rejected": -108.57071685791016, "logps/rejected": -340.831787109375, "loss": 0.9373, "margin_dpo/margin_mean": 88.1395263671875, "margin_dpo/margin_std": 97.56378173828125, "step": 392 }, { "epoch": 0.5770925110132159, "fcm_dpo/beta": 0.0078022656962275505, "fcm_dpo/delta": -0.027617137879133224, "fcm_dpo/margin": 80.21553802490234, "fcm_dpo/q_t": 0.3618813455104828, "grad_norm": 10.270767211914062, "learning_rate": 2.2821106431308543e-07, "logits/chosen": -1.5781824588775635, "logits/rejected": -1.2512006759643555, "logps/chosen": -205.15457153320312, "logps/ref_chosen": -76.56381225585938, "logps/ref_rejected": -79.72618103027344, "logps/rejected": -288.5325012207031, "loss": 0.9551, "margin_dpo/margin_mean": 80.21553802490234, "margin_dpo/margin_std": 85.68644714355469, "step": 393 }, { "epoch": 0.57856093979442, "fcm_dpo/beta": 0.007711788639426231, "fcm_dpo/delta": -0.06800325214862823, "fcm_dpo/margin": 86.02757263183594, "fcm_dpo/q_t": 0.3508681356906891, "grad_norm": 10.564422607421875, "learning_rate": 2.2693291013417452e-07, "logits/chosen": -1.5955944061279297, "logits/rejected": -1.3630306720733643, "logps/chosen": -219.39657592773438, "logps/ref_chosen": -84.69866180419922, "logps/ref_rejected": -97.457763671875, "logps/rejected": -318.1832275390625, "loss": 0.9127, "margin_dpo/margin_mean": 86.02757263183594, "margin_dpo/margin_std": 78.89590454101562, "step": 394 }, { "epoch": 0.580029368575624, "fcm_dpo/beta": 0.00774542149156332, "fcm_dpo/delta": 0.08366027474403381, "fcm_dpo/margin": 67.20747375488281, "fcm_dpo/q_t": 0.3928409814834595, "grad_norm": 11.868879318237305, "learning_rate": 2.2565536379453404e-07, "logits/chosen": -2.6224658489227295, "logits/rejected": -2.231447219848633, "logps/chosen": -229.33065795898438, "logps/ref_chosen": -88.88763427734375, "logps/ref_rejected": -91.0455322265625, "logps/rejected": -298.696044921875, "loss": 1.1162, "margin_dpo/margin_mean": 67.20747375488281, "margin_dpo/margin_std": 117.15966796875, "step": 395 }, { "epoch": 0.5814977973568282, "fcm_dpo/beta": 0.007993882521986961, "fcm_dpo/delta": 0.14788684248924255, "fcm_dpo/margin": 57.40891647338867, "fcm_dpo/q_t": 0.3995135426521301, "grad_norm": 15.381148338317871, "learning_rate": 2.2437845895869825e-07, "logits/chosen": -1.979184627532959, "logits/rejected": -1.4956262111663818, "logps/chosen": -252.1164093017578, "logps/ref_chosen": -102.47132873535156, "logps/ref_rejected": -93.23478698730469, "logps/rejected": -300.28875732421875, "loss": 1.0905, "margin_dpo/margin_mean": 57.40891647338867, "margin_dpo/margin_std": 87.18148803710938, "step": 396 }, { "epoch": 0.5829662261380323, "fcm_dpo/beta": 0.0077405949123203754, "fcm_dpo/delta": -0.2481895387172699, "fcm_dpo/margin": 106.80502319335938, "fcm_dpo/q_t": 0.3245747983455658, "grad_norm": 11.806785583496094, "learning_rate": 2.2310222927429716e-07, "logits/chosen": -2.333134889602661, "logits/rejected": -1.7398998737335205, "logps/chosen": -234.05245971679688, "logps/ref_chosen": -101.04441833496094, "logps/ref_rejected": -108.8511734008789, "logps/rejected": -348.66424560546875, "loss": 0.8711, "margin_dpo/margin_mean": 106.80502319335938, "margin_dpo/margin_std": 107.98983001708984, "step": 397 }, { "epoch": 0.5844346549192364, "fcm_dpo/beta": 0.00766055379062891, "fcm_dpo/delta": 0.006077233701944351, "fcm_dpo/margin": 77.572265625, "fcm_dpo/q_t": 0.3723902702331543, "grad_norm": 11.528839111328125, "learning_rate": 2.2182670837116972e-07, "logits/chosen": -2.045728921890259, "logits/rejected": -1.755936861038208, "logps/chosen": -235.32550048828125, "logps/ref_chosen": -95.04922485351562, "logps/ref_rejected": -114.00563049316406, "logps/rejected": -331.85418701171875, "loss": 1.0144, "margin_dpo/margin_mean": 77.572265625, "margin_dpo/margin_std": 103.19320678710938, "step": 398 }, { "epoch": 0.5859030837004405, "fcm_dpo/beta": 0.0075286137871444225, "fcm_dpo/delta": -0.15626700222492218, "fcm_dpo/margin": 98.94232940673828, "fcm_dpo/q_t": 0.3343579173088074, "grad_norm": 10.617576599121094, "learning_rate": 2.2055192986047804e-07, "logits/chosen": -2.003676414489746, "logits/rejected": -1.1966228485107422, "logps/chosen": -222.55133056640625, "logps/ref_chosen": -96.63854217529297, "logps/ref_rejected": -83.53645324707031, "logps/rejected": -308.3916015625, "loss": 0.8622, "margin_dpo/margin_mean": 98.94233703613281, "margin_dpo/margin_std": 83.51806640625, "step": 399 }, { "epoch": 0.5873715124816447, "fcm_dpo/beta": 0.007274920120835304, "fcm_dpo/delta": -0.14528706669807434, "fcm_dpo/margin": 100.96365356445312, "fcm_dpo/q_t": 0.3401756286621094, "grad_norm": 14.20820426940918, "learning_rate": 2.192779273338215e-07, "logits/chosen": -2.292538642883301, "logits/rejected": -1.8154387474060059, "logps/chosen": -234.29095458984375, "logps/ref_chosen": -105.30314636230469, "logps/ref_rejected": -108.6555404663086, "logps/rejected": -338.60699462890625, "loss": 0.9031, "margin_dpo/margin_mean": 100.96365356445312, "margin_dpo/margin_std": 101.42951965332031, "step": 400 }, { "epoch": 0.5873715124816447, "eval_fcm_dpo/beta": 0.0072255851700901985, "eval_logits/chosen": -1.9563562870025635, "eval_logits/rejected": -1.5862782001495361, "eval_logps/chosen": -255.01856994628906, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -314.7674865722656, "eval_loss": 0.5320852398872375, "eval_margin_dpo/margin_mean": 66.17474365234375, "eval_margin_dpo/margin_std": 89.82697296142578, "eval_runtime": 43.7256, "eval_samples_per_second": 53.493, "eval_steps_per_second": 1.692, "step": 400 }, { "epoch": 0.5888399412628488, "fcm_dpo/beta": 0.007348657585680485, "fcm_dpo/delta": 0.13697752356529236, "fcm_dpo/margin": 63.91277313232422, "fcm_dpo/q_t": 0.40259644389152527, "grad_norm": 14.34986686706543, "learning_rate": 2.1800473436235136e-07, "logits/chosen": -1.7043498754501343, "logits/rejected": -1.4674514532089233, "logps/chosen": -227.89923095703125, "logps/ref_chosen": -84.28916931152344, "logps/ref_rejected": -90.943115234375, "logps/rejected": -298.4659423828125, "loss": 1.1053, "margin_dpo/margin_mean": 63.91277313232422, "margin_dpo/margin_std": 107.23680114746094, "step": 401 }, { "epoch": 0.5903083700440529, "fcm_dpo/beta": 0.007161559071391821, "fcm_dpo/delta": -0.25808143615722656, "fcm_dpo/margin": 116.8514404296875, "fcm_dpo/q_t": 0.3132210671901703, "grad_norm": 11.211475372314453, "learning_rate": 2.1673238449588665e-07, "logits/chosen": -1.7466460466384888, "logits/rejected": -1.2662107944488525, "logps/chosen": -213.04330444335938, "logps/ref_chosen": -83.59312438964844, "logps/ref_rejected": -87.81027221679688, "logps/rejected": -334.11187744140625, "loss": 0.8053, "margin_dpo/margin_mean": 116.8514404296875, "margin_dpo/margin_std": 88.61773681640625, "step": 402 }, { "epoch": 0.591776798825257, "fcm_dpo/beta": 0.007018306292593479, "fcm_dpo/delta": -0.011818572878837585, "fcm_dpo/margin": 87.03970336914062, "fcm_dpo/q_t": 0.3649277985095978, "grad_norm": 10.342928886413574, "learning_rate": 2.154609112620295e-07, "logits/chosen": -2.358241081237793, "logits/rejected": -1.9733853340148926, "logps/chosen": -205.4605712890625, "logps/ref_chosen": -73.75308227539062, "logps/ref_rejected": -83.92012786865234, "logps/rejected": -302.6673278808594, "loss": 0.9675, "margin_dpo/margin_mean": 87.03970336914062, "margin_dpo/margin_std": 95.28384399414062, "step": 403 }, { "epoch": 0.593245227606461, "fcm_dpo/beta": 0.007098081987351179, "fcm_dpo/delta": 0.07683449983596802, "fcm_dpo/margin": 74.28379821777344, "fcm_dpo/q_t": 0.38582703471183777, "grad_norm": 15.550983428955078, "learning_rate": 2.1419034816528218e-07, "logits/chosen": -2.0557281970977783, "logits/rejected": -1.7230305671691895, "logps/chosen": -237.1645050048828, "logps/ref_chosen": -79.67617797851562, "logps/ref_rejected": -84.280517578125, "logps/rejected": -316.0526428222656, "loss": 1.0557, "margin_dpo/margin_mean": 74.28379821777344, "margin_dpo/margin_std": 107.16883087158203, "step": 404 }, { "epoch": 0.5947136563876652, "fcm_dpo/beta": 0.007242183201014996, "fcm_dpo/delta": 0.046799372881650925, "fcm_dpo/margin": 76.58912658691406, "fcm_dpo/q_t": 0.3775564432144165, "grad_norm": 9.922403335571289, "learning_rate": 2.129207286861638e-07, "logits/chosen": -1.4873124361038208, "logits/rejected": -1.0152088403701782, "logps/chosen": -255.55612182617188, "logps/ref_chosen": -96.46195220947266, "logps/ref_rejected": -92.87071228027344, "logps/rejected": -328.55401611328125, "loss": 0.9945, "margin_dpo/margin_mean": 76.58912658691406, "margin_dpo/margin_std": 85.40648651123047, "step": 405 }, { "epoch": 0.5961820851688693, "fcm_dpo/beta": 0.007322330959141254, "fcm_dpo/delta": 0.09602566063404083, "fcm_dpo/margin": 69.48136138916016, "fcm_dpo/q_t": 0.3899584114551544, "grad_norm": 17.966711044311523, "learning_rate": 2.1165208628032861e-07, "logits/chosen": -2.0658977031707764, "logits/rejected": -1.8964214324951172, "logps/chosen": -228.33712768554688, "logps/ref_chosen": -78.13396453857422, "logps/ref_rejected": -98.28359985351562, "logps/rejected": -317.9681396484375, "loss": 1.0736, "margin_dpo/margin_mean": 69.48136138916016, "margin_dpo/margin_std": 104.44835662841797, "step": 406 }, { "epoch": 0.5976505139500734, "fcm_dpo/beta": 0.007518821395933628, "fcm_dpo/delta": 0.14913788437843323, "fcm_dpo/margin": 60.868133544921875, "fcm_dpo/q_t": 0.39967426657676697, "grad_norm": 13.635825157165527, "learning_rate": 2.1038445437768375e-07, "logits/chosen": -2.2239134311676025, "logits/rejected": -1.7089815139770508, "logps/chosen": -243.0015869140625, "logps/ref_chosen": -84.01283264160156, "logps/ref_rejected": -82.78103637695312, "logps/rejected": -302.6379089355469, "loss": 1.0895, "margin_dpo/margin_mean": 60.868133544921875, "margin_dpo/margin_std": 93.1767578125, "step": 407 }, { "epoch": 0.5991189427312775, "fcm_dpo/beta": 0.007688861805945635, "fcm_dpo/delta": 0.11370319128036499, "fcm_dpo/margin": 63.99238204956055, "fcm_dpo/q_t": 0.39551639556884766, "grad_norm": 13.247913360595703, "learning_rate": 2.0911786638150872e-07, "logits/chosen": -2.340407133102417, "logits/rejected": -1.736910343170166, "logps/chosen": -254.63917541503906, "logps/ref_chosen": -104.46174621582031, "logps/ref_rejected": -96.37217712402344, "logps/rejected": -310.5419921875, "loss": 1.0843, "margin_dpo/margin_mean": 63.99237823486328, "margin_dpo/margin_std": 99.77020263671875, "step": 408 }, { "epoch": 0.6005873715124816, "fcm_dpo/beta": 0.007855923846364021, "fcm_dpo/delta": 0.098321832716465, "fcm_dpo/margin": 64.50284576416016, "fcm_dpo/q_t": 0.3866105079650879, "grad_norm": 11.660709381103516, "learning_rate": 2.0785235566757517e-07, "logits/chosen": -1.5960277318954468, "logits/rejected": -1.1638531684875488, "logps/chosen": -259.2583923339844, "logps/ref_chosen": -97.66830444335938, "logps/ref_rejected": -90.04583740234375, "logps/rejected": -316.13873291015625, "loss": 1.0741, "margin_dpo/margin_mean": 64.50284576416016, "margin_dpo/margin_std": 95.21458435058594, "step": 409 }, { "epoch": 0.6020558002936858, "fcm_dpo/beta": 0.007921447977423668, "fcm_dpo/delta": 0.01372436247766018, "fcm_dpo/margin": 74.1120834350586, "fcm_dpo/q_t": 0.3695540428161621, "grad_norm": 11.11070442199707, "learning_rate": 2.065879555832674e-07, "logits/chosen": -1.7564365863800049, "logits/rejected": -1.6230534315109253, "logps/chosen": -235.60093688964844, "logps/ref_chosen": -76.46923828125, "logps/ref_rejected": -88.64064025878906, "logps/rejected": -321.8843994140625, "loss": 0.9988, "margin_dpo/margin_mean": 74.11209106445312, "margin_dpo/margin_std": 90.84548950195312, "step": 410 }, { "epoch": 0.6035242290748899, "fcm_dpo/beta": 0.007924282923340797, "fcm_dpo/delta": 0.025149427354335785, "fcm_dpo/margin": 72.70220184326172, "fcm_dpo/q_t": 0.37517866492271423, "grad_norm": 11.753535270690918, "learning_rate": 2.0532469944670343e-07, "logits/chosen": -2.482978343963623, "logits/rejected": -2.093186140060425, "logps/chosen": -254.4764862060547, "logps/ref_chosen": -87.16630554199219, "logps/ref_rejected": -87.09603118896484, "logps/rejected": -327.1083984375, "loss": 0.9966, "margin_dpo/margin_mean": 72.70220184326172, "margin_dpo/margin_std": 86.63870239257812, "step": 411 }, { "epoch": 0.604992657856094, "fcm_dpo/beta": 0.007970331236720085, "fcm_dpo/delta": 0.03290639817714691, "fcm_dpo/margin": 71.35936737060547, "fcm_dpo/q_t": 0.38153523206710815, "grad_norm": 12.409209251403809, "learning_rate": 2.0406262054585738e-07, "logits/chosen": -2.412627696990967, "logits/rejected": -2.191441059112549, "logps/chosen": -236.4732208251953, "logps/ref_chosen": -78.94734191894531, "logps/ref_rejected": -106.10554504394531, "logps/rejected": -334.99078369140625, "loss": 1.0581, "margin_dpo/margin_mean": 71.35936737060547, "margin_dpo/margin_std": 107.4913330078125, "step": 412 }, { "epoch": 0.6064610866372981, "fcm_dpo/beta": 0.0079441387206316, "fcm_dpo/delta": -0.033406391739845276, "fcm_dpo/margin": 79.40829467773438, "fcm_dpo/q_t": 0.3635585904121399, "grad_norm": 11.994926452636719, "learning_rate": 2.0280175213768205e-07, "logits/chosen": -1.2549076080322266, "logits/rejected": -0.9982696771621704, "logps/chosen": -265.3006591796875, "logps/ref_chosen": -95.69470977783203, "logps/ref_rejected": -107.96085357666016, "logps/rejected": -356.97509765625, "loss": 0.9942, "margin_dpo/margin_mean": 79.40829467773438, "margin_dpo/margin_std": 102.14022827148438, "step": 413 }, { "epoch": 0.6079295154185022, "fcm_dpo/beta": 0.007994389161467552, "fcm_dpo/delta": 0.021429970860481262, "fcm_dpo/margin": 72.52461242675781, "fcm_dpo/q_t": 0.37816792726516724, "grad_norm": 16.40660858154297, "learning_rate": 2.0154212744723247e-07, "logits/chosen": -1.648315668106079, "logits/rejected": -1.2972140312194824, "logps/chosen": -245.08993530273438, "logps/ref_chosen": -88.27667236328125, "logps/ref_rejected": -92.87004089355469, "logps/rejected": -322.2078857421875, "loss": 1.0534, "margin_dpo/margin_mean": 72.52461242675781, "margin_dpo/margin_std": 109.23208618164062, "step": 414 }, { "epoch": 0.6093979441997063, "fcm_dpo/beta": 0.008126772940158844, "fcm_dpo/delta": 0.09716188907623291, "fcm_dpo/margin": 62.45792007446289, "fcm_dpo/q_t": 0.39152491092681885, "grad_norm": 13.653444290161133, "learning_rate": 2.002837796667909e-07, "logits/chosen": -1.602043867111206, "logits/rejected": -1.2246946096420288, "logps/chosen": -278.5191955566406, "logps/ref_chosen": -108.91590118408203, "logps/ref_rejected": -107.47135925292969, "logps/rejected": -339.5325622558594, "loss": 1.0767, "margin_dpo/margin_mean": 62.45792007446289, "margin_dpo/margin_std": 95.18862915039062, "step": 415 }, { "epoch": 0.6108663729809104, "fcm_dpo/beta": 0.008111116476356983, "fcm_dpo/delta": -0.1124371886253357, "fcm_dpo/margin": 86.70777893066406, "fcm_dpo/q_t": 0.3507331311702728, "grad_norm": 11.193100929260254, "learning_rate": 1.990267419549914e-07, "logits/chosen": -1.8388745784759521, "logits/rejected": -1.336613655090332, "logps/chosen": -259.5759582519531, "logps/ref_chosen": -93.39888000488281, "logps/ref_rejected": -97.6729736328125, "logps/rejected": -350.5578308105469, "loss": 0.9505, "margin_dpo/margin_mean": 86.70777893066406, "margin_dpo/margin_std": 98.13165283203125, "step": 416 }, { "epoch": 0.6123348017621145, "fcm_dpo/beta": 0.007943946868181229, "fcm_dpo/delta": -0.04033021628856659, "fcm_dpo/margin": 80.280517578125, "fcm_dpo/q_t": 0.36008456349372864, "grad_norm": 11.344613075256348, "learning_rate": 1.9777104743594686e-07, "logits/chosen": -2.1126794815063477, "logits/rejected": -1.1838053464889526, "logps/chosen": -239.34365844726562, "logps/ref_chosen": -83.53533172607422, "logps/ref_rejected": -74.44184875488281, "logps/rejected": -310.53070068359375, "loss": 0.9601, "margin_dpo/margin_mean": 80.280517578125, "margin_dpo/margin_std": 89.32632446289062, "step": 417 }, { "epoch": 0.6138032305433186, "fcm_dpo/beta": 0.007776966318488121, "fcm_dpo/delta": -0.20287159085273743, "fcm_dpo/margin": 101.0872802734375, "fcm_dpo/q_t": 0.3297736644744873, "grad_norm": 14.166253089904785, "learning_rate": 1.965167291983757e-07, "logits/chosen": -2.374035358428955, "logits/rejected": -1.742133378982544, "logps/chosen": -255.0244598388672, "logps/ref_chosen": -108.22152709960938, "logps/ref_rejected": -111.8646469116211, "logps/rejected": -359.7548828125, "loss": 0.8604, "margin_dpo/margin_mean": 101.0872802734375, "margin_dpo/margin_std": 87.68946838378906, "step": 418 }, { "epoch": 0.6152716593245228, "fcm_dpo/beta": 0.007567869499325752, "fcm_dpo/delta": -0.06575581431388855, "fcm_dpo/margin": 87.35041809082031, "fcm_dpo/q_t": 0.3550986051559448, "grad_norm": 11.697413444519043, "learning_rate": 1.9526382029472988e-07, "logits/chosen": -1.4137977361679077, "logits/rejected": -0.960525631904602, "logps/chosen": -251.655517578125, "logps/ref_chosen": -97.18328094482422, "logps/ref_rejected": -98.18531799316406, "logps/rejected": -340.0079345703125, "loss": 0.9383, "margin_dpo/margin_mean": 87.35041809082031, "margin_dpo/margin_std": 90.81410217285156, "step": 419 }, { "epoch": 0.6167400881057269, "fcm_dpo/beta": 0.007550552487373352, "fcm_dpo/delta": 0.013763319700956345, "fcm_dpo/margin": 77.66873931884766, "fcm_dpo/q_t": 0.37317919731140137, "grad_norm": 16.0394344329834, "learning_rate": 1.9401235374032425e-07, "logits/chosen": -2.1781749725341797, "logits/rejected": -1.0760204792022705, "logps/chosen": -272.736572265625, "logps/ref_chosen": -114.30848693847656, "logps/ref_rejected": -75.68355560302734, "logps/rejected": -311.7803955078125, "loss": 1.0063, "margin_dpo/margin_mean": 77.66874694824219, "margin_dpo/margin_std": 97.64486694335938, "step": 420 }, { "epoch": 0.618208516886931, "fcm_dpo/beta": 0.007453433237969875, "fcm_dpo/delta": -0.04194428399205208, "fcm_dpo/margin": 85.74388885498047, "fcm_dpo/q_t": 0.35998445749282837, "grad_norm": 10.710517883300781, "learning_rate": 1.9276236251246653e-07, "logits/chosen": -1.8413752317428589, "logits/rejected": -1.4720008373260498, "logps/chosen": -239.76809692382812, "logps/ref_chosen": -85.87985229492188, "logps/ref_rejected": -96.33648681640625, "logps/rejected": -335.9686279296875, "loss": 0.9597, "margin_dpo/margin_mean": 85.743896484375, "margin_dpo/margin_std": 95.09271240234375, "step": 421 }, { "epoch": 0.6196769456681351, "fcm_dpo/beta": 0.007458934560418129, "fcm_dpo/delta": -0.008748026564717293, "fcm_dpo/margin": 81.53353881835938, "fcm_dpo/q_t": 0.3671715557575226, "grad_norm": 12.771909713745117, "learning_rate": 1.9151387954958792e-07, "logits/chosen": -2.399049758911133, "logits/rejected": -2.0303146839141846, "logps/chosen": -251.56912231445312, "logps/ref_chosen": -100.48060607910156, "logps/ref_rejected": -94.40821838378906, "logps/rejected": -327.0303039550781, "loss": 0.9712, "margin_dpo/margin_mean": 81.53353881835938, "margin_dpo/margin_std": 92.20245361328125, "step": 422 }, { "epoch": 0.6211453744493393, "fcm_dpo/beta": 0.00747031532227993, "fcm_dpo/delta": -0.004844721406698227, "fcm_dpo/margin": 80.87257385253906, "fcm_dpo/q_t": 0.3732362687587738, "grad_norm": 12.241605758666992, "learning_rate": 1.902669377503756e-07, "logits/chosen": -1.9506278038024902, "logits/rejected": -1.833554744720459, "logps/chosen": -219.573486328125, "logps/ref_chosen": -78.44993591308594, "logps/ref_rejected": -92.04652404785156, "logps/rejected": -314.04266357421875, "loss": 1.0157, "margin_dpo/margin_mean": 80.8725814819336, "margin_dpo/margin_std": 108.63528442382812, "step": 423 }, { "epoch": 0.6226138032305433, "fcm_dpo/beta": 0.007418894208967686, "fcm_dpo/delta": -0.012280648574233055, "fcm_dpo/margin": 82.428466796875, "fcm_dpo/q_t": 0.3642910420894623, "grad_norm": 10.28632640838623, "learning_rate": 1.890215699729057e-07, "logits/chosen": -1.6936588287353516, "logits/rejected": -1.089149832725525, "logps/chosen": -245.31788635253906, "logps/ref_chosen": -87.64236450195312, "logps/ref_rejected": -72.36566162109375, "logps/rejected": -312.46966552734375, "loss": 0.973, "margin_dpo/margin_mean": 82.428466796875, "margin_dpo/margin_std": 93.11031341552734, "step": 424 }, { "epoch": 0.6240822320117474, "fcm_dpo/beta": 0.007599799428135157, "fcm_dpo/delta": 0.1809379905462265, "fcm_dpo/margin": 56.19786071777344, "fcm_dpo/q_t": 0.41294217109680176, "grad_norm": 13.262221336364746, "learning_rate": 1.8777780903377732e-07, "logits/chosen": -1.6487901210784912, "logits/rejected": -1.5741477012634277, "logps/chosen": -241.32943725585938, "logps/ref_chosen": -78.51979064941406, "logps/ref_rejected": -102.74864196777344, "logps/rejected": -321.75616455078125, "loss": 1.1604, "margin_dpo/margin_mean": 56.19786071777344, "margin_dpo/margin_std": 110.29562377929688, "step": 425 }, { "epoch": 0.6255506607929515, "fcm_dpo/beta": 0.007676470093429089, "fcm_dpo/delta": -0.026276282966136932, "fcm_dpo/margin": 81.36367797851562, "fcm_dpo/q_t": 0.36279982328414917, "grad_norm": 14.7294282913208, "learning_rate": 1.8653568770724803e-07, "logits/chosen": -1.8185904026031494, "logits/rejected": -1.1083682775497437, "logps/chosen": -247.37643432617188, "logps/ref_chosen": -108.50582885742188, "logps/ref_rejected": -88.300048828125, "logps/rejected": -308.53436279296875, "loss": 0.9699, "margin_dpo/margin_mean": 81.36367797851562, "margin_dpo/margin_std": 92.48495483398438, "step": 426 }, { "epoch": 0.6270190895741556, "fcm_dpo/beta": 0.007682330906391144, "fcm_dpo/delta": 0.018056850880384445, "fcm_dpo/margin": 75.87571716308594, "fcm_dpo/q_t": 0.3702627420425415, "grad_norm": 10.52995491027832, "learning_rate": 1.8529523872436977e-07, "logits/chosen": -2.527616262435913, "logits/rejected": -1.911233901977539, "logps/chosen": -213.69146728515625, "logps/ref_chosen": -99.12046813964844, "logps/ref_rejected": -85.724609375, "logps/rejected": -276.17132568359375, "loss": 0.9796, "margin_dpo/margin_mean": 75.87571716308594, "margin_dpo/margin_std": 83.9215316772461, "step": 427 }, { "epoch": 0.6284875183553598, "fcm_dpo/beta": 0.007750866934657097, "fcm_dpo/delta": 0.1085626631975174, "fcm_dpo/margin": 64.08061981201172, "fcm_dpo/q_t": 0.3931909203529358, "grad_norm": 11.369186401367188, "learning_rate": 1.8405649477212697e-07, "logits/chosen": -1.6850427389144897, "logits/rejected": -1.4209712743759155, "logps/chosen": -267.068115234375, "logps/ref_chosen": -105.96925354003906, "logps/ref_rejected": -109.10218048095703, "logps/rejected": -334.28167724609375, "loss": 1.0875, "margin_dpo/margin_mean": 64.08061981201172, "margin_dpo/margin_std": 99.4907455444336, "step": 428 }, { "epoch": 0.6299559471365639, "fcm_dpo/beta": 0.0078573664650321, "fcm_dpo/delta": -0.057729579508304596, "fcm_dpo/margin": 83.04490661621094, "fcm_dpo/q_t": 0.3629707396030426, "grad_norm": 11.991065979003906, "learning_rate": 1.828194884925749e-07, "logits/chosen": -2.019653797149658, "logits/rejected": -1.3182734251022339, "logps/chosen": -259.6905517578125, "logps/ref_chosen": -113.54487609863281, "logps/ref_rejected": -98.24201965332031, "logps/rejected": -327.4326171875, "loss": 0.9858, "margin_dpo/margin_mean": 83.04490661621094, "margin_dpo/margin_std": 102.54899597167969, "step": 429 }, { "epoch": 0.631424375917768, "fcm_dpo/beta": 0.00791855901479721, "fcm_dpo/delta": 0.11582081019878387, "fcm_dpo/margin": 61.77325439453125, "fcm_dpo/q_t": 0.39517396688461304, "grad_norm": 11.80427074432373, "learning_rate": 1.8158425248197928e-07, "logits/chosen": -1.3399078845977783, "logits/rejected": -1.298473596572876, "logps/chosen": -235.0428009033203, "logps/ref_chosen": -91.31936645507812, "logps/ref_rejected": -110.1096420288086, "logps/rejected": -315.6063232421875, "loss": 1.0881, "margin_dpo/margin_mean": 61.773258209228516, "margin_dpo/margin_std": 96.4741439819336, "step": 430 }, { "epoch": 0.6328928046989721, "fcm_dpo/beta": 0.007927455008029938, "fcm_dpo/delta": 0.0042446693405508995, "fcm_dpo/margin": 75.17669677734375, "fcm_dpo/q_t": 0.371150940656662, "grad_norm": 12.48121166229248, "learning_rate": 1.8035081928995788e-07, "logits/chosen": -1.9271609783172607, "logits/rejected": -1.5382022857666016, "logps/chosen": -235.30166625976562, "logps/ref_chosen": -93.18122100830078, "logps/ref_rejected": -98.13226318359375, "logps/rejected": -315.4294128417969, "loss": 1.0076, "margin_dpo/margin_mean": 75.17669677734375, "margin_dpo/margin_std": 96.68666076660156, "step": 431 }, { "epoch": 0.6343612334801763, "fcm_dpo/beta": 0.007823891006410122, "fcm_dpo/delta": -0.13029845058918, "fcm_dpo/margin": 92.16277313232422, "fcm_dpo/q_t": 0.34236711263656616, "grad_norm": 10.779977798461914, "learning_rate": 1.791192214186223e-07, "logits/chosen": -1.6222333908081055, "logits/rejected": -1.0763202905654907, "logps/chosen": -229.423828125, "logps/ref_chosen": -104.43478393554688, "logps/ref_rejected": -105.08955383300781, "logps/rejected": -322.24139404296875, "loss": 0.9015, "margin_dpo/margin_mean": 92.16278076171875, "margin_dpo/margin_std": 89.09420776367188, "step": 432 }, { "epoch": 0.6358296622613803, "fcm_dpo/beta": 0.007831670343875885, "fcm_dpo/delta": 0.033429112285375595, "fcm_dpo/margin": 72.43510437011719, "fcm_dpo/q_t": 0.3780878186225891, "grad_norm": 11.7706880569458, "learning_rate": 1.7788949132172193e-07, "logits/chosen": -1.7953574657440186, "logits/rejected": -1.4741289615631104, "logps/chosen": -237.90676879882812, "logps/ref_chosen": -89.84322357177734, "logps/ref_rejected": -101.73345947265625, "logps/rejected": -322.23211669921875, "loss": 1.0302, "margin_dpo/margin_mean": 72.43511199951172, "margin_dpo/margin_std": 97.10591125488281, "step": 433 }, { "epoch": 0.6372980910425844, "fcm_dpo/beta": 0.007778637111186981, "fcm_dpo/delta": -0.019643208011984825, "fcm_dpo/margin": 79.50172424316406, "fcm_dpo/q_t": 0.37190669775009155, "grad_norm": 11.87337875366211, "learning_rate": 1.7666166140378853e-07, "logits/chosen": -2.532245635986328, "logits/rejected": -1.888089656829834, "logps/chosen": -229.92483520507812, "logps/ref_chosen": -97.6925277709961, "logps/ref_rejected": -84.09129333496094, "logps/rejected": -295.8253173828125, "loss": 1.0079, "margin_dpo/margin_mean": 79.50172424316406, "margin_dpo/margin_std": 108.16815185546875, "step": 434 }, { "epoch": 0.6387665198237885, "fcm_dpo/beta": 0.007717899512499571, "fcm_dpo/delta": -0.0468808189034462, "fcm_dpo/margin": 83.42731475830078, "fcm_dpo/q_t": 0.36141249537467957, "grad_norm": 12.820802688598633, "learning_rate": 1.7543576401928218e-07, "logits/chosen": -2.351439952850342, "logits/rejected": -1.8964674472808838, "logps/chosen": -214.12286376953125, "logps/ref_chosen": -86.17192077636719, "logps/ref_rejected": -93.751708984375, "logps/rejected": -305.12994384765625, "loss": 0.9617, "margin_dpo/margin_mean": 83.42730712890625, "margin_dpo/margin_std": 96.24213409423828, "step": 435 }, { "epoch": 0.6402349486049926, "fcm_dpo/beta": 0.00758952833712101, "fcm_dpo/delta": -0.08244708925485611, "fcm_dpo/margin": 89.17771911621094, "fcm_dpo/q_t": 0.3484029769897461, "grad_norm": 11.64830493927002, "learning_rate": 1.742118314717391e-07, "logits/chosen": -1.9208111763000488, "logits/rejected": -1.0720462799072266, "logps/chosen": -211.42588806152344, "logps/ref_chosen": -105.78710174560547, "logps/ref_rejected": -88.62471008300781, "logps/rejected": -283.44122314453125, "loss": 0.9044, "margin_dpo/margin_mean": 89.17771911621094, "margin_dpo/margin_std": 80.94717407226562, "step": 436 }, { "epoch": 0.6417033773861968, "fcm_dpo/beta": 0.007655961439013481, "fcm_dpo/delta": 0.07603086531162262, "fcm_dpo/margin": 68.94998168945312, "fcm_dpo/q_t": 0.38024958968162537, "grad_norm": 11.85803508758545, "learning_rate": 1.7298989601292036e-07, "logits/chosen": -1.9982421398162842, "logits/rejected": -1.3477120399475098, "logps/chosen": -220.46435546875, "logps/ref_chosen": -96.06204223632812, "logps/ref_rejected": -89.01220703125, "logps/rejected": -282.364501953125, "loss": 1.0144, "margin_dpo/margin_mean": 68.94998168945312, "margin_dpo/margin_std": 80.42849731445312, "step": 437 }, { "epoch": 0.6431718061674009, "fcm_dpo/beta": 0.00769776338711381, "fcm_dpo/delta": 0.05381428450345993, "fcm_dpo/margin": 71.3000717163086, "fcm_dpo/q_t": 0.3784894645214081, "grad_norm": 12.725931167602539, "learning_rate": 1.7176998984196144e-07, "logits/chosen": -2.1903576850891113, "logits/rejected": -1.3835668563842773, "logps/chosen": -227.7496337890625, "logps/ref_chosen": -101.85537719726562, "logps/ref_rejected": -89.44766235351562, "logps/rejected": -286.6419677734375, "loss": 1.0157, "margin_dpo/margin_mean": 71.3000717163086, "margin_dpo/margin_std": 86.569091796875, "step": 438 }, { "epoch": 0.644640234948605, "fcm_dpo/beta": 0.007735989056527615, "fcm_dpo/delta": 0.037646569311618805, "fcm_dpo/margin": 72.81299591064453, "fcm_dpo/q_t": 0.3774023652076721, "grad_norm": 13.056401252746582, "learning_rate": 1.7055214510452458e-07, "logits/chosen": -1.570444941520691, "logits/rejected": -1.3217337131500244, "logps/chosen": -219.38943481445312, "logps/ref_chosen": -81.75563049316406, "logps/ref_rejected": -90.58636474609375, "logps/rejected": -301.03314208984375, "loss": 1.0128, "margin_dpo/margin_mean": 72.81299591064453, "margin_dpo/margin_std": 88.81855773925781, "step": 439 }, { "epoch": 0.6461086637298091, "fcm_dpo/beta": 0.007678865920752287, "fcm_dpo/delta": -0.07341113686561584, "fcm_dpo/margin": 86.8636703491211, "fcm_dpo/q_t": 0.35673287510871887, "grad_norm": 10.554864883422852, "learning_rate": 1.6933639389195134e-07, "logits/chosen": -2.4091434478759766, "logits/rejected": -1.9146919250488281, "logps/chosen": -229.0126190185547, "logps/ref_chosen": -105.64108276367188, "logps/ref_rejected": -103.4010009765625, "logps/rejected": -313.63623046875, "loss": 0.9517, "margin_dpo/margin_mean": 86.86366271972656, "margin_dpo/margin_std": 97.84516143798828, "step": 440 }, { "epoch": 0.6475770925110133, "fcm_dpo/beta": 0.007652001921087503, "fcm_dpo/delta": -0.02822795696556568, "fcm_dpo/margin": 81.84053039550781, "fcm_dpo/q_t": 0.36398565769195557, "grad_norm": 11.060223579406738, "learning_rate": 1.681227682404166e-07, "logits/chosen": -2.021010398864746, "logits/rejected": -1.6652534008026123, "logps/chosen": -243.90728759765625, "logps/ref_chosen": -91.529541015625, "logps/ref_rejected": -103.619384765625, "logps/rejected": -337.8376770019531, "loss": 0.9485, "margin_dpo/margin_mean": 81.84053039550781, "margin_dpo/margin_std": 87.4005126953125, "step": 441 }, { "epoch": 0.6490455212922174, "fcm_dpo/beta": 0.007508729584515095, "fcm_dpo/delta": -0.13639724254608154, "fcm_dpo/margin": 96.75260162353516, "fcm_dpo/q_t": 0.33930784463882446, "grad_norm": 11.458242416381836, "learning_rate": 1.669113001300851e-07, "logits/chosen": -1.8102037906646729, "logits/rejected": -1.3520574569702148, "logps/chosen": -236.66015625, "logps/ref_chosen": -84.77755737304688, "logps/ref_rejected": -83.82415771484375, "logps/rejected": -332.4593505859375, "loss": 0.8827, "margin_dpo/margin_mean": 96.75260162353516, "margin_dpo/margin_std": 88.31156158447266, "step": 442 }, { "epoch": 0.6505139500734214, "fcm_dpo/beta": 0.007541971281170845, "fcm_dpo/delta": 0.12367384880781174, "fcm_dpo/margin": 63.93056869506836, "fcm_dpo/q_t": 0.3926345407962799, "grad_norm": 11.647642135620117, "learning_rate": 1.6570202148426815e-07, "logits/chosen": -2.023066997528076, "logits/rejected": -1.4911339282989502, "logps/chosen": -249.26025390625, "logps/ref_chosen": -102.64927673339844, "logps/ref_rejected": -93.03807067871094, "logps/rejected": -303.5796203613281, "loss": 1.057, "margin_dpo/margin_mean": 63.930564880371094, "margin_dpo/margin_std": 82.83998107910156, "step": 443 }, { "epoch": 0.6519823788546255, "fcm_dpo/beta": 0.0077362204901874065, "fcm_dpo/delta": 0.07435087114572525, "fcm_dpo/margin": 68.44133758544922, "fcm_dpo/q_t": 0.3861270546913147, "grad_norm": 12.801765441894531, "learning_rate": 1.6449496416858282e-07, "logits/chosen": -1.6765544414520264, "logits/rejected": -1.5086448192596436, "logps/chosen": -242.2406463623047, "logps/ref_chosen": -87.91971588134766, "logps/ref_rejected": -103.32345581054688, "logps/rejected": -326.08575439453125, "loss": 1.0539, "margin_dpo/margin_mean": 68.44134521484375, "margin_dpo/margin_std": 97.85394287109375, "step": 444 }, { "epoch": 0.6534508076358296, "fcm_dpo/beta": 0.007698110304772854, "fcm_dpo/delta": -0.021563369780778885, "fcm_dpo/margin": 80.50859832763672, "fcm_dpo/q_t": 0.3627839982509613, "grad_norm": 11.485919952392578, "learning_rate": 1.6329015999011182e-07, "logits/chosen": -1.9858006238937378, "logits/rejected": -1.5744715929031372, "logps/chosen": -236.77423095703125, "logps/ref_chosen": -101.40086364746094, "logps/ref_rejected": -99.03790283203125, "logps/rejected": -314.91986083984375, "loss": 0.9579, "margin_dpo/margin_mean": 80.50859832763672, "margin_dpo/margin_std": 85.82842254638672, "step": 445 }, { "epoch": 0.6549192364170338, "fcm_dpo/beta": 0.007764734327793121, "fcm_dpo/delta": 0.028329739347100258, "fcm_dpo/margin": 73.82595825195312, "fcm_dpo/q_t": 0.3735341429710388, "grad_norm": 14.396162986755371, "learning_rate": 1.6208764069656578e-07, "logits/chosen": -1.9688878059387207, "logits/rejected": -1.821672797203064, "logps/chosen": -217.6196746826172, "logps/ref_chosen": -87.4223403930664, "logps/ref_rejected": -106.70075988769531, "logps/rejected": -310.72406005859375, "loss": 0.9884, "margin_dpo/margin_mean": 73.82595825195312, "margin_dpo/margin_std": 83.8984603881836, "step": 446 }, { "epoch": 0.6563876651982379, "fcm_dpo/beta": 0.007610865402966738, "fcm_dpo/delta": -0.12789002060890198, "fcm_dpo/margin": 94.40399169921875, "fcm_dpo/q_t": 0.34479495882987976, "grad_norm": 11.825496673583984, "learning_rate": 1.608874379754465e-07, "logits/chosen": -2.1955559253692627, "logits/rejected": -2.148773670196533, "logps/chosen": -214.28981018066406, "logps/ref_chosen": -83.6152572631836, "logps/ref_rejected": -104.91239929199219, "logps/rejected": -329.990966796875, "loss": 0.9191, "margin_dpo/margin_mean": 94.40399169921875, "margin_dpo/margin_std": 100.11077880859375, "step": 447 }, { "epoch": 0.657856093979442, "fcm_dpo/beta": 0.007517419755458832, "fcm_dpo/delta": -0.09112964570522308, "fcm_dpo/margin": 91.07563781738281, "fcm_dpo/q_t": 0.35592204332351685, "grad_norm": 12.4849271774292, "learning_rate": 1.5968958345321177e-07, "logits/chosen": -1.9084062576293945, "logits/rejected": -1.6441185474395752, "logps/chosen": -232.83050537109375, "logps/ref_chosen": -92.5757827758789, "logps/ref_rejected": -107.68977355957031, "logps/rejected": -339.0201110839844, "loss": 0.9746, "margin_dpo/margin_mean": 91.07563781738281, "margin_dpo/margin_std": 112.18666076660156, "step": 448 }, { "epoch": 0.6593245227606461, "fcm_dpo/beta": 0.007335230708122253, "fcm_dpo/delta": -0.09747156500816345, "fcm_dpo/margin": 94.17698669433594, "fcm_dpo/q_t": 0.34920936822891235, "grad_norm": 10.423608779907227, "learning_rate": 1.584941086944423e-07, "logits/chosen": -1.7010339498519897, "logits/rejected": -1.2772597074508667, "logps/chosen": -240.6586151123047, "logps/ref_chosen": -102.39893341064453, "logps/ref_rejected": -95.14886474609375, "logps/rejected": -327.58551025390625, "loss": 0.9312, "margin_dpo/margin_mean": 94.17698669433594, "margin_dpo/margin_std": 100.80087280273438, "step": 449 }, { "epoch": 0.6607929515418502, "fcm_dpo/beta": 0.007234559860080481, "fcm_dpo/delta": -0.046480074524879456, "fcm_dpo/margin": 88.93949127197266, "fcm_dpo/q_t": 0.35815033316612244, "grad_norm": 11.052569389343262, "learning_rate": 1.573010452010098e-07, "logits/chosen": -2.46598482131958, "logits/rejected": -2.271839141845703, "logps/chosen": -210.14553833007812, "logps/ref_chosen": -86.99285888671875, "logps/ref_rejected": -108.53203582763672, "logps/rejected": -320.62420654296875, "loss": 0.9593, "margin_dpo/margin_mean": 88.93949890136719, "margin_dpo/margin_std": 99.60723876953125, "step": 450 }, { "epoch": 0.6622613803230544, "fcm_dpo/beta": 0.007310773245990276, "fcm_dpo/delta": 0.062415413558483124, "fcm_dpo/margin": 73.95240020751953, "fcm_dpo/q_t": 0.3855394721031189, "grad_norm": 19.515153884887695, "learning_rate": 1.5611042441124687e-07, "logits/chosen": -2.4347171783447266, "logits/rejected": -1.9352984428405762, "logps/chosen": -226.25865173339844, "logps/ref_chosen": -86.811279296875, "logps/ref_rejected": -79.8555908203125, "logps/rejected": -293.25537109375, "loss": 1.0629, "margin_dpo/margin_mean": 73.952392578125, "margin_dpo/margin_std": 110.59036254882812, "step": 451 }, { "epoch": 0.6637298091042585, "fcm_dpo/beta": 0.007255923468619585, "fcm_dpo/delta": 0.012697272002696991, "fcm_dpo/margin": 80.79637145996094, "fcm_dpo/q_t": 0.36846262216567993, "grad_norm": 10.28548812866211, "learning_rate": 1.549222776991186e-07, "logits/chosen": -1.8156119585037231, "logits/rejected": -1.7515356540679932, "logps/chosen": -201.019287109375, "logps/ref_chosen": -79.379638671875, "logps/ref_rejected": -103.71539306640625, "logps/rejected": -306.15142822265625, "loss": 0.9648, "margin_dpo/margin_mean": 80.79637145996094, "margin_dpo/margin_std": 77.09156799316406, "step": 452 }, { "epoch": 0.6651982378854625, "fcm_dpo/beta": 0.007229423150420189, "fcm_dpo/delta": -0.10436852276325226, "fcm_dpo/margin": 96.43700408935547, "fcm_dpo/q_t": 0.34717243909835815, "grad_norm": 14.366576194763184, "learning_rate": 1.5373663637339584e-07, "logits/chosen": -2.0531060695648193, "logits/rejected": -1.5909814834594727, "logps/chosen": -234.8137664794922, "logps/ref_chosen": -87.6951904296875, "logps/ref_rejected": -90.0582275390625, "logps/rejected": -333.6138000488281, "loss": 0.9015, "margin_dpo/margin_mean": 96.43701171875, "margin_dpo/margin_std": 90.89103698730469, "step": 453 }, { "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.007256688550114632, "fcm_dpo/delta": 0.03500938415527344, "fcm_dpo/margin": 78.03349304199219, "fcm_dpo/q_t": 0.3794366121292114, "grad_norm": 12.678661346435547, "learning_rate": 1.5255353167683017e-07, "logits/chosen": -1.8184224367141724, "logits/rejected": -1.3561699390411377, "logps/chosen": -246.05104064941406, "logps/ref_chosen": -89.56623840332031, "logps/ref_rejected": -92.92105102539062, "logps/rejected": -327.4393310546875, "loss": 1.0581, "margin_dpo/margin_mean": 78.03349304199219, "margin_dpo/margin_std": 115.84363555908203, "step": 454 }, { "epoch": 0.6681350954478708, "fcm_dpo/beta": 0.0073183816857635975, "fcm_dpo/delta": 0.04829338192939758, "fcm_dpo/margin": 75.66259765625, "fcm_dpo/q_t": 0.37961339950561523, "grad_norm": 11.24074935913086, "learning_rate": 1.5137299478533064e-07, "logits/chosen": -2.125541925430298, "logits/rejected": -2.0756983757019043, "logps/chosen": -222.00599670410156, "logps/ref_chosen": -77.62998962402344, "logps/ref_rejected": -118.97795104980469, "logps/rejected": -339.0165710449219, "loss": 1.0307, "margin_dpo/margin_mean": 75.66259765625, "margin_dpo/margin_std": 100.04164123535156, "step": 455 }, { "epoch": 0.6696035242290749, "fcm_dpo/beta": 0.00738774798810482, "fcm_dpo/delta": 0.056851405650377274, "fcm_dpo/margin": 73.88893127441406, "fcm_dpo/q_t": 0.37978994846343994, "grad_norm": 13.592082023620605, "learning_rate": 1.5019505680714232e-07, "logits/chosen": -1.9894077777862549, "logits/rejected": -1.8558869361877441, "logps/chosen": -236.3739013671875, "logps/ref_chosen": -89.61686706542969, "logps/ref_rejected": -109.5597152709961, "logps/rejected": -330.2056884765625, "loss": 1.0154, "margin_dpo/margin_mean": 73.88893127441406, "margin_dpo/margin_std": 90.75523376464844, "step": 456 }, { "epoch": 0.671071953010279, "fcm_dpo/beta": 0.007179590407758951, "fcm_dpo/delta": -0.12014248967170715, "fcm_dpo/margin": 98.66180419921875, "fcm_dpo/q_t": 0.33989059925079346, "grad_norm": 12.305672645568848, "learning_rate": 1.4901974878202627e-07, "logits/chosen": -2.244877338409424, "logits/rejected": -1.9393136501312256, "logps/chosen": -209.8995819091797, "logps/ref_chosen": -87.32168579101562, "logps/ref_rejected": -90.76660919189453, "logps/rejected": -312.0063171386719, "loss": 0.8772, "margin_dpo/margin_mean": 98.66180419921875, "margin_dpo/margin_std": 78.503662109375, "step": 457 }, { "epoch": 0.6725403817914831, "fcm_dpo/beta": 0.00729702552780509, "fcm_dpo/delta": 0.06375245004892349, "fcm_dpo/margin": 73.92202758789062, "fcm_dpo/q_t": 0.38597458600997925, "grad_norm": 12.738073348999023, "learning_rate": 1.4784710168044212e-07, "logits/chosen": -1.7707667350769043, "logits/rejected": -1.5331363677978516, "logps/chosen": -233.474365234375, "logps/ref_chosen": -93.52044677734375, "logps/ref_rejected": -103.36898803710938, "logps/rejected": -317.24493408203125, "loss": 1.0455, "margin_dpo/margin_mean": 73.92202758789062, "margin_dpo/margin_std": 105.11477661132812, "step": 458 }, { "epoch": 0.6740088105726872, "fcm_dpo/beta": 0.007197215687483549, "fcm_dpo/delta": -0.05717407539486885, "fcm_dpo/margin": 90.64775085449219, "fcm_dpo/q_t": 0.35962265729904175, "grad_norm": 9.387142181396484, "learning_rate": 1.466771464027316e-07, "logits/chosen": -1.5463656187057495, "logits/rejected": -1.285736322402954, "logps/chosen": -208.82803344726562, "logps/ref_chosen": -75.68820190429688, "logps/ref_rejected": -92.17048645019531, "logps/rejected": -315.95806884765625, "loss": 0.9552, "margin_dpo/margin_mean": 90.64775085449219, "margin_dpo/margin_std": 101.39945983886719, "step": 459 }, { "epoch": 0.6754772393538914, "fcm_dpo/beta": 0.007193270605057478, "fcm_dpo/delta": 0.00503632053732872, "fcm_dpo/margin": 82.70344543457031, "fcm_dpo/q_t": 0.37113451957702637, "grad_norm": 13.572725296020508, "learning_rate": 1.4550991377830423e-07, "logits/chosen": -1.9946367740631104, "logits/rejected": -2.009538412094116, "logps/chosen": -219.1749267578125, "logps/ref_chosen": -81.11788940429688, "logps/ref_rejected": -110.31238555908203, "logps/rejected": -331.0728759765625, "loss": 0.9981, "margin_dpo/margin_mean": 82.70344543457031, "margin_dpo/margin_std": 101.98342895507812, "step": 460 }, { "epoch": 0.6769456681350955, "fcm_dpo/beta": 0.007337551563978195, "fcm_dpo/delta": 0.12816324830055237, "fcm_dpo/margin": 65.16386413574219, "fcm_dpo/q_t": 0.3958069682121277, "grad_norm": 10.882566452026367, "learning_rate": 1.4434543456482518e-07, "logits/chosen": -1.553586483001709, "logits/rejected": -1.35811448097229, "logps/chosen": -250.552734375, "logps/ref_chosen": -81.58352661132812, "logps/ref_rejected": -93.87710571289062, "logps/rejected": -328.01019287109375, "loss": 1.0685, "margin_dpo/margin_mean": 65.16386413574219, "margin_dpo/margin_std": 90.7364501953125, "step": 461 }, { "epoch": 0.6784140969162996, "fcm_dpo/beta": 0.007375137880444527, "fcm_dpo/delta": -0.05297492444515228, "fcm_dpo/margin": 88.07096862792969, "fcm_dpo/q_t": 0.3572496771812439, "grad_norm": 10.971365928649902, "learning_rate": 1.4318373944740484e-07, "logits/chosen": -2.2870004177093506, "logits/rejected": -1.787416696548462, "logps/chosen": -221.18679809570312, "logps/ref_chosen": -94.19854736328125, "logps/ref_rejected": -85.63162994384766, "logps/rejected": -300.69085693359375, "loss": 0.9407, "margin_dpo/margin_mean": 88.07096862792969, "margin_dpo/margin_std": 91.79960632324219, "step": 462 }, { "epoch": 0.6798825256975036, "fcm_dpo/beta": 0.007232788018882275, "fcm_dpo/delta": -0.1597738415002823, "fcm_dpo/margin": 103.35470581054688, "fcm_dpo/q_t": 0.33559200167655945, "grad_norm": 14.430492401123047, "learning_rate": 1.4202485903778976e-07, "logits/chosen": -1.9360418319702148, "logits/rejected": -1.5115666389465332, "logps/chosen": -210.00100708007812, "logps/ref_chosen": -85.92474365234375, "logps/ref_rejected": -96.90184020996094, "logps/rejected": -324.33282470703125, "loss": 0.8987, "margin_dpo/margin_mean": 103.35470581054688, "margin_dpo/margin_std": 98.86663818359375, "step": 463 }, { "epoch": 0.6813509544787077, "fcm_dpo/beta": 0.007100261747837067, "fcm_dpo/delta": 0.008855840191245079, "fcm_dpo/margin": 83.32122802734375, "fcm_dpo/q_t": 0.37016385793685913, "grad_norm": 14.36965274810791, "learning_rate": 1.4086882387355658e-07, "logits/chosen": -2.0437088012695312, "logits/rejected": -2.1165390014648438, "logps/chosen": -232.46543884277344, "logps/ref_chosen": -79.68919372558594, "logps/ref_rejected": -107.29232025146484, "logps/rejected": -343.3898010253906, "loss": 0.9944, "margin_dpo/margin_mean": 83.32122802734375, "margin_dpo/margin_std": 101.45448303222656, "step": 464 }, { "epoch": 0.6828193832599119, "fcm_dpo/beta": 0.006998377852141857, "fcm_dpo/delta": -0.09421905875205994, "fcm_dpo/margin": 98.20915222167969, "fcm_dpo/q_t": 0.3482474386692047, "grad_norm": 12.179363250732422, "learning_rate": 1.3971566441730714e-07, "logits/chosen": -1.6612213850021362, "logits/rejected": -1.3575751781463623, "logps/chosen": -235.50955200195312, "logps/ref_chosen": -91.8602294921875, "logps/ref_rejected": -118.71001434326172, "logps/rejected": -360.5684814453125, "loss": 0.9084, "margin_dpo/margin_mean": 98.20915222167969, "margin_dpo/margin_std": 94.13998413085938, "step": 465 }, { "epoch": 0.684287812041116, "fcm_dpo/beta": 0.006858200766146183, "fcm_dpo/delta": -0.10749548673629761, "fcm_dpo/margin": 102.01173400878906, "fcm_dpo/q_t": 0.3455418348312378, "grad_norm": 8.609355926513672, "learning_rate": 1.3856541105586545e-07, "logits/chosen": -1.733262300491333, "logits/rejected": -1.417356252670288, "logps/chosen": -213.08628845214844, "logps/ref_chosen": -84.70140838623047, "logps/ref_rejected": -96.05084228515625, "logps/rejected": -326.44744873046875, "loss": 0.9046, "margin_dpo/margin_mean": 102.01173400878906, "margin_dpo/margin_std": 98.38445281982422, "step": 466 }, { "epoch": 0.6857562408223201, "fcm_dpo/beta": 0.006808539852499962, "fcm_dpo/delta": 0.01279231533408165, "fcm_dpo/margin": 86.26240539550781, "fcm_dpo/q_t": 0.3721468448638916, "grad_norm": 13.353436470031738, "learning_rate": 1.3741809409947729e-07, "logits/chosen": -2.1667706966400146, "logits/rejected": -1.576673984527588, "logps/chosen": -269.39898681640625, "logps/ref_chosen": -109.29832458496094, "logps/ref_rejected": -108.8436508178711, "logps/rejected": -355.20672607421875, "loss": 1.0157, "margin_dpo/margin_mean": 86.26240539550781, "margin_dpo/margin_std": 111.6978759765625, "step": 467 }, { "epoch": 0.6872246696035242, "fcm_dpo/beta": 0.006919668987393379, "fcm_dpo/delta": 0.07496071606874466, "fcm_dpo/margin": 76.45877075195312, "fcm_dpo/q_t": 0.38796383142471313, "grad_norm": 11.61962604522705, "learning_rate": 1.362737437810114e-07, "logits/chosen": -1.8730924129486084, "logits/rejected": -1.6779046058654785, "logps/chosen": -226.63394165039062, "logps/ref_chosen": -98.32164764404297, "logps/ref_rejected": -106.68048095703125, "logps/rejected": -311.4515380859375, "loss": 1.0509, "margin_dpo/margin_mean": 76.45877075195312, "margin_dpo/margin_std": 109.24627685546875, "step": 468 }, { "epoch": 0.6886930983847284, "fcm_dpo/beta": 0.006858726963400841, "fcm_dpo/delta": -0.05379898101091385, "fcm_dpo/margin": 94.6535873413086, "fcm_dpo/q_t": 0.35775983333587646, "grad_norm": 13.751016616821289, "learning_rate": 1.351323902551631e-07, "logits/chosen": -2.398892879486084, "logits/rejected": -1.9867908954620361, "logps/chosen": -252.68528747558594, "logps/ref_chosen": -96.76420593261719, "logps/ref_rejected": -109.59500885009766, "logps/rejected": -360.169677734375, "loss": 0.9539, "margin_dpo/margin_mean": 94.6535873413086, "margin_dpo/margin_std": 103.79216003417969, "step": 469 }, { "epoch": 0.6901615271659325, "fcm_dpo/beta": 0.00688820518553257, "fcm_dpo/delta": 0.0024176451843231916, "fcm_dpo/margin": 86.77494812011719, "fcm_dpo/q_t": 0.3669547140598297, "grad_norm": 10.803339004516602, "learning_rate": 1.339940635976592e-07, "logits/chosen": -1.6873784065246582, "logits/rejected": -1.2658147811889648, "logps/chosen": -228.20758056640625, "logps/ref_chosen": -83.49665832519531, "logps/ref_rejected": -88.48578643798828, "logps/rejected": -319.9716796875, "loss": 0.962, "margin_dpo/margin_mean": 86.77494812011719, "margin_dpo/margin_std": 90.58299255371094, "step": 470 }, { "epoch": 0.6916299559471366, "fcm_dpo/beta": 0.00690112542361021, "fcm_dpo/delta": 0.02034003473818302, "fcm_dpo/margin": 84.1663589477539, "fcm_dpo/q_t": 0.36976000666618347, "grad_norm": 11.01422119140625, "learning_rate": 1.3285879380446563e-07, "logits/chosen": -1.8118414878845215, "logits/rejected": -1.4235670566558838, "logps/chosen": -255.23658752441406, "logps/ref_chosen": -88.47430419921875, "logps/ref_rejected": -90.48171997070312, "logps/rejected": -341.41033935546875, "loss": 0.973, "margin_dpo/margin_mean": 84.16635131835938, "margin_dpo/margin_std": 89.08399963378906, "step": 471 }, { "epoch": 0.6930983847283406, "fcm_dpo/beta": 0.006775230169296265, "fcm_dpo/delta": -0.1039826050400734, "fcm_dpo/margin": 102.69633483886719, "fcm_dpo/q_t": 0.3456118404865265, "grad_norm": 9.285819053649902, "learning_rate": 1.317266107909975e-07, "logits/chosen": -2.095188617706299, "logits/rejected": -1.393007516860962, "logps/chosen": -276.9514465332031, "logps/ref_chosen": -125.23370361328125, "logps/ref_rejected": -121.05349731445312, "logps/rejected": -375.46759033203125, "loss": 0.8906, "margin_dpo/margin_mean": 102.69633483886719, "margin_dpo/margin_std": 90.67658233642578, "step": 472 }, { "epoch": 0.6945668135095447, "fcm_dpo/beta": 0.0069533102214336395, "fcm_dpo/delta": 0.16166609525680542, "fcm_dpo/margin": 64.00907897949219, "fcm_dpo/q_t": 0.4053763747215271, "grad_norm": 12.549628257751465, "learning_rate": 1.3059754439133002e-07, "logits/chosen": -1.8435192108154297, "logits/rejected": -1.3511978387832642, "logps/chosen": -268.58892822265625, "logps/ref_chosen": -95.61137390136719, "logps/ref_rejected": -88.15115356445312, "logps/rejected": -325.1377868652344, "loss": 1.1087, "margin_dpo/margin_mean": 64.00907897949219, "margin_dpo/margin_std": 105.01445770263672, "step": 473 }, { "epoch": 0.6960352422907489, "fcm_dpo/beta": 0.007215796038508415, "fcm_dpo/delta": 0.14754922688007355, "fcm_dpo/margin": 63.232147216796875, "fcm_dpo/q_t": 0.4007920026779175, "grad_norm": 14.762158393859863, "learning_rate": 1.2947162435741277e-07, "logits/chosen": -1.478015422821045, "logits/rejected": -1.284541368484497, "logps/chosen": -245.74856567382812, "logps/ref_chosen": -81.47975158691406, "logps/ref_rejected": -96.46562957763672, "logps/rejected": -323.9665832519531, "loss": 1.112, "margin_dpo/margin_mean": 63.23215103149414, "margin_dpo/margin_std": 100.12031555175781, "step": 474 }, { "epoch": 0.697503671071953, "fcm_dpo/beta": 0.007110551930963993, "fcm_dpo/delta": -0.09296688437461853, "fcm_dpo/margin": 96.56515502929688, "fcm_dpo/q_t": 0.35164913535118103, "grad_norm": 10.715070724487305, "learning_rate": 1.2834888035828596e-07, "logits/chosen": -2.3761649131774902, "logits/rejected": -2.3209524154663086, "logps/chosen": -197.6427001953125, "logps/ref_chosen": -74.19598388671875, "logps/ref_rejected": -94.69242095947266, "logps/rejected": -314.70428466796875, "loss": 0.9244, "margin_dpo/margin_mean": 96.56515502929688, "margin_dpo/margin_std": 101.15115356445312, "step": 475 }, { "epoch": 0.6989720998531571, "fcm_dpo/beta": 0.007045960985124111, "fcm_dpo/delta": 0.005972862243652344, "fcm_dpo/margin": 84.30451965332031, "fcm_dpo/q_t": 0.3728053569793701, "grad_norm": 14.778757095336914, "learning_rate": 1.2722934197929802e-07, "logits/chosen": -2.1326091289520264, "logits/rejected": -1.7579033374786377, "logps/chosen": -218.2198944091797, "logps/ref_chosen": -71.97109985351562, "logps/ref_rejected": -80.26224517822266, "logps/rejected": -310.8155517578125, "loss": 0.9929, "margin_dpo/margin_mean": 84.30451965332031, "margin_dpo/margin_std": 102.45613098144531, "step": 476 }, { "epoch": 0.7004405286343612, "fcm_dpo/beta": 0.007064092438668013, "fcm_dpo/delta": -0.026253901422023773, "fcm_dpo/margin": 88.41944885253906, "fcm_dpo/q_t": 0.3624919652938843, "grad_norm": 11.6608304977417, "learning_rate": 1.2611303872132631e-07, "logits/chosen": -2.2218666076660156, "logits/rejected": -1.326930284500122, "logps/chosen": -249.8019561767578, "logps/ref_chosen": -105.00555419921875, "logps/ref_rejected": -81.87843322753906, "logps/rejected": -315.09429931640625, "loss": 0.9793, "margin_dpo/margin_mean": 88.41944885253906, "margin_dpo/margin_std": 104.81526947021484, "step": 477 }, { "epoch": 0.7019089574155654, "fcm_dpo/beta": 0.007006206549704075, "fcm_dpo/delta": -0.0406154990196228, "fcm_dpo/margin": 91.0680923461914, "fcm_dpo/q_t": 0.3596052825450897, "grad_norm": 10.416852951049805, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.7539746761322021, "logits/rejected": -1.58416748046875, "logps/chosen": -215.23123168945312, "logps/ref_chosen": -76.7882080078125, "logps/ref_rejected": -90.43994140625, "logps/rejected": -319.9510498046875, "loss": 0.9565, "margin_dpo/margin_mean": 91.06809997558594, "margin_dpo/margin_std": 100.75262451171875, "step": 478 }, { "epoch": 0.7033773861967695, "fcm_dpo/beta": 0.007025442086160183, "fcm_dpo/delta": 0.00436464324593544, "fcm_dpo/margin": 84.7488784790039, "fcm_dpo/q_t": 0.36682647466659546, "grad_norm": 13.093260765075684, "learning_rate": 1.2389025514492456e-07, "logits/chosen": -1.6674690246582031, "logits/rejected": -1.5295199155807495, "logps/chosen": -226.80955505371094, "logps/ref_chosen": -81.3623046875, "logps/ref_rejected": -101.09114074707031, "logps/rejected": -331.2872619628906, "loss": 0.9753, "margin_dpo/margin_mean": 84.7488784790039, "margin_dpo/margin_std": 92.19709014892578, "step": 479 }, { "epoch": 0.7048458149779736, "fcm_dpo/beta": 0.007094934582710266, "fcm_dpo/delta": 0.09958771616220474, "fcm_dpo/margin": 71.2434310913086, "fcm_dpo/q_t": 0.3897866904735565, "grad_norm": 13.344147682189941, "learning_rate": 1.227838333989088e-07, "logits/chosen": -1.7132782936096191, "logits/rejected": -1.0284159183502197, "logps/chosen": -262.8040771484375, "logps/ref_chosen": -96.7739028930664, "logps/ref_rejected": -86.40473937988281, "logps/rejected": -323.6783447265625, "loss": 1.0558, "margin_dpo/margin_mean": 71.2434310913086, "margin_dpo/margin_std": 99.02870178222656, "step": 480 }, { "epoch": 0.7063142437591777, "fcm_dpo/beta": 0.007131170481443405, "fcm_dpo/delta": 0.0027018017135560513, "fcm_dpo/margin": 83.7805404663086, "fcm_dpo/q_t": 0.3683137893676758, "grad_norm": 14.994175910949707, "learning_rate": 1.2168076391719489e-07, "logits/chosen": -2.007399559020996, "logits/rejected": -1.5706861019134521, "logps/chosen": -255.43331909179688, "logps/ref_chosen": -91.670166015625, "logps/ref_rejected": -98.69490051269531, "logps/rejected": -346.2386169433594, "loss": 0.9915, "margin_dpo/margin_mean": 83.7805404663086, "margin_dpo/margin_std": 100.96444702148438, "step": 481 }, { "epoch": 0.7077826725403817, "fcm_dpo/beta": 0.00711690541356802, "fcm_dpo/delta": 0.00016288645565509796, "fcm_dpo/margin": 84.26334381103516, "fcm_dpo/q_t": 0.3666335344314575, "grad_norm": 14.931113243103027, "learning_rate": 1.2058107576668938e-07, "logits/chosen": -1.6981770992279053, "logits/rejected": -1.342320442199707, "logps/chosen": -254.57460021972656, "logps/ref_chosen": -98.52011108398438, "logps/ref_rejected": -94.82945251464844, "logps/rejected": -335.14727783203125, "loss": 0.9789, "margin_dpo/margin_mean": 84.26334381103516, "margin_dpo/margin_std": 95.54747009277344, "step": 482 }, { "epoch": 0.7092511013215859, "fcm_dpo/beta": 0.007158739026635885, "fcm_dpo/delta": -0.03249619901180267, "fcm_dpo/margin": 87.9349365234375, "fcm_dpo/q_t": 0.3601437211036682, "grad_norm": 9.465551376342773, "learning_rate": 1.194847979251979e-07, "logits/chosen": -1.9062191247940063, "logits/rejected": -1.2519001960754395, "logps/chosen": -269.0156555175781, "logps/ref_chosen": -107.11860656738281, "logps/ref_rejected": -101.11499786376953, "logps/rejected": -350.9469909667969, "loss": 0.9553, "margin_dpo/margin_mean": 87.9349365234375, "margin_dpo/margin_std": 92.60682678222656, "step": 483 }, { "epoch": 0.71071953010279, "fcm_dpo/beta": 0.007042921148240566, "fcm_dpo/delta": -0.07625816017389297, "fcm_dpo/margin": 95.28274536132812, "fcm_dpo/q_t": 0.3532974123954773, "grad_norm": 10.351212501525879, "learning_rate": 1.1839195928066101e-07, "logits/chosen": -2.5401973724365234, "logits/rejected": -2.068960189819336, "logps/chosen": -238.0645751953125, "logps/ref_chosen": -86.97991943359375, "logps/ref_rejected": -90.72367095947266, "logps/rejected": -337.091064453125, "loss": 0.9457, "margin_dpo/margin_mean": 95.28274536132812, "margin_dpo/margin_std": 104.58644104003906, "step": 484 }, { "epoch": 0.7121879588839941, "fcm_dpo/beta": 0.006840351969003677, "fcm_dpo/delta": -0.14380024373531342, "fcm_dpo/margin": 107.2056884765625, "fcm_dpo/q_t": 0.3391224145889282, "grad_norm": 10.769954681396484, "learning_rate": 1.1730258863039347e-07, "logits/chosen": -1.80538809299469, "logits/rejected": -1.378275990486145, "logps/chosen": -233.96734619140625, "logps/ref_chosen": -94.05874633789062, "logps/ref_rejected": -108.56298828125, "logps/rejected": -355.67724609375, "loss": 0.881, "margin_dpo/margin_mean": 107.2056884765625, "margin_dpo/margin_std": 98.861328125, "step": 485 }, { "epoch": 0.7136563876651982, "fcm_dpo/beta": 0.006765207275748253, "fcm_dpo/delta": -0.04138827323913574, "fcm_dpo/margin": 94.40589904785156, "fcm_dpo/q_t": 0.3596769869327545, "grad_norm": 12.390087127685547, "learning_rate": 1.1621671468032493e-07, "logits/chosen": -1.9631407260894775, "logits/rejected": -1.2379772663116455, "logps/chosen": -255.2804718017578, "logps/ref_chosen": -93.74588012695312, "logps/ref_rejected": -98.07064819335938, "logps/rejected": -354.0111389160156, "loss": 0.9607, "margin_dpo/margin_mean": 94.40589904785156, "margin_dpo/margin_std": 104.61402893066406, "step": 486 }, { "epoch": 0.7151248164464024, "fcm_dpo/beta": 0.006735553964972496, "fcm_dpo/delta": 0.016399431973695755, "fcm_dpo/margin": 86.7838134765625, "fcm_dpo/q_t": 0.3709105849266052, "grad_norm": 11.133604049682617, "learning_rate": 1.1513436604424378e-07, "logits/chosen": -1.875716209411621, "logits/rejected": -1.5496511459350586, "logps/chosen": -243.07168579101562, "logps/ref_chosen": -88.0335693359375, "logps/ref_rejected": -98.47209930419922, "logps/rejected": -340.2940673828125, "loss": 0.9787, "margin_dpo/margin_mean": 86.78382110595703, "margin_dpo/margin_std": 96.44297790527344, "step": 487 }, { "epoch": 0.7165932452276065, "fcm_dpo/beta": 0.006739957258105278, "fcm_dpo/delta": -0.01644238643348217, "fcm_dpo/margin": 91.31090545654297, "fcm_dpo/q_t": 0.3608691692352295, "grad_norm": 12.61165714263916, "learning_rate": 1.1405557124304335e-07, "logits/chosen": -1.6958119869232178, "logits/rejected": -1.4399720430374146, "logps/chosen": -229.36561584472656, "logps/ref_chosen": -84.78964233398438, "logps/ref_rejected": -90.2734603881836, "logps/rejected": -326.16033935546875, "loss": 0.9391, "margin_dpo/margin_mean": 91.3109130859375, "margin_dpo/margin_std": 84.56864929199219, "step": 488 }, { "epoch": 0.7180616740088106, "fcm_dpo/beta": 0.006757946684956551, "fcm_dpo/delta": 0.025971170514822006, "fcm_dpo/margin": 85.16073608398438, "fcm_dpo/q_t": 0.3735937476158142, "grad_norm": 11.42379093170166, "learning_rate": 1.1298035870396985e-07, "logits/chosen": -1.6805338859558105, "logits/rejected": -1.1919324398040771, "logps/chosen": -250.7588653564453, "logps/ref_chosen": -90.46929931640625, "logps/ref_rejected": -86.39761352539062, "logps/rejected": -331.847900390625, "loss": 0.9907, "margin_dpo/margin_mean": 85.16073608398438, "margin_dpo/margin_std": 98.77789306640625, "step": 489 }, { "epoch": 0.7195301027900147, "fcm_dpo/beta": 0.006898378022015095, "fcm_dpo/delta": 0.06882701814174652, "fcm_dpo/margin": 77.1955795288086, "fcm_dpo/q_t": 0.38805437088012695, "grad_norm": 12.629081726074219, "learning_rate": 1.1190875675987355e-07, "logits/chosen": -2.1147122383117676, "logits/rejected": -2.184906482696533, "logps/chosen": -248.49630737304688, "logps/ref_chosen": -85.32012939453125, "logps/ref_rejected": -115.99385070800781, "logps/rejected": -356.3656311035156, "loss": 1.091, "margin_dpo/margin_mean": 77.1955795288086, "margin_dpo/margin_std": 123.41070556640625, "step": 490 }, { "epoch": 0.7209985315712188, "fcm_dpo/beta": 0.006943402346223593, "fcm_dpo/delta": 0.0842859223484993, "fcm_dpo/margin": 74.90750122070312, "fcm_dpo/q_t": 0.38507401943206787, "grad_norm": 11.258004188537598, "learning_rate": 1.1084079364846241e-07, "logits/chosen": -1.9617629051208496, "logits/rejected": -1.4733606576919556, "logps/chosen": -249.405517578125, "logps/ref_chosen": -86.14352416992188, "logps/ref_rejected": -80.67945861816406, "logps/rejected": -318.8489685058594, "loss": 1.0404, "margin_dpo/margin_mean": 74.9074935913086, "margin_dpo/margin_std": 99.26988220214844, "step": 491 }, { "epoch": 0.7224669603524229, "fcm_dpo/beta": 0.0071650012396276, "fcm_dpo/delta": 0.1844560205936432, "fcm_dpo/margin": 59.10064697265625, "fcm_dpo/q_t": 0.40985268354415894, "grad_norm": 16.328256607055664, "learning_rate": 1.097764975115576e-07, "logits/chosen": -2.2341504096984863, "logits/rejected": -1.856196641921997, "logps/chosen": -232.42469787597656, "logps/ref_chosen": -81.10757446289062, "logps/ref_rejected": -80.75199890136719, "logps/rejected": -291.1697692871094, "loss": 1.1711, "margin_dpo/margin_mean": 59.10065460205078, "margin_dpo/margin_std": 119.12300872802734, "step": 492 }, { "epoch": 0.723935389133627, "fcm_dpo/beta": 0.007225475274026394, "fcm_dpo/delta": 0.03164070099592209, "fcm_dpo/margin": 78.82869720458984, "fcm_dpo/q_t": 0.37277752161026, "grad_norm": 15.412538528442383, "learning_rate": 1.0871589639435203e-07, "logits/chosen": -2.5492968559265137, "logits/rejected": -1.8315892219543457, "logps/chosen": -262.19671630859375, "logps/ref_chosen": -112.20733642578125, "logps/ref_rejected": -93.60719299316406, "logps/rejected": -322.42523193359375, "loss": 0.9933, "margin_dpo/margin_mean": 78.82870483398438, "margin_dpo/margin_std": 88.707275390625, "step": 493 }, { "epoch": 0.7254038179148311, "fcm_dpo/beta": 0.007072390988469124, "fcm_dpo/delta": -0.1455509066581726, "fcm_dpo/margin": 103.59664916992188, "fcm_dpo/q_t": 0.3390905261039734, "grad_norm": 12.173338890075684, "learning_rate": 1.0765901824467166e-07, "logits/chosen": -1.4517457485198975, "logits/rejected": -1.3680830001831055, "logps/chosen": -218.484375, "logps/ref_chosen": -73.11489868164062, "logps/ref_rejected": -92.16300201416016, "logps/rejected": -341.1291198730469, "loss": 0.8945, "margin_dpo/margin_mean": 103.59664916992188, "margin_dpo/margin_std": 99.80902862548828, "step": 494 }, { "epoch": 0.7268722466960352, "fcm_dpo/beta": 0.0070437416434288025, "fcm_dpo/delta": -0.02011135406792164, "fcm_dpo/margin": 87.85247802734375, "fcm_dpo/q_t": 0.36548200249671936, "grad_norm": 12.353002548217773, "learning_rate": 1.0660589091223854e-07, "logits/chosen": -2.263014316558838, "logits/rejected": -1.8048030138015747, "logps/chosen": -248.50387573242188, "logps/ref_chosen": -99.52031707763672, "logps/ref_rejected": -97.93089294433594, "logps/rejected": -334.76690673828125, "loss": 0.9874, "margin_dpo/margin_mean": 87.85247802734375, "margin_dpo/margin_std": 108.39643859863281, "step": 495 }, { "epoch": 0.7283406754772394, "fcm_dpo/beta": 0.007146789226680994, "fcm_dpo/delta": 0.0695204809308052, "fcm_dpo/margin": 74.6668701171875, "fcm_dpo/q_t": 0.3810140788555145, "grad_norm": 11.570090293884277, "learning_rate": 1.0555654214793722e-07, "logits/chosen": -2.214664936065674, "logits/rejected": -1.5754952430725098, "logps/chosen": -272.01220703125, "logps/ref_chosen": -107.85675048828125, "logps/ref_rejected": -92.77056121826172, "logps/rejected": -331.5928955078125, "loss": 1.0234, "margin_dpo/margin_mean": 74.6668701171875, "margin_dpo/margin_std": 91.9163818359375, "step": 496 }, { "epoch": 0.7298091042584435, "fcm_dpo/beta": 0.007371243089437485, "fcm_dpo/delta": 0.22922512888908386, "fcm_dpo/margin": 51.54722595214844, "fcm_dpo/q_t": 0.4190720319747925, "grad_norm": 11.735376358032227, "learning_rate": 1.0451099960308374e-07, "logits/chosen": -1.9056165218353271, "logits/rejected": -1.4678566455841064, "logps/chosen": -271.3009033203125, "logps/ref_chosen": -92.08322143554688, "logps/ref_rejected": -81.79503631591797, "logps/rejected": -312.5599365234375, "loss": 1.1688, "margin_dpo/margin_mean": 51.54722213745117, "margin_dpo/margin_std": 99.53231048583984, "step": 497 }, { "epoch": 0.7312775330396476, "fcm_dpo/beta": 0.007443058304488659, "fcm_dpo/delta": -0.029292959719896317, "fcm_dpo/margin": 84.29811096191406, "fcm_dpo/q_t": 0.3616979122161865, "grad_norm": 13.830591201782227, "learning_rate": 1.0346929082869641e-07, "logits/chosen": -1.9414262771606445, "logits/rejected": -1.4592478275299072, "logps/chosen": -244.99317932128906, "logps/ref_chosen": -98.19437408447266, "logps/ref_rejected": -90.68745422363281, "logps/rejected": -321.7843933105469, "loss": 0.9608, "margin_dpo/margin_mean": 84.29811096191406, "margin_dpo/margin_std": 93.23235321044922, "step": 498 }, { "epoch": 0.7327459618208517, "fcm_dpo/beta": 0.0074178483337163925, "fcm_dpo/delta": -0.029103467240929604, "fcm_dpo/margin": 84.56077575683594, "fcm_dpo/q_t": 0.36542099714279175, "grad_norm": 15.131891250610352, "learning_rate": 1.0243144327477013e-07, "logits/chosen": -1.9305214881896973, "logits/rejected": -1.8768131732940674, "logps/chosen": -239.75906372070312, "logps/ref_chosen": -81.0399169921875, "logps/ref_rejected": -106.92170715332031, "logps/rejected": -350.2016296386719, "loss": 0.992, "margin_dpo/margin_mean": 84.56077575683594, "margin_dpo/margin_std": 106.96080017089844, "step": 499 }, { "epoch": 0.7342143906020558, "fcm_dpo/beta": 0.007414363324642181, "fcm_dpo/delta": 0.03089684061706066, "fcm_dpo/margin": 76.994140625, "fcm_dpo/q_t": 0.381319522857666, "grad_norm": 14.555001258850098, "learning_rate": 1.0139748428955333e-07, "logits/chosen": -1.7026747465133667, "logits/rejected": -1.5297117233276367, "logps/chosen": -260.76824951171875, "logps/ref_chosen": -89.248046875, "logps/ref_rejected": -100.41021728515625, "logps/rejected": -348.9245300292969, "loss": 1.0372, "margin_dpo/margin_mean": 76.994140625, "margin_dpo/margin_std": 109.05560302734375, "step": 500 }, { "epoch": 0.7342143906020558, "eval_fcm_dpo/beta": 0.0074382987804710865, "eval_logits/chosen": -1.973371148109436, "eval_logits/rejected": -1.5978766679763794, "eval_logps/chosen": -264.56976318359375, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -330.363525390625, "eval_loss": 0.5229191184043884, "eval_margin_dpo/margin_mean": 72.21954345703125, "eval_margin_dpo/margin_std": 99.1610107421875, "eval_runtime": 43.7228, "eval_samples_per_second": 53.496, "eval_steps_per_second": 1.692, "step": 500 }, { "epoch": 0.73568281938326, "fcm_dpo/beta": 0.007509919814765453, "fcm_dpo/delta": -0.011263996362686157, "fcm_dpo/margin": 80.9957046508789, "fcm_dpo/q_t": 0.3736144006252289, "grad_norm": 14.008468627929688, "learning_rate": 1.0036744111882672e-07, "logits/chosen": -1.6845519542694092, "logits/rejected": -1.2528064250946045, "logps/chosen": -241.5672607421875, "logps/ref_chosen": -90.596923828125, "logps/ref_rejected": -85.2972640991211, "logps/rejected": -317.2633361816406, "loss": 1.0472, "margin_dpo/margin_mean": 80.99571228027344, "margin_dpo/margin_std": 117.20719909667969, "step": 501 }, { "epoch": 0.737151248164464, "fcm_dpo/beta": 0.007532845251262188, "fcm_dpo/delta": 0.05232547968626022, "fcm_dpo/margin": 72.90962982177734, "fcm_dpo/q_t": 0.38146716356277466, "grad_norm": 12.48260498046875, "learning_rate": 9.934134090518592e-08, "logits/chosen": -1.9113637208938599, "logits/rejected": -1.1727447509765625, "logps/chosen": -257.06707763671875, "logps/ref_chosen": -103.47073364257812, "logps/ref_rejected": -89.04415893554688, "logps/rejected": -315.55010986328125, "loss": 1.0651, "margin_dpo/margin_mean": 72.90962982177734, "margin_dpo/margin_std": 107.84392547607422, "step": 502 }, { "epoch": 0.7386196769456681, "fcm_dpo/beta": 0.007508490234613419, "fcm_dpo/delta": -0.042639248073101044, "fcm_dpo/margin": 85.12161254882812, "fcm_dpo/q_t": 0.36175140738487244, "grad_norm": 11.972295761108398, "learning_rate": 9.831921068732571e-08, "logits/chosen": -1.7123284339904785, "logits/rejected": -1.2041406631469727, "logps/chosen": -243.31890869140625, "logps/ref_chosen": -93.72998809814453, "logps/ref_rejected": -87.60896301269531, "logps/rejected": -322.3194885253906, "loss": 0.9774, "margin_dpo/margin_mean": 85.12161254882812, "margin_dpo/margin_std": 100.05804443359375, "step": 503 }, { "epoch": 0.7400881057268722, "fcm_dpo/beta": 0.007257069461047649, "fcm_dpo/delta": -0.17726479470729828, "fcm_dpo/margin": 105.27082824707031, "fcm_dpo/q_t": 0.3317434787750244, "grad_norm": 15.06264877319336, "learning_rate": 9.730107739932805e-08, "logits/chosen": -2.0711169242858887, "logits/rejected": -1.7322639226913452, "logps/chosen": -242.6743927001953, "logps/ref_chosen": -92.2918701171875, "logps/ref_rejected": -109.03765106201172, "logps/rejected": -364.6910095214844, "loss": 0.8702, "margin_dpo/margin_mean": 105.27082824707031, "margin_dpo/margin_std": 96.85734558105469, "step": 504 }, { "epoch": 0.7415565345080763, "fcm_dpo/beta": 0.007209095172584057, "fcm_dpo/delta": 0.027257125824689865, "fcm_dpo/margin": 79.66241455078125, "fcm_dpo/q_t": 0.3737871050834656, "grad_norm": 11.139007568359375, "learning_rate": 9.628696786995188e-08, "logits/chosen": -2.458643674850464, "logits/rejected": -1.9072757959365845, "logps/chosen": -274.4703063964844, "logps/ref_chosen": -106.27693176269531, "logps/ref_rejected": -94.20611572265625, "logps/rejected": -342.0618896484375, "loss": 0.9951, "margin_dpo/margin_mean": 79.66241455078125, "margin_dpo/margin_std": 93.06168365478516, "step": 505 }, { "epoch": 0.7430249632892805, "fcm_dpo/beta": 0.007173755206167698, "fcm_dpo/delta": -0.01147904247045517, "fcm_dpo/margin": 85.08673095703125, "fcm_dpo/q_t": 0.3622966408729553, "grad_norm": 11.042329788208008, "learning_rate": 9.527690882192635e-08, "logits/chosen": -2.1348423957824707, "logits/rejected": -1.7820146083831787, "logps/chosen": -245.61795043945312, "logps/ref_chosen": -81.44086456298828, "logps/ref_rejected": -85.66439819335938, "logps/rejected": -334.92822265625, "loss": 0.9651, "margin_dpo/margin_mean": 85.08673095703125, "margin_dpo/margin_std": 90.8485107421875, "step": 506 }, { "epoch": 0.7444933920704846, "fcm_dpo/beta": 0.007161298766732216, "fcm_dpo/delta": -0.0466289147734642, "fcm_dpo/margin": 89.87873077392578, "fcm_dpo/q_t": 0.3620367646217346, "grad_norm": 11.227087020874023, "learning_rate": 9.427092687124691e-08, "logits/chosen": -2.110520362854004, "logits/rejected": -1.6617255210876465, "logps/chosen": -269.304931640625, "logps/ref_chosen": -99.42694854736328, "logps/ref_rejected": -102.25296020507812, "logps/rejected": -362.00970458984375, "loss": 0.9832, "margin_dpo/margin_mean": 89.87872314453125, "margin_dpo/margin_std": 110.67439270019531, "step": 507 }, { "epoch": 0.7459618208516887, "fcm_dpo/beta": 0.007206980139017105, "fcm_dpo/delta": 0.07583308964967728, "fcm_dpo/margin": 73.293212890625, "fcm_dpo/q_t": 0.3894771933555603, "grad_norm": 15.699549674987793, "learning_rate": 9.326904852647344e-08, "logits/chosen": -2.079139232635498, "logits/rejected": -1.599036455154419, "logps/chosen": -264.4894714355469, "logps/ref_chosen": -97.9156494140625, "logps/ref_rejected": -103.79782104492188, "logps/rejected": -343.66485595703125, "loss": 1.0835, "margin_dpo/margin_mean": 73.293212890625, "margin_dpo/margin_std": 114.59175872802734, "step": 508 }, { "epoch": 0.7474302496328928, "fcm_dpo/beta": 0.007344736717641354, "fcm_dpo/delta": 0.07915890216827393, "fcm_dpo/margin": 71.44049072265625, "fcm_dpo/q_t": 0.3829602003097534, "grad_norm": 17.265066146850586, "learning_rate": 9.227130018803195e-08, "logits/chosen": -2.11910080909729, "logits/rejected": -1.4653677940368652, "logps/chosen": -257.30712890625, "logps/ref_chosen": -100.72984313964844, "logps/ref_rejected": -86.943359375, "logps/rejected": -314.961181640625, "loss": 1.0586, "margin_dpo/margin_mean": 71.44050598144531, "margin_dpo/margin_std": 101.78379821777344, "step": 509 }, { "epoch": 0.748898678414097, "fcm_dpo/beta": 0.0073591480031609535, "fcm_dpo/delta": -0.015946993604302406, "fcm_dpo/margin": 83.5523452758789, "fcm_dpo/q_t": 0.36579430103302, "grad_norm": 14.097626686096191, "learning_rate": 9.127770814751932e-08, "logits/chosen": -1.7049915790557861, "logits/rejected": -1.5141171216964722, "logps/chosen": -272.86090087890625, "logps/ref_chosen": -89.16938781738281, "logps/ref_rejected": -107.91939544677734, "logps/rejected": -375.1632385253906, "loss": 0.9857, "margin_dpo/margin_mean": 83.55235290527344, "margin_dpo/margin_std": 98.11227416992188, "step": 510 }, { "epoch": 0.750367107195301, "fcm_dpo/beta": 0.007278499659150839, "fcm_dpo/delta": -0.07066594809293747, "fcm_dpo/margin": 91.5009765625, "fcm_dpo/q_t": 0.3567890226840973, "grad_norm": 12.136595726013184, "learning_rate": 9.028829858700973e-08, "logits/chosen": -2.2216238975524902, "logits/rejected": -1.6600124835968018, "logps/chosen": -255.50558471679688, "logps/ref_chosen": -89.74979400634766, "logps/ref_rejected": -97.73127746582031, "logps/rejected": -354.988037109375, "loss": 0.9465, "margin_dpo/margin_mean": 91.5009765625, "margin_dpo/margin_std": 102.25430297851562, "step": 511 }, { "epoch": 0.7518355359765051, "fcm_dpo/beta": 0.0072515252977609634, "fcm_dpo/delta": -0.02720758691430092, "fcm_dpo/margin": 86.15798950195312, "fcm_dpo/q_t": 0.3647354543209076, "grad_norm": 16.666259765625, "learning_rate": 8.930309757836516e-08, "logits/chosen": -1.8731253147125244, "logits/rejected": -1.4305176734924316, "logps/chosen": -244.8845977783203, "logps/ref_chosen": -85.73515319824219, "logps/ref_rejected": -87.6248779296875, "logps/rejected": -332.93231201171875, "loss": 0.9965, "margin_dpo/margin_mean": 86.1579818725586, "margin_dpo/margin_std": 108.06047058105469, "step": 512 }, { "epoch": 0.7533039647577092, "fcm_dpo/beta": 0.007211950141936541, "fcm_dpo/delta": 0.06090360879898071, "fcm_dpo/margin": 75.15151977539062, "fcm_dpo/q_t": 0.37852686643600464, "grad_norm": 12.995072364807129, "learning_rate": 8.832213108254863e-08, "logits/chosen": -2.311227798461914, "logits/rejected": -1.6142702102661133, "logps/chosen": -264.4798583984375, "logps/ref_chosen": -96.19050598144531, "logps/ref_rejected": -82.91773986816406, "logps/rejected": -326.3586120605469, "loss": 1.0292, "margin_dpo/margin_mean": 75.15151977539062, "margin_dpo/margin_std": 95.23574829101562, "step": 513 }, { "epoch": 0.7547723935389133, "fcm_dpo/beta": 0.007290856912732124, "fcm_dpo/delta": 0.017535630613565445, "fcm_dpo/margin": 80.029296875, "fcm_dpo/q_t": 0.37469518184661865, "grad_norm": 14.053443908691406, "learning_rate": 8.734542494893954e-08, "logits/chosen": -1.6758110523223877, "logits/rejected": -1.0973663330078125, "logps/chosen": -259.1905822753906, "logps/ref_chosen": -97.82363891601562, "logps/ref_rejected": -85.36982727050781, "logps/rejected": -326.76605224609375, "loss": 1.0028, "margin_dpo/margin_mean": 80.029296875, "margin_dpo/margin_std": 99.76383972167969, "step": 514 }, { "epoch": 0.7562408223201175, "fcm_dpo/beta": 0.00737146707251668, "fcm_dpo/delta": 0.06299495697021484, "fcm_dpo/margin": 73.31365203857422, "fcm_dpo/q_t": 0.381864994764328, "grad_norm": 17.193218231201172, "learning_rate": 8.637300491465272e-08, "logits/chosen": -1.7850375175476074, "logits/rejected": -1.5526132583618164, "logps/chosen": -240.5751953125, "logps/ref_chosen": -81.93355560302734, "logps/ref_rejected": -93.57656860351562, "logps/rejected": -325.5318603515625, "loss": 1.0369, "margin_dpo/margin_mean": 73.31365203857422, "margin_dpo/margin_std": 97.8846206665039, "step": 515 }, { "epoch": 0.7577092511013216, "fcm_dpo/beta": 0.00745687261223793, "fcm_dpo/delta": 0.0405082106590271, "fcm_dpo/margin": 75.3163833618164, "fcm_dpo/q_t": 0.3833583891391754, "grad_norm": 13.243304252624512, "learning_rate": 8.540489660386064e-08, "logits/chosen": -2.4003708362579346, "logits/rejected": -2.211030960083008, "logps/chosen": -257.3096923828125, "logps/ref_chosen": -97.09166717529297, "logps/ref_rejected": -118.55667114257812, "logps/rejected": -354.0910949707031, "loss": 1.0819, "margin_dpo/margin_mean": 75.3163833618164, "margin_dpo/margin_std": 121.37615966796875, "step": 516 }, { "epoch": 0.7591776798825257, "fcm_dpo/beta": 0.007305598817765713, "fcm_dpo/delta": -0.07763826102018356, "fcm_dpo/margin": 91.7294921875, "fcm_dpo/q_t": 0.3567124009132385, "grad_norm": 12.040420532226562, "learning_rate": 8.444112552711752e-08, "logits/chosen": -1.9732433557510376, "logits/rejected": -1.5516693592071533, "logps/chosen": -264.7029113769531, "logps/ref_chosen": -93.18656921386719, "logps/ref_rejected": -95.5927963256836, "logps/rejected": -358.838623046875, "loss": 0.9666, "margin_dpo/margin_mean": 91.7294921875, "margin_dpo/margin_std": 106.93476104736328, "step": 517 }, { "epoch": 0.7606461086637298, "fcm_dpo/beta": 0.007446288131177425, "fcm_dpo/delta": 0.09759702533483505, "fcm_dpo/margin": 68.14192199707031, "fcm_dpo/q_t": 0.3923153877258301, "grad_norm": 15.57699966430664, "learning_rate": 8.348171708068747e-08, "logits/chosen": -2.0085532665252686, "logits/rejected": -1.6788051128387451, "logps/chosen": -266.467041015625, "logps/ref_chosen": -87.0213623046875, "logps/ref_rejected": -91.24456024169922, "logps/rejected": -338.8321533203125, "loss": 1.0852, "margin_dpo/margin_mean": 68.14192199707031, "margin_dpo/margin_std": 108.119873046875, "step": 518 }, { "epoch": 0.762114537444934, "fcm_dpo/beta": 0.007489525713026524, "fcm_dpo/delta": 0.032412007451057434, "fcm_dpo/margin": 75.99348449707031, "fcm_dpo/q_t": 0.37510281801223755, "grad_norm": 12.590337753295898, "learning_rate": 8.25266965458755e-08, "logits/chosen": -1.9630098342895508, "logits/rejected": -1.5859870910644531, "logps/chosen": -266.3951416015625, "logps/ref_chosen": -100.68755340576172, "logps/ref_rejected": -109.03042602539062, "logps/rejected": -350.73150634765625, "loss": 0.9882, "margin_dpo/margin_mean": 75.99348449707031, "margin_dpo/margin_std": 85.33174133300781, "step": 519 }, { "epoch": 0.7635829662261381, "fcm_dpo/beta": 0.007443825248628855, "fcm_dpo/delta": -0.06601864099502563, "fcm_dpo/margin": 88.84642028808594, "fcm_dpo/q_t": 0.3531188368797302, "grad_norm": 11.373416900634766, "learning_rate": 8.15760890883607e-08, "logits/chosen": -1.54036545753479, "logits/rejected": -1.0939218997955322, "logps/chosen": -250.6941375732422, "logps/ref_chosen": -98.77762603759766, "logps/ref_rejected": -106.9457015991211, "logps/rejected": -347.7086486816406, "loss": 0.9295, "margin_dpo/margin_mean": 88.84642028808594, "margin_dpo/margin_std": 88.97753143310547, "step": 520 }, { "epoch": 0.7650513950073421, "fcm_dpo/beta": 0.0074075451120734215, "fcm_dpo/delta": -0.06705917418003082, "fcm_dpo/margin": 89.41288757324219, "fcm_dpo/q_t": 0.35395270586013794, "grad_norm": 12.826030731201172, "learning_rate": 8.062991975753378e-08, "logits/chosen": -2.1257705688476562, "logits/rejected": -1.7991995811462402, "logps/chosen": -244.10650634765625, "logps/ref_chosen": -86.35108947753906, "logps/ref_rejected": -90.39317321777344, "logps/rejected": -337.56146240234375, "loss": 0.9291, "margin_dpo/margin_mean": 89.41287994384766, "margin_dpo/margin_std": 89.05442810058594, "step": 521 }, { "epoch": 0.7665198237885462, "fcm_dpo/beta": 0.007335927337408066, "fcm_dpo/delta": -0.02064390294253826, "fcm_dpo/margin": 84.41744232177734, "fcm_dpo/q_t": 0.36679452657699585, "grad_norm": 12.274141311645508, "learning_rate": 7.968821348583643e-08, "logits/chosen": -1.7425016164779663, "logits/rejected": -1.3717331886291504, "logps/chosen": -236.16275024414062, "logps/ref_chosen": -76.008056640625, "logps/ref_rejected": -74.44125366210938, "logps/rejected": -319.0133972167969, "loss": 0.9823, "margin_dpo/margin_mean": 84.41744995117188, "margin_dpo/margin_std": 101.350341796875, "step": 522 }, { "epoch": 0.7679882525697503, "fcm_dpo/beta": 0.007474465761333704, "fcm_dpo/delta": 0.16874706745147705, "fcm_dpo/margin": 58.72737121582031, "fcm_dpo/q_t": 0.40729784965515137, "grad_norm": 14.717813491821289, "learning_rate": 7.875099508810484e-08, "logits/chosen": -1.9797134399414062, "logits/rejected": -1.2677710056304932, "logps/chosen": -276.0887451171875, "logps/ref_chosen": -96.49627685546875, "logps/ref_rejected": -91.0152816772461, "logps/rejected": -329.3351135253906, "loss": 1.1446, "margin_dpo/margin_mean": 58.72737503051758, "margin_dpo/margin_std": 108.98553466796875, "step": 523 }, { "epoch": 0.7694566813509545, "fcm_dpo/beta": 0.007531831972301006, "fcm_dpo/delta": -0.0187044907361269, "fcm_dpo/margin": 81.98780822753906, "fcm_dpo/q_t": 0.36835557222366333, "grad_norm": 16.366233825683594, "learning_rate": 7.781828926091535e-08, "logits/chosen": -2.228161334991455, "logits/rejected": -1.5479540824890137, "logps/chosen": -268.5673828125, "logps/ref_chosen": -103.40550994873047, "logps/ref_rejected": -87.91015625, "logps/rejected": -335.059814453125, "loss": 0.9845, "margin_dpo/margin_mean": 81.98780822753906, "margin_dpo/margin_std": 100.14673614501953, "step": 524 }, { "epoch": 0.7709251101321586, "fcm_dpo/beta": 0.007332179695367813, "fcm_dpo/delta": -0.08620290458202362, "fcm_dpo/margin": 92.11900329589844, "fcm_dpo/q_t": 0.3595668077468872, "grad_norm": 12.162001609802246, "learning_rate": 7.689012058193384e-08, "logits/chosen": -1.6582214832305908, "logits/rejected": -1.572461724281311, "logps/chosen": -239.363525390625, "logps/ref_chosen": -84.20648193359375, "logps/ref_rejected": -106.26158905029297, "logps/rejected": -353.53765869140625, "loss": 0.9656, "margin_dpo/margin_mean": 92.11900329589844, "margin_dpo/margin_std": 107.05292510986328, "step": 525 }, { "epoch": 0.7723935389133627, "fcm_dpo/beta": 0.007367183454334736, "fcm_dpo/delta": -0.03810001537203789, "fcm_dpo/margin": 86.28730773925781, "fcm_dpo/q_t": 0.36337071657180786, "grad_norm": 10.43152141571045, "learning_rate": 7.596651350926836e-08, "logits/chosen": -1.8134328126907349, "logits/rejected": -1.1642718315124512, "logps/chosen": -264.8533935546875, "logps/ref_chosen": -102.85411071777344, "logps/ref_rejected": -93.65145874023438, "logps/rejected": -341.9380187988281, "loss": 0.9824, "margin_dpo/margin_mean": 86.28730773925781, "margin_dpo/margin_std": 105.58222961425781, "step": 526 }, { "epoch": 0.7738619676945668, "fcm_dpo/beta": 0.007302050478756428, "fcm_dpo/delta": -0.04052596166729927, "fcm_dpo/margin": 87.3643798828125, "fcm_dpo/q_t": 0.3573962152004242, "grad_norm": 12.692599296569824, "learning_rate": 7.504749238082414e-08, "logits/chosen": -3.0793709754943848, "logits/rejected": -2.2557575702667236, "logps/chosen": -257.8551025390625, "logps/ref_chosen": -96.76543426513672, "logps/ref_rejected": -85.39518737792969, "logps/rejected": -333.8492431640625, "loss": 0.9283, "margin_dpo/margin_mean": 87.36438751220703, "margin_dpo/margin_std": 82.9266357421875, "step": 527 }, { "epoch": 0.775330396475771, "fcm_dpo/beta": 0.0072532519698143005, "fcm_dpo/delta": -0.04537132754921913, "fcm_dpo/margin": 88.57562255859375, "fcm_dpo/q_t": 0.3604862689971924, "grad_norm": 12.81235122680664, "learning_rate": 7.413308141366254e-08, "logits/chosen": -2.018289089202881, "logits/rejected": -1.5019323825836182, "logps/chosen": -244.18003845214844, "logps/ref_chosen": -94.1649169921875, "logps/ref_rejected": -100.372314453125, "logps/rejected": -338.96307373046875, "loss": 0.9672, "margin_dpo/margin_mean": 88.57562255859375, "margin_dpo/margin_std": 102.981201171875, "step": 528 }, { "epoch": 0.7767988252569751, "fcm_dpo/beta": 0.007266518659889698, "fcm_dpo/delta": 0.034581415355205536, "fcm_dpo/margin": 78.07312774658203, "fcm_dpo/q_t": 0.3804980516433716, "grad_norm": 12.89798641204834, "learning_rate": 7.322330470336313e-08, "logits/chosen": -1.6607023477554321, "logits/rejected": -1.4771153926849365, "logps/chosen": -255.24560546875, "logps/ref_chosen": -80.77680206298828, "logps/ref_rejected": -97.191162109375, "logps/rejected": -349.73309326171875, "loss": 1.0404, "margin_dpo/margin_mean": 78.0731201171875, "margin_dpo/margin_std": 111.84912109375, "step": 529 }, { "epoch": 0.7782672540381792, "fcm_dpo/beta": 0.007262610364705324, "fcm_dpo/delta": 0.023171119391918182, "fcm_dpo/margin": 79.56475830078125, "fcm_dpo/q_t": 0.37278905510902405, "grad_norm": 11.019624710083008, "learning_rate": 7.231818622338822e-08, "logits/chosen": -1.5359987020492554, "logits/rejected": -1.2066240310668945, "logps/chosen": -239.765869140625, "logps/ref_chosen": -79.7757339477539, "logps/ref_rejected": -92.05390167236328, "logps/rejected": -331.6087646484375, "loss": 0.977, "margin_dpo/margin_mean": 79.56475830078125, "margin_dpo/margin_std": 86.05628967285156, "step": 530 }, { "epoch": 0.7797356828193832, "fcm_dpo/beta": 0.007175838574767113, "fcm_dpo/delta": -0.11706428229808807, "fcm_dpo/margin": 98.76272583007812, "fcm_dpo/q_t": 0.3418416380882263, "grad_norm": 11.516433715820312, "learning_rate": 7.141774982445147e-08, "logits/chosen": -1.8055527210235596, "logits/rejected": -1.1960854530334473, "logps/chosen": -235.0717315673828, "logps/ref_chosen": -79.31919860839844, "logps/ref_rejected": -78.10832214355469, "logps/rejected": -332.62359619140625, "loss": 0.8993, "margin_dpo/margin_mean": 98.76272583007812, "margin_dpo/margin_std": 93.84896850585938, "step": 531 }, { "epoch": 0.7812041116005873, "fcm_dpo/beta": 0.006985160522162914, "fcm_dpo/delta": -0.16617754101753235, "fcm_dpo/margin": 107.93487548828125, "fcm_dpo/q_t": 0.334781676530838, "grad_norm": 11.529740333557129, "learning_rate": 7.052201923388953e-08, "logits/chosen": -1.7615710496902466, "logits/rejected": -1.1453497409820557, "logps/chosen": -266.70928955078125, "logps/ref_chosen": -101.63691711425781, "logps/ref_rejected": -93.34539031982422, "logps/rejected": -366.3526611328125, "loss": 0.8779, "margin_dpo/margin_mean": 107.93487548828125, "margin_dpo/margin_std": 99.59811401367188, "step": 532 }, { "epoch": 0.7826725403817915, "fcm_dpo/beta": 0.007072297390550375, "fcm_dpo/delta": 0.15511363744735718, "fcm_dpo/margin": 63.86847686767578, "fcm_dpo/q_t": 0.4039548635482788, "grad_norm": 12.298042297363281, "learning_rate": 6.963101805503646e-08, "logits/chosen": -2.4961955547332764, "logits/rejected": -1.911888837814331, "logps/chosen": -251.64068603515625, "logps/ref_chosen": -90.74664306640625, "logps/ref_rejected": -82.88848876953125, "logps/rejected": -307.6510009765625, "loss": 1.1014, "margin_dpo/margin_mean": 63.86847686767578, "margin_dpo/margin_std": 102.30418395996094, "step": 533 }, { "epoch": 0.7841409691629956, "fcm_dpo/beta": 0.007072807289659977, "fcm_dpo/delta": -0.053309373557567596, "fcm_dpo/margin": 91.88386535644531, "fcm_dpo/q_t": 0.3540825843811035, "grad_norm": 10.521297454833984, "learning_rate": 6.874476976660184e-08, "logits/chosen": -2.0540428161621094, "logits/rejected": -1.6685407161712646, "logps/chosen": -231.44000244140625, "logps/ref_chosen": -86.6308364868164, "logps/ref_rejected": -84.87196350097656, "logps/rejected": -321.56500244140625, "loss": 0.9216, "margin_dpo/margin_mean": 91.88386535644531, "margin_dpo/margin_std": 85.76850128173828, "step": 534 }, { "epoch": 0.7856093979441997, "fcm_dpo/beta": 0.0070062074810266495, "fcm_dpo/delta": -0.07865796238183975, "fcm_dpo/margin": 96.07039642333984, "fcm_dpo/q_t": 0.3533538579940796, "grad_norm": 10.109777450561523, "learning_rate": 6.786329772205246e-08, "logits/chosen": -1.8793187141418457, "logits/rejected": -1.6919231414794922, "logps/chosen": -236.0832061767578, "logps/ref_chosen": -90.34539031982422, "logps/ref_rejected": -100.7779769897461, "logps/rejected": -342.586181640625, "loss": 0.939, "margin_dpo/margin_mean": 96.07039642333984, "margin_dpo/margin_std": 102.51802062988281, "step": 535 }, { "epoch": 0.7870778267254038, "fcm_dpo/beta": 0.006879711989313364, "fcm_dpo/delta": -0.037070631980895996, "fcm_dpo/margin": 92.2296371459961, "fcm_dpo/q_t": 0.3588649034500122, "grad_norm": 10.414005279541016, "learning_rate": 6.698662514899638e-08, "logits/chosen": -1.927714228630066, "logits/rejected": -1.678464651107788, "logps/chosen": -227.02041625976562, "logps/ref_chosen": -78.95956420898438, "logps/ref_rejected": -93.08779907226562, "logps/rejected": -333.3782958984375, "loss": 0.9498, "margin_dpo/margin_mean": 92.22964477539062, "margin_dpo/margin_std": 97.07234954833984, "step": 536 }, { "epoch": 0.788546255506608, "fcm_dpo/beta": 0.006824993994086981, "fcm_dpo/delta": -0.11446214467287064, "fcm_dpo/margin": 103.43373107910156, "fcm_dpo/q_t": 0.34388312697410583, "grad_norm": 9.952900886535645, "learning_rate": 6.611477514857114e-08, "logits/chosen": -1.647277593612671, "logits/rejected": -0.9549525380134583, "logps/chosen": -214.62451171875, "logps/ref_chosen": -86.70939636230469, "logps/ref_rejected": -78.90766906738281, "logps/rejected": -310.25653076171875, "loss": 0.8996, "margin_dpo/margin_mean": 103.43373107910156, "margin_dpo/margin_std": 94.62214660644531, "step": 537 }, { "epoch": 0.7900146842878121, "fcm_dpo/beta": 0.006767315790057182, "fcm_dpo/delta": 0.02974122203886509, "fcm_dpo/margin": 84.50604248046875, "fcm_dpo/q_t": 0.37143373489379883, "grad_norm": 11.350445747375488, "learning_rate": 6.524777069483525e-08, "logits/chosen": -1.9193155765533447, "logits/rejected": -1.3912053108215332, "logps/chosen": -269.3667297363281, "logps/ref_chosen": -98.44214630126953, "logps/ref_rejected": -91.09465026855469, "logps/rejected": -346.5252990722656, "loss": 0.9744, "margin_dpo/margin_mean": 84.50605010986328, "margin_dpo/margin_std": 87.87483215332031, "step": 538 }, { "epoch": 0.7914831130690162, "fcm_dpo/beta": 0.006792771629989147, "fcm_dpo/delta": 0.016240041702985764, "fcm_dpo/margin": 86.07090759277344, "fcm_dpo/q_t": 0.3723304867744446, "grad_norm": 10.20103645324707, "learning_rate": 6.438563463416221e-08, "logits/chosen": -2.425452709197998, "logits/rejected": -1.947152018547058, "logps/chosen": -242.98489379882812, "logps/ref_chosen": -90.93211364746094, "logps/ref_rejected": -97.44546508789062, "logps/rejected": -335.56915283203125, "loss": 1.005, "margin_dpo/margin_mean": 86.07090759277344, "margin_dpo/margin_std": 107.96048736572266, "step": 539 }, { "epoch": 0.7929515418502202, "fcm_dpo/beta": 0.006723560392856598, "fcm_dpo/delta": -0.10000798851251602, "fcm_dpo/margin": 103.08011627197266, "fcm_dpo/q_t": 0.3510266840457916, "grad_norm": 12.132619857788086, "learning_rate": 6.352838968463919e-08, "logits/chosen": -2.134742498397827, "logits/rejected": -1.8956007957458496, "logps/chosen": -265.84393310546875, "logps/ref_chosen": -107.86990356445312, "logps/ref_rejected": -121.39350128173828, "logps/rejected": -382.4476318359375, "loss": 0.9372, "margin_dpo/margin_mean": 103.08011627197266, "margin_dpo/margin_std": 114.15684509277344, "step": 540 }, { "epoch": 0.7944199706314243, "fcm_dpo/beta": 0.00676216883584857, "fcm_dpo/delta": 0.15339398384094238, "fcm_dpo/margin": 67.08287048339844, "fcm_dpo/q_t": 0.4000394940376282, "grad_norm": 11.626500129699707, "learning_rate": 6.267605843546767e-08, "logits/chosen": -2.422119140625, "logits/rejected": -1.958017349243164, "logps/chosen": -283.84246826171875, "logps/ref_chosen": -106.56748962402344, "logps/ref_rejected": -109.06633758544922, "logps/rejected": -353.4242248535156, "loss": 1.0758, "margin_dpo/margin_mean": 67.08287048339844, "margin_dpo/margin_std": 90.2316665649414, "step": 541 }, { "epoch": 0.7958883994126285, "fcm_dpo/beta": 0.006774522364139557, "fcm_dpo/delta": -0.09346829354763031, "fcm_dpo/margin": 101.42529296875, "fcm_dpo/q_t": 0.3474395275115967, "grad_norm": 11.156708717346191, "learning_rate": 6.182866334636888e-08, "logits/chosen": -2.3203277587890625, "logits/rejected": -2.1964328289031982, "logps/chosen": -237.198486328125, "logps/ref_chosen": -87.36929321289062, "logps/ref_rejected": -102.82034301757812, "logps/rejected": -354.0748291015625, "loss": 0.9173, "margin_dpo/margin_mean": 101.42529296875, "margin_dpo/margin_std": 100.42752838134766, "step": 542 }, { "epoch": 0.7973568281938326, "fcm_dpo/beta": 0.006940866820514202, "fcm_dpo/delta": 0.23537762463092804, "fcm_dpo/margin": 53.89453887939453, "fcm_dpo/q_t": 0.422155499458313, "grad_norm": 14.22153377532959, "learning_rate": 6.098622674699147e-08, "logits/chosen": -1.7961983680725098, "logits/rejected": -1.828367829322815, "logps/chosen": -261.8563232421875, "logps/ref_chosen": -87.152587890625, "logps/ref_rejected": -111.53172302246094, "logps/rejected": -340.1300048828125, "loss": 1.1992, "margin_dpo/margin_mean": 53.89453887939453, "margin_dpo/margin_std": 116.86991882324219, "step": 543 }, { "epoch": 0.7988252569750367, "fcm_dpo/beta": 0.007195335812866688, "fcm_dpo/delta": 0.13186031579971313, "fcm_dpo/margin": 65.94219207763672, "fcm_dpo/q_t": 0.3974402844905853, "grad_norm": 13.812807083129883, "learning_rate": 6.01487708363232e-08, "logits/chosen": -2.0716145038604736, "logits/rejected": -1.9726595878601074, "logps/chosen": -265.1961669921875, "logps/ref_chosen": -86.14691162109375, "logps/ref_rejected": -106.3939208984375, "logps/rejected": -351.3853759765625, "loss": 1.0679, "margin_dpo/margin_mean": 65.94218444824219, "margin_dpo/margin_std": 94.23214721679688, "step": 544 }, { "epoch": 0.8002936857562408, "fcm_dpo/beta": 0.007269757799804211, "fcm_dpo/delta": 0.005659213289618492, "fcm_dpo/margin": 81.78932189941406, "fcm_dpo/q_t": 0.37121522426605225, "grad_norm": 12.46208667755127, "learning_rate": 5.9316317682106294e-08, "logits/chosen": -1.712825059890747, "logits/rejected": -1.2983132600784302, "logps/chosen": -258.543701171875, "logps/ref_chosen": -96.42424011230469, "logps/ref_rejected": -100.73405456542969, "logps/rejected": -344.642822265625, "loss": 1.0252, "margin_dpo/margin_mean": 81.7893295288086, "margin_dpo/margin_std": 111.28877258300781, "step": 545 }, { "epoch": 0.801762114537445, "fcm_dpo/beta": 0.007279456127434969, "fcm_dpo/delta": 0.041319385170936584, "fcm_dpo/margin": 77.05686950683594, "fcm_dpo/q_t": 0.37605637311935425, "grad_norm": 12.261453628540039, "learning_rate": 5.848888922025552e-08, "logits/chosen": -1.5237065553665161, "logits/rejected": -1.1611510515213013, "logps/chosen": -252.166259765625, "logps/ref_chosen": -86.02112579345703, "logps/ref_rejected": -87.37263488769531, "logps/rejected": -330.5746154785156, "loss": 1.0274, "margin_dpo/margin_mean": 77.05686950683594, "margin_dpo/margin_std": 102.12185668945312, "step": 546 }, { "epoch": 0.8032305433186491, "fcm_dpo/beta": 0.007295276969671249, "fcm_dpo/delta": -0.05277668312191963, "fcm_dpo/margin": 88.98194885253906, "fcm_dpo/q_t": 0.3542296886444092, "grad_norm": 11.10513973236084, "learning_rate": 5.7666507254280265e-08, "logits/chosen": -1.8882625102996826, "logits/rejected": -1.4319205284118652, "logps/chosen": -260.6878662109375, "logps/ref_chosen": -100.6510009765625, "logps/ref_rejected": -97.9117431640625, "logps/rejected": -346.9305419921875, "loss": 0.9216, "margin_dpo/margin_mean": 88.98194885253906, "margin_dpo/margin_std": 82.17151641845703, "step": 547 }, { "epoch": 0.8046989720998532, "fcm_dpo/beta": 0.00735054537653923, "fcm_dpo/delta": 0.093236543238163, "fcm_dpo/margin": 69.57292175292969, "fcm_dpo/q_t": 0.39055800437927246, "grad_norm": 13.908289909362793, "learning_rate": 5.684919345471029e-08, "logits/chosen": -2.291468858718872, "logits/rejected": -1.938688039779663, "logps/chosen": -255.26414489746094, "logps/ref_chosen": -96.03010559082031, "logps/ref_rejected": -98.41322326660156, "logps/rejected": -327.2201843261719, "loss": 1.1036, "margin_dpo/margin_mean": 69.57292938232422, "margin_dpo/margin_std": 116.0625, "step": 548 }, { "epoch": 0.8061674008810573, "fcm_dpo/beta": 0.007444350048899651, "fcm_dpo/delta": 0.020497113466262817, "fcm_dpo/margin": 77.89366149902344, "fcm_dpo/q_t": 0.3747206926345825, "grad_norm": 14.169367790222168, "learning_rate": 5.603696935852426e-08, "logits/chosen": -1.9140429496765137, "logits/rejected": -1.494286060333252, "logps/chosen": -231.77978515625, "logps/ref_chosen": -83.45095825195312, "logps/ref_rejected": -81.38023376464844, "logps/rejected": -307.60272216796875, "loss": 1.0385, "margin_dpo/margin_mean": 77.89366149902344, "margin_dpo/margin_std": 108.47966003417969, "step": 549 }, { "epoch": 0.8076358296622613, "fcm_dpo/beta": 0.007483618333935738, "fcm_dpo/delta": 0.04548526555299759, "fcm_dpo/margin": 74.3836898803711, "fcm_dpo/q_t": 0.3776836395263672, "grad_norm": 12.159344673156738, "learning_rate": 5.5229856368582376e-08, "logits/chosen": -1.6402368545532227, "logits/rejected": -1.4186840057373047, "logps/chosen": -255.8831329345703, "logps/ref_chosen": -89.59434509277344, "logps/ref_rejected": -101.31549835205078, "logps/rejected": -341.98797607421875, "loss": 1.0083, "margin_dpo/margin_mean": 74.38369750976562, "margin_dpo/margin_std": 88.22019958496094, "step": 550 }, { "epoch": 0.8091042584434655, "fcm_dpo/beta": 0.007342686876654625, "fcm_dpo/delta": -0.06708859652280807, "fcm_dpo/margin": 90.0142593383789, "fcm_dpo/q_t": 0.3585403561592102, "grad_norm": 14.69068717956543, "learning_rate": 5.4427875753062734e-08, "logits/chosen": -1.7167030572891235, "logits/rejected": -1.5703403949737549, "logps/chosen": -241.60235595703125, "logps/ref_chosen": -95.08216094970703, "logps/ref_rejected": -117.7192611694336, "logps/rejected": -354.25372314453125, "loss": 0.942, "margin_dpo/margin_mean": 90.0142593383789, "margin_dpo/margin_std": 97.10916137695312, "step": 551 }, { "epoch": 0.8105726872246696, "fcm_dpo/beta": 0.0069796196185052395, "fcm_dpo/delta": -0.29933837056159973, "fcm_dpo/margin": 124.37358093261719, "fcm_dpo/q_t": 0.31151920557022095, "grad_norm": 10.166218757629395, "learning_rate": 5.363104864490034e-08, "logits/chosen": -2.0635464191436768, "logits/rejected": -1.6969006061553955, "logps/chosen": -239.97976684570312, "logps/ref_chosen": -101.40918731689453, "logps/ref_rejected": -110.83843994140625, "logps/rejected": -373.7825927734375, "loss": 0.8214, "margin_dpo/margin_mean": 124.37358093261719, "margin_dpo/margin_std": 107.26200866699219, "step": 552 }, { "epoch": 0.8120411160058737, "fcm_dpo/beta": 0.007029990665614605, "fcm_dpo/delta": 0.06483317166566849, "fcm_dpo/margin": 76.58863830566406, "fcm_dpo/q_t": 0.3810678720474243, "grad_norm": 11.677241325378418, "learning_rate": 5.2839396041230415e-08, "logits/chosen": -1.5280718803405762, "logits/rejected": -1.200793981552124, "logps/chosen": -251.30337524414062, "logps/ref_chosen": -97.96388244628906, "logps/ref_rejected": -103.57719421386719, "logps/rejected": -333.50531005859375, "loss": 1.0384, "margin_dpo/margin_mean": 76.58863830566406, "margin_dpo/margin_std": 102.57685089111328, "step": 553 }, { "epoch": 0.8135095447870778, "fcm_dpo/beta": 0.007085290737450123, "fcm_dpo/delta": 0.005978398025035858, "fcm_dpo/margin": 83.78956604003906, "fcm_dpo/q_t": 0.37405747175216675, "grad_norm": 11.701247215270996, "learning_rate": 5.205293880283551e-08, "logits/chosen": -1.9315733909606934, "logits/rejected": -1.1649553775787354, "logps/chosen": -265.73052978515625, "logps/ref_chosen": -103.68680572509766, "logps/ref_rejected": -92.99210357666016, "logps/rejected": -338.82537841796875, "loss": 1.0497, "margin_dpo/margin_mean": 83.78956604003906, "margin_dpo/margin_std": 123.03929138183594, "step": 554 }, { "epoch": 0.8149779735682819, "fcm_dpo/beta": 0.007088308688253164, "fcm_dpo/delta": 0.04228687286376953, "fcm_dpo/margin": 79.01618957519531, "fcm_dpo/q_t": 0.38176795840263367, "grad_norm": 12.377523422241211, "learning_rate": 5.127169765359515e-08, "logits/chosen": -1.7843549251556396, "logits/rejected": -1.688063144683838, "logps/chosen": -255.608154296875, "logps/ref_chosen": -86.20362854003906, "logps/ref_rejected": -114.54719543457031, "logps/rejected": -362.9678955078125, "loss": 1.0493, "margin_dpo/margin_mean": 79.01618957519531, "margin_dpo/margin_std": 114.8555908203125, "step": 555 }, { "epoch": 0.8164464023494861, "fcm_dpo/beta": 0.007074237335473299, "fcm_dpo/delta": -0.01300017535686493, "fcm_dpo/margin": 86.51844024658203, "fcm_dpo/q_t": 0.36618727445602417, "grad_norm": 12.069602012634277, "learning_rate": 5.049569317994012e-08, "logits/chosen": -1.5851771831512451, "logits/rejected": -1.2715528011322021, "logps/chosen": -238.14895629882812, "logps/ref_chosen": -88.40348815917969, "logps/ref_rejected": -106.63911437988281, "logps/rejected": -342.90301513671875, "loss": 0.9853, "margin_dpo/margin_mean": 86.5184326171875, "margin_dpo/margin_std": 104.031494140625, "step": 556 }, { "epoch": 0.8179148311306902, "fcm_dpo/beta": 0.006955698132514954, "fcm_dpo/delta": -0.08430520445108414, "fcm_dpo/margin": 97.38102722167969, "fcm_dpo/q_t": 0.3552389144897461, "grad_norm": 10.077157974243164, "learning_rate": 4.9724945830310144e-08, "logits/chosen": -1.8743904829025269, "logits/rejected": -1.6990039348602295, "logps/chosen": -259.66729736328125, "logps/ref_chosen": -98.83523559570312, "logps/ref_rejected": -115.64888763427734, "logps/rejected": -373.8619689941406, "loss": 0.9496, "margin_dpo/margin_mean": 97.38102722167969, "margin_dpo/margin_std": 110.20993041992188, "step": 557 }, { "epoch": 0.8193832599118943, "fcm_dpo/beta": 0.006852718070149422, "fcm_dpo/delta": -0.09594091773033142, "fcm_dpo/margin": 100.5401840209961, "fcm_dpo/q_t": 0.346677303314209, "grad_norm": 12.043631553649902, "learning_rate": 4.8959475914614554e-08, "logits/chosen": -2.3513083457946777, "logits/rejected": -1.7356767654418945, "logps/chosen": -267.2415771484375, "logps/ref_chosen": -106.01183319091797, "logps/ref_rejected": -108.39131927490234, "logps/rejected": -370.1612548828125, "loss": 0.9025, "margin_dpo/margin_mean": 100.5401840209961, "margin_dpo/margin_std": 92.63386535644531, "step": 558 }, { "epoch": 0.8208516886930984, "fcm_dpo/beta": 0.006862045265734196, "fcm_dpo/delta": 0.020558176562190056, "fcm_dpo/margin": 84.61210632324219, "fcm_dpo/q_t": 0.3711848556995392, "grad_norm": 13.065723419189453, "learning_rate": 4.8199303603697614e-08, "logits/chosen": -2.5563955307006836, "logits/rejected": -2.0908899307250977, "logps/chosen": -271.6568908691406, "logps/ref_chosen": -100.42474365234375, "logps/ref_rejected": -100.41270446777344, "logps/rejected": -356.2569580078125, "loss": 1.0026, "margin_dpo/margin_mean": 84.61210632324219, "margin_dpo/margin_std": 103.49603271484375, "step": 559 }, { "epoch": 0.8223201174743024, "fcm_dpo/beta": 0.006905478425323963, "fcm_dpo/delta": 0.04596704989671707, "fcm_dpo/margin": 80.60478973388672, "fcm_dpo/q_t": 0.37765079736709595, "grad_norm": 13.970322608947754, "learning_rate": 4.7444448928806615e-08, "logits/chosen": -1.1586146354675293, "logits/rejected": -0.6924932599067688, "logps/chosen": -254.54705810546875, "logps/ref_chosen": -87.63551330566406, "logps/ref_rejected": -87.37985229492188, "logps/rejected": -334.89617919921875, "loss": 1.0214, "margin_dpo/margin_mean": 80.60478210449219, "margin_dpo/margin_std": 103.96781921386719, "step": 560 }, { "epoch": 0.8237885462555066, "fcm_dpo/beta": 0.0070221684873104095, "fcm_dpo/delta": 0.06730255484580994, "fcm_dpo/margin": 76.30671691894531, "fcm_dpo/q_t": 0.38102492690086365, "grad_norm": 10.298686981201172, "learning_rate": 4.669493178106432e-08, "logits/chosen": -1.9619371891021729, "logits/rejected": -1.9289535284042358, "logps/chosen": -249.20779418945312, "logps/ref_chosen": -74.88140106201172, "logps/ref_rejected": -106.04796600341797, "logps/rejected": -356.68109130859375, "loss": 1.0044, "margin_dpo/margin_mean": 76.30671691894531, "margin_dpo/margin_std": 86.78927612304688, "step": 561 }, { "epoch": 0.8252569750367107, "fcm_dpo/beta": 0.0070929634384810925, "fcm_dpo/delta": 0.0866638720035553, "fcm_dpo/margin": 73.01620483398438, "fcm_dpo/q_t": 0.38909482955932617, "grad_norm": 15.064519882202148, "learning_rate": 4.5950771910944596e-08, "logits/chosen": -2.21871280670166, "logits/rejected": -1.770498514175415, "logps/chosen": -259.5347900390625, "logps/ref_chosen": -100.66322326660156, "logps/ref_rejected": -102.24087524414062, "logps/rejected": -334.1286315917969, "loss": 1.0786, "margin_dpo/margin_mean": 73.01620483398438, "margin_dpo/margin_std": 114.18824768066406, "step": 562 }, { "epoch": 0.8267254038179148, "fcm_dpo/beta": 0.007309372071176767, "fcm_dpo/delta": 0.12443551421165466, "fcm_dpo/margin": 65.76155090332031, "fcm_dpo/q_t": 0.3946084678173065, "grad_norm": 12.58653736114502, "learning_rate": 4.521198892775202e-08, "logits/chosen": -1.7142508029937744, "logits/rejected": -1.3739852905273438, "logps/chosen": -265.404296875, "logps/ref_chosen": -96.25921630859375, "logps/ref_rejected": -101.20503997802734, "logps/rejected": -336.1116943359375, "loss": 1.0736, "margin_dpo/margin_mean": 65.76155090332031, "margin_dpo/margin_std": 95.51123046875, "step": 563 }, { "epoch": 0.8281938325991189, "fcm_dpo/beta": 0.007428737357258797, "fcm_dpo/delta": 0.06579644232988358, "fcm_dpo/margin": 72.30021667480469, "fcm_dpo/q_t": 0.3801688849925995, "grad_norm": 13.077413558959961, "learning_rate": 4.447860229910544e-08, "logits/chosen": -2.345478057861328, "logits/rejected": -1.7775757312774658, "logps/chosen": -277.65167236328125, "logps/ref_chosen": -112.96040344238281, "logps/ref_rejected": -99.52803802490234, "logps/rejected": -336.51953125, "loss": 1.0098, "margin_dpo/margin_mean": 72.30021667480469, "margin_dpo/margin_std": 83.55976867675781, "step": 564 }, { "epoch": 0.8296622613803231, "fcm_dpo/beta": 0.007322209887206554, "fcm_dpo/delta": -0.07626369595527649, "fcm_dpo/margin": 91.62553405761719, "fcm_dpo/q_t": 0.3552612066268921, "grad_norm": 11.750452995300293, "learning_rate": 4.375063135042445e-08, "logits/chosen": -1.7991347312927246, "logits/rejected": -1.296679139137268, "logps/chosen": -253.31402587890625, "logps/ref_chosen": -95.36558532714844, "logps/ref_rejected": -92.21737670898438, "logps/rejected": -341.7913513183594, "loss": 0.9521, "margin_dpo/margin_mean": 91.62553405761719, "margin_dpo/margin_std": 103.9658203125, "step": 565 }, { "epoch": 0.8311306901615272, "fcm_dpo/beta": 0.007290005683898926, "fcm_dpo/delta": -0.03795613348484039, "fcm_dpo/margin": 87.17108154296875, "fcm_dpo/q_t": 0.36021772027015686, "grad_norm": 10.284100532531738, "learning_rate": 4.3028095264420525e-08, "logits/chosen": -2.10237455368042, "logits/rejected": -1.6776249408721924, "logps/chosen": -264.3515625, "logps/ref_chosen": -98.97320556640625, "logps/ref_rejected": -110.82090759277344, "logps/rejected": -363.3703308105469, "loss": 0.9705, "margin_dpo/margin_mean": 87.17108154296875, "margin_dpo/margin_std": 100.97877502441406, "step": 566 }, { "epoch": 0.8325991189427313, "fcm_dpo/beta": 0.007268569432199001, "fcm_dpo/delta": 0.050699688494205475, "fcm_dpo/margin": 75.91093444824219, "fcm_dpo/q_t": 0.3769448399543762, "grad_norm": 13.014694213867188, "learning_rate": 4.231101308059165e-08, "logits/chosen": -2.2701256275177, "logits/rejected": -1.6002724170684814, "logps/chosen": -259.28155517578125, "logps/ref_chosen": -92.7035903930664, "logps/ref_rejected": -91.22431945800781, "logps/rejected": -333.7132568359375, "loss": 1.0209, "margin_dpo/margin_mean": 75.91093444824219, "margin_dpo/margin_std": 94.99116516113281, "step": 567 }, { "epoch": 0.8340675477239354, "fcm_dpo/beta": 0.007200797088444233, "fcm_dpo/delta": -0.09029556810855865, "fcm_dpo/margin": 94.91342163085938, "fcm_dpo/q_t": 0.35067832469940186, "grad_norm": 10.619403839111328, "learning_rate": 4.1599403694720145e-08, "logits/chosen": -1.4319794178009033, "logits/rejected": -1.2937116622924805, "logps/chosen": -230.70367431640625, "logps/ref_chosen": -73.13226318359375, "logps/ref_rejected": -94.95265197753906, "logps/rejected": -347.4375, "loss": 0.9189, "margin_dpo/margin_mean": 94.91342163085938, "margin_dpo/margin_std": 95.35987091064453, "step": 568 }, { "epoch": 0.8355359765051396, "fcm_dpo/beta": 0.007249847054481506, "fcm_dpo/delta": 0.02976367063820362, "fcm_dpo/margin": 78.85527801513672, "fcm_dpo/q_t": 0.37523531913757324, "grad_norm": 17.318199157714844, "learning_rate": 4.089328585837512e-08, "logits/chosen": -2.0498688220977783, "logits/rejected": -1.6809823513031006, "logps/chosen": -256.4047546386719, "logps/ref_chosen": -89.21708679199219, "logps/ref_rejected": -86.41105651855469, "logps/rejected": -332.4539794921875, "loss": 1.0482, "margin_dpo/margin_mean": 78.85527801513672, "margin_dpo/margin_std": 112.83515930175781, "step": 569 }, { "epoch": 0.8370044052863436, "fcm_dpo/beta": 0.00708900997415185, "fcm_dpo/delta": -0.13322904706001282, "fcm_dpo/margin": 102.04464721679688, "fcm_dpo/q_t": 0.3389042615890503, "grad_norm": 11.147782325744629, "learning_rate": 4.019267817841834e-08, "logits/chosen": -2.2344322204589844, "logits/rejected": -1.5383317470550537, "logps/chosen": -243.58880615234375, "logps/ref_chosen": -94.76277160644531, "logps/ref_rejected": -88.846923828125, "logps/rejected": -339.71759033203125, "loss": 0.8871, "margin_dpo/margin_mean": 102.04464721679688, "margin_dpo/margin_std": 94.04344177246094, "step": 570 }, { "epoch": 0.8384728340675477, "fcm_dpo/beta": 0.00701398029923439, "fcm_dpo/delta": -0.053493283689022064, "fcm_dpo/margin": 92.67195129394531, "fcm_dpo/q_t": 0.3594912588596344, "grad_norm": 14.784183502197266, "learning_rate": 3.9497599116513705e-08, "logits/chosen": -1.6442519426345825, "logits/rejected": -1.4821221828460693, "logps/chosen": -234.88565063476562, "logps/ref_chosen": -80.8212890625, "logps/ref_rejected": -97.01528930664062, "logps/rejected": -343.7516174316406, "loss": 0.9712, "margin_dpo/margin_mean": 92.67195129394531, "margin_dpo/margin_std": 109.70027160644531, "step": 571 }, { "epoch": 0.8399412628487518, "fcm_dpo/beta": 0.00692150229588151, "fcm_dpo/delta": -0.035205624997615814, "fcm_dpo/margin": 91.42375183105469, "fcm_dpo/q_t": 0.36460578441619873, "grad_norm": 11.84706974029541, "learning_rate": 3.880806698864086e-08, "logits/chosen": -1.739823341369629, "logits/rejected": -1.5698119401931763, "logps/chosen": -235.00161743164062, "logps/ref_chosen": -77.37992858886719, "logps/ref_rejected": -90.56597900390625, "logps/rejected": -339.6114196777344, "loss": 0.9675, "margin_dpo/margin_mean": 91.42375183105469, "margin_dpo/margin_std": 106.2401123046875, "step": 572 }, { "epoch": 0.8414096916299559, "fcm_dpo/beta": 0.006989034358412027, "fcm_dpo/delta": -0.0046057626605033875, "fcm_dpo/margin": 86.20860290527344, "fcm_dpo/q_t": 0.3665117621421814, "grad_norm": 10.864632606506348, "learning_rate": 3.812409996461275e-08, "logits/chosen": -2.1315410137176514, "logits/rejected": -1.9030604362487793, "logps/chosen": -236.8482208251953, "logps/ref_chosen": -83.7060775756836, "logps/ref_rejected": -90.69746398925781, "logps/rejected": -330.0482177734375, "loss": 0.9925, "margin_dpo/margin_mean": 86.20860290527344, "margin_dpo/margin_std": 100.88923645019531, "step": 573 }, { "epoch": 0.8428781204111601, "fcm_dpo/beta": 0.006914999336004257, "fcm_dpo/delta": -0.02837441675364971, "fcm_dpo/margin": 90.59143829345703, "fcm_dpo/q_t": 0.36350974440574646, "grad_norm": 13.734319686889648, "learning_rate": 3.74457160675965e-08, "logits/chosen": -2.0994553565979004, "logits/rejected": -1.7679946422576904, "logps/chosen": -246.83309936523438, "logps/ref_chosen": -85.12948608398438, "logps/ref_rejected": -99.05403137207031, "logps/rejected": -351.3490905761719, "loss": 0.9786, "margin_dpo/margin_mean": 90.59143829345703, "margin_dpo/margin_std": 107.298828125, "step": 574 }, { "epoch": 0.8443465491923642, "fcm_dpo/beta": 0.006864962168037891, "fcm_dpo/delta": -0.005945529788732529, "fcm_dpo/margin": 88.20824432373047, "fcm_dpo/q_t": 0.3688689172267914, "grad_norm": 15.33983039855957, "learning_rate": 3.677293317363864e-08, "logits/chosen": -1.398423433303833, "logits/rejected": -0.9362001419067383, "logps/chosen": -272.59100341796875, "logps/ref_chosen": -105.77084350585938, "logps/ref_rejected": -100.94944763183594, "logps/rejected": -355.9778747558594, "loss": 1.0027, "margin_dpo/margin_mean": 88.20824432373047, "margin_dpo/margin_std": 113.10353088378906, "step": 575 }, { "epoch": 0.8458149779735683, "fcm_dpo/beta": 0.006903272587805986, "fcm_dpo/delta": 0.013031164184212685, "fcm_dpo/margin": 85.11408996582031, "fcm_dpo/q_t": 0.37200257182121277, "grad_norm": 12.17516803741455, "learning_rate": 3.6105769011194224e-08, "logits/chosen": -2.0569982528686523, "logits/rejected": -1.9845412969589233, "logps/chosen": -248.39340209960938, "logps/ref_chosen": -81.20452880859375, "logps/ref_rejected": -105.40940856933594, "logps/rejected": -357.7123718261719, "loss": 1.0148, "margin_dpo/margin_mean": 85.11408996582031, "margin_dpo/margin_std": 110.67799377441406, "step": 576 }, { "epoch": 0.8472834067547724, "fcm_dpo/beta": 0.00673964899033308, "fcm_dpo/delta": -0.11396480351686478, "fcm_dpo/margin": 104.58753204345703, "fcm_dpo/q_t": 0.34595972299575806, "grad_norm": 9.555338859558105, "learning_rate": 3.5444241160659304e-08, "logits/chosen": -1.9512568712234497, "logits/rejected": -1.2471516132354736, "logps/chosen": -249.70469665527344, "logps/ref_chosen": -101.46419525146484, "logps/ref_rejected": -89.32637023925781, "logps/rejected": -342.1544189453125, "loss": 0.9266, "margin_dpo/margin_mean": 104.58753204345703, "margin_dpo/margin_std": 111.21625518798828, "step": 577 }, { "epoch": 0.8487518355359766, "fcm_dpo/beta": 0.006825780496001244, "fcm_dpo/delta": 0.08758461475372314, "fcm_dpo/margin": 75.72772979736328, "fcm_dpo/q_t": 0.38606128096580505, "grad_norm": 16.20516586303711, "learning_rate": 3.478836705390808e-08, "logits/chosen": -1.3706796169281006, "logits/rejected": -1.131410837173462, "logps/chosen": -253.5721435546875, "logps/ref_chosen": -77.29241943359375, "logps/ref_rejected": -88.24701690673828, "logps/rejected": -340.25445556640625, "loss": 1.0516, "margin_dpo/margin_mean": 75.72772979736328, "margin_dpo/margin_std": 105.21282196044922, "step": 578 }, { "epoch": 0.8502202643171806, "fcm_dpo/beta": 0.006886948365718126, "fcm_dpo/delta": 0.06752566248178482, "fcm_dpo/margin": 77.81400299072266, "fcm_dpo/q_t": 0.3831830620765686, "grad_norm": 11.248205184936523, "learning_rate": 3.41381639738331e-08, "logits/chosen": -1.7072319984436035, "logits/rejected": -1.3880934715270996, "logps/chosen": -245.95306396484375, "logps/ref_chosen": -87.34305572509766, "logps/ref_rejected": -102.12100982666016, "logps/rejected": -338.5450439453125, "loss": 1.027, "margin_dpo/margin_mean": 77.81400299072266, "margin_dpo/margin_std": 96.24092102050781, "step": 579 }, { "epoch": 0.8516886930983847, "fcm_dpo/beta": 0.006863095331937075, "fcm_dpo/delta": -0.040312498807907104, "fcm_dpo/margin": 92.7848129272461, "fcm_dpo/q_t": 0.3631054759025574, "grad_norm": 11.509198188781738, "learning_rate": 3.349364905389032e-08, "logits/chosen": -1.6127009391784668, "logits/rejected": -1.3059601783752441, "logps/chosen": -242.31298828125, "logps/ref_chosen": -88.92879486083984, "logps/ref_rejected": -88.69661712646484, "logps/rejected": -334.8656311035156, "loss": 0.9711, "margin_dpo/margin_mean": 92.7848129272461, "margin_dpo/margin_std": 108.14720153808594, "step": 580 }, { "epoch": 0.8531571218795888, "fcm_dpo/beta": 0.006930704694241285, "fcm_dpo/delta": 0.026802916079759598, "fcm_dpo/margin": 82.91061401367188, "fcm_dpo/q_t": 0.3752003312110901, "grad_norm": 13.487149238586426, "learning_rate": 3.285483927764726e-08, "logits/chosen": -2.016202449798584, "logits/rejected": -1.6698317527770996, "logps/chosen": -254.0625762939453, "logps/ref_chosen": -92.22323608398438, "logps/ref_rejected": -99.38943481445312, "logps/rejected": -344.139404296875, "loss": 1.0399, "margin_dpo/margin_mean": 82.91061401367188, "margin_dpo/margin_std": 116.20519256591797, "step": 581 }, { "epoch": 0.8546255506607929, "fcm_dpo/beta": 0.007061149924993515, "fcm_dpo/delta": 0.04960351437330246, "fcm_dpo/margin": 77.88203430175781, "fcm_dpo/q_t": 0.3824002146720886, "grad_norm": 11.220653533935547, "learning_rate": 3.222175147833556e-08, "logits/chosen": -1.9758110046386719, "logits/rejected": -1.9616727828979492, "logps/chosen": -229.19464111328125, "logps/ref_chosen": -87.33561706542969, "logps/ref_rejected": -115.1544189453125, "logps/rejected": -334.8954772949219, "loss": 1.039, "margin_dpo/margin_mean": 77.88203430175781, "margin_dpo/margin_std": 104.54690551757812, "step": 582 }, { "epoch": 0.856093979441997, "fcm_dpo/beta": 0.006990609224885702, "fcm_dpo/delta": 0.02330578863620758, "fcm_dpo/margin": 82.64878845214844, "fcm_dpo/q_t": 0.37635916471481323, "grad_norm": 12.386331558227539, "learning_rate": 3.159440233840763e-08, "logits/chosen": -1.5934879779815674, "logits/rejected": -1.364257574081421, "logps/chosen": -257.1393737792969, "logps/ref_chosen": -83.71707153320312, "logps/ref_rejected": -94.58465576171875, "logps/rejected": -350.65576171875, "loss": 1.0101, "margin_dpo/margin_mean": 82.64878845214844, "margin_dpo/margin_std": 105.64147186279297, "step": 583 }, { "epoch": 0.8575624082232012, "fcm_dpo/beta": 0.007066408172249794, "fcm_dpo/delta": -0.031313344836235046, "fcm_dpo/margin": 88.83890533447266, "fcm_dpo/q_t": 0.3632817268371582, "grad_norm": 10.694197654724121, "learning_rate": 3.0972808389096635e-08, "logits/chosen": -2.124901294708252, "logits/rejected": -1.6727513074874878, "logps/chosen": -246.2718963623047, "logps/ref_chosen": -94.25955963134766, "logps/ref_rejected": -104.48269653320312, "logps/rejected": -345.33392333984375, "loss": 0.9905, "margin_dpo/margin_mean": 88.83891296386719, "margin_dpo/margin_std": 106.91671752929688, "step": 584 }, { "epoch": 0.8590308370044053, "fcm_dpo/beta": 0.00690193846821785, "fcm_dpo/delta": -0.11694584041833878, "fcm_dpo/margin": 102.68148040771484, "fcm_dpo/q_t": 0.350594162940979, "grad_norm": 11.292356491088867, "learning_rate": 3.035698600998121e-08, "logits/chosen": -1.7038193941116333, "logits/rejected": -1.4119065999984741, "logps/chosen": -247.07659912109375, "logps/ref_chosen": -85.63676452636719, "logps/ref_rejected": -90.28547668457031, "logps/rejected": -354.40679931640625, "loss": 0.955, "margin_dpo/margin_mean": 102.68147277832031, "margin_dpo/margin_std": 123.2242431640625, "step": 585 }, { "epoch": 0.8604992657856094, "fcm_dpo/beta": 0.006955640856176615, "fcm_dpo/delta": 0.1244577169418335, "fcm_dpo/margin": 69.21916198730469, "fcm_dpo/q_t": 0.3929838240146637, "grad_norm": 13.513263702392578, "learning_rate": 2.974695142855388e-08, "logits/chosen": -1.7758688926696777, "logits/rejected": -1.507314682006836, "logps/chosen": -263.46990966796875, "logps/ref_chosen": -83.68114471435547, "logps/ref_rejected": -98.35916137695312, "logps/rejected": -347.3670654296875, "loss": 1.0502, "margin_dpo/margin_mean": 69.21916198730469, "margin_dpo/margin_std": 89.19205474853516, "step": 586 }, { "epoch": 0.8619676945668135, "fcm_dpo/beta": 0.0070916032418608665, "fcm_dpo/delta": 0.06968455016613007, "fcm_dpo/margin": 75.24751281738281, "fcm_dpo/q_t": 0.3832997679710388, "grad_norm": 11.80888557434082, "learning_rate": 2.9142720719793122e-08, "logits/chosen": -2.346747636795044, "logits/rejected": -2.054532527923584, "logps/chosen": -222.94000244140625, "logps/ref_chosen": -77.57750701904297, "logps/ref_rejected": -89.32032775878906, "logps/rejected": -309.93035888671875, "loss": 1.0274, "margin_dpo/margin_mean": 75.24751281738281, "margin_dpo/margin_std": 96.26628875732422, "step": 587 }, { "epoch": 0.8634361233480177, "fcm_dpo/beta": 0.00717865489423275, "fcm_dpo/delta": 0.10119879245758057, "fcm_dpo/margin": 70.20915222167969, "fcm_dpo/q_t": 0.39323359727859497, "grad_norm": 14.625384330749512, "learning_rate": 2.8544309805740018e-08, "logits/chosen": -1.971228003501892, "logits/rejected": -1.9192923307418823, "logps/chosen": -241.52024841308594, "logps/ref_chosen": -83.80426025390625, "logps/ref_rejected": -113.11943054199219, "logps/rejected": -341.0445861816406, "loss": 1.0898, "margin_dpo/margin_mean": 70.20915222167969, "margin_dpo/margin_std": 111.68272399902344, "step": 588 }, { "epoch": 0.8649045521292217, "fcm_dpo/beta": 0.007164877839386463, "fcm_dpo/delta": -0.06353701651096344, "fcm_dpo/margin": 92.01499938964844, "fcm_dpo/q_t": 0.35298681259155273, "grad_norm": 12.12359619140625, "learning_rate": 2.7951734455078786e-08, "logits/chosen": -1.600557565689087, "logits/rejected": -1.1946024894714355, "logps/chosen": -266.53143310546875, "logps/ref_chosen": -96.46501922607422, "logps/ref_rejected": -116.03719329833984, "logps/rejected": -378.11859130859375, "loss": 0.9228, "margin_dpo/margin_mean": 92.01498413085938, "margin_dpo/margin_std": 88.87408447265625, "step": 589 }, { "epoch": 0.8663729809104258, "fcm_dpo/beta": 0.007136746309697628, "fcm_dpo/delta": -0.05460678040981293, "fcm_dpo/margin": 91.158203125, "fcm_dpo/q_t": 0.3565479516983032, "grad_norm": 13.865069389343262, "learning_rate": 2.736501028272095e-08, "logits/chosen": -1.906883716583252, "logits/rejected": -1.7435002326965332, "logps/chosen": -242.1221160888672, "logps/ref_chosen": -88.4278564453125, "logps/ref_rejected": -110.19876861572266, "logps/rejected": -355.0512390136719, "loss": 0.9316, "margin_dpo/margin_mean": 91.158203125, "margin_dpo/margin_std": 90.19902038574219, "step": 590 }, { "epoch": 0.8678414096916299, "fcm_dpo/beta": 0.00695719663053751, "fcm_dpo/delta": -0.08778176456689835, "fcm_dpo/margin": 97.93551635742188, "fcm_dpo/q_t": 0.3495037257671356, "grad_norm": 8.954543113708496, "learning_rate": 2.678415274939408e-08, "logits/chosen": -2.069058418273926, "logits/rejected": -1.4569966793060303, "logps/chosen": -236.16778564453125, "logps/ref_chosen": -87.84968566894531, "logps/ref_rejected": -89.57919311523438, "logps/rejected": -335.83282470703125, "loss": 0.9107, "margin_dpo/margin_mean": 97.93551635742188, "margin_dpo/margin_std": 93.53607177734375, "step": 591 }, { "epoch": 0.869309838472834, "fcm_dpo/beta": 0.007020566612482071, "fcm_dpo/delta": 0.06493528187274933, "fcm_dpo/margin": 76.70272827148438, "fcm_dpo/q_t": 0.3827671408653259, "grad_norm": 13.41107177734375, "learning_rate": 2.6209177161234442e-08, "logits/chosen": -1.726281762123108, "logits/rejected": -1.3931875228881836, "logps/chosen": -250.2864990234375, "logps/ref_chosen": -74.97573852539062, "logps/ref_rejected": -81.55262756347656, "logps/rejected": -333.5661315917969, "loss": 1.0491, "margin_dpo/margin_mean": 76.70272827148438, "margin_dpo/margin_std": 108.41436004638672, "step": 592 }, { "epoch": 0.8707782672540382, "fcm_dpo/beta": 0.007145709358155727, "fcm_dpo/delta": 0.1387026607990265, "fcm_dpo/margin": 65.48255157470703, "fcm_dpo/q_t": 0.399899959564209, "grad_norm": 17.311737060546875, "learning_rate": 2.564009866938349e-08, "logits/chosen": -0.939947247505188, "logits/rejected": -0.6396098136901855, "logps/chosen": -234.7220458984375, "logps/ref_chosen": -72.23902893066406, "logps/ref_rejected": -70.09652709960938, "logps/rejected": -298.0621337890625, "loss": 1.099, "margin_dpo/margin_mean": 65.48255157470703, "margin_dpo/margin_std": 103.57083129882812, "step": 593 }, { "epoch": 0.8722466960352423, "fcm_dpo/beta": 0.0071470364928245544, "fcm_dpo/delta": -0.11245208233594894, "fcm_dpo/margin": 98.58993530273438, "fcm_dpo/q_t": 0.3443548083305359, "grad_norm": 10.641993522644043, "learning_rate": 2.5076932269588708e-08, "logits/chosen": -1.9720404148101807, "logits/rejected": -1.4504088163375854, "logps/chosen": -236.50711059570312, "logps/ref_chosen": -90.43771362304688, "logps/ref_rejected": -92.267578125, "logps/rejected": -336.92694091796875, "loss": 0.8928, "margin_dpo/margin_mean": 98.58993530273438, "margin_dpo/margin_std": 89.36225891113281, "step": 594 }, { "epoch": 0.8737151248164464, "fcm_dpo/beta": 0.007122883573174477, "fcm_dpo/delta": 0.020702892914414406, "fcm_dpo/margin": 81.45152282714844, "fcm_dpo/q_t": 0.3758925795555115, "grad_norm": 12.234829902648926, "learning_rate": 2.451969280180849e-08, "logits/chosen": -1.6966605186462402, "logits/rejected": -1.2726609706878662, "logps/chosen": -236.58111572265625, "logps/ref_chosen": -81.51480102539062, "logps/ref_rejected": -86.52047729492188, "logps/rejected": -323.038330078125, "loss": 1.0138, "margin_dpo/margin_mean": 81.45152282714844, "margin_dpo/margin_std": 105.23150634765625, "step": 595 }, { "epoch": 0.8751835535976505, "fcm_dpo/beta": 0.00720050185918808, "fcm_dpo/delta": 0.08340231329202652, "fcm_dpo/margin": 72.33407592773438, "fcm_dpo/q_t": 0.3886866271495819, "grad_norm": 15.935720443725586, "learning_rate": 2.396839494982103e-08, "logits/chosen": -1.7379255294799805, "logits/rejected": -1.1650152206420898, "logps/chosen": -248.61962890625, "logps/ref_chosen": -88.20805358886719, "logps/ref_rejected": -86.09375, "logps/rejected": -318.83941650390625, "loss": 1.0738, "margin_dpo/margin_mean": 72.33407592773438, "margin_dpo/margin_std": 109.93353271484375, "step": 596 }, { "epoch": 0.8766519823788547, "fcm_dpo/beta": 0.007105865050107241, "fcm_dpo/delta": -0.09072263538837433, "fcm_dpo/margin": 96.26971435546875, "fcm_dpo/q_t": 0.3528348505496979, "grad_norm": 10.61738109588623, "learning_rate": 2.3423053240837514e-08, "logits/chosen": -1.3981832265853882, "logits/rejected": -1.2777040004730225, "logps/chosen": -240.2840576171875, "logps/ref_chosen": -83.58998107910156, "logps/ref_rejected": -106.94439697265625, "logps/rejected": -359.908203125, "loss": 0.9408, "margin_dpo/margin_mean": 96.26971435546875, "margin_dpo/margin_std": 107.11369323730469, "step": 597 }, { "epoch": 0.8781204111600588, "fcm_dpo/beta": 0.007089652121067047, "fcm_dpo/delta": 0.03473873436450958, "fcm_dpo/margin": 79.942138671875, "fcm_dpo/q_t": 0.37478604912757874, "grad_norm": 11.30370807647705, "learning_rate": 2.2883682045119062e-08, "logits/chosen": -2.0580103397369385, "logits/rejected": -1.748920202255249, "logps/chosen": -239.97232055664062, "logps/ref_chosen": -90.18262481689453, "logps/ref_rejected": -94.79856872558594, "logps/rejected": -324.5303955078125, "loss": 1.021, "margin_dpo/margin_mean": 79.942138671875, "margin_dpo/margin_std": 102.13404846191406, "step": 598 }, { "epoch": 0.8795888399412628, "fcm_dpo/beta": 0.007190364878624678, "fcm_dpo/delta": 0.09109681099653244, "fcm_dpo/margin": 71.37487030029297, "fcm_dpo/q_t": 0.38869598507881165, "grad_norm": 12.38364028930664, "learning_rate": 2.2350295575598367e-08, "logits/chosen": -1.799156665802002, "logits/rejected": -1.6733660697937012, "logps/chosen": -232.6363067626953, "logps/ref_chosen": -77.84226989746094, "logps/ref_rejected": -89.1976318359375, "logps/rejected": -315.36651611328125, "loss": 1.0679, "margin_dpo/margin_mean": 71.37487030029297, "margin_dpo/margin_std": 102.9161148071289, "step": 599 }, { "epoch": 0.8810572687224669, "fcm_dpo/beta": 0.007273955270648003, "fcm_dpo/delta": -0.006405937019735575, "fcm_dpo/margin": 83.31228637695312, "fcm_dpo/q_t": 0.3687729835510254, "grad_norm": 10.347834587097168, "learning_rate": 2.1822907887504932e-08, "logits/chosen": -2.4206528663635254, "logits/rejected": -2.0758280754089355, "logps/chosen": -257.964111328125, "logps/ref_chosen": -92.18781280517578, "logps/ref_rejected": -92.95429992675781, "logps/rejected": -342.0428771972656, "loss": 0.9868, "margin_dpo/margin_mean": 83.31228637695312, "margin_dpo/margin_std": 99.32923889160156, "step": 600 }, { "epoch": 0.8810572687224669, "eval_fcm_dpo/beta": 0.0072656371630728245, "eval_logits/chosen": -1.840718150138855, "eval_logits/rejected": -1.4554615020751953, "eval_logps/chosen": -265.1147155761719, "eval_logps/ref_chosen": -100.49356842041016, "eval_logps/ref_rejected": -94.06775665283203, "eval_logps/rejected": -332.2911376953125, "eval_loss": 0.5223754644393921, "eval_margin_dpo/margin_mean": 73.60225677490234, "eval_margin_dpo/margin_std": 100.21179962158203, "eval_runtime": 43.7553, "eval_samples_per_second": 53.456, "eval_steps_per_second": 1.691, "step": 600 }, { "epoch": 0.882525697503671, "fcm_dpo/beta": 0.00720963254570961, "fcm_dpo/delta": -0.023707086220383644, "fcm_dpo/margin": 86.24479675292969, "fcm_dpo/q_t": 0.3629060983657837, "grad_norm": 11.4024019241333, "learning_rate": 2.1301532877994742e-08, "logits/chosen": -1.7290210723876953, "logits/rejected": -1.4024295806884766, "logps/chosen": -259.41070556640625, "logps/ref_chosen": -89.21614074707031, "logps/ref_rejected": -100.17054748535156, "logps/rejected": -356.60992431640625, "loss": 0.9523, "margin_dpo/margin_mean": 86.24479675292969, "margin_dpo/margin_std": 90.01136016845703, "step": 601 }, { "epoch": 0.8839941262848752, "fcm_dpo/beta": 0.007033023051917553, "fcm_dpo/delta": -0.185970276594162, "fcm_dpo/margin": 109.70604705810547, "fcm_dpo/q_t": 0.3343247175216675, "grad_norm": 10.687939643859863, "learning_rate": 2.0786184285784298e-08, "logits/chosen": -2.0846588611602783, "logits/rejected": -1.826683759689331, "logps/chosen": -214.779052734375, "logps/ref_chosen": -80.05760192871094, "logps/ref_rejected": -93.197509765625, "logps/rejected": -337.625, "loss": 0.8887, "margin_dpo/margin_mean": 109.70604705810547, "margin_dpo/margin_std": 112.28135681152344, "step": 602 }, { "epoch": 0.8854625550660793, "fcm_dpo/beta": 0.006929402239620686, "fcm_dpo/delta": -0.0485604926943779, "fcm_dpo/margin": 93.14605712890625, "fcm_dpo/q_t": 0.3590378165245056, "grad_norm": 10.695706367492676, "learning_rate": 2.0276875690788204e-08, "logits/chosen": -1.9837377071380615, "logits/rejected": -1.3861087560653687, "logps/chosen": -247.59893798828125, "logps/ref_chosen": -102.30958557128906, "logps/ref_rejected": -108.06884765625, "logps/rejected": -346.5042724609375, "loss": 0.9534, "margin_dpo/margin_mean": 93.14605712890625, "margin_dpo/margin_std": 104.23910522460938, "step": 603 }, { "epoch": 0.8869309838472834, "fcm_dpo/beta": 0.006826011463999748, "fcm_dpo/delta": -0.04624610394239426, "fcm_dpo/margin": 94.17044067382812, "fcm_dpo/q_t": 0.35456353425979614, "grad_norm": 10.141494750976562, "learning_rate": 1.977362051376158e-08, "logits/chosen": -1.782772183418274, "logits/rejected": -1.626922845840454, "logps/chosen": -228.49160766601562, "logps/ref_chosen": -78.17408752441406, "logps/ref_rejected": -99.49610900878906, "logps/rejected": -343.98406982421875, "loss": 0.9365, "margin_dpo/margin_mean": 94.17044067382812, "margin_dpo/margin_std": 92.8728256225586, "step": 604 }, { "epoch": 0.8883994126284875, "fcm_dpo/beta": 0.006894152611494064, "fcm_dpo/delta": 0.06398770958185196, "fcm_dpo/margin": 78.25582122802734, "fcm_dpo/q_t": 0.38114598393440247, "grad_norm": 10.904273986816406, "learning_rate": 1.9276432015946446e-08, "logits/chosen": -1.7464425563812256, "logits/rejected": -1.2376749515533447, "logps/chosen": -258.21099853515625, "logps/ref_chosen": -94.77332305908203, "logps/ref_rejected": -107.30490112304688, "logps/rejected": -348.9984130859375, "loss": 1.0158, "margin_dpo/margin_mean": 78.25582885742188, "margin_dpo/margin_std": 95.33082580566406, "step": 605 }, { "epoch": 0.8898678414096917, "fcm_dpo/beta": 0.007000097073614597, "fcm_dpo/delta": 0.02021496742963791, "fcm_dpo/margin": 82.80071258544922, "fcm_dpo/q_t": 0.3707614541053772, "grad_norm": 11.527751922607422, "learning_rate": 1.8785323298722093e-08, "logits/chosen": -1.9277280569076538, "logits/rejected": -1.524000883102417, "logps/chosen": -245.60330200195312, "logps/ref_chosen": -87.7533950805664, "logps/ref_rejected": -104.2422103881836, "logps/rejected": -344.892822265625, "loss": 0.9999, "margin_dpo/margin_mean": 82.80071258544922, "margin_dpo/margin_std": 97.99324035644531, "step": 606 }, { "epoch": 0.8913362701908958, "fcm_dpo/beta": 0.0069069629535079, "fcm_dpo/delta": -0.04535501077771187, "fcm_dpo/margin": 93.01445007324219, "fcm_dpo/q_t": 0.36220258474349976, "grad_norm": 14.510759353637695, "learning_rate": 1.8300307303259904e-08, "logits/chosen": -1.7667630910873413, "logits/rejected": -1.2318663597106934, "logps/chosen": -243.1822509765625, "logps/ref_chosen": -88.32904052734375, "logps/ref_rejected": -86.76810455322266, "logps/rejected": -334.6357421875, "loss": 1.0038, "margin_dpo/margin_mean": 93.01445007324219, "margin_dpo/margin_std": 122.94918823242188, "step": 607 }, { "epoch": 0.8928046989720999, "fcm_dpo/beta": 0.006772700231522322, "fcm_dpo/delta": -0.05694134533405304, "fcm_dpo/margin": 96.16629028320312, "fcm_dpo/q_t": 0.35457879304885864, "grad_norm": 12.842971801757812, "learning_rate": 1.7821396810182437e-08, "logits/chosen": -1.7402276992797852, "logits/rejected": -1.4407744407653809, "logps/chosen": -226.5384521484375, "logps/ref_chosen": -85.76937866210938, "logps/ref_rejected": -100.23281860351562, "logps/rejected": -337.1681823730469, "loss": 0.9581, "margin_dpo/margin_mean": 96.16629028320312, "margin_dpo/margin_std": 104.13595581054688, "step": 608 }, { "epoch": 0.8942731277533039, "fcm_dpo/beta": 0.006742427125573158, "fcm_dpo/delta": -0.037574149668216705, "fcm_dpo/margin": 94.12151336669922, "fcm_dpo/q_t": 0.36158236861228943, "grad_norm": 13.74590015411377, "learning_rate": 1.7348604439226617e-08, "logits/chosen": -2.4941606521606445, "logits/rejected": -2.0870494842529297, "logps/chosen": -246.841796875, "logps/ref_chosen": -92.9665756225586, "logps/ref_rejected": -95.91818237304688, "logps/rejected": -343.9149169921875, "loss": 0.9629, "margin_dpo/margin_mean": 94.12152099609375, "margin_dpo/margin_std": 102.90473937988281, "step": 609 }, { "epoch": 0.895741556534508, "fcm_dpo/beta": 0.00668362807482481, "fcm_dpo/delta": -0.060604121536016464, "fcm_dpo/margin": 98.20524597167969, "fcm_dpo/q_t": 0.3524959683418274, "grad_norm": 10.179794311523438, "learning_rate": 1.6881942648911074e-08, "logits/chosen": -1.8149501085281372, "logits/rejected": -1.2743051052093506, "logps/chosen": -239.31069946289062, "logps/ref_chosen": -94.70028686523438, "logps/ref_rejected": -89.68739318847656, "logps/rejected": -332.5030517578125, "loss": 0.934, "margin_dpo/margin_mean": 98.20524597167969, "margin_dpo/margin_std": 99.45082092285156, "step": 610 }, { "epoch": 0.8972099853157122, "fcm_dpo/beta": 0.00667162798345089, "fcm_dpo/delta": -0.047587163746356964, "fcm_dpo/margin": 96.54525756835938, "fcm_dpo/q_t": 0.35953009128570557, "grad_norm": 12.610694885253906, "learning_rate": 1.6421423736208e-08, "logits/chosen": -2.17702317237854, "logits/rejected": -1.8332421779632568, "logps/chosen": -237.6696319580078, "logps/ref_chosen": -86.78334045410156, "logps/ref_rejected": -89.84307861328125, "logps/rejected": -337.274658203125, "loss": 0.9441, "margin_dpo/margin_mean": 96.54525756835938, "margin_dpo/margin_std": 100.653076171875, "step": 611 }, { "epoch": 0.8986784140969163, "fcm_dpo/beta": 0.006550832651555538, "fcm_dpo/delta": -0.049320608377456665, "fcm_dpo/margin": 98.5962142944336, "fcm_dpo/q_t": 0.35573434829711914, "grad_norm": 10.573701858520508, "learning_rate": 1.5967059836219042e-08, "logits/chosen": -2.050720453262329, "logits/rejected": -1.197265625, "logps/chosen": -265.29229736328125, "logps/ref_chosen": -101.02015686035156, "logps/ref_rejected": -93.78302764892578, "logps/rejected": -356.6513671875, "loss": 0.9267, "margin_dpo/margin_mean": 98.5962142944336, "margin_dpo/margin_std": 94.86685180664062, "step": 612 }, { "epoch": 0.9001468428781204, "fcm_dpo/beta": 0.00644554290920496, "fcm_dpo/delta": -0.09590654820203781, "fcm_dpo/margin": 106.90154266357422, "fcm_dpo/q_t": 0.34533023834228516, "grad_norm": 10.872828483581543, "learning_rate": 1.551886292185553e-08, "logits/chosen": -1.9289722442626953, "logits/rejected": -1.728402853012085, "logps/chosen": -224.5867919921875, "logps/ref_chosen": -88.98861694335938, "logps/ref_rejected": -109.99551391601562, "logps/rejected": -352.4952392578125, "loss": 0.8885, "margin_dpo/margin_mean": 106.90153503417969, "margin_dpo/margin_std": 91.88870239257812, "step": 613 }, { "epoch": 0.9016152716593245, "fcm_dpo/beta": 0.0064603416249156, "fcm_dpo/delta": 0.04875949025154114, "fcm_dpo/margin": 85.74708557128906, "fcm_dpo/q_t": 0.37675923109054565, "grad_norm": 12.27489948272705, "learning_rate": 1.507684480352292e-08, "logits/chosen": -1.6303455829620361, "logits/rejected": -1.6167235374450684, "logps/chosen": -247.53366088867188, "logps/ref_chosen": -80.20005798339844, "logps/ref_rejected": -109.86239624023438, "logps/rejected": -362.94305419921875, "loss": 1.0172, "margin_dpo/margin_mean": 85.74708557128906, "margin_dpo/margin_std": 105.97724151611328, "step": 614 }, { "epoch": 0.9030837004405287, "fcm_dpo/beta": 0.0066406624391674995, "fcm_dpo/delta": 0.09489738941192627, "fcm_dpo/margin": 76.43423461914062, "fcm_dpo/q_t": 0.38481491804122925, "grad_norm": 11.700821876525879, "learning_rate": 1.4641017128809801e-08, "logits/chosen": -1.9045593738555908, "logits/rejected": -1.5077332258224487, "logps/chosen": -256.5118408203125, "logps/ref_chosen": -100.43525695800781, "logps/ref_rejected": -101.1800537109375, "logps/rejected": -333.6908874511719, "loss": 1.0219, "margin_dpo/margin_mean": 76.43423461914062, "margin_dpo/margin_std": 84.58399963378906, "step": 615 }, { "epoch": 0.9045521292217328, "fcm_dpo/beta": 0.006577151361852884, "fcm_dpo/delta": -0.0695214495062828, "fcm_dpo/margin": 101.07377624511719, "fcm_dpo/q_t": 0.35348814725875854, "grad_norm": 10.295190811157227, "learning_rate": 1.4211391382180637e-08, "logits/chosen": -1.8900903463363647, "logits/rejected": -1.4016035795211792, "logps/chosen": -250.1123504638672, "logps/ref_chosen": -92.49292755126953, "logps/ref_rejected": -82.06065368652344, "logps/rejected": -340.7538757324219, "loss": 0.937, "margin_dpo/margin_mean": 101.07378387451172, "margin_dpo/margin_std": 106.29083251953125, "step": 616 }, { "epoch": 0.9060205580029369, "fcm_dpo/beta": 0.006596277467906475, "fcm_dpo/delta": 0.07905448973178864, "fcm_dpo/margin": 79.61018371582031, "fcm_dpo/q_t": 0.3824957013130188, "grad_norm": 10.861001014709473, "learning_rate": 1.378797888467345e-08, "logits/chosen": -1.6198773384094238, "logits/rejected": -0.9793082475662231, "logps/chosen": -255.09112548828125, "logps/ref_chosen": -91.09699249267578, "logps/ref_rejected": -70.41004943847656, "logps/rejected": -314.01434326171875, "loss": 1.0275, "margin_dpo/margin_mean": 79.61019897460938, "margin_dpo/margin_std": 98.98773193359375, "step": 617 }, { "epoch": 0.9074889867841409, "fcm_dpo/beta": 0.006635970436036587, "fcm_dpo/delta": 0.026904039084911346, "fcm_dpo/margin": 86.58537292480469, "fcm_dpo/q_t": 0.37111544609069824, "grad_norm": 11.58753776550293, "learning_rate": 1.3370790793601371e-08, "logits/chosen": -1.829303503036499, "logits/rejected": -1.4636094570159912, "logps/chosen": -266.91119384765625, "logps/ref_chosen": -102.02059936523438, "logps/ref_rejected": -99.80119323730469, "logps/rejected": -351.27716064453125, "loss": 1.0048, "margin_dpo/margin_mean": 86.58537292480469, "margin_dpo/margin_std": 104.85649108886719, "step": 618 }, { "epoch": 0.908957415565345, "fcm_dpo/beta": 0.006731549743562937, "fcm_dpo/delta": 0.07085485756397247, "fcm_dpo/margin": 79.16950988769531, "fcm_dpo/q_t": 0.38630783557891846, "grad_norm": 11.098875999450684, "learning_rate": 1.2959838102258535e-08, "logits/chosen": -1.452412724494934, "logits/rejected": -1.0741287469863892, "logps/chosen": -248.23838806152344, "logps/ref_chosen": -89.74136352539062, "logps/ref_rejected": -99.90138244628906, "logps/rejected": -337.56793212890625, "loss": 1.0501, "margin_dpo/margin_mean": 79.16950988769531, "margin_dpo/margin_std": 113.035400390625, "step": 619 }, { "epoch": 0.9104258443465492, "fcm_dpo/beta": 0.006684327032417059, "fcm_dpo/delta": 0.012951068580150604, "fcm_dpo/margin": 87.47955322265625, "fcm_dpo/q_t": 0.3729515075683594, "grad_norm": 11.075997352600098, "learning_rate": 1.2555131639630567e-08, "logits/chosen": -2.05159068107605, "logits/rejected": -1.514744758605957, "logps/chosen": -238.4269256591797, "logps/ref_chosen": -85.12431335449219, "logps/ref_rejected": -85.41253662109375, "logps/rejected": -326.1947021484375, "loss": 1.0182, "margin_dpo/margin_mean": 87.47955322265625, "margin_dpo/margin_std": 107.8419189453125, "step": 620 }, { "epoch": 0.9118942731277533, "fcm_dpo/beta": 0.006873922888189554, "fcm_dpo/delta": 0.017858199775218964, "fcm_dpo/margin": 84.41682434082031, "fcm_dpo/q_t": 0.3753046989440918, "grad_norm": 11.407662391662598, "learning_rate": 1.2156682070109086e-08, "logits/chosen": -1.8650214672088623, "logits/rejected": -1.598212480545044, "logps/chosen": -245.00628662109375, "logps/ref_chosen": -89.24842071533203, "logps/ref_rejected": -95.46463775634766, "logps/rejected": -335.6393127441406, "loss": 1.0371, "margin_dpo/margin_mean": 84.41682434082031, "margin_dpo/margin_std": 113.9649658203125, "step": 621 }, { "epoch": 0.9133627019089574, "fcm_dpo/beta": 0.0067746308632195, "fcm_dpo/delta": -0.006477778777480125, "fcm_dpo/margin": 89.44147491455078, "fcm_dpo/q_t": 0.3655218780040741, "grad_norm": 12.676264762878418, "learning_rate": 1.1764499893210878e-08, "logits/chosen": -1.4255738258361816, "logits/rejected": -0.7507696747779846, "logps/chosen": -255.17373657226562, "logps/ref_chosen": -99.79412841796875, "logps/ref_rejected": -90.82821655273438, "logps/rejected": -335.6492919921875, "loss": 0.9635, "margin_dpo/margin_mean": 89.44148254394531, "margin_dpo/margin_std": 94.65943908691406, "step": 622 }, { "epoch": 0.9148311306901615, "fcm_dpo/beta": 0.006937139667570591, "fcm_dpo/delta": 0.16734254360198975, "fcm_dpo/margin": 63.47694396972656, "fcm_dpo/q_t": 0.40689754486083984, "grad_norm": 13.347342491149902, "learning_rate": 1.1378595443300998e-08, "logits/chosen": -2.32456111907959, "logits/rejected": -1.8910653591156006, "logps/chosen": -251.2813262939453, "logps/ref_chosen": -90.45555114746094, "logps/ref_rejected": -91.32276153564453, "logps/rejected": -315.62548828125, "loss": 1.138, "margin_dpo/margin_mean": 63.47693634033203, "margin_dpo/margin_std": 114.36991882324219, "step": 623 }, { "epoch": 0.9162995594713657, "fcm_dpo/beta": 0.006922868080437183, "fcm_dpo/delta": -0.09390503168106079, "fcm_dpo/margin": 99.29826354980469, "fcm_dpo/q_t": 0.35036206245422363, "grad_norm": 12.173335075378418, "learning_rate": 1.0998978889320582e-08, "logits/chosen": -2.5920774936676025, "logits/rejected": -1.668988585472107, "logps/chosen": -272.736083984375, "logps/ref_chosen": -109.87522888183594, "logps/ref_rejected": -104.77320861816406, "logps/rejected": -366.93231201171875, "loss": 0.9295, "margin_dpo/margin_mean": 99.29826354980469, "margin_dpo/margin_std": 105.40048217773438, "step": 624 }, { "epoch": 0.9177679882525698, "fcm_dpo/beta": 0.006994827184826136, "fcm_dpo/delta": 0.08261488378047943, "fcm_dpo/margin": 74.53196716308594, "fcm_dpo/q_t": 0.38579094409942627, "grad_norm": 10.665149688720703, "learning_rate": 1.0625660234518913e-08, "logits/chosen": -1.9561140537261963, "logits/rejected": -1.590733289718628, "logps/chosen": -259.02740478515625, "logps/ref_chosen": -87.16815948486328, "logps/ref_rejected": -91.86148071289062, "logps/rejected": -338.252685546875, "loss": 1.0348, "margin_dpo/margin_mean": 74.53196716308594, "margin_dpo/margin_std": 97.0251693725586, "step": 625 }, { "epoch": 0.9192364170337739, "fcm_dpo/beta": 0.007120197638869286, "fcm_dpo/delta": 0.08808340132236481, "fcm_dpo/margin": 72.4712142944336, "fcm_dpo/q_t": 0.39089781045913696, "grad_norm": 13.34603214263916, "learning_rate": 1.0258649316189721e-08, "logits/chosen": -1.5797321796417236, "logits/rejected": -1.1557903289794922, "logps/chosen": -267.87542724609375, "logps/ref_chosen": -104.22421264648438, "logps/ref_rejected": -104.1774673461914, "logps/rejected": -340.29986572265625, "loss": 1.0775, "margin_dpo/margin_mean": 72.47120666503906, "margin_dpo/margin_std": 112.5821533203125, "step": 626 }, { "epoch": 0.920704845814978, "fcm_dpo/beta": 0.007080005016177893, "fcm_dpo/delta": -0.04163035377860069, "fcm_dpo/margin": 90.24308776855469, "fcm_dpo/q_t": 0.36456674337387085, "grad_norm": 9.200769424438477, "learning_rate": 9.897955805412e-09, "logits/chosen": -1.5551083087921143, "logits/rejected": -1.6972510814666748, "logps/chosen": -221.11859130859375, "logps/ref_chosen": -74.93461608886719, "logps/ref_rejected": -112.57289123535156, "logps/rejected": -348.9999694824219, "loss": 0.9833, "margin_dpo/margin_mean": 90.24308776855469, "margin_dpo/margin_std": 113.31100463867188, "step": 627 }, { "epoch": 0.922173274596182, "fcm_dpo/beta": 0.006989369168877602, "fcm_dpo/delta": -0.0777251347899437, "fcm_dpo/margin": 96.21630859375, "fcm_dpo/q_t": 0.3529139757156372, "grad_norm": 12.8143892288208, "learning_rate": 9.543589206795238e-09, "logits/chosen": -2.0264172554016113, "logits/rejected": -1.7620849609375, "logps/chosen": -246.24087524414062, "logps/ref_chosen": -93.69107818603516, "logps/ref_rejected": -107.34395599365234, "logps/rejected": -356.11004638671875, "loss": 0.9296, "margin_dpo/margin_mean": 96.21630096435547, "margin_dpo/margin_std": 98.98152923583984, "step": 628 }, { "epoch": 0.9236417033773862, "fcm_dpo/beta": 0.0069330655969679356, "fcm_dpo/delta": -0.0012096241116523743, "fcm_dpo/margin": 86.66081237792969, "fcm_dpo/q_t": 0.36651545763015747, "grad_norm": 11.387245178222656, "learning_rate": 9.19555885822887e-09, "logits/chosen": -2.008543014526367, "logits/rejected": -1.5725985765457153, "logps/chosen": -266.6309814453125, "logps/ref_chosen": -103.23037719726562, "logps/ref_rejected": -97.16841888427734, "logps/rejected": -347.2298278808594, "loss": 0.9856, "margin_dpo/margin_mean": 86.66081237792969, "margin_dpo/margin_std": 100.66366577148438, "step": 629 }, { "epoch": 0.9251101321585903, "fcm_dpo/beta": 0.007189781405031681, "fcm_dpo/delta": 0.1778494417667389, "fcm_dpo/margin": 59.536293029785156, "fcm_dpo/q_t": 0.40600132942199707, "grad_norm": 15.609967231750488, "learning_rate": 8.85387393063622e-09, "logits/chosen": -2.0184245109558105, "logits/rejected": -1.4799931049346924, "logps/chosen": -244.27438354492188, "logps/ref_chosen": -93.89755249023438, "logps/ref_rejected": -89.3743896484375, "logps/rejected": -299.28753662109375, "loss": 1.1223, "margin_dpo/margin_mean": 59.536293029785156, "margin_dpo/margin_std": 98.0714111328125, "step": 630 }, { "epoch": 0.9265785609397944, "fcm_dpo/beta": 0.007201473228633404, "fcm_dpo/delta": -0.001188160851597786, "fcm_dpo/margin": 83.4664535522461, "fcm_dpo/q_t": 0.36686837673187256, "grad_norm": 11.061304092407227, "learning_rate": 8.518543427732949e-09, "logits/chosen": -2.5078396797180176, "logits/rejected": -2.078655958175659, "logps/chosen": -235.43612670898438, "logps/ref_chosen": -87.77082061767578, "logps/ref_rejected": -88.68241882324219, "logps/rejected": -319.81414794921875, "loss": 0.9793, "margin_dpo/margin_mean": 83.4664535522461, "margin_dpo/margin_std": 95.37765502929688, "step": 631 }, { "epoch": 0.9280469897209985, "fcm_dpo/beta": 0.007080842275172472, "fcm_dpo/delta": -0.06554602086544037, "fcm_dpo/margin": 93.11445617675781, "fcm_dpo/q_t": 0.35878896713256836, "grad_norm": 12.773686408996582, "learning_rate": 8.189576185789637e-09, "logits/chosen": -1.9766130447387695, "logits/rejected": -1.5455785989761353, "logps/chosen": -233.4774169921875, "logps/ref_chosen": -88.62652587890625, "logps/ref_rejected": -91.45091247558594, "logps/rejected": -329.416259765625, "loss": 0.9759, "margin_dpo/margin_mean": 93.11445617675781, "margin_dpo/margin_std": 112.72152709960938, "step": 632 }, { "epoch": 0.9295154185022027, "fcm_dpo/beta": 0.007372128311544657, "fcm_dpo/delta": 0.18217022716999054, "fcm_dpo/margin": 57.35297393798828, "fcm_dpo/q_t": 0.41445010900497437, "grad_norm": 14.506038665771484, "learning_rate": 7.866980873399015e-09, "logits/chosen": -2.240860939025879, "logits/rejected": -2.0498335361480713, "logps/chosen": -260.7783203125, "logps/ref_chosen": -81.37442016601562, "logps/ref_rejected": -98.62571716308594, "logps/rejected": -335.3825988769531, "loss": 1.1966, "margin_dpo/margin_mean": 57.35297393798828, "margin_dpo/margin_std": 124.81423950195312, "step": 633 }, { "epoch": 0.9309838472834068, "fcm_dpo/beta": 0.007527489215135574, "fcm_dpo/delta": 0.04844316840171814, "fcm_dpo/margin": 73.15084838867188, "fcm_dpo/q_t": 0.3781312108039856, "grad_norm": 13.961141586303711, "learning_rate": 7.550765991247654e-09, "logits/chosen": -1.4511160850524902, "logits/rejected": -1.2699205875396729, "logps/chosen": -263.0788879394531, "logps/ref_chosen": -96.12284851074219, "logps/ref_rejected": -112.84780883789062, "logps/rejected": -352.9547119140625, "loss": 1.0141, "margin_dpo/margin_mean": 73.15084838867188, "margin_dpo/margin_std": 84.39659881591797, "step": 634 }, { "epoch": 0.9324522760646109, "fcm_dpo/beta": 0.007684573996812105, "fcm_dpo/delta": 0.16829924285411835, "fcm_dpo/margin": 56.93890380859375, "fcm_dpo/q_t": 0.40733087062835693, "grad_norm": 16.105562210083008, "learning_rate": 7.240939871891699e-09, "logits/chosen": -1.7153419256210327, "logits/rejected": -1.1771185398101807, "logps/chosen": -269.976806640625, "logps/ref_chosen": -98.68411254882812, "logps/ref_rejected": -89.89920043945312, "logps/rejected": -318.13079833984375, "loss": 1.1449, "margin_dpo/margin_mean": 56.93890380859375, "margin_dpo/margin_std": 105.87216186523438, "step": 635 }, { "epoch": 0.933920704845815, "fcm_dpo/beta": 0.007606941275298595, "fcm_dpo/delta": -0.09959021955728531, "fcm_dpo/margin": 91.07327270507812, "fcm_dpo/q_t": 0.35048407316207886, "grad_norm": 12.00423526763916, "learning_rate": 6.937510679537628e-09, "logits/chosen": -1.446263074874878, "logits/rejected": -0.9623331427574158, "logps/chosen": -246.98480224609375, "logps/ref_chosen": -90.41796112060547, "logps/ref_rejected": -87.70687866210938, "logps/rejected": -335.34698486328125, "loss": 0.9282, "margin_dpo/margin_mean": 91.07327270507812, "margin_dpo/margin_std": 97.26042175292969, "step": 636 }, { "epoch": 0.9353891336270191, "fcm_dpo/beta": 0.0074293059296905994, "fcm_dpo/delta": -0.1272699534893036, "fcm_dpo/margin": 96.68272399902344, "fcm_dpo/q_t": 0.34441307187080383, "grad_norm": 10.729310989379883, "learning_rate": 6.640486409826785e-09, "logits/chosen": -2.0841870307922363, "logits/rejected": -1.981583595275879, "logps/chosen": -240.16139221191406, "logps/ref_chosen": -82.44971466064453, "logps/ref_rejected": -104.02860260009766, "logps/rejected": -358.4230041503906, "loss": 0.9195, "margin_dpo/margin_mean": 96.68272399902344, "margin_dpo/margin_std": 101.56159973144531, "step": 637 }, { "epoch": 0.9368575624082232, "fcm_dpo/beta": 0.007293237838894129, "fcm_dpo/delta": -0.11687527596950531, "fcm_dpo/margin": 97.12511444091797, "fcm_dpo/q_t": 0.3418697118759155, "grad_norm": 10.414422988891602, "learning_rate": 6.349874889624962e-09, "logits/chosen": -1.5123820304870605, "logits/rejected": -0.8182043433189392, "logps/chosen": -238.3617401123047, "logps/ref_chosen": -91.92498779296875, "logps/ref_rejected": -86.28703308105469, "logps/rejected": -329.8489074707031, "loss": 0.8832, "margin_dpo/margin_mean": 97.12511444091797, "margin_dpo/margin_std": 81.02644348144531, "step": 638 }, { "epoch": 0.9383259911894273, "fcm_dpo/beta": 0.007484002038836479, "fcm_dpo/delta": 0.23280690610408783, "fcm_dpo/margin": 50.03688430786133, "fcm_dpo/q_t": 0.41923975944519043, "grad_norm": 14.05245304107666, "learning_rate": 6.065683776815933e-09, "logits/chosen": -1.3220126628875732, "logits/rejected": -0.5478463768959045, "logps/chosen": -296.0784912109375, "logps/ref_chosen": -104.52755737304688, "logps/ref_rejected": -81.4803466796875, "logps/rejected": -323.06817626953125, "loss": 1.2022, "margin_dpo/margin_mean": 50.036888122558594, "margin_dpo/margin_std": 110.06419372558594, "step": 639 }, { "epoch": 0.9397944199706314, "fcm_dpo/beta": 0.00756090460345149, "fcm_dpo/delta": -0.06717022508382797, "fcm_dpo/margin": 87.34074401855469, "fcm_dpo/q_t": 0.3554985821247101, "grad_norm": 12.982050895690918, "learning_rate": 5.7879205600998296e-09, "logits/chosen": -1.7683886289596558, "logits/rejected": -1.2369413375854492, "logps/chosen": -261.49896240234375, "logps/ref_chosen": -97.88526916503906, "logps/ref_rejected": -112.70502471923828, "logps/rejected": -363.65948486328125, "loss": 0.9568, "margin_dpo/margin_mean": 87.34074401855469, "margin_dpo/margin_std": 92.46745300292969, "step": 640 }, { "epoch": 0.9412628487518355, "fcm_dpo/beta": 0.007357077207416296, "fcm_dpo/delta": -0.08074723184108734, "fcm_dpo/margin": 91.79606628417969, "fcm_dpo/q_t": 0.3599125146865845, "grad_norm": 10.860076904296875, "learning_rate": 5.516592558795746e-09, "logits/chosen": -2.443859338760376, "logits/rejected": -1.9180916547775269, "logps/chosen": -245.45460510253906, "logps/ref_chosen": -96.4456787109375, "logps/ref_rejected": -95.13568878173828, "logps/rejected": -335.940673828125, "loss": 0.9735, "margin_dpo/margin_mean": 91.79606628417969, "margin_dpo/margin_std": 117.18472290039062, "step": 641 }, { "epoch": 0.9427312775330396, "fcm_dpo/beta": 0.007323989644646645, "fcm_dpo/delta": -0.019782835617661476, "fcm_dpo/margin": 84.43093872070312, "fcm_dpo/q_t": 0.3701496422290802, "grad_norm": 12.883811950683594, "learning_rate": 5.251706922648868e-09, "logits/chosen": -1.569532036781311, "logits/rejected": -1.235536813735962, "logps/chosen": -269.09429931640625, "logps/ref_chosen": -100.75984954833984, "logps/ref_rejected": -114.7076416015625, "logps/rejected": -367.4729919433594, "loss": 1.0067, "margin_dpo/margin_mean": 84.43093872070312, "margin_dpo/margin_std": 113.8532485961914, "step": 642 }, { "epoch": 0.9441997063142438, "fcm_dpo/beta": 0.007224624045193195, "fcm_dpo/delta": 0.009218744933605194, "fcm_dpo/margin": 81.56900787353516, "fcm_dpo/q_t": 0.37049052119255066, "grad_norm": 13.749687194824219, "learning_rate": 4.993270631642038e-09, "logits/chosen": -2.2235236167907715, "logits/rejected": -1.8363418579101562, "logps/chosen": -241.62869262695312, "logps/ref_chosen": -84.74365997314453, "logps/ref_rejected": -94.31842041015625, "logps/rejected": -332.7724609375, "loss": 1.0051, "margin_dpo/margin_mean": 81.56900787353516, "margin_dpo/margin_std": 98.04161071777344, "step": 643 }, { "epoch": 0.9456681350954479, "fcm_dpo/beta": 0.0074260118417441845, "fcm_dpo/delta": 0.10681871324777603, "fcm_dpo/margin": 67.11778259277344, "fcm_dpo/q_t": 0.38967859745025635, "grad_norm": 14.644425392150879, "learning_rate": 4.741290495811873e-09, "logits/chosen": -1.818040370941162, "logits/rejected": -1.5412225723266602, "logps/chosen": -246.3582763671875, "logps/ref_chosen": -85.32274627685547, "logps/ref_rejected": -94.60861206054688, "logps/rejected": -322.76190185546875, "loss": 1.0502, "margin_dpo/margin_mean": 67.11778259277344, "margin_dpo/margin_std": 89.98631286621094, "step": 644 }, { "epoch": 0.947136563876652, "fcm_dpo/beta": 0.007773152552545071, "fcm_dpo/delta": 0.2508290708065033, "fcm_dpo/margin": 45.93043518066406, "fcm_dpo/q_t": 0.425441175699234, "grad_norm": 14.240182876586914, "learning_rate": 4.495773155069299e-09, "logits/chosen": -1.446195363998413, "logits/rejected": -1.2669878005981445, "logps/chosen": -257.103515625, "logps/ref_chosen": -82.59024047851562, "logps/ref_rejected": -103.00375366210938, "logps/rejected": -323.4474792480469, "loss": 1.2211, "margin_dpo/margin_mean": 45.93043518066406, "margin_dpo/margin_std": 107.12353515625, "step": 645 }, { "epoch": 0.9486049926578561, "fcm_dpo/beta": 0.007703070994466543, "fcm_dpo/delta": -0.08048854768276215, "fcm_dpo/margin": 87.47425842285156, "fcm_dpo/q_t": 0.3519340753555298, "grad_norm": 11.884034156799316, "learning_rate": 4.256725079024553e-09, "logits/chosen": -1.8835480213165283, "logits/rejected": -1.2794466018676758, "logps/chosen": -241.96484375, "logps/ref_chosen": -91.1920394897461, "logps/ref_rejected": -83.77833557128906, "logps/rejected": -322.025390625, "loss": 0.9277, "margin_dpo/margin_mean": 87.47425842285156, "margin_dpo/margin_std": 88.36001586914062, "step": 646 }, { "epoch": 0.9500734214390602, "fcm_dpo/beta": 0.0077326856553554535, "fcm_dpo/delta": 0.018357396125793457, "fcm_dpo/margin": 75.35665130615234, "fcm_dpo/q_t": 0.37991610169410706, "grad_norm": 12.295222282409668, "learning_rate": 4.024152566816791e-09, "logits/chosen": -1.3169585466384888, "logits/rejected": -1.1735953092575073, "logps/chosen": -254.99407958984375, "logps/ref_chosen": -88.8444595336914, "logps/ref_rejected": -99.49833679199219, "logps/rejected": -341.00457763671875, "loss": 1.0523, "margin_dpo/margin_mean": 75.35665130615234, "margin_dpo/margin_std": 113.26918029785156, "step": 647 }, { "epoch": 0.9515418502202643, "fcm_dpo/beta": 0.007531121838837862, "fcm_dpo/delta": -0.2151474952697754, "fcm_dpo/margin": 106.015625, "fcm_dpo/q_t": 0.3290584087371826, "grad_norm": 12.194585800170898, "learning_rate": 3.798061746947995e-09, "logits/chosen": -1.9958875179290771, "logits/rejected": -1.9187660217285156, "logps/chosen": -240.71719360351562, "logps/ref_chosen": -87.84810638427734, "logps/ref_rejected": -104.67005920410156, "logps/rejected": -363.5547790527344, "loss": 0.8804, "margin_dpo/margin_mean": 106.015625, "margin_dpo/margin_std": 108.19938659667969, "step": 648 }, { "epoch": 0.9530102790014684, "fcm_dpo/beta": 0.007324899546802044, "fcm_dpo/delta": -0.042867887765169144, "fcm_dpo/margin": 87.23211669921875, "fcm_dpo/q_t": 0.35840868949890137, "grad_norm": 13.264395713806152, "learning_rate": 3.5784585771215235e-09, "logits/chosen": -2.3595848083496094, "logits/rejected": -1.9544248580932617, "logps/chosen": -229.560302734375, "logps/ref_chosen": -89.6925048828125, "logps/ref_rejected": -88.70658111572266, "logps/rejected": -315.8064880371094, "loss": 0.9392, "margin_dpo/margin_mean": 87.23211669921875, "margin_dpo/margin_std": 85.69928741455078, "step": 649 }, { "epoch": 0.9544787077826725, "fcm_dpo/beta": 0.007353082299232483, "fcm_dpo/delta": -0.015879113227128983, "fcm_dpo/margin": 83.62142181396484, "fcm_dpo/q_t": 0.3669193983078003, "grad_norm": 12.675481796264648, "learning_rate": 3.3653488440851253e-09, "logits/chosen": -1.3255770206451416, "logits/rejected": -1.12962007522583, "logps/chosen": -255.35781860351562, "logps/ref_chosen": -89.93060302734375, "logps/ref_rejected": -102.61282348632812, "logps/rejected": -351.66143798828125, "loss": 1.0064, "margin_dpo/margin_mean": 83.62141418457031, "margin_dpo/margin_std": 110.35063934326172, "step": 650 }, { "epoch": 0.9559471365638766, "fcm_dpo/beta": 0.007182122208178043, "fcm_dpo/delta": -0.1339079886674881, "fcm_dpo/margin": 100.80975341796875, "fcm_dpo/q_t": 0.3390667736530304, "grad_norm": 11.261962890625, "learning_rate": 3.158738163478475e-09, "logits/chosen": -2.088024854660034, "logits/rejected": -1.914245367050171, "logps/chosen": -218.3494873046875, "logps/ref_chosen": -79.18731689453125, "logps/ref_rejected": -105.93333435058594, "logps/rejected": -345.9052734375, "loss": 0.8862, "margin_dpo/margin_mean": 100.80976104736328, "margin_dpo/margin_std": 92.29314422607422, "step": 651 }, { "epoch": 0.9574155653450808, "fcm_dpo/beta": 0.007138779386878014, "fcm_dpo/delta": -0.021067023277282715, "fcm_dpo/margin": 86.80287170410156, "fcm_dpo/q_t": 0.36111682653427124, "grad_norm": 10.429312705993652, "learning_rate": 2.9586319796851555e-09, "logits/chosen": -2.4728636741638184, "logits/rejected": -2.168954849243164, "logps/chosen": -246.78160095214844, "logps/ref_chosen": -101.79022216796875, "logps/ref_rejected": -116.3245849609375, "logps/rejected": -348.11883544921875, "loss": 0.9352, "margin_dpo/margin_mean": 86.80288696289062, "margin_dpo/margin_std": 81.80032348632812, "step": 652 }, { "epoch": 0.9588839941262849, "fcm_dpo/beta": 0.007149814628064632, "fcm_dpo/delta": 0.018516050651669502, "fcm_dpo/margin": 81.45347595214844, "fcm_dpo/q_t": 0.36881303787231445, "grad_norm": 12.834737777709961, "learning_rate": 2.7650355656892166e-09, "logits/chosen": -2.072822093963623, "logits/rejected": -1.819368600845337, "logps/chosen": -257.77667236328125, "logps/ref_chosen": -93.35359191894531, "logps/ref_rejected": -109.12324523925781, "logps/rejected": -354.99981689453125, "loss": 0.9894, "margin_dpo/margin_mean": 81.45347595214844, "margin_dpo/margin_std": 93.0560073852539, "step": 653 }, { "epoch": 0.960352422907489, "fcm_dpo/beta": 0.007231119088828564, "fcm_dpo/delta": 0.08581791818141937, "fcm_dpo/margin": 71.71923828125, "fcm_dpo/q_t": 0.3922192454338074, "grad_norm": 15.544170379638672, "learning_rate": 2.577954022936174e-09, "logits/chosen": -2.3582053184509277, "logits/rejected": -2.17484712600708, "logps/chosen": -254.76153564453125, "logps/ref_chosen": -89.11553955078125, "logps/ref_rejected": -104.91995239257812, "logps/rejected": -342.28515625, "loss": 1.0873, "margin_dpo/margin_mean": 71.71923828125, "margin_dpo/margin_std": 116.78224182128906, "step": 654 }, { "epoch": 0.9618208516886931, "fcm_dpo/beta": 0.007248979061841965, "fcm_dpo/delta": -0.02873397059738636, "fcm_dpo/margin": 86.4774169921875, "fcm_dpo/q_t": 0.362434983253479, "grad_norm": 11.312616348266602, "learning_rate": 2.397392281198729e-09, "logits/chosen": -2.129537582397461, "logits/rejected": -2.125460147857666, "logps/chosen": -233.4827423095703, "logps/ref_chosen": -81.03610229492188, "logps/ref_rejected": -102.80233764648438, "logps/rejected": -341.72637939453125, "loss": 0.9844, "margin_dpo/margin_mean": 86.4774169921875, "margin_dpo/margin_std": 104.30917358398438, "step": 655 }, { "epoch": 0.9632892804698973, "fcm_dpo/beta": 0.007026261650025845, "fcm_dpo/delta": -0.16489456593990326, "fcm_dpo/margin": 107.01776123046875, "fcm_dpo/q_t": 0.3365343511104584, "grad_norm": 12.231975555419922, "learning_rate": 2.223355098446622e-09, "logits/chosen": -1.2732338905334473, "logits/rejected": -1.3455628156661987, "logps/chosen": -237.61703491210938, "logps/ref_chosen": -85.32534790039062, "logps/ref_rejected": -118.33866119384766, "logps/rejected": -377.6481018066406, "loss": 0.8735, "margin_dpo/margin_mean": 107.01776123046875, "margin_dpo/margin_std": 99.96194458007812, "step": 656 }, { "epoch": 0.9647577092511013, "fcm_dpo/beta": 0.006966191343963146, "fcm_dpo/delta": 0.019626092165708542, "fcm_dpo/margin": 83.40191650390625, "fcm_dpo/q_t": 0.3764026165008545, "grad_norm": 14.666167259216309, "learning_rate": 2.055847060721566e-09, "logits/chosen": -1.931036114692688, "logits/rejected": -1.757138729095459, "logps/chosen": -242.91915893554688, "logps/ref_chosen": -80.19772338867188, "logps/ref_rejected": -102.581298828125, "logps/rejected": -348.70465087890625, "loss": 1.0244, "margin_dpo/margin_mean": 83.40192413330078, "margin_dpo/margin_std": 111.77230834960938, "step": 657 }, { "epoch": 0.9662261380323054, "fcm_dpo/beta": 0.007131836377084255, "fcm_dpo/delta": 0.08433780819177628, "fcm_dpo/margin": 72.8197250366211, "fcm_dpo/q_t": 0.38564208149909973, "grad_norm": 11.62498664855957, "learning_rate": 1.8948725820160662e-09, "logits/chosen": -2.0923075675964355, "logits/rejected": -1.626826286315918, "logps/chosen": -262.4381103515625, "logps/ref_chosen": -94.634521484375, "logps/ref_rejected": -101.6316146850586, "logps/rejected": -342.25494384765625, "loss": 1.0369, "margin_dpo/margin_mean": 72.8197250366211, "margin_dpo/margin_std": 94.01678466796875, "step": 658 }, { "epoch": 0.9676945668135095, "fcm_dpo/beta": 0.007036149501800537, "fcm_dpo/delta": -0.056030645966529846, "fcm_dpo/margin": 92.60414123535156, "fcm_dpo/q_t": 0.36230194568634033, "grad_norm": 11.66299819946289, "learning_rate": 1.7404359041573723e-09, "logits/chosen": -2.0208144187927246, "logits/rejected": -1.0372681617736816, "logps/chosen": -259.408447265625, "logps/ref_chosen": -112.55587005615234, "logps/ref_rejected": -93.9216079711914, "logps/rejected": -333.37835693359375, "loss": 1.0067, "margin_dpo/margin_mean": 92.6041488647461, "margin_dpo/margin_std": 122.59965515136719, "step": 659 }, { "epoch": 0.9691629955947136, "fcm_dpo/beta": 0.006981555372476578, "fcm_dpo/delta": -0.06499287486076355, "fcm_dpo/margin": 94.63003540039062, "fcm_dpo/q_t": 0.35565003752708435, "grad_norm": 14.255919456481934, "learning_rate": 1.592541096695571e-09, "logits/chosen": -1.8492063283920288, "logits/rejected": -1.2232909202575684, "logps/chosen": -247.27349853515625, "logps/ref_chosen": -93.37742614746094, "logps/ref_rejected": -81.39482116699219, "logps/rejected": -329.9209289550781, "loss": 0.9439, "margin_dpo/margin_mean": 94.63003540039062, "margin_dpo/margin_std": 103.12388610839844, "step": 660 }, { "epoch": 0.9706314243759178, "fcm_dpo/beta": 0.006860889494419098, "fcm_dpo/delta": -0.1109163761138916, "fcm_dpo/margin": 102.49998474121094, "fcm_dpo/q_t": 0.34864702820777893, "grad_norm": 11.033123970031738, "learning_rate": 1.4511920567963908e-09, "logits/chosen": -1.9187567234039307, "logits/rejected": -1.4068787097930908, "logps/chosen": -245.22006225585938, "logps/ref_chosen": -87.85516357421875, "logps/ref_rejected": -92.40330505371094, "logps/rejected": -352.2681884765625, "loss": 0.931, "margin_dpo/margin_mean": 102.49998474121094, "margin_dpo/margin_std": 111.98723602294922, "step": 661 }, { "epoch": 0.9720998531571219, "fcm_dpo/beta": 0.006795001216232777, "fcm_dpo/delta": -0.02861689031124115, "fcm_dpo/margin": 92.24244689941406, "fcm_dpo/q_t": 0.3603462278842926, "grad_norm": 10.01567554473877, "learning_rate": 1.3163925091384532e-09, "logits/chosen": -1.5263056755065918, "logits/rejected": -0.9408450722694397, "logps/chosen": -255.88613891601562, "logps/ref_chosen": -102.77980041503906, "logps/ref_rejected": -95.22531127929688, "logps/rejected": -340.5740966796875, "loss": 0.942, "margin_dpo/margin_mean": 92.24244689941406, "margin_dpo/margin_std": 90.77671813964844, "step": 662 }, { "epoch": 0.973568281938326, "fcm_dpo/beta": 0.006806028075516224, "fcm_dpo/delta": 0.015626247972249985, "fcm_dpo/margin": 85.96340942382812, "fcm_dpo/q_t": 0.3726150095462799, "grad_norm": 11.392067909240723, "learning_rate": 1.1881460058152382e-09, "logits/chosen": -2.2485814094543457, "logits/rejected": -2.024899959564209, "logps/chosen": -249.08914184570312, "logps/ref_chosen": -96.34658813476562, "logps/ref_rejected": -120.52645874023438, "logps/rejected": -359.232421875, "loss": 1.0095, "margin_dpo/margin_mean": 85.96340942382812, "margin_dpo/margin_std": 108.69853210449219, "step": 663 }, { "epoch": 0.9750367107195301, "fcm_dpo/beta": 0.006872647441923618, "fcm_dpo/delta": 0.062802754342556, "fcm_dpo/margin": 78.6165771484375, "fcm_dpo/q_t": 0.37933892011642456, "grad_norm": 11.565393447875977, "learning_rate": 1.066455926241383e-09, "logits/chosen": -1.6666076183319092, "logits/rejected": -1.4859673976898193, "logps/chosen": -259.26226806640625, "logps/ref_chosen": -91.84242248535156, "logps/ref_rejected": -111.8366928100586, "logps/rejected": -357.8731384277344, "loss": 1.0112, "margin_dpo/margin_mean": 78.6165771484375, "margin_dpo/margin_std": 91.88618469238281, "step": 664 }, { "epoch": 0.9765051395007343, "fcm_dpo/beta": 0.006901168264448643, "fcm_dpo/delta": 0.007144978269934654, "fcm_dpo/margin": 85.94855499267578, "fcm_dpo/q_t": 0.36612850427627563, "grad_norm": 9.935822486877441, "learning_rate": 9.513254770636137e-10, "logits/chosen": -2.3310766220092773, "logits/rejected": -1.89298677444458, "logps/chosen": -232.2900848388672, "logps/ref_chosen": -88.18618774414062, "logps/ref_rejected": -91.9120101928711, "logps/rejected": -321.9644775390625, "loss": 0.9693, "margin_dpo/margin_mean": 85.94855499267578, "margin_dpo/margin_std": 91.62937927246094, "step": 665 }, { "epoch": 0.9779735682819384, "fcm_dpo/beta": 0.006875505670905113, "fcm_dpo/delta": 0.01688658446073532, "fcm_dpo/margin": 84.91444396972656, "fcm_dpo/q_t": 0.37375444173812866, "grad_norm": 14.775041580200195, "learning_rate": 8.427576920763956e-10, "logits/chosen": -1.6999645233154297, "logits/rejected": -1.3265091180801392, "logps/chosen": -261.86273193359375, "logps/ref_chosen": -100.97459411621094, "logps/ref_rejected": -101.24992370605469, "logps/rejected": -347.052490234375, "loss": 1.0104, "margin_dpo/margin_mean": 84.91444396972656, "margin_dpo/margin_std": 106.82335662841797, "step": 666 }, { "epoch": 0.9794419970631424, "fcm_dpo/beta": 0.006852601654827595, "fcm_dpo/delta": -0.03697332367300987, "fcm_dpo/margin": 92.57352447509766, "fcm_dpo/q_t": 0.3577842712402344, "grad_norm": 11.277308464050293, "learning_rate": 7.407554321417764e-10, "logits/chosen": -1.651948094367981, "logits/rejected": -1.0065885782241821, "logps/chosen": -270.1557922363281, "logps/ref_chosen": -97.5711669921875, "logps/ref_rejected": -93.58476257324219, "logps/rejected": -358.742919921875, "loss": 0.9343, "margin_dpo/margin_mean": 92.57352447509766, "margin_dpo/margin_std": 89.03514099121094, "step": 667 }, { "epoch": 0.9809104258443465, "fcm_dpo/beta": 0.006754533387720585, "fcm_dpo/delta": -0.11870689690113068, "fcm_dpo/margin": 105.17715454101562, "fcm_dpo/q_t": 0.3479474186897278, "grad_norm": 14.153767585754395, "learning_rate": 6.453213851142225e-10, "logits/chosen": -2.2566616535186768, "logits/rejected": -1.7255539894104004, "logps/chosen": -255.34854125976562, "logps/ref_chosen": -102.5750503540039, "logps/ref_rejected": -108.81768798828125, "logps/rejected": -366.7683410644531, "loss": 0.911, "margin_dpo/margin_mean": 105.17715454101562, "margin_dpo/margin_std": 108.63129425048828, "step": 668 }, { "epoch": 0.9823788546255506, "fcm_dpo/beta": 0.006622363813221455, "fcm_dpo/delta": -0.0791027769446373, "fcm_dpo/margin": 101.74537658691406, "fcm_dpo/q_t": 0.350632905960083, "grad_norm": 11.706164360046387, "learning_rate": 5.564580657695939e-10, "logits/chosen": -1.6500535011291504, "logits/rejected": -1.2420462369918823, "logps/chosen": -229.16888427734375, "logps/ref_chosen": -89.49478149414062, "logps/ref_rejected": -82.51950073242188, "logps/rejected": -323.9389953613281, "loss": 0.9178, "margin_dpo/margin_mean": 101.74537658691406, "margin_dpo/margin_std": 98.94613647460938, "step": 669 }, { "epoch": 0.9838472834067548, "fcm_dpo/beta": 0.006471183151006699, "fcm_dpo/delta": -0.12788358330726624, "fcm_dpo/margin": 111.06494140625, "fcm_dpo/q_t": 0.3385842442512512, "grad_norm": 9.694023132324219, "learning_rate": 4.741678157389739e-10, "logits/chosen": -1.7507426738739014, "logits/rejected": -1.2104510068893433, "logps/chosen": -234.94271850585938, "logps/ref_chosen": -95.45459747314453, "logps/ref_rejected": -101.53292846679688, "logps/rejected": -352.08599853515625, "loss": 0.8892, "margin_dpo/margin_mean": 111.06494140625, "margin_dpo/margin_std": 101.67810821533203, "step": 670 }, { "epoch": 0.9853157121879589, "fcm_dpo/beta": 0.006587146315723658, "fcm_dpo/delta": 0.11642096936702728, "fcm_dpo/margin": 73.98999786376953, "fcm_dpo/q_t": 0.3927046060562134, "grad_norm": 11.580408096313477, "learning_rate": 3.9845280344705245e-10, "logits/chosen": -1.8810558319091797, "logits/rejected": -1.6646201610565186, "logps/chosen": -254.25238037109375, "logps/ref_chosen": -82.12312316894531, "logps/ref_rejected": -90.21969604492188, "logps/rejected": -336.33892822265625, "loss": 1.0615, "margin_dpo/margin_mean": 73.98999786376953, "margin_dpo/margin_std": 101.05743408203125, "step": 671 }, { "epoch": 0.986784140969163, "fcm_dpo/beta": 0.0067345621064305305, "fcm_dpo/delta": 0.16344712674617767, "fcm_dpo/margin": 65.91778564453125, "fcm_dpo/q_t": 0.4018493890762329, "grad_norm": 13.346258163452148, "learning_rate": 3.293150240547549e-10, "logits/chosen": -1.9560139179229736, "logits/rejected": -1.6140336990356445, "logps/chosen": -260.7767333984375, "logps/ref_chosen": -90.0619125366211, "logps/ref_rejected": -100.45323181152344, "logps/rejected": -337.0858154296875, "loss": 1.0877, "margin_dpo/margin_mean": 65.91779327392578, "margin_dpo/margin_std": 93.67379760742188, "step": 672 }, { "epoch": 0.9882525697503671, "fcm_dpo/beta": 0.006898353807628155, "fcm_dpo/delta": 0.08046399801969528, "fcm_dpo/margin": 75.86769104003906, "fcm_dpo/q_t": 0.38369953632354736, "grad_norm": 13.707597732543945, "learning_rate": 2.6675629940689504e-10, "logits/chosen": -2.0346531867980957, "logits/rejected": -1.667980432510376, "logps/chosen": -230.58885192871094, "logps/ref_chosen": -79.26315307617188, "logps/ref_rejected": -91.34925079345703, "logps/rejected": -318.5426330566406, "loss": 1.0206, "margin_dpo/margin_mean": 75.86769104003906, "margin_dpo/margin_std": 91.65807342529297, "step": 673 }, { "epoch": 0.9897209985315712, "fcm_dpo/beta": 0.006784925237298012, "fcm_dpo/delta": -0.10350759327411652, "fcm_dpo/margin": 102.5726318359375, "fcm_dpo/q_t": 0.3477708697319031, "grad_norm": 10.508623123168945, "learning_rate": 2.1077827798404725e-10, "logits/chosen": -1.6836049556732178, "logits/rejected": -1.2868695259094238, "logps/chosen": -219.0989990234375, "logps/ref_chosen": -75.45831298828125, "logps/ref_rejected": -76.20362854003906, "logps/rejected": -322.41693115234375, "loss": 0.9032, "margin_dpo/margin_mean": 102.57262420654297, "margin_dpo/margin_std": 99.2423095703125, "step": 674 }, { "epoch": 0.9911894273127754, "fcm_dpo/beta": 0.006704601459205151, "fcm_dpo/delta": -0.0026360079646110535, "fcm_dpo/margin": 89.63999938964844, "fcm_dpo/q_t": 0.3685849606990814, "grad_norm": 13.317923545837402, "learning_rate": 1.6138243485910863e-10, "logits/chosen": -1.458611249923706, "logits/rejected": -1.1178425550460815, "logps/chosen": -239.28143310546875, "logps/ref_chosen": -79.90953063964844, "logps/ref_rejected": -81.21824645996094, "logps/rejected": -330.23016357421875, "loss": 1.0169, "margin_dpo/margin_mean": 89.63999938964844, "margin_dpo/margin_std": 117.47451782226562, "step": 675 }, { "epoch": 0.9926578560939795, "fcm_dpo/beta": 0.006719764322042465, "fcm_dpo/delta": -0.015834422782063484, "fcm_dpo/margin": 91.44828796386719, "fcm_dpo/q_t": 0.36421921849250793, "grad_norm": 11.21393871307373, "learning_rate": 1.1857007165852472e-10, "logits/chosen": -1.9717350006103516, "logits/rejected": -1.388458251953125, "logps/chosen": -268.1275329589844, "logps/ref_chosen": -98.17111206054688, "logps/ref_rejected": -95.024658203125, "logps/rejected": -356.42938232421875, "loss": 0.9797, "margin_dpo/margin_mean": 91.44828796386719, "margin_dpo/margin_std": 105.38261413574219, "step": 676 }, { "epoch": 0.9941262848751835, "fcm_dpo/beta": 0.006692584604024887, "fcm_dpo/delta": -0.030026260763406754, "fcm_dpo/margin": 93.82452392578125, "fcm_dpo/q_t": 0.35947829484939575, "grad_norm": 10.283641815185547, "learning_rate": 8.23423165278725e-11, "logits/chosen": -1.81768798828125, "logits/rejected": -1.1459150314331055, "logps/chosen": -243.63877868652344, "logps/ref_chosen": -91.37928771972656, "logps/ref_rejected": -82.87776947021484, "logps/rejected": -328.9617919921875, "loss": 0.9365, "margin_dpo/margin_mean": 93.82452392578125, "margin_dpo/margin_std": 89.39640808105469, "step": 677 }, { "epoch": 0.9955947136563876, "fcm_dpo/beta": 0.00660483306273818, "fcm_dpo/delta": -0.14169199764728546, "fcm_dpo/margin": 110.73104858398438, "fcm_dpo/q_t": 0.33739006519317627, "grad_norm": 10.643210411071777, "learning_rate": 5.270012410216185e-11, "logits/chosen": -1.7930124998092651, "logits/rejected": -1.4681642055511475, "logps/chosen": -219.61293029785156, "logps/ref_chosen": -75.6458740234375, "logps/ref_rejected": -86.96611022949219, "logps/rejected": -341.66424560546875, "loss": 0.88, "margin_dpo/margin_mean": 110.73104858398438, "margin_dpo/margin_std": 97.85108947753906, "step": 678 }, { "epoch": 0.9970631424375918, "fcm_dpo/beta": 0.006523388437926769, "fcm_dpo/delta": 0.013953792862594128, "fcm_dpo/margin": 89.96353149414062, "fcm_dpo/q_t": 0.36797723174095154, "grad_norm": 10.297441482543945, "learning_rate": 2.9644275480772416e-11, "logits/chosen": -2.299855947494507, "logits/rejected": -1.7864303588867188, "logps/chosen": -245.36917114257812, "logps/ref_chosen": -80.77344512939453, "logps/ref_rejected": -82.87850189208984, "logps/rejected": -337.4377746582031, "loss": 0.9515, "margin_dpo/margin_mean": 89.96353912353516, "margin_dpo/margin_std": 85.39958190917969, "step": 679 }, { "epoch": 0.9985315712187959, "fcm_dpo/beta": 0.006581049412488937, "fcm_dpo/delta": -0.020787008106708527, "fcm_dpo/margin": 93.83221435546875, "fcm_dpo/q_t": 0.36515846848487854, "grad_norm": 9.382950782775879, "learning_rate": 1.31753782067201e-11, "logits/chosen": -1.9891002178192139, "logits/rejected": -1.6667735576629639, "logps/chosen": -276.03668212890625, "logps/ref_chosen": -107.68292999267578, "logps/ref_rejected": -116.0948715209961, "logps/rejected": -378.2808837890625, "loss": 0.9739, "margin_dpo/margin_mean": 93.83221435546875, "margin_dpo/margin_std": 103.33714294433594, "step": 680 }, { "epoch": 1.0, "fcm_dpo/beta": 0.006566454656422138, "fcm_dpo/delta": 0.05062846094369888, "fcm_dpo/margin": 84.06166076660156, "fcm_dpo/q_t": 0.3779335916042328, "grad_norm": 10.81689167022705, "learning_rate": 3.2938662507808745e-12, "logits/chosen": -2.369196653366089, "logits/rejected": -2.0650794506073, "logps/chosen": -243.9755859375, "logps/ref_chosen": -93.01106262207031, "logps/ref_rejected": -94.8221664428711, "logps/rejected": -329.84832763671875, "loss": 1.0069, "margin_dpo/margin_mean": 84.06166076660156, "margin_dpo/margin_std": 100.67030334472656, "step": 681 }, { "epoch": 1.0, "step": 681, "total_flos": 0.0, "train_loss": 1.0153061533114283, "train_runtime": 2239.7507, "train_samples_per_second": 19.466, "train_steps_per_second": 0.304 } ], "logging_steps": 1, "max_steps": 681, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }