jackf857/qwen3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
qwen3-8b-base-new-dpo-hh-he…/trainer_state.json
ModelHub XC 74414d18a2 初始化项目，由ModelHub XC社区提供模型 
Model: jackf857/qwen3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.6
Source: Original Platform
2026-06-18 20:14:22 +08:00

13085 lines
481 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06123492121696472,
"fcm_dpo/q_t": 0.4984707534313202,
"grad_norm": 38.81780242919922,
"learning_rate": 0.0,
"logits/chosen": 0.2677389979362488,
"logits/rejected": 0.6191319227218628,
"logps/chosen": -85.37664031982422,
"logps/ref_chosen": -85.43083190917969,
"logps/ref_rejected": -79.90458679199219,
"logps/rejected": -79.91163635253906,
"loss": 1.3807,
"margin_dpo/margin_mean": 0.061235010623931885,
"margin_dpo/margin_std": 0.4424324035644531,
"step": 1
},
{
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.12442219257354736,
"fcm_dpo/q_t": 0.5031089186668396,
"grad_norm": 29.75464630126953,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": 0.47771453857421875,
"logits/rejected": 0.5908545851707458,
"logps/chosen": -82.11383056640625,
"logps/ref_chosen": -82.06893157958984,
"logps/ref_rejected": -81.65457916259766,
"logps/rejected": -81.57505798339844,
"loss": 1.3992,
"margin_dpo/margin_mean": -0.12442246079444885,
"margin_dpo/margin_std": 0.41358208656311035,
"step": 2
},
{
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.014965459704399109,
"fcm_dpo/q_t": 0.4996257424354553,
"grad_norm": 26.07487678527832,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": 0.4268788695335388,
"logits/rejected": 0.6501424312591553,
"logps/chosen": -93.79121398925781,
"logps/ref_chosen": -93.81098937988281,
"logps/ref_rejected": -74.22950744628906,
"logps/rejected": -74.22470092773438,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.014965400099754333,
"margin_dpo/margin_std": 0.45240455865859985,
"step": 3
},
{
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02085861563682556,
"fcm_dpo/q_t": 0.4994773864746094,
"grad_norm": 32.71585464477539,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": 0.6463205814361572,
"logits/rejected": 0.7779839038848877,
"logps/chosen": -87.30850219726562,
"logps/ref_chosen": -87.29246520996094,
"logps/ref_rejected": -93.82425689697266,
"logps/rejected": -93.86114501953125,
"loss": 1.3846,
"margin_dpo/margin_mean": 0.020858705043792725,
"margin_dpo/margin_std": 0.39117318391799927,
"step": 4
},
{
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.050249382853507996,
"fcm_dpo/q_t": 0.4987432062625885,
"grad_norm": 35.20225143432617,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": 0.4468464255332947,
"logits/rejected": 0.6582976579666138,
"logps/chosen": -89.32503509521484,
"logps/ref_chosen": -89.33676147460938,
"logps/ref_rejected": -88.74783325195312,
"logps/rejected": -88.78636169433594,
"loss": 1.3818,
"margin_dpo/margin_mean": 0.050250083208084106,
"margin_dpo/margin_std": 0.44784289598464966,
"step": 5
},
{
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.01285526156425476,
"fcm_dpo/q_t": 0.5003200173377991,
"grad_norm": 36.15748596191406,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": 0.48705005645751953,
"logits/rejected": 0.803049623966217,
"logps/chosen": -97.32769775390625,
"logps/ref_chosen": -97.32147216796875,
"logps/ref_rejected": -97.88345336914062,
"logps/rejected": -97.87681579589844,
"loss": 1.3881,
"margin_dpo/margin_mean": -0.012855678796768188,
"margin_dpo/margin_std": 0.42675212025642395,
"step": 6
},
{
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06436635553836823,
"fcm_dpo/q_t": 0.49839529395103455,
"grad_norm": 37.96323776245117,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": 0.49750369787216187,
"logits/rejected": 0.5453470945358276,
"logps/chosen": -86.62813568115234,
"logps/ref_chosen": -86.64852905273438,
"logps/ref_rejected": -109.61618041992188,
"logps/rejected": -109.66015625,
"loss": 1.3806,
"margin_dpo/margin_mean": 0.06436613202095032,
"margin_dpo/margin_std": 0.5262112021446228,
"step": 7
},
{
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.1121322363615036,
"fcm_dpo/q_t": 0.5028019547462463,
"grad_norm": 32.900169372558594,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": 0.436749666929245,
"logits/rejected": 0.7514491677284241,
"logps/chosen": -90.00808715820312,
"logps/ref_chosen": -89.9236831665039,
"logps/ref_rejected": -86.22804260253906,
"logps/rejected": -86.20030975341797,
"loss": 1.398,
"margin_dpo/margin_mean": -0.11213211715221405,
"margin_dpo/margin_std": 0.4109366536140442,
"step": 8
},
{
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.02080167829990387,
"fcm_dpo/q_t": 0.499479740858078,
"grad_norm": 36.028873443603516,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": 0.661954939365387,
"logits/rejected": 0.8412786722183228,
"logps/chosen": -103.87676239013672,
"logps/ref_chosen": -103.85713195800781,
"logps/ref_rejected": -104.31932067871094,
"logps/rejected": -104.35975646972656,
"loss": 1.3848,
"margin_dpo/margin_mean": 0.020801380276679993,
"margin_dpo/margin_std": 0.4883061647415161,
"step": 9
},
{
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07037302851676941,
"fcm_dpo/q_t": 0.5017586350440979,
"grad_norm": 33.41664505004883,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": 0.6513626575469971,
"logits/rejected": 0.870223879814148,
"logps/chosen": -76.17118072509766,
"logps/ref_chosen": -76.20436096191406,
"logps/ref_rejected": -87.152099609375,
"logps/rejected": -87.04855346679688,
"loss": 1.3939,
"margin_dpo/margin_mean": -0.07037210464477539,
"margin_dpo/margin_std": 0.4575299322605133,
"step": 10
},
{
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.053068071603775024,
"fcm_dpo/q_t": 0.49867451190948486,
"grad_norm": 36.69655990600586,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": 0.277323842048645,
"logits/rejected": 0.4923889636993408,
"logps/chosen": -82.32759094238281,
"logps/ref_chosen": -82.36650085449219,
"logps/ref_rejected": -94.26461791992188,
"logps/rejected": -94.2787857055664,
"loss": 1.3814,
"margin_dpo/margin_mean": 0.053068608045578,
"margin_dpo/margin_std": 0.41766154766082764,
"step": 11
},
{
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01172947883605957,
"fcm_dpo/q_t": 0.4997062683105469,
"grad_norm": 40.26313781738281,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": 0.40038958191871643,
"logits/rejected": 0.5286105871200562,
"logps/chosen": -99.0933837890625,
"logps/ref_chosen": -99.10549926757812,
"logps/ref_rejected": -110.27140808105469,
"logps/rejected": -110.27102661132812,
"loss": 1.3858,
"margin_dpo/margin_mean": 0.011729925870895386,
"margin_dpo/margin_std": 0.5183511972427368,
"step": 12
},
{
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.059926122426986694,
"fcm_dpo/q_t": 0.501498281955719,
"grad_norm": 41.93380355834961,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": 0.5050075054168701,
"logits/rejected": 0.7639053463935852,
"logps/chosen": -90.6314697265625,
"logps/ref_chosen": -90.55972290039062,
"logps/ref_rejected": -93.69110107421875,
"logps/rejected": -93.70291137695312,
"loss": 1.3928,
"margin_dpo/margin_mean": -0.05992648005485535,
"margin_dpo/margin_std": 0.4534745216369629,
"step": 13
},
{
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.019990622997283936,
"fcm_dpo/q_t": 0.500498354434967,
"grad_norm": 35.67725372314453,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": 1.0646815299987793,
"logits/rejected": 1.1590429544448853,
"logps/chosen": -99.85965728759766,
"logps/ref_chosen": -99.82717895507812,
"logps/ref_rejected": -108.94200134277344,
"logps/rejected": -108.95449829101562,
"loss": 1.3888,
"margin_dpo/margin_mean": -0.01999080181121826,
"margin_dpo/margin_std": 0.4524303078651428,
"step": 14
},
{
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06978121399879456,
"fcm_dpo/q_t": 0.49825772643089294,
"grad_norm": 31.65287971496582,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": 0.4620896875858307,
"logits/rejected": 0.6329752802848816,
"logps/chosen": -78.88731384277344,
"logps/ref_chosen": -78.90997314453125,
"logps/ref_rejected": -90.06234741210938,
"logps/rejected": -90.10946655273438,
"loss": 1.3799,
"margin_dpo/margin_mean": 0.06978106498718262,
"margin_dpo/margin_std": 0.4777568280696869,
"step": 15
},
{
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.01587851345539093,
"fcm_dpo/q_t": 0.49960121512413025,
"grad_norm": 34.593482971191406,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": 0.7190598249435425,
"logits/rejected": 0.9916927814483643,
"logps/chosen": -97.41981506347656,
"logps/ref_chosen": -97.42327880859375,
"logps/ref_rejected": -90.59945678710938,
"logps/rejected": -90.61186981201172,
"loss": 1.3855,
"margin_dpo/margin_mean": 0.015878498554229736,
"margin_dpo/margin_std": 0.5510396361351013,
"step": 16
},
{
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08168734610080719,
"fcm_dpo/q_t": 0.49796023964881897,
"grad_norm": 35.512939453125,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": 0.4151402413845062,
"logits/rejected": 0.6361797451972961,
"logps/chosen": -104.31832122802734,
"logps/ref_chosen": -104.36430358886719,
"logps/ref_rejected": -90.46772766113281,
"logps/rejected": -90.50341796875,
"loss": 1.3787,
"margin_dpo/margin_mean": 0.08168637752532959,
"margin_dpo/margin_std": 0.47371718287467957,
"step": 17
},
{
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03281433880329132,
"fcm_dpo/q_t": 0.5008206367492676,
"grad_norm": 42.083641052246094,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": 0.47482335567474365,
"logits/rejected": 0.8290694355964661,
"logps/chosen": -87.07789611816406,
"logps/ref_chosen": -87.09195709228516,
"logps/ref_rejected": -81.85072326660156,
"logps/rejected": -81.80384826660156,
"loss": 1.3903,
"margin_dpo/margin_mean": -0.03281354904174805,
"margin_dpo/margin_std": 0.543687105178833,
"step": 18
},
{
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.07938456535339355,
"fcm_dpo/q_t": 0.501983642578125,
"grad_norm": 31.655494689941406,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": 0.5405235886573792,
"logits/rejected": 0.8114136457443237,
"logps/chosen": -105.96595764160156,
"logps/ref_chosen": -105.87354278564453,
"logps/ref_rejected": -96.93023681640625,
"logps/rejected": -96.94326782226562,
"loss": 1.3946,
"margin_dpo/margin_mean": -0.07938426733016968,
"margin_dpo/margin_std": 0.3811599910259247,
"step": 19
},
{
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.025069117546081543,
"fcm_dpo/q_t": 0.4993741512298584,
"grad_norm": 34.66242599487305,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": 0.23079314827919006,
"logits/rejected": 0.5058917999267578,
"logps/chosen": -90.76351928710938,
"logps/ref_chosen": -90.75811767578125,
"logps/ref_rejected": -85.91232299804688,
"logps/rejected": -85.94279479980469,
"loss": 1.3844,
"margin_dpo/margin_mean": 0.025068938732147217,
"margin_dpo/margin_std": 0.4899085462093353,
"step": 20
},
{
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.091633141040802,
"fcm_dpo/q_t": 0.4977106750011444,
"grad_norm": 31.416053771972656,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": 0.5428293347358704,
"logits/rejected": 0.7209779620170593,
"logps/chosen": -80.27669525146484,
"logps/ref_chosen": -80.33345794677734,
"logps/ref_rejected": -83.93373107910156,
"logps/rejected": -83.9686050415039,
"loss": 1.3776,
"margin_dpo/margin_mean": 0.09163343906402588,
"margin_dpo/margin_std": 0.40345489978790283,
"step": 21
},
{
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0237371027469635,
"fcm_dpo/q_t": 0.4994067847728729,
"grad_norm": 42.76321029663086,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": 0.4888147711753845,
"logits/rejected": 0.6019254326820374,
"logps/chosen": -95.3747329711914,
"logps/ref_chosen": -95.39530181884766,
"logps/ref_rejected": -103.47351837158203,
"logps/rejected": -103.47669219970703,
"loss": 1.3843,
"margin_dpo/margin_mean": 0.02373659610748291,
"margin_dpo/margin_std": 0.41101372241973877,
"step": 22
},
{
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.054152026772499084,
"fcm_dpo/q_t": 0.5013521909713745,
"grad_norm": 31.53017807006836,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": 0.7403199076652527,
"logits/rejected": 0.9192092418670654,
"logps/chosen": -90.61726379394531,
"logps/ref_chosen": -90.63751220703125,
"logps/ref_rejected": -86.59425354003906,
"logps/rejected": -86.51985168457031,
"loss": 1.3921,
"margin_dpo/margin_mean": -0.05415254831314087,
"margin_dpo/margin_std": 0.4045397639274597,
"step": 23
},
{
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.038981035351753235,
"fcm_dpo/q_t": 0.4990260601043701,
"grad_norm": 44.209590911865234,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": 0.6239376068115234,
"logits/rejected": 0.6544588804244995,
"logps/chosen": -69.91582489013672,
"logps/ref_chosen": -69.91728973388672,
"logps/ref_rejected": -106.60990142822266,
"logps/rejected": -106.64741516113281,
"loss": 1.383,
"margin_dpo/margin_mean": 0.03898155689239502,
"margin_dpo/margin_std": 0.4552841782569885,
"step": 24
},
{
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.050283268094062805,
"fcm_dpo/q_t": 0.4987441301345825,
"grad_norm": 36.85416030883789,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": 0.5321314334869385,
"logits/rejected": 0.7012242078781128,
"logps/chosen": -80.85140991210938,
"logps/ref_chosen": -80.82548522949219,
"logps/ref_rejected": -95.95710754394531,
"logps/rejected": -96.0333251953125,
"loss": 1.3816,
"margin_dpo/margin_mean": 0.05028343200683594,
"margin_dpo/margin_std": 0.37170085310935974,
"step": 25
},
{
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.07252025604248047,
"fcm_dpo/q_t": 0.4981876611709595,
"grad_norm": 40.67619705200195,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": 0.36407941579818726,
"logits/rejected": 0.5750868320465088,
"logps/chosen": -88.88731384277344,
"logps/ref_chosen": -88.90116882324219,
"logps/ref_rejected": -109.82819366455078,
"logps/rejected": -109.88685607910156,
"loss": 1.3795,
"margin_dpo/margin_mean": 0.072520911693573,
"margin_dpo/margin_std": 0.4113643765449524,
"step": 26
},
{
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0897430032491684,
"fcm_dpo/q_t": 0.4977561831474304,
"grad_norm": 44.181434631347656,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": 0.2009836733341217,
"logits/rejected": 0.38905927538871765,
"logps/chosen": -77.58397674560547,
"logps/ref_chosen": -77.59600830078125,
"logps/ref_rejected": -103.93850708007812,
"logps/rejected": -104.0162124633789,
"loss": 1.3779,
"margin_dpo/margin_mean": 0.08974292874336243,
"margin_dpo/margin_std": 0.4670272469520569,
"step": 27
},
{
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12937551736831665,
"fcm_dpo/q_t": 0.49676674604415894,
"grad_norm": 36.718238830566406,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": 0.7008620500564575,
"logits/rejected": 0.9329044818878174,
"logps/chosen": -102.15823364257812,
"logps/ref_chosen": -102.22856903076172,
"logps/ref_rejected": -96.9594955444336,
"logps/rejected": -97.01853942871094,
"loss": 1.3738,
"margin_dpo/margin_mean": 0.12937626242637634,
"margin_dpo/margin_std": 0.42088428139686584,
"step": 28
},
{
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11628834903240204,
"fcm_dpo/q_t": 0.49709412455558777,
"grad_norm": 42.49556350708008,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": 0.37622809410095215,
"logits/rejected": 0.638656497001648,
"logps/chosen": -88.61930084228516,
"logps/ref_chosen": -88.64704895019531,
"logps/ref_rejected": -102.96011352539062,
"logps/rejected": -103.04864501953125,
"loss": 1.3753,
"margin_dpo/margin_mean": 0.11628936231136322,
"margin_dpo/margin_std": 0.48608508706092834,
"step": 29
},
{
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.11767734587192535,
"fcm_dpo/q_t": 0.49706411361694336,
"grad_norm": 39.036170959472656,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": 0.8277263045310974,
"logits/rejected": 0.9531229734420776,
"logps/chosen": -88.38198852539062,
"logps/ref_chosen": -88.38838195800781,
"logps/ref_rejected": -102.31889343261719,
"logps/rejected": -102.43018341064453,
"loss": 1.3752,
"margin_dpo/margin_mean": 0.11767733097076416,
"margin_dpo/margin_std": 0.5070103406906128,
"step": 30
},
{
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.05853770673274994,
"fcm_dpo/q_t": 0.49853795766830444,
"grad_norm": 31.66895866394043,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": 0.718494713306427,
"logits/rejected": 0.9628149271011353,
"logps/chosen": -101.16627502441406,
"logps/ref_chosen": -101.12565612792969,
"logps/ref_rejected": -79.8584213256836,
"logps/rejected": -79.95758056640625,
"loss": 1.381,
"margin_dpo/margin_mean": 0.05853772163391113,
"margin_dpo/margin_std": 0.4458237588405609,
"step": 31
},
{
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14585043489933014,
"fcm_dpo/q_t": 0.4963546395301819,
"grad_norm": 35.37748718261719,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": 0.3660997748374939,
"logits/rejected": 0.5037564635276794,
"logps/chosen": -96.66069030761719,
"logps/ref_chosen": -96.61702728271484,
"logps/ref_rejected": -96.224365234375,
"logps/rejected": -96.41387176513672,
"loss": 1.3723,
"margin_dpo/margin_mean": 0.14585140347480774,
"margin_dpo/margin_std": 0.44825515151023865,
"step": 32
},
{
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.16191767156124115,
"fcm_dpo/q_t": 0.4959545135498047,
"grad_norm": 33.53567123413086,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": 0.5710635185241699,
"logits/rejected": 0.7510501742362976,
"logps/chosen": -81.49163055419922,
"logps/ref_chosen": -81.52101135253906,
"logps/ref_rejected": -93.80595397949219,
"logps/rejected": -93.93849182128906,
"loss": 1.3709,
"margin_dpo/margin_mean": 0.1619182527065277,
"margin_dpo/margin_std": 0.5155723094940186,
"step": 33
},
{
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19678400456905365,
"fcm_dpo/q_t": 0.4950863718986511,
"grad_norm": 40.77206802368164,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": 0.5938290357589722,
"logits/rejected": 0.7664215564727783,
"logps/chosen": -77.15574645996094,
"logps/ref_chosen": -77.20204162597656,
"logps/ref_rejected": -106.71876525878906,
"logps/rejected": -106.86925506591797,
"loss": 1.3673,
"margin_dpo/margin_mean": 0.19678422808647156,
"margin_dpo/margin_std": 0.4836421608924866,
"step": 34
},
{
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1951250433921814,
"fcm_dpo/q_t": 0.4951246976852417,
"grad_norm": 42.55626678466797,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": 0.1299372911453247,
"logits/rejected": 0.2869094908237457,
"logps/chosen": -77.56372833251953,
"logps/ref_chosen": -77.57035064697266,
"logps/ref_rejected": -112.18855285644531,
"logps/rejected": -112.3770523071289,
"loss": 1.3673,
"margin_dpo/margin_mean": 0.1951252818107605,
"margin_dpo/margin_std": 0.41524142026901245,
"step": 35
},
{
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12170365452766418,
"fcm_dpo/q_t": 0.4969629645347595,
"grad_norm": 32.65164566040039,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": 0.40253281593322754,
"logits/rejected": 0.5915440320968628,
"logps/chosen": -83.32685089111328,
"logps/ref_chosen": -83.28823852539062,
"logps/ref_rejected": -91.75741577148438,
"logps/rejected": -91.91773223876953,
"loss": 1.375,
"margin_dpo/margin_mean": 0.12170398235321045,
"margin_dpo/margin_std": 0.55162113904953,
"step": 36
},
{
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3155759572982788,
"fcm_dpo/q_t": 0.4921198785305023,
"grad_norm": 36.134918212890625,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": 0.5907583236694336,
"logits/rejected": 0.7441189885139465,
"logps/chosen": -94.78392028808594,
"logps/ref_chosen": -94.77108764648438,
"logps/ref_rejected": -85.71726989746094,
"logps/rejected": -86.04566955566406,
"loss": 1.3558,
"margin_dpo/margin_mean": 0.3155764639377594,
"margin_dpo/margin_std": 0.5700342655181885,
"step": 37
},
{
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.19760102033615112,
"fcm_dpo/q_t": 0.4950709939002991,
"grad_norm": 37.12024688720703,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": 0.5169904232025146,
"logits/rejected": 0.6563962697982788,
"logps/chosen": -75.88797760009766,
"logps/ref_chosen": -75.97850036621094,
"logps/ref_rejected": -104.10401916503906,
"logps/rejected": -104.21109008789062,
"loss": 1.3675,
"margin_dpo/margin_mean": 0.197601780295372,
"margin_dpo/margin_std": 0.5769727230072021,
"step": 38
},
{
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3096907138824463,
"fcm_dpo/q_t": 0.492266446352005,
"grad_norm": 36.1661376953125,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": 0.6212589740753174,
"logits/rejected": 0.818722128868103,
"logps/chosen": -81.21092987060547,
"logps/ref_chosen": -81.18577575683594,
"logps/ref_rejected": -84.1959228515625,
"logps/rejected": -84.53076171875,
"loss": 1.3563,
"margin_dpo/margin_mean": 0.30969101190567017,
"margin_dpo/margin_std": 0.5184357762336731,
"step": 39
},
{
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24696290493011475,
"fcm_dpo/q_t": 0.49383074045181274,
"grad_norm": 29.007984161376953,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": 0.6268089413642883,
"logits/rejected": 0.8235390186309814,
"logps/chosen": -83.29570007324219,
"logps/ref_chosen": -83.33257293701172,
"logps/ref_rejected": -80.25591278076172,
"logps/rejected": -80.46599578857422,
"loss": 1.3624,
"margin_dpo/margin_mean": 0.24696248769760132,
"margin_dpo/margin_std": 0.489422082901001,
"step": 40
},
{
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3290652930736542,
"fcm_dpo/q_t": 0.4917981028556824,
"grad_norm": 33.2258415222168,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": 0.6201198697090149,
"logits/rejected": 0.8052632808685303,
"logps/chosen": -93.12638854980469,
"logps/ref_chosen": -93.14866638183594,
"logps/ref_rejected": -102.07920837402344,
"logps/rejected": -102.38600158691406,
"loss": 1.3551,
"margin_dpo/margin_mean": 0.32906582951545715,
"margin_dpo/margin_std": 0.7185872197151184,
"step": 41
},
{
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4246216416358948,
"fcm_dpo/q_t": 0.4893936514854431,
"grad_norm": 40.833499908447266,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": 0.3932719826698303,
"logits/rejected": 0.5345179438591003,
"logps/chosen": -90.62908172607422,
"logps/ref_chosen": -90.70162200927734,
"logps/ref_rejected": -113.8646469116211,
"logps/rejected": -114.21672821044922,
"loss": 1.3449,
"margin_dpo/margin_mean": 0.4246218204498291,
"margin_dpo/margin_std": 0.5078005790710449,
"step": 42
},
{
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4403374493122101,
"fcm_dpo/q_t": 0.4890177845954895,
"grad_norm": 37.972869873046875,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": 0.5793946981430054,
"logits/rejected": 0.7695983052253723,
"logps/chosen": -89.6876220703125,
"logps/ref_chosen": -89.64402770996094,
"logps/ref_rejected": -103.95185852050781,
"logps/rejected": -104.435791015625,
"loss": 1.3441,
"margin_dpo/margin_mean": 0.440337598323822,
"margin_dpo/margin_std": 0.7237739562988281,
"step": 43
},
{
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3891373872756958,
"fcm_dpo/q_t": 0.49028751254081726,
"grad_norm": 34.31340026855469,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": 0.8794394135475159,
"logits/rejected": 1.000215768814087,
"logps/chosen": -81.93470764160156,
"logps/ref_chosen": -81.8783187866211,
"logps/ref_rejected": -113.421630859375,
"logps/rejected": -113.86715698242188,
"loss": 1.3487,
"margin_dpo/margin_mean": 0.3891381621360779,
"margin_dpo/margin_std": 0.6131525039672852,
"step": 44
},
{
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1822262555360794,
"fcm_dpo/q_t": 0.49545037746429443,
"grad_norm": 24.454357147216797,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": 0.6332857608795166,
"logits/rejected": 0.7449449300765991,
"logps/chosen": -77.44635009765625,
"logps/ref_chosen": -77.34459686279297,
"logps/ref_rejected": -84.24774169921875,
"logps/rejected": -84.53172302246094,
"loss": 1.3691,
"margin_dpo/margin_mean": 0.18222709000110626,
"margin_dpo/margin_std": 0.5951623916625977,
"step": 45
},
{
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.48641544580459595,
"fcm_dpo/q_t": 0.4878547787666321,
"grad_norm": 32.93342208862305,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": 0.5085247755050659,
"logits/rejected": 0.7518465518951416,
"logps/chosen": -90.32666015625,
"logps/ref_chosen": -90.3408203125,
"logps/ref_rejected": -100.676513671875,
"logps/rejected": -101.14876556396484,
"loss": 1.339,
"margin_dpo/margin_mean": 0.4864157736301422,
"margin_dpo/margin_std": 0.5446327328681946,
"step": 46
},
{
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3937411308288574,
"fcm_dpo/q_t": 0.49017661809921265,
"grad_norm": 34.04849624633789,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": 0.45205074548721313,
"logits/rejected": 0.6340922117233276,
"logps/chosen": -104.42767333984375,
"logps/ref_chosen": -104.41130065917969,
"logps/ref_rejected": -101.25489807128906,
"logps/rejected": -101.6650161743164,
"loss": 1.3487,
"margin_dpo/margin_mean": 0.39373981952667236,
"margin_dpo/margin_std": 0.7378332614898682,
"step": 47
},
{
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6404982805252075,
"fcm_dpo/q_t": 0.4840244948863983,
"grad_norm": 38.5645751953125,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": 0.29659873247146606,
"logits/rejected": 0.5025741457939148,
"logps/chosen": -91.0909423828125,
"logps/ref_chosen": -91.10027313232422,
"logps/ref_rejected": -95.08058166503906,
"logps/rejected": -95.71174621582031,
"loss": 1.3251,
"margin_dpo/margin_mean": 0.6404985189437866,
"margin_dpo/margin_std": 0.856590986251831,
"step": 48
},
{
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7378369569778442,
"fcm_dpo/q_t": 0.4816128611564636,
"grad_norm": 39.67837905883789,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": 0.10201732814311981,
"logits/rejected": 0.5736548900604248,
"logps/chosen": -93.02400207519531,
"logps/ref_chosen": -93.00367736816406,
"logps/ref_rejected": -91.74899291992188,
"logps/rejected": -92.50715637207031,
"loss": 1.316,
"margin_dpo/margin_mean": 0.7378373742103577,
"margin_dpo/margin_std": 0.9256041646003723,
"step": 49
},
{
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7001360654830933,
"fcm_dpo/q_t": 0.4825558662414551,
"grad_norm": 33.148658752441406,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": 0.5446861982345581,
"logits/rejected": 0.7148386240005493,
"logps/chosen": -94.72885131835938,
"logps/ref_chosen": -94.62681579589844,
"logps/ref_rejected": -103.57435607910156,
"logps/rejected": -104.37653350830078,
"loss": 1.3197,
"margin_dpo/margin_mean": 0.7001360654830933,
"margin_dpo/margin_std": 0.9148711562156677,
"step": 50
},
{
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6637753248214722,
"fcm_dpo/q_t": 0.48347365856170654,
"grad_norm": 30.014331817626953,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": 0.7485964298248291,
"logits/rejected": 0.8959745168685913,
"logps/chosen": -87.53605651855469,
"logps/ref_chosen": -87.50727844238281,
"logps/ref_rejected": -83.47235107421875,
"logps/rejected": -84.16490936279297,
"loss": 1.3236,
"margin_dpo/margin_mean": 0.6637749671936035,
"margin_dpo/margin_std": 1.0224616527557373,
"step": 51
},
{
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.873137354850769,
"fcm_dpo/q_t": 0.47824588418006897,
"grad_norm": 32.00587844848633,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": 0.49871087074279785,
"logits/rejected": 0.7171226739883423,
"logps/chosen": -90.64942169189453,
"logps/ref_chosen": -90.63026428222656,
"logps/ref_rejected": -87.0390625,
"logps/rejected": -87.93136596679688,
"loss": 1.3029,
"margin_dpo/margin_mean": 0.8731374740600586,
"margin_dpo/margin_std": 0.9018590450286865,
"step": 52
},
{
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1096992492675781,
"fcm_dpo/q_t": 0.47249969840049744,
"grad_norm": 39.590824127197266,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": 0.07211381196975708,
"logits/rejected": 0.1840796172618866,
"logps/chosen": -81.551025390625,
"logps/ref_chosen": -81.58307647705078,
"logps/ref_rejected": -95.66152954101562,
"logps/rejected": -96.73918151855469,
"loss": 1.2831,
"margin_dpo/margin_mean": 1.1096991300582886,
"margin_dpo/margin_std": 1.3239152431488037,
"step": 53
},
{
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1452147960662842,
"fcm_dpo/q_t": 0.4715806841850281,
"grad_norm": 39.977752685546875,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": 0.47239190340042114,
"logits/rejected": 0.6545497179031372,
"logps/chosen": -88.91471099853516,
"logps/ref_chosen": -88.91017150878906,
"logps/ref_rejected": -99.1175537109375,
"logps/rejected": -100.2673110961914,
"loss": 1.2792,
"margin_dpo/margin_mean": 1.1452147960662842,
"margin_dpo/margin_std": 1.2928842306137085,
"step": 54
},
{
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.180344820022583,
"fcm_dpo/q_t": 0.4708316922187805,
"grad_norm": 34.956790924072266,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": 0.10857650637626648,
"logits/rejected": 0.4178081750869751,
"logps/chosen": -92.31873321533203,
"logps/ref_chosen": -92.45591735839844,
"logps/ref_rejected": -92.97093200683594,
"logps/rejected": -94.01409149169922,
"loss": 1.2787,
"margin_dpo/margin_mean": 1.180344581604004,
"margin_dpo/margin_std": 1.6435117721557617,
"step": 55
},
{
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.060053825378418,
"fcm_dpo/q_t": 0.4738434851169586,
"grad_norm": 33.11833572387695,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": 0.2020319402217865,
"logits/rejected": 0.5419068336486816,
"logps/chosen": -87.31375122070312,
"logps/ref_chosen": -87.23665618896484,
"logps/ref_rejected": -100.64553833007812,
"logps/rejected": -101.78268432617188,
"loss": 1.2901,
"margin_dpo/margin_mean": 1.0600528717041016,
"margin_dpo/margin_std": 1.6760348081588745,
"step": 56
},
{
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1467325687408447,
"fcm_dpo/q_t": 0.4716096520423889,
"grad_norm": 33.74794387817383,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": 0.31785082817077637,
"logits/rejected": 0.5342439413070679,
"logps/chosen": -98.2012939453125,
"logps/ref_chosen": -98.15074157714844,
"logps/ref_rejected": -101.05284118652344,
"logps/rejected": -102.25011444091797,
"loss": 1.2808,
"margin_dpo/margin_mean": 1.146733283996582,
"margin_dpo/margin_std": 1.5416290760040283,
"step": 57
},
{
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5004881620407104,
"fcm_dpo/q_t": 0.46311402320861816,
"grad_norm": 37.92626190185547,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": 0.29364755749702454,
"logits/rejected": 0.6253389120101929,
"logps/chosen": -99.32469177246094,
"logps/ref_chosen": -99.58097076416016,
"logps/ref_rejected": -91.22227478027344,
"logps/rejected": -92.46648406982422,
"loss": 1.2512,
"margin_dpo/margin_mean": 1.50048828125,
"margin_dpo/margin_std": 1.8517991304397583,
"step": 58
},
{
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2302589416503906,
"fcm_dpo/q_t": 0.469511479139328,
"grad_norm": 34.269100189208984,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": 0.13759316504001617,
"logits/rejected": 0.3323509693145752,
"logps/chosen": -89.81268310546875,
"logps/ref_chosen": -89.80232238769531,
"logps/ref_rejected": -94.27667236328125,
"logps/rejected": -95.51730346679688,
"loss": 1.2723,
"margin_dpo/margin_mean": 1.230258584022522,
"margin_dpo/margin_std": 1.4574449062347412,
"step": 59
},
{
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1427745819091797,
"fcm_dpo/q_t": 0.4716527462005615,
"grad_norm": 28.718473434448242,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": 0.3863717317581177,
"logits/rejected": 0.6563723087310791,
"logps/chosen": -95.13877868652344,
"logps/ref_chosen": -95.15571594238281,
"logps/ref_rejected": -91.3724365234375,
"logps/rejected": -92.4982681274414,
"loss": 1.2804,
"margin_dpo/margin_mean": 1.1427747011184692,
"margin_dpo/margin_std": 1.4296643733978271,
"step": 60
},
{
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1766782999038696,
"fcm_dpo/q_t": 0.47107481956481934,
"grad_norm": 28.268402099609375,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": 0.2091524302959442,
"logits/rejected": 0.34982138872146606,
"logps/chosen": -85.9404296875,
"logps/ref_chosen": -85.73230743408203,
"logps/ref_rejected": -97.96575927734375,
"logps/rejected": -99.3505630493164,
"loss": 1.2807,
"margin_dpo/margin_mean": 1.1766784191131592,
"margin_dpo/margin_std": 1.8282430171966553,
"step": 61
},
{
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2114932537078857,
"fcm_dpo/q_t": 0.47012367844581604,
"grad_norm": 28.394248962402344,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": 0.222091406583786,
"logits/rejected": 0.4194578528404236,
"logps/chosen": -81.77252197265625,
"logps/ref_chosen": -81.63538360595703,
"logps/ref_rejected": -84.03831481933594,
"logps/rejected": -85.38694763183594,
"loss": 1.2787,
"margin_dpo/margin_mean": 1.2114924192428589,
"margin_dpo/margin_std": 1.857269048690796,
"step": 62
},
{
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.4869378805160522,
"fcm_dpo/q_t": 0.46316325664520264,
"grad_norm": 35.04859924316406,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": 0.33780792355537415,
"logits/rejected": 0.5467942357063293,
"logps/chosen": -103.6246337890625,
"logps/ref_chosen": -103.62405395507812,
"logps/ref_rejected": -103.40303039550781,
"logps/rejected": -104.89054870605469,
"loss": 1.2495,
"margin_dpo/margin_mean": 1.4869377613067627,
"margin_dpo/margin_std": 1.5714600086212158,
"step": 63
},
{
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.10389302670955658,
"fcm_dpo/delta": 0.19095781445503235,
"fcm_dpo/margin": 1.7406915426254272,
"fcm_dpo/q_t": 0.45685333013534546,
"grad_norm": 35.974021911621094,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": 0.3727140724658966,
"logits/rejected": 0.584496259689331,
"logps/chosen": -87.18478393554688,
"logps/ref_chosen": -87.0015869140625,
"logps/ref_rejected": -100.5854721069336,
"logps/rejected": -102.50936126708984,
"loss": 1.2273,
"margin_dpo/margin_mean": 1.7406916618347168,
"margin_dpo/margin_std": 2.1689281463623047,
"step": 64
},
{
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.10389302670955658,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.7801563739776611,
"fcm_dpo/q_t": 0.4546605348587036,
"grad_norm": 39.58053970336914,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": 0.08340189605951309,
"logits/rejected": 0.22737503051757812,
"logps/chosen": -91.27607727050781,
"logps/ref_chosen": -91.22191619873047,
"logps/ref_rejected": -115.33553314208984,
"logps/rejected": -117.16984558105469,
"loss": 1.2216,
"margin_dpo/margin_mean": 1.780156135559082,
"margin_dpo/margin_std": 2.092562437057495,
"step": 65
},
{
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.10389302670955658,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2560508251190186,
"fcm_dpo/q_t": 0.46791407465934753,
"grad_norm": 26.43943977355957,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": 0.25161832571029663,
"logits/rejected": 0.460490345954895,
"logps/chosen": -84.09396362304688,
"logps/ref_chosen": -83.78422546386719,
"logps/ref_rejected": -82.7520980834961,
"logps/rejected": -84.31788635253906,
"loss": 1.2702,
"margin_dpo/margin_mean": 1.2560513019561768,
"margin_dpo/margin_std": 1.885561466217041,
"step": 66
},
{
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.10389302670955658,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1451356410980225,
"fcm_dpo/q_t": 0.47134000062942505,
"grad_norm": 25.080429077148438,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": 0.3801344335079193,
"logits/rejected": 0.6603550910949707,
"logps/chosen": -88.52891540527344,
"logps/ref_chosen": -87.67295837402344,
"logps/ref_rejected": -79.08674621582031,
"logps/rejected": -81.08783721923828,
"loss": 1.2902,
"margin_dpo/margin_mean": 1.1451354026794434,
"margin_dpo/margin_std": 2.6883256435394287,
"step": 67
},
{
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.10986920446157455,
"fcm_dpo/delta": 0.3722814917564392,
"fcm_dpo/margin": 2.152301788330078,
"fcm_dpo/q_t": 0.44556739926338196,
"grad_norm": 29.909442901611328,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.003267938271164894,
"logits/rejected": 0.2983437776565552,
"logps/chosen": -97.34490966796875,
"logps/ref_chosen": -97.07884216308594,
"logps/ref_rejected": -84.11872863769531,
"logps/rejected": -86.53709411621094,
"loss": 1.1887,
"margin_dpo/margin_mean": 2.15230131149292,
"margin_dpo/margin_std": 2.906233072280884,
"step": 68
},
{
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.1179037019610405,
"fcm_dpo/delta": 0.3364790081977844,
"fcm_dpo/margin": 2.307511806488037,
"fcm_dpo/q_t": 0.4372074604034424,
"grad_norm": 33.9486083984375,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": 0.2297275960445404,
"logits/rejected": 0.3395928740501404,
"logps/chosen": -86.02600860595703,
"logps/ref_chosen": -85.71971130371094,
"logps/ref_rejected": -109.48020935058594,
"logps/rejected": -112.0940170288086,
"loss": 1.1623,
"margin_dpo/margin_mean": 2.307512044906616,
"margin_dpo/margin_std": 3.0080556869506836,
"step": 69
},
{
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.12486476451158524,
"fcm_dpo/delta": 0.2788800001144409,
"fcm_dpo/margin": 2.644542694091797,
"fcm_dpo/q_t": 0.4251058101654053,
"grad_norm": 33.3270378112793,
"learning_rate": 5e-07,
"logits/chosen": 0.3216286301612854,
"logits/rejected": 0.5021553635597229,
"logps/chosen": -95.56564331054688,
"logps/ref_chosen": -95.00994873046875,
"logps/ref_rejected": -96.21272277832031,
"logps/rejected": -99.4129638671875,
"loss": 1.1246,
"margin_dpo/margin_mean": 2.644542932510376,
"margin_dpo/margin_std": 3.2647337913513184,
"step": 70
},
{
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.12874360382556915,
"fcm_dpo/delta": 0.13646240532398224,
"fcm_dpo/margin": 3.6519854068756104,
"fcm_dpo/q_t": 0.3953820765018463,
"grad_norm": 39.30204772949219,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.15218183398246765,
"logits/rejected": 0.21293151378631592,
"logps/chosen": -91.0079345703125,
"logps/ref_chosen": -90.97735595703125,
"logps/ref_rejected": -102.59103393554688,
"logps/rejected": -106.27359771728516,
"loss": 1.0304,
"margin_dpo/margin_mean": 3.6519851684570312,
"margin_dpo/margin_std": 4.069307804107666,
"step": 71
},
{
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.13415689766407013,
"fcm_dpo/delta": 0.2152521312236786,
"fcm_dpo/margin": 2.9328012466430664,
"fcm_dpo/q_t": 0.4104134142398834,
"grad_norm": 35.07579803466797,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.1298740804195404,
"logits/rejected": 0.19889742136001587,
"logps/chosen": -98.41287231445312,
"logps/ref_chosen": -97.89152526855469,
"logps/ref_rejected": -100.19171142578125,
"logps/rejected": -103.64585876464844,
"loss": 1.0781,
"margin_dpo/margin_mean": 2.932800769805908,
"margin_dpo/margin_std": 3.356008529663086,
"step": 72
},
{
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.1394435167312622,
"fcm_dpo/delta": 0.17690719664096832,
"fcm_dpo/margin": 3.0887374877929688,
"fcm_dpo/q_t": 0.4099673926830292,
"grad_norm": 33.995853424072266,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.2099972814321518,
"logits/rejected": 0.10542194545269012,
"logps/chosen": -96.5327377319336,
"logps/ref_chosen": -95.7690200805664,
"logps/ref_rejected": -95.93243408203125,
"logps/rejected": -99.78489685058594,
"loss": 1.1024,
"margin_dpo/margin_mean": 3.0887370109558105,
"margin_dpo/margin_std": 4.92803430557251,
"step": 73
},
{
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.14254900813102722,
"fcm_dpo/delta": 0.1032918393611908,
"fcm_dpo/margin": 3.521709442138672,
"fcm_dpo/q_t": 0.3906998634338379,
"grad_norm": 36.36503219604492,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.10034316778182983,
"logits/rejected": 0.0426066517829895,
"logps/chosen": -79.76939392089844,
"logps/ref_chosen": -78.80839538574219,
"logps/ref_rejected": -101.64677429199219,
"logps/rejected": -106.12947082519531,
"loss": 1.0323,
"margin_dpo/margin_mean": 3.5217092037200928,
"margin_dpo/margin_std": 4.384693145751953,
"step": 74
},
{
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.14341631531715393,
"fcm_dpo/delta": -0.055621419101953506,
"fcm_dpo/margin": 4.54258918762207,
"fcm_dpo/q_t": 0.3650940954685211,
"grad_norm": 36.794151306152344,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.20140711963176727,
"logits/rejected": -0.000715896487236023,
"logps/chosen": -79.31031799316406,
"logps/ref_chosen": -78.28185272216797,
"logps/ref_rejected": -115.40311431884766,
"logps/rejected": -120.97416687011719,
"loss": 0.959,
"margin_dpo/margin_mean": 4.542590141296387,
"margin_dpo/margin_std": 5.413492679595947,
"step": 75
},
{
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.14552651345729828,
"fcm_dpo/delta": 0.16189850866794586,
"fcm_dpo/margin": 3.059915542602539,
"fcm_dpo/q_t": 0.40516766905784607,
"grad_norm": 30.741098403930664,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.614639401435852,
"logits/rejected": -0.23801718652248383,
"logps/chosen": -89.27581787109375,
"logps/ref_chosen": -87.87713623046875,
"logps/ref_rejected": -85.71968078613281,
"logps/rejected": -90.17826843261719,
"loss": 1.0865,
"margin_dpo/margin_mean": 3.059915542602539,
"margin_dpo/margin_std": 4.615878105163574,
"step": 76
},
{
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.14861461520195007,
"fcm_dpo/delta": 0.10367655009031296,
"fcm_dpo/margin": 3.3754987716674805,
"fcm_dpo/q_t": 0.3928101658821106,
"grad_norm": 37.48255920410156,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.18063946068286896,
"logits/rejected": 0.18999367952346802,
"logps/chosen": -101.2906723022461,
"logps/ref_chosen": -99.70034790039062,
"logps/ref_rejected": -98.20576477050781,
"logps/rejected": -103.17158508300781,
"loss": 1.0692,
"margin_dpo/margin_mean": 3.3754987716674805,
"margin_dpo/margin_std": 5.01793098449707,
"step": 77
},
{
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.15083855390548706,
"fcm_dpo/delta": 0.049115367233753204,
"fcm_dpo/margin": 3.670267105102539,
"fcm_dpo/q_t": 0.3794415593147278,
"grad_norm": 31.02741241455078,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": 0.14838413894176483,
"logits/rejected": 0.38533467054367065,
"logps/chosen": -90.91835021972656,
"logps/ref_chosen": -90.29670715332031,
"logps/ref_rejected": -91.13772583007812,
"logps/rejected": -95.42964172363281,
"loss": 0.9953,
"margin_dpo/margin_mean": 3.670267105102539,
"margin_dpo/margin_std": 4.296913146972656,
"step": 78
},
{
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.14908111095428467,
"fcm_dpo/delta": -0.10986006259918213,
"fcm_dpo/margin": 4.710598945617676,
"fcm_dpo/q_t": 0.355399489402771,
"grad_norm": 32.22175598144531,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6625052690505981,
"logits/rejected": -0.28548330068588257,
"logps/chosen": -87.22914123535156,
"logps/ref_chosen": -86.37832641601562,
"logps/ref_rejected": -94.10777282714844,
"logps/rejected": -99.66917419433594,
"loss": 0.9479,
"margin_dpo/margin_mean": 4.710598945617676,
"margin_dpo/margin_std": 5.8075103759765625,
"step": 79
},
{
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.1489923596382141,
"fcm_dpo/delta": 0.04605354368686676,
"fcm_dpo/margin": 3.7351717948913574,
"fcm_dpo/q_t": 0.3830299377441406,
"grad_norm": 33.21147537231445,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.6652204990386963,
"logits/rejected": -0.30884838104248047,
"logps/chosen": -95.46414947509766,
"logps/ref_chosen": -93.97032165527344,
"logps/ref_rejected": -92.57441711425781,
"logps/rejected": -97.80342102050781,
"loss": 1.0157,
"margin_dpo/margin_mean": 3.7351717948913574,
"margin_dpo/margin_std": 4.898221015930176,
"step": 80
},
{
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.14419203996658325,
"fcm_dpo/delta": -0.18252533674240112,
"fcm_dpo/margin": 5.311590194702148,
"fcm_dpo/q_t": 0.3500846028327942,
"grad_norm": 32.420406341552734,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.641904890537262,
"logits/rejected": -0.22355474531650543,
"logps/chosen": -100.79706573486328,
"logps/ref_chosen": -99.83012390136719,
"logps/ref_rejected": -133.67247009277344,
"logps/rejected": -139.95098876953125,
"loss": 0.9325,
"margin_dpo/margin_mean": 5.311590194702148,
"margin_dpo/margin_std": 6.772265434265137,
"step": 81
},
{
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.14681574702262878,
"fcm_dpo/delta": 0.07446617633104324,
"fcm_dpo/margin": 3.593234062194824,
"fcm_dpo/q_t": 0.38746753334999084,
"grad_norm": 33.34585952758789,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.10603200644254684,
"logits/rejected": 0.11097083985805511,
"logps/chosen": -84.93232727050781,
"logps/ref_chosen": -83.04598236083984,
"logps/ref_rejected": -94.52595520019531,
"logps/rejected": -100.00553131103516,
"loss": 1.0399,
"margin_dpo/margin_mean": 3.593233823776245,
"margin_dpo/margin_std": 4.823699951171875,
"step": 82
},
{
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.14609777927398682,
"fcm_dpo/delta": -0.0029450468719005585,
"fcm_dpo/margin": 4.125781536102295,
"fcm_dpo/q_t": 0.38040691614151,
"grad_norm": 39.993614196777344,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.4488382339477539,
"logits/rejected": -0.025571677833795547,
"logps/chosen": -94.34185028076172,
"logps/ref_chosen": -92.17621612548828,
"logps/ref_rejected": -107.74464416503906,
"logps/rejected": -114.03605651855469,
"loss": 1.0479,
"margin_dpo/margin_mean": 4.125782012939453,
"margin_dpo/margin_std": 6.356573104858398,
"step": 83
},
{
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.1460931897163391,
"fcm_dpo/delta": 0.0006008504424244165,
"fcm_dpo/margin": 2.527191638946533,
"fcm_dpo/q_t": 0.422789990901947,
"grad_norm": 44.41518020629883,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.4026920795440674,
"logits/rejected": -0.021030515432357788,
"logps/chosen": -104.27578735351562,
"logps/ref_chosen": -101.5264892578125,
"logps/ref_rejected": -92.42608642578125,
"logps/rejected": -97.70257568359375,
"loss": 1.1681,
"margin_dpo/margin_mean": 2.5271923542022705,
"margin_dpo/margin_std": 4.771922588348389,
"step": 84
},
{
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.1492038369178772,
"fcm_dpo/delta": 0.09993791580200195,
"fcm_dpo/margin": 3.376296043395996,
"fcm_dpo/q_t": 0.3976307511329651,
"grad_norm": 42.52838134765625,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.22517216205596924,
"logits/rejected": 0.10873828083276749,
"logps/chosen": -98.53033447265625,
"logps/ref_chosen": -96.12738037109375,
"logps/ref_rejected": -85.05519104003906,
"logps/rejected": -90.83444213867188,
"loss": 1.1171,
"margin_dpo/margin_mean": 3.3762965202331543,
"margin_dpo/margin_std": 5.977930545806885,
"step": 85
},
{
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.14783477783203125,
"fcm_dpo/delta": -0.044383928179740906,
"fcm_dpo/margin": 4.338801383972168,
"fcm_dpo/q_t": 0.37967485189437866,
"grad_norm": 30.416973114013672,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.4074563682079315,
"logits/rejected": -0.16365355253219604,
"logps/chosen": -83.54771423339844,
"logps/ref_chosen": -81.70426177978516,
"logps/ref_rejected": -93.65550231933594,
"logps/rejected": -99.8377456665039,
"loss": 1.0355,
"margin_dpo/margin_mean": 4.338800430297852,
"margin_dpo/margin_std": 6.941473960876465,
"step": 86
},
{
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.1455276608467102,
"fcm_dpo/delta": -0.07910901308059692,
"fcm_dpo/margin": 4.626860618591309,
"fcm_dpo/q_t": 0.36312583088874817,
"grad_norm": 38.379615783691406,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.8469661474227905,
"logits/rejected": -0.48443603515625,
"logps/chosen": -94.81890869140625,
"logps/ref_chosen": -92.68596649169922,
"logps/ref_rejected": -102.91818237304688,
"logps/rejected": -109.67797088623047,
"loss": 1.0375,
"margin_dpo/margin_mean": 4.626861572265625,
"margin_dpo/margin_std": 7.198031425476074,
"step": 87
},
{
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.14544734358787537,
"fcm_dpo/delta": 0.05892753228545189,
"fcm_dpo/margin": 3.7355780601501465,
"fcm_dpo/q_t": 0.39093679189682007,
"grad_norm": 35.78076171875,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.09812933206558228,
"logits/rejected": 0.08861222863197327,
"logps/chosen": -103.42122650146484,
"logps/ref_chosen": -100.76298522949219,
"logps/ref_rejected": -113.15037536621094,
"logps/rejected": -119.54420471191406,
"loss": 1.087,
"margin_dpo/margin_mean": 3.7355780601501465,
"margin_dpo/margin_std": 6.172787666320801,
"step": 88
},
{
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.147188201546669,
"fcm_dpo/delta": 0.028476351872086525,
"fcm_dpo/margin": 3.892774820327759,
"fcm_dpo/q_t": 0.3894566297531128,
"grad_norm": 30.757919311523438,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.6192222833633423,
"logits/rejected": -0.28894031047821045,
"logps/chosen": -91.76663208007812,
"logps/ref_chosen": -89.69108581542969,
"logps/ref_rejected": -88.56832885742188,
"logps/rejected": -94.53666687011719,
"loss": 1.0588,
"margin_dpo/margin_mean": 3.892774820327759,
"margin_dpo/margin_std": 6.543310165405273,
"step": 89
},
{
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.14558328688144684,
"fcm_dpo/delta": -0.14587749540805817,
"fcm_dpo/margin": 5.050919532775879,
"fcm_dpo/q_t": 0.35632702708244324,
"grad_norm": 32.816978454589844,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.28241512179374695,
"logits/rejected": -0.16540709137916565,
"logps/chosen": -83.23228454589844,
"logps/ref_chosen": -81.38255310058594,
"logps/ref_rejected": -117.77714538574219,
"logps/rejected": -124.67778015136719,
"loss": 0.9688,
"margin_dpo/margin_mean": 5.050920486450195,
"margin_dpo/margin_std": 6.877577781677246,
"step": 90
},
{
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.14302682876586914,
"fcm_dpo/delta": -0.03883608058094978,
"fcm_dpo/margin": 4.4493865966796875,
"fcm_dpo/q_t": 0.37537193298339844,
"grad_norm": 35.73554611206055,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.2560047507286072,
"logits/rejected": 0.15471355617046356,
"logps/chosen": -99.27584838867188,
"logps/ref_chosen": -97.22187805175781,
"logps/ref_rejected": -92.97674560546875,
"logps/rejected": -99.4801025390625,
"loss": 1.0129,
"margin_dpo/margin_mean": 4.449386119842529,
"margin_dpo/margin_std": 6.499499320983887,
"step": 91
},
{
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.14428263902664185,
"fcm_dpo/delta": 0.06443801522254944,
"fcm_dpo/margin": 3.7344517707824707,
"fcm_dpo/q_t": 0.39002060890197754,
"grad_norm": 30.80695343017578,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.8915022611618042,
"logits/rejected": -0.5445601344108582,
"logps/chosen": -86.91651153564453,
"logps/ref_chosen": -84.5302734375,
"logps/ref_rejected": -84.5013198852539,
"logps/rejected": -90.62200927734375,
"loss": 1.0648,
"margin_dpo/margin_mean": 3.7344512939453125,
"margin_dpo/margin_std": 5.879533290863037,
"step": 92
},
{
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.14631253480911255,
"fcm_dpo/delta": 0.07103022187948227,
"fcm_dpo/margin": 3.6395092010498047,
"fcm_dpo/q_t": 0.39281517267227173,
"grad_norm": 39.31964111328125,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.8906035423278809,
"logits/rejected": -0.34005117416381836,
"logps/chosen": -92.74388122558594,
"logps/ref_chosen": -90.25043487548828,
"logps/ref_rejected": -84.0942153930664,
"logps/rejected": -90.2271728515625,
"loss": 1.1067,
"margin_dpo/margin_mean": 3.6395092010498047,
"margin_dpo/margin_std": 6.321407318115234,
"step": 93
},
{
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.14079803228378296,
"fcm_dpo/delta": -0.1881101429462433,
"fcm_dpo/margin": 5.460287094116211,
"fcm_dpo/q_t": 0.3449600040912628,
"grad_norm": 34.704139709472656,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.45207756757736206,
"logits/rejected": -0.15455085039138794,
"logps/chosen": -94.94246673583984,
"logps/ref_chosen": -92.4542236328125,
"logps/ref_rejected": -105.2472915649414,
"logps/rejected": -113.19581604003906,
"loss": 0.954,
"margin_dpo/margin_mean": 5.4602861404418945,
"margin_dpo/margin_std": 6.94245719909668,
"step": 94
},
{
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.14194484055042267,
"fcm_dpo/delta": -0.0035935528576374054,
"fcm_dpo/margin": 4.245328903198242,
"fcm_dpo/q_t": 0.38038820028305054,
"grad_norm": 29.971302032470703,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.8304082155227661,
"logits/rejected": -0.5099495053291321,
"logps/chosen": -80.91264343261719,
"logps/ref_chosen": -78.87370300292969,
"logps/ref_rejected": -83.59121704101562,
"logps/rejected": -89.87547302246094,
"loss": 1.029,
"margin_dpo/margin_mean": 4.245327949523926,
"margin_dpo/margin_std": 6.151725769042969,
"step": 95
},
{
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.13753080368041992,
"fcm_dpo/delta": -0.15204164385795593,
"fcm_dpo/margin": 5.382932186126709,
"fcm_dpo/q_t": 0.35132813453674316,
"grad_norm": 29.244211196899414,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.7205407619476318,
"logits/rejected": -0.30231958627700806,
"logps/chosen": -108.13746643066406,
"logps/ref_chosen": -106.5921630859375,
"logps/ref_rejected": -101.76802825927734,
"logps/rejected": -108.6962661743164,
"loss": 0.9483,
"margin_dpo/margin_mean": 5.382932662963867,
"margin_dpo/margin_std": 6.747956275939941,
"step": 96
},
{
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.1367703527212143,
"fcm_dpo/delta": -0.06670509278774261,
"fcm_dpo/margin": 4.833524703979492,
"fcm_dpo/q_t": 0.36010098457336426,
"grad_norm": 32.554283142089844,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.327857106924057,
"logits/rejected": 0.07270251214504242,
"logps/chosen": -91.24253845214844,
"logps/ref_chosen": -88.67988586425781,
"logps/ref_rejected": -84.81229400634766,
"logps/rejected": -92.20846557617188,
"loss": 0.9517,
"margin_dpo/margin_mean": 4.833524227142334,
"margin_dpo/margin_std": 5.391106605529785,
"step": 97
},
{
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.1318550407886505,
"fcm_dpo/delta": -0.11207875609397888,
"fcm_dpo/margin": 5.328702926635742,
"fcm_dpo/q_t": 0.35097742080688477,
"grad_norm": 30.32093620300293,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.5091695785522461,
"logits/rejected": -0.2531248927116394,
"logps/chosen": -94.11245727539062,
"logps/ref_chosen": -92.24249267578125,
"logps/ref_rejected": -101.51948547363281,
"logps/rejected": -108.7181396484375,
"loss": 0.917,
"margin_dpo/margin_mean": 5.328701972961426,
"margin_dpo/margin_std": 5.660694122314453,
"step": 98
},
{
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.12901289761066437,
"fcm_dpo/delta": -0.14893479645252228,
"fcm_dpo/margin": 5.719096660614014,
"fcm_dpo/q_t": 0.35661765933036804,
"grad_norm": 27.99370002746582,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.3547890782356262,
"logits/rejected": -0.14507462084293365,
"logps/chosen": -73.60296630859375,
"logps/ref_chosen": -72.18464660644531,
"logps/ref_rejected": -91.88131713867188,
"logps/rejected": -99.01873016357422,
"loss": 0.9419,
"margin_dpo/margin_mean": 5.719097137451172,
"margin_dpo/margin_std": 7.474823474884033,
"step": 99
},
{
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.1281561702489853,
"fcm_dpo/delta": 0.03391501307487488,
"fcm_dpo/margin": 4.430019378662109,
"fcm_dpo/q_t": 0.3762722611427307,
"grad_norm": 26.248151779174805,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6022966504096985,
"logits/rejected": -0.14130988717079163,
"logps/chosen": -91.0530014038086,
"logps/ref_chosen": -89.51382446289062,
"logps/ref_rejected": -81.21713256835938,
"logps/rejected": -87.18632507324219,
"loss": 0.9831,
"margin_dpo/margin_mean": 4.430019378662109,
"margin_dpo/margin_std": 4.9434709548950195,
"step": 100
},
{
"epoch": 0.14684287812041116,
"eval_fcm_dpo/beta": 0.12897978723049164,
"eval_logits/chosen": 0.03100498393177986,
"eval_logits/rejected": 0.2742968201637268,
"eval_logps/chosen": -103.41716766357422,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -100.0186767578125,
"eval_loss": 0.5851320028305054,
"eval_margin_dpo/margin_mean": 3.027308940887451,
"eval_margin_dpo/margin_std": 6.074099063873291,
"eval_runtime": 43.7521,
"eval_samples_per_second": 53.46,
"eval_steps_per_second": 1.691,
"step": 100
},
{
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.13015779852867126,
"fcm_dpo/delta": 0.09972133487462997,
"fcm_dpo/margin": 3.881521224975586,
"fcm_dpo/q_t": 0.39363157749176025,
"grad_norm": 33.8118896484375,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.4824643135070801,
"logits/rejected": -0.21651335060596466,
"logps/chosen": -97.09152221679688,
"logps/ref_chosen": -94.4031982421875,
"logps/ref_rejected": -89.91001892089844,
"logps/rejected": -96.47987365722656,
"loss": 1.0872,
"margin_dpo/margin_mean": 3.881521224975586,
"margin_dpo/margin_std": 6.2039475440979,
"step": 101
},
{
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.1360168159008026,
"fcm_dpo/delta": 0.1453799605369568,
"fcm_dpo/margin": 3.3603551387786865,
"fcm_dpo/q_t": 0.4019954204559326,
"grad_norm": 43.768089294433594,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.7954562902450562,
"logits/rejected": -0.4590097665786743,
"logps/chosen": -94.66258239746094,
"logps/ref_chosen": -91.60942077636719,
"logps/ref_rejected": -75.7689208984375,
"logps/rejected": -82.18241882324219,
"loss": 1.128,
"margin_dpo/margin_mean": 3.3603553771972656,
"margin_dpo/margin_std": 5.937575340270996,
"step": 102
},
{
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.13238456845283508,
"fcm_dpo/delta": -0.1370110958814621,
"fcm_dpo/margin": 5.488382339477539,
"fcm_dpo/q_t": 0.34756556153297424,
"grad_norm": 32.67784118652344,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6738690733909607,
"logits/rejected": -0.30804526805877686,
"logps/chosen": -99.65579223632812,
"logps/ref_chosen": -97.40238952636719,
"logps/ref_rejected": -98.981689453125,
"logps/rejected": -106.72347259521484,
"loss": 0.9394,
"margin_dpo/margin_mean": 5.4883832931518555,
"margin_dpo/margin_std": 6.411886215209961,
"step": 103
},
{
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.12842795252799988,
"fcm_dpo/delta": -0.16619333624839783,
"fcm_dpo/margin": 5.866588115692139,
"fcm_dpo/q_t": 0.34973686933517456,
"grad_norm": 25.817495346069336,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.8139798045158386,
"logits/rejected": -0.4907284379005432,
"logps/chosen": -96.42156982421875,
"logps/ref_chosen": -94.85636138916016,
"logps/ref_rejected": -97.02669525146484,
"logps/rejected": -104.45848846435547,
"loss": 0.9237,
"margin_dpo/margin_mean": 5.866589069366455,
"margin_dpo/margin_std": 7.296744346618652,
"step": 104
},
{
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.12592965364456177,
"fcm_dpo/delta": -0.12789560854434967,
"fcm_dpo/margin": 5.70680046081543,
"fcm_dpo/q_t": 0.3505769968032837,
"grad_norm": 30.186168670654297,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.8461470603942871,
"logits/rejected": -0.40368765592575073,
"logps/chosen": -98.9314956665039,
"logps/ref_chosen": -96.75790405273438,
"logps/ref_rejected": -101.5774917602539,
"logps/rejected": -109.4578857421875,
"loss": 0.9491,
"margin_dpo/margin_mean": 5.70680046081543,
"margin_dpo/margin_std": 6.950338840484619,
"step": 105
},
{
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.12308678776025772,
"fcm_dpo/delta": -0.1014535203576088,
"fcm_dpo/margin": 5.641180515289307,
"fcm_dpo/q_t": 0.3579603433609009,
"grad_norm": 26.967708587646484,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.3590039014816284,
"logits/rejected": 0.06661949306726456,
"logps/chosen": -94.51701354980469,
"logps/ref_chosen": -92.43791198730469,
"logps/ref_rejected": -91.00970458984375,
"logps/rejected": -98.72999572753906,
"loss": 0.9482,
"margin_dpo/margin_mean": 5.641180038452148,
"margin_dpo/margin_std": 6.846166133880615,
"step": 106
},
{
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.11860156059265137,
"fcm_dpo/delta": -0.1794869601726532,
"fcm_dpo/margin": 6.456604480743408,
"fcm_dpo/q_t": 0.3544139862060547,
"grad_norm": 27.01357650756836,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.43201225996017456,
"logits/rejected": 0.07057125866413116,
"logps/chosen": -106.36869812011719,
"logps/ref_chosen": -105.12500762939453,
"logps/ref_rejected": -101.1824951171875,
"logps/rejected": -108.8827896118164,
"loss": 0.9456,
"margin_dpo/margin_mean": 6.456604480743408,
"margin_dpo/margin_std": 9.23145580291748,
"step": 107
},
{
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.11540443450212479,
"fcm_dpo/delta": -0.08156667649745941,
"fcm_dpo/margin": 5.848906993865967,
"fcm_dpo/q_t": 0.3599376976490021,
"grad_norm": 25.21524429321289,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.8443419933319092,
"logits/rejected": -0.35467660427093506,
"logps/chosen": -91.43235778808594,
"logps/ref_chosen": -89.20936584472656,
"logps/ref_rejected": -101.83784484863281,
"logps/rejected": -109.90974426269531,
"loss": 0.9559,
"margin_dpo/margin_mean": 5.848907470703125,
"margin_dpo/margin_std": 6.974748134613037,
"step": 108
},
{
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.11124076694250107,
"fcm_dpo/delta": -0.23742878437042236,
"fcm_dpo/margin": 7.345666408538818,
"fcm_dpo/q_t": 0.33463311195373535,
"grad_norm": 25.55326271057129,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.5116323232650757,
"logits/rejected": -0.30790773034095764,
"logps/chosen": -101.7799072265625,
"logps/ref_chosen": -100.18708801269531,
"logps/ref_rejected": -103.53739929199219,
"logps/rejected": -112.47588348388672,
"loss": 0.895,
"margin_dpo/margin_mean": 7.34566593170166,
"margin_dpo/margin_std": 8.431303024291992,
"step": 109
},
{
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.11126747727394104,
"fcm_dpo/delta": 0.09853193908929825,
"fcm_dpo/margin": 4.551311492919922,
"fcm_dpo/q_t": 0.3907678723335266,
"grad_norm": 26.554285049438477,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.20916850864887238,
"logits/rejected": 0.15151086449623108,
"logps/chosen": -109.06941223144531,
"logps/ref_chosen": -106.89572143554688,
"logps/ref_rejected": -85.81120300292969,
"logps/rejected": -92.53620910644531,
"loss": 1.0485,
"margin_dpo/margin_mean": 4.551311492919922,
"margin_dpo/margin_std": 6.169561386108398,
"step": 110
},
{
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.10644402354955673,
"fcm_dpo/delta": -0.31627288460731506,
"fcm_dpo/margin": 8.312642097473145,
"fcm_dpo/q_t": 0.3186783790588379,
"grad_norm": 23.229040145874023,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.9531800746917725,
"logits/rejected": -0.6935971975326538,
"logps/chosen": -95.22097778320312,
"logps/ref_chosen": -93.3293685913086,
"logps/ref_rejected": -104.177490234375,
"logps/rejected": -114.38174438476562,
"loss": 0.8389,
"margin_dpo/margin_mean": 8.312643051147461,
"margin_dpo/margin_std": 8.214632034301758,
"step": 111
},
{
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.10744242370128632,
"fcm_dpo/delta": 0.12501248717308044,
"fcm_dpo/margin": 4.4774169921875,
"fcm_dpo/q_t": 0.39411237835884094,
"grad_norm": 24.20155906677246,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.9408313632011414,
"logits/rejected": -0.488766074180603,
"logps/chosen": -84.89042663574219,
"logps/ref_chosen": -82.74703979492188,
"logps/ref_rejected": -71.5584487915039,
"logps/rejected": -78.17925262451172,
"loss": 1.0479,
"margin_dpo/margin_mean": 4.4774169921875,
"margin_dpo/margin_std": 5.751065254211426,
"step": 112
},
{
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.10971181094646454,
"fcm_dpo/delta": 0.08851583302021027,
"fcm_dpo/margin": 4.70300817489624,
"fcm_dpo/q_t": 0.38598090410232544,
"grad_norm": 28.11321449279785,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.6415950059890747,
"logits/rejected": -0.14356252551078796,
"logps/chosen": -100.6945571899414,
"logps/ref_chosen": -97.89949035644531,
"logps/ref_rejected": -84.41960144042969,
"logps/rejected": -91.91766357421875,
"loss": 1.0218,
"margin_dpo/margin_mean": 4.703007698059082,
"margin_dpo/margin_std": 5.460232257843018,
"step": 113
},
{
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.10798073559999466,
"fcm_dpo/delta": -0.10012990236282349,
"fcm_dpo/margin": 6.41129732131958,
"fcm_dpo/q_t": 0.36218881607055664,
"grad_norm": 28.432401657104492,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -1.0188771486282349,
"logits/rejected": -0.4950348734855652,
"logps/chosen": -99.10498046875,
"logps/ref_chosen": -96.4951171875,
"logps/ref_rejected": -92.01536560058594,
"logps/rejected": -101.0365219116211,
"loss": 0.9806,
"margin_dpo/margin_mean": 6.411297798156738,
"margin_dpo/margin_std": 8.566350936889648,
"step": 114
},
{
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.10620786994695663,
"fcm_dpo/delta": -0.11520249396562576,
"fcm_dpo/margin": 6.658008098602295,
"fcm_dpo/q_t": 0.3513981103897095,
"grad_norm": 23.633058547973633,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.5462427139282227,
"logits/rejected": -0.1587030589580536,
"logps/chosen": -105.50727844238281,
"logps/ref_chosen": -103.18013000488281,
"logps/ref_rejected": -105.71595764160156,
"logps/rejected": -114.70111846923828,
"loss": 0.9284,
"margin_dpo/margin_mean": 6.658008575439453,
"margin_dpo/margin_std": 7.443235874176025,
"step": 115
},
{
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.1062956303358078,
"fcm_dpo/delta": 0.024708664044737816,
"fcm_dpo/margin": 5.4217939376831055,
"fcm_dpo/q_t": 0.37565532326698303,
"grad_norm": 22.831968307495117,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.5053553581237793,
"logits/rejected": -0.07657349109649658,
"logps/chosen": -83.38106536865234,
"logps/ref_chosen": -81.15837097167969,
"logps/ref_rejected": -86.30586242675781,
"logps/rejected": -93.95034790039062,
"loss": 0.9908,
"margin_dpo/margin_mean": 5.421794414520264,
"margin_dpo/margin_std": 6.337811470031738,
"step": 116
},
{
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.1065780371427536,
"fcm_dpo/delta": 0.09205998480319977,
"fcm_dpo/margin": 4.804106712341309,
"fcm_dpo/q_t": 0.38822945952415466,
"grad_norm": 31.471729278564453,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.6579692363739014,
"logits/rejected": -0.30281364917755127,
"logps/chosen": -99.8343505859375,
"logps/ref_chosen": -97.0809555053711,
"logps/ref_rejected": -75.1399154663086,
"logps/rejected": -82.69741821289062,
"loss": 1.0457,
"margin_dpo/margin_mean": 4.804106712341309,
"margin_dpo/margin_std": 6.34288215637207,
"step": 117
},
{
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.10656137019395828,
"fcm_dpo/delta": -0.0756685882806778,
"fcm_dpo/margin": 6.291032791137695,
"fcm_dpo/q_t": 0.35226526856422424,
"grad_norm": 22.64267349243164,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -1.0219924449920654,
"logits/rejected": -0.7220288515090942,
"logps/chosen": -88.30572509765625,
"logps/ref_chosen": -86.53623962402344,
"logps/ref_rejected": -82.93563842773438,
"logps/rejected": -90.99617004394531,
"loss": 0.9216,
"margin_dpo/margin_mean": 6.291032791137695,
"margin_dpo/margin_std": 6.30666446685791,
"step": 118
},
{
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.10483279079198837,
"fcm_dpo/delta": -0.08183208853006363,
"fcm_dpo/margin": 6.448603630065918,
"fcm_dpo/q_t": 0.3583568036556244,
"grad_norm": 26.867589950561523,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.9083903431892395,
"logits/rejected": -0.6197102665901184,
"logps/chosen": -107.41993713378906,
"logps/ref_chosen": -104.60908508300781,
"logps/ref_rejected": -109.15780639648438,
"logps/rejected": -118.41727447509766,
"loss": 0.9898,
"margin_dpo/margin_mean": 6.448603630065918,
"margin_dpo/margin_std": 8.662832260131836,
"step": 119
},
{
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.10017947107553482,
"fcm_dpo/delta": -0.2905012369155884,
"fcm_dpo/margin": 8.636421203613281,
"fcm_dpo/q_t": 0.33309102058410645,
"grad_norm": 30.011741638183594,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -1.0279607772827148,
"logits/rejected": -0.5229378938674927,
"logps/chosen": -116.57168579101562,
"logps/ref_chosen": -114.68742370605469,
"logps/ref_rejected": -111.18586730957031,
"logps/rejected": -121.70654296875,
"loss": 0.915,
"margin_dpo/margin_mean": 8.636421203613281,
"margin_dpo/margin_std": 10.763944625854492,
"step": 120
},
{
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.0964190736413002,
"fcm_dpo/delta": -0.12846185266971588,
"fcm_dpo/margin": 7.4496049880981445,
"fcm_dpo/q_t": 0.3460380434989929,
"grad_norm": 25.585208892822266,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.7847539186477661,
"logits/rejected": -0.4723082184791565,
"logps/chosen": -93.91836547851562,
"logps/ref_chosen": -92.11666107177734,
"logps/ref_rejected": -114.972900390625,
"logps/rejected": -124.22420501708984,
"loss": 0.9003,
"margin_dpo/margin_mean": 7.4496049880981445,
"margin_dpo/margin_std": 7.533651351928711,
"step": 121
},
{
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.09395934641361237,
"fcm_dpo/delta": -0.18832112848758698,
"fcm_dpo/margin": 8.238999366760254,
"fcm_dpo/q_t": 0.3329077959060669,
"grad_norm": 26.141273498535156,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.47960373759269714,
"logits/rejected": -0.35823744535446167,
"logps/chosen": -87.03057861328125,
"logps/ref_chosen": -84.87757110595703,
"logps/ref_rejected": -111.0829849243164,
"logps/rejected": -121.47499084472656,
"loss": 0.8771,
"margin_dpo/margin_mean": 8.238998413085938,
"margin_dpo/margin_std": 8.030372619628906,
"step": 122
},
{
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.09212273359298706,
"fcm_dpo/delta": 0.013646617531776428,
"fcm_dpo/margin": 6.361550807952881,
"fcm_dpo/q_t": 0.3727787137031555,
"grad_norm": 22.423507690429688,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.9911929368972778,
"logits/rejected": -0.7056489586830139,
"logps/chosen": -100.84278869628906,
"logps/ref_chosen": -98.47874450683594,
"logps/ref_rejected": -93.85417175292969,
"logps/rejected": -102.57976531982422,
"loss": 0.9885,
"margin_dpo/margin_mean": 6.361550331115723,
"margin_dpo/margin_std": 7.391707420349121,
"step": 123
},
{
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.09153375774621964,
"fcm_dpo/delta": -0.10555803775787354,
"fcm_dpo/margin": 7.628256320953369,
"fcm_dpo/q_t": 0.36510169506073,
"grad_norm": 22.723674774169922,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.6390783786773682,
"logits/rejected": -0.385051965713501,
"logps/chosen": -104.82085418701172,
"logps/ref_chosen": -102.76290130615234,
"logps/ref_rejected": -123.005615234375,
"logps/rejected": -132.69183349609375,
"loss": 0.9851,
"margin_dpo/margin_mean": 7.628256320953369,
"margin_dpo/margin_std": 10.736438751220703,
"step": 124
},
{
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.09196379780769348,
"fcm_dpo/delta": 0.07185746729373932,
"fcm_dpo/margin": 5.784401893615723,
"fcm_dpo/q_t": 0.38678333163261414,
"grad_norm": 24.231779098510742,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.6529036164283752,
"logits/rejected": -0.4633956253528595,
"logps/chosen": -93.97822570800781,
"logps/ref_chosen": -90.42627716064453,
"logps/ref_rejected": -89.24901580810547,
"logps/rejected": -98.58536529541016,
"loss": 1.0442,
"margin_dpo/margin_mean": 5.784401893615723,
"margin_dpo/margin_std": 8.095062255859375,
"step": 125
},
{
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.09028153121471405,
"fcm_dpo/delta": -0.15866564214229584,
"fcm_dpo/margin": 8.273946762084961,
"fcm_dpo/q_t": 0.3347667157649994,
"grad_norm": 22.696521759033203,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.7492173910140991,
"logits/rejected": -0.49526458978652954,
"logps/chosen": -96.82389831542969,
"logps/ref_chosen": -94.42288208007812,
"logps/ref_rejected": -103.88156127929688,
"logps/rejected": -114.55652618408203,
"loss": 0.8628,
"margin_dpo/margin_mean": 8.273946762084961,
"margin_dpo/margin_std": 7.15079402923584,
"step": 126
},
{
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.09020580351352692,
"fcm_dpo/delta": 0.022558212280273438,
"fcm_dpo/margin": 6.407971382141113,
"fcm_dpo/q_t": 0.377363920211792,
"grad_norm": 23.82652473449707,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.6647278070449829,
"logits/rejected": -0.35185229778289795,
"logps/chosen": -87.22705841064453,
"logps/ref_chosen": -84.75105285644531,
"logps/ref_rejected": -83.54523468017578,
"logps/rejected": -92.42921447753906,
"loss": 1.0158,
"margin_dpo/margin_mean": 6.407971382141113,
"margin_dpo/margin_std": 8.475217819213867,
"step": 127
},
{
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.09033428132534027,
"fcm_dpo/delta": 0.012706171721220016,
"fcm_dpo/margin": 6.505819320678711,
"fcm_dpo/q_t": 0.37230080366134644,
"grad_norm": 25.748743057250977,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.5167370438575745,
"logits/rejected": -0.09917855262756348,
"logps/chosen": -93.13040161132812,
"logps/ref_chosen": -90.61964416503906,
"logps/ref_rejected": -83.10347747802734,
"logps/rejected": -92.12004852294922,
"loss": 1.0246,
"margin_dpo/margin_mean": 6.505819320678711,
"margin_dpo/margin_std": 9.225418090820312,
"step": 128
},
{
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.09042790532112122,
"fcm_dpo/delta": -0.0049272216856479645,
"fcm_dpo/margin": 6.681199550628662,
"fcm_dpo/q_t": 0.37444210052490234,
"grad_norm": 21.542911529541016,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.42197108268737793,
"logits/rejected": -0.1338576376438141,
"logps/chosen": -101.74089050292969,
"logps/ref_chosen": -98.91870880126953,
"logps/ref_rejected": -103.28419494628906,
"logps/rejected": -112.78758239746094,
"loss": 0.9973,
"margin_dpo/margin_mean": 6.681199073791504,
"margin_dpo/margin_std": 8.355775833129883,
"step": 129
},
{
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.08857293426990509,
"fcm_dpo/delta": -0.17602676153182983,
"fcm_dpo/margin": 8.605558395385742,
"fcm_dpo/q_t": 0.3452799320220947,
"grad_norm": 20.744892120361328,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.8332382440567017,
"logits/rejected": -0.5825516581535339,
"logps/chosen": -75.53247833251953,
"logps/ref_chosen": -73.18368530273438,
"logps/ref_rejected": -88.02154541015625,
"logps/rejected": -98.97589111328125,
"loss": 0.9116,
"margin_dpo/margin_mean": 8.605558395385742,
"margin_dpo/margin_std": 9.510639190673828,
"step": 130
},
{
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.08698062598705292,
"fcm_dpo/delta": -0.04303303733468056,
"fcm_dpo/margin": 7.350835800170898,
"fcm_dpo/q_t": 0.3662865161895752,
"grad_norm": 21.23581886291504,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.792221188545227,
"logits/rejected": -0.576511800289154,
"logps/chosen": -75.73124694824219,
"logps/ref_chosen": -73.9564437866211,
"logps/ref_rejected": -94.45582580566406,
"logps/rejected": -103.58146667480469,
"loss": 0.9788,
"margin_dpo/margin_mean": 7.350835800170898,
"margin_dpo/margin_std": 9.003894805908203,
"step": 131
},
{
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.08460888266563416,
"fcm_dpo/delta": -0.08415042608976364,
"fcm_dpo/margin": 8.008447647094727,
"fcm_dpo/q_t": 0.3516160249710083,
"grad_norm": 23.11528205871582,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.7575106620788574,
"logits/rejected": -0.360861212015152,
"logps/chosen": -95.101318359375,
"logps/ref_chosen": -92.43948364257812,
"logps/ref_rejected": -94.13252258300781,
"logps/rejected": -104.80279541015625,
"loss": 0.9311,
"margin_dpo/margin_mean": 8.008447647094727,
"margin_dpo/margin_std": 8.533775329589844,
"step": 132
},
{
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.08362139761447906,
"fcm_dpo/delta": -0.1000952273607254,
"fcm_dpo/margin": 8.29056167602539,
"fcm_dpo/q_t": 0.35254591703414917,
"grad_norm": 21.20782470703125,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -1.1061053276062012,
"logits/rejected": -0.7356460690498352,
"logps/chosen": -94.83768463134766,
"logps/ref_chosen": -93.11666107177734,
"logps/ref_rejected": -102.09490966796875,
"logps/rejected": -112.10649108886719,
"loss": 0.9185,
"margin_dpo/margin_mean": 8.29056167602539,
"margin_dpo/margin_std": 8.746664047241211,
"step": 133
},
{
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.08186972141265869,
"fcm_dpo/delta": -0.14321613311767578,
"fcm_dpo/margin": 8.948598861694336,
"fcm_dpo/q_t": 0.3416699171066284,
"grad_norm": 19.530412673950195,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -1.1816595792770386,
"logits/rejected": -0.8657940626144409,
"logps/chosen": -85.77832794189453,
"logps/ref_chosen": -83.55012512207031,
"logps/ref_rejected": -93.8815689086914,
"logps/rejected": -105.05838012695312,
"loss": 0.8911,
"margin_dpo/margin_mean": 8.948599815368652,
"margin_dpo/margin_std": 8.551406860351562,
"step": 134
},
{
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.0804293155670166,
"fcm_dpo/delta": -0.045598309487104416,
"fcm_dpo/margin": 7.988539695739746,
"fcm_dpo/q_t": 0.3668595552444458,
"grad_norm": 26.672277450561523,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.9140989780426025,
"logits/rejected": -0.7617922425270081,
"logps/chosen": -79.91949462890625,
"logps/ref_chosen": -77.58262634277344,
"logps/ref_rejected": -121.05648040771484,
"logps/rejected": -131.38189697265625,
"loss": 0.9718,
"margin_dpo/margin_mean": 7.9885406494140625,
"margin_dpo/margin_std": 9.902532577514648,
"step": 135
},
{
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.0786333680152893,
"fcm_dpo/delta": -0.09831453859806061,
"fcm_dpo/margin": 8.791913986206055,
"fcm_dpo/q_t": 0.34778937697410583,
"grad_norm": 19.27442169189453,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -1.0831172466278076,
"logits/rejected": -0.6637728214263916,
"logps/chosen": -96.29593658447266,
"logps/ref_chosen": -94.1727294921875,
"logps/ref_rejected": -103.26428985595703,
"logps/rejected": -114.17940521240234,
"loss": 0.8993,
"margin_dpo/margin_mean": 8.791913986206055,
"margin_dpo/margin_std": 8.21432113647461,
"step": 136
},
{
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.07321903109550476,
"fcm_dpo/delta": -0.3953135013580322,
"fcm_dpo/margin": 13.009666442871094,
"fcm_dpo/q_t": 0.302865207195282,
"grad_norm": 21.34975814819336,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.7808685302734375,
"logits/rejected": -0.5483202934265137,
"logps/chosen": -79.78280639648438,
"logps/ref_chosen": -78.11874389648438,
"logps/ref_rejected": -117.92237091064453,
"logps/rejected": -132.59609985351562,
"loss": 0.7814,
"margin_dpo/margin_mean": 13.00966739654541,
"margin_dpo/margin_std": 11.704439163208008,
"step": 137
},
{
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.07211971282958984,
"fcm_dpo/delta": -0.01127841230481863,
"fcm_dpo/margin": 8.464805603027344,
"fcm_dpo/q_t": 0.36739087104797363,
"grad_norm": 18.32305908203125,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.5027905702590942,
"logits/rejected": -0.2673172950744629,
"logps/chosen": -96.51560974121094,
"logps/ref_chosen": -92.82994079589844,
"logps/ref_rejected": -98.64840698242188,
"logps/rejected": -110.79888153076172,
"loss": 0.96,
"margin_dpo/margin_mean": 8.464805603027344,
"margin_dpo/margin_std": 9.157770156860352,
"step": 138
},
{
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.07099495828151703,
"fcm_dpo/delta": -0.09988156706094742,
"fcm_dpo/margin": 9.755861282348633,
"fcm_dpo/q_t": 0.3473873734474182,
"grad_norm": 18.42373275756836,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.8951112031936646,
"logits/rejected": -0.577903687953949,
"logps/chosen": -81.29713439941406,
"logps/ref_chosen": -79.00301361083984,
"logps/ref_rejected": -85.82733154296875,
"logps/rejected": -97.87730407714844,
"loss": 0.9312,
"margin_dpo/margin_mean": 9.755861282348633,
"margin_dpo/margin_std": 10.73771858215332,
"step": 139
},
{
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.07040712982416153,
"fcm_dpo/delta": -0.03950156271457672,
"fcm_dpo/margin": 9.047286987304688,
"fcm_dpo/q_t": 0.36051124334335327,
"grad_norm": 17.700237274169922,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -1.4827024936676025,
"logits/rejected": -1.119741439819336,
"logps/chosen": -90.76173400878906,
"logps/ref_chosen": -88.35244750976562,
"logps/ref_rejected": -90.57476806640625,
"logps/rejected": -102.03134155273438,
"loss": 0.9434,
"margin_dpo/margin_mean": 9.047286033630371,
"margin_dpo/margin_std": 9.427406311035156,
"step": 140
},
{
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.06885601580142975,
"fcm_dpo/delta": -0.14049942791461945,
"fcm_dpo/margin": 10.606624603271484,
"fcm_dpo/q_t": 0.3436024785041809,
"grad_norm": 19.141874313354492,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -1.1690764427185059,
"logits/rejected": -0.8353248834609985,
"logps/chosen": -110.29083251953125,
"logps/ref_chosen": -107.22016143798828,
"logps/ref_rejected": -104.71223449707031,
"logps/rejected": -118.3895263671875,
"loss": 0.8996,
"margin_dpo/margin_mean": 10.606623649597168,
"margin_dpo/margin_std": 10.912744522094727,
"step": 141
},
{
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.067325659096241,
"fcm_dpo/delta": -0.07276973128318787,
"fcm_dpo/margin": 9.913378715515137,
"fcm_dpo/q_t": 0.3580227196216583,
"grad_norm": 18.446134567260742,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.8606932759284973,
"logits/rejected": -0.6145029067993164,
"logps/chosen": -92.71931457519531,
"logps/ref_chosen": -88.72894287109375,
"logps/ref_rejected": -99.73038482666016,
"logps/rejected": -113.63412475585938,
"loss": 0.9319,
"margin_dpo/margin_mean": 9.91337776184082,
"margin_dpo/margin_std": 10.864763259887695,
"step": 142
},
{
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.06714846193790436,
"fcm_dpo/delta": 0.021465405821800232,
"fcm_dpo/margin": 8.62674331665039,
"fcm_dpo/q_t": 0.37440505623817444,
"grad_norm": 17.625202178955078,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -1.2188315391540527,
"logits/rejected": -0.7806566953659058,
"logps/chosen": -87.72565460205078,
"logps/ref_chosen": -85.50457763671875,
"logps/ref_rejected": -85.8056640625,
"logps/rejected": -96.65348815917969,
"loss": 0.9974,
"margin_dpo/margin_mean": 8.62674331665039,
"margin_dpo/margin_std": 10.493571281433105,
"step": 143
},
{
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.06795937567949295,
"fcm_dpo/delta": -0.022204145789146423,
"fcm_dpo/margin": 9.117193222045898,
"fcm_dpo/q_t": 0.37021785974502563,
"grad_norm": 22.30422019958496,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -1.0173625946044922,
"logits/rejected": -0.5171209573745728,
"logps/chosen": -107.23023223876953,
"logps/ref_chosen": -104.68708801269531,
"logps/ref_rejected": -87.71908569335938,
"logps/rejected": -99.37940979003906,
"loss": 1.0019,
"margin_dpo/margin_mean": 9.117193222045898,
"margin_dpo/margin_std": 12.013388633728027,
"step": 144
},
{
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.0664219856262207,
"fcm_dpo/delta": -0.08367905020713806,
"fcm_dpo/margin": 10.206480026245117,
"fcm_dpo/q_t": 0.35269540548324585,
"grad_norm": 18.23365592956543,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.9210905432701111,
"logits/rejected": -0.36352017521858215,
"logps/chosen": -90.94596862792969,
"logps/ref_chosen": -88.51988983154297,
"logps/ref_rejected": -83.76513671875,
"logps/rejected": -96.397705078125,
"loss": 0.9285,
"margin_dpo/margin_mean": 10.206480979919434,
"margin_dpo/margin_std": 10.862861633300781,
"step": 145
},
{
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.06422297656536102,
"fcm_dpo/delta": -0.15030890703201294,
"fcm_dpo/margin": 11.474971771240234,
"fcm_dpo/q_t": 0.3387315571308136,
"grad_norm": 17.679115295410156,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -1.0683330297470093,
"logits/rejected": -0.4716656804084778,
"logps/chosen": -105.42335510253906,
"logps/ref_chosen": -102.90586853027344,
"logps/ref_rejected": -88.9493637084961,
"logps/rejected": -102.94181823730469,
"loss": 0.8824,
"margin_dpo/margin_mean": 11.474971771240234,
"margin_dpo/margin_std": 10.659229278564453,
"step": 146
},
{
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.06264235824346542,
"fcm_dpo/delta": -0.1676965206861496,
"fcm_dpo/margin": 12.052690505981445,
"fcm_dpo/q_t": 0.334587037563324,
"grad_norm": 16.055294036865234,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -1.0700526237487793,
"logits/rejected": -0.5048679113388062,
"logps/chosen": -91.99482727050781,
"logps/ref_chosen": -89.26219177246094,
"logps/ref_rejected": -76.50011444091797,
"logps/rejected": -91.28544616699219,
"loss": 0.8674,
"margin_dpo/margin_mean": 12.052690505981445,
"margin_dpo/margin_std": 10.964826583862305,
"step": 147
},
{
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.06236677244305611,
"fcm_dpo/delta": -0.03803453594446182,
"fcm_dpo/margin": 10.167755126953125,
"fcm_dpo/q_t": 0.3664420247077942,
"grad_norm": 18.700401306152344,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -1.050333023071289,
"logits/rejected": -0.6528257131576538,
"logps/chosen": -93.0657958984375,
"logps/ref_chosen": -91.76287841796875,
"logps/ref_rejected": -110.38552856445312,
"logps/rejected": -121.856201171875,
"loss": 0.966,
"margin_dpo/margin_mean": 10.167754173278809,
"margin_dpo/margin_std": 11.42441177368164,
"step": 148
},
{
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.0613025538623333,
"fcm_dpo/delta": -0.013467121869325638,
"fcm_dpo/margin": 9.98930835723877,
"fcm_dpo/q_t": 0.37364453077316284,
"grad_norm": 17.756973266601562,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.9625898003578186,
"logits/rejected": -0.6630960702896118,
"logps/chosen": -97.20555114746094,
"logps/ref_chosen": -93.48401641845703,
"logps/ref_rejected": -105.6436996459961,
"logps/rejected": -119.35454559326172,
"loss": 0.991,
"margin_dpo/margin_mean": 9.989309310913086,
"margin_dpo/margin_std": 12.843564987182617,
"step": 149
},
{
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.060672082006931305,
"fcm_dpo/delta": -0.07046893239021301,
"fcm_dpo/margin": 10.968360900878906,
"fcm_dpo/q_t": 0.35937947034835815,
"grad_norm": 15.75436782836914,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.8642300367355347,
"logits/rejected": -0.5881214141845703,
"logps/chosen": -78.63943481445312,
"logps/ref_chosen": -75.7786865234375,
"logps/ref_rejected": -90.02676391601562,
"logps/rejected": -103.85586547851562,
"loss": 0.9473,
"margin_dpo/margin_mean": 10.968360900878906,
"margin_dpo/margin_std": 12.808120727539062,
"step": 150
},
{
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.06035682559013367,
"fcm_dpo/delta": -0.029848817735910416,
"fcm_dpo/margin": 10.404455184936523,
"fcm_dpo/q_t": 0.3672201931476593,
"grad_norm": 19.685895919799805,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -1.7692604064941406,
"logits/rejected": -1.2120516300201416,
"logps/chosen": -106.98040771484375,
"logps/ref_chosen": -102.73422241210938,
"logps/ref_rejected": -106.03781127929688,
"logps/rejected": -120.68846130371094,
"loss": 0.9769,
"margin_dpo/margin_mean": 10.404455184936523,
"margin_dpo/margin_std": 12.831315994262695,
"step": 151
},
{
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.05888533219695091,
"fcm_dpo/delta": -0.14253246784210205,
"fcm_dpo/margin": 12.427541732788086,
"fcm_dpo/q_t": 0.34149640798568726,
"grad_norm": 17.039073944091797,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -1.1869471073150635,
"logits/rejected": -0.8218509554862976,
"logps/chosen": -99.31748962402344,
"logps/ref_chosen": -97.59074401855469,
"logps/ref_rejected": -110.49913024902344,
"logps/rejected": -124.6534194946289,
"loss": 0.8913,
"margin_dpo/margin_mean": 12.427543640136719,
"margin_dpo/margin_std": 12.471309661865234,
"step": 152
},
{
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.05911749601364136,
"fcm_dpo/delta": 0.06782151758670807,
"fcm_dpo/margin": 9.064018249511719,
"fcm_dpo/q_t": 0.3846895396709442,
"grad_norm": 17.052255630493164,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -1.4898900985717773,
"logits/rejected": -1.1749351024627686,
"logps/chosen": -91.19923400878906,
"logps/ref_chosen": -87.02560424804688,
"logps/ref_rejected": -91.06854248046875,
"logps/rejected": -104.30618286132812,
"loss": 1.0342,
"margin_dpo/margin_mean": 9.064018249511719,
"margin_dpo/margin_std": 12.064422607421875,
"step": 153
},
{
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.05774786323308945,
"fcm_dpo/delta": -0.2577056586742401,
"fcm_dpo/margin": 14.486139297485352,
"fcm_dpo/q_t": 0.319768488407135,
"grad_norm": 16.834199905395508,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -1.1176913976669312,
"logits/rejected": -0.7909313440322876,
"logps/chosen": -93.70020294189453,
"logps/ref_chosen": -91.10566711425781,
"logps/ref_rejected": -111.37238311767578,
"logps/rejected": -128.45306396484375,
"loss": 0.8249,
"margin_dpo/margin_mean": 14.486139297485352,
"margin_dpo/margin_std": 12.080253601074219,
"step": 154
},
{
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.05554405599832535,
"fcm_dpo/delta": -0.05215900391340256,
"fcm_dpo/margin": 11.656644821166992,
"fcm_dpo/q_t": 0.35945552587509155,
"grad_norm": 20.919530868530273,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -1.0245808362960815,
"logits/rejected": -0.6745032072067261,
"logps/chosen": -98.85678100585938,
"logps/ref_chosen": -94.56097412109375,
"logps/ref_rejected": -84.77359008789062,
"logps/rejected": -100.72604370117188,
"loss": 0.9947,
"margin_dpo/margin_mean": 11.656644821166992,
"margin_dpo/margin_std": 15.20389175415039,
"step": 155
},
{
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.055864423513412476,
"fcm_dpo/delta": 0.03552495688199997,
"fcm_dpo/margin": 10.137336730957031,
"fcm_dpo/q_t": 0.37930774688720703,
"grad_norm": 18.78835105895996,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -1.3272318840026855,
"logits/rejected": -0.8460725545883179,
"logps/chosen": -100.33680725097656,
"logps/ref_chosen": -94.65538787841797,
"logps/ref_rejected": -84.41934204101562,
"logps/rejected": -100.23809051513672,
"loss": 1.0171,
"margin_dpo/margin_mean": 10.137334823608398,
"margin_dpo/margin_std": 13.136527061462402,
"step": 156
},
{
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.05504686385393143,
"fcm_dpo/delta": -0.15157991647720337,
"fcm_dpo/margin": 13.453115463256836,
"fcm_dpo/q_t": 0.33803731203079224,
"grad_norm": 17.70085334777832,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -1.6043157577514648,
"logits/rejected": -1.1462740898132324,
"logps/chosen": -102.32630920410156,
"logps/ref_chosen": -97.01239013671875,
"logps/ref_rejected": -111.70133972167969,
"logps/rejected": -130.4683837890625,
"loss": 0.8766,
"margin_dpo/margin_mean": 13.453115463256836,
"margin_dpo/margin_std": 12.435213088989258,
"step": 157
},
{
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.053367845714092255,
"fcm_dpo/delta": -0.1598518192768097,
"fcm_dpo/margin": 14.018548011779785,
"fcm_dpo/q_t": 0.3366357088088989,
"grad_norm": 17.31310272216797,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -1.1085999011993408,
"logits/rejected": -0.8174431920051575,
"logps/chosen": -99.63768005371094,
"logps/ref_chosen": -94.71296691894531,
"logps/ref_rejected": -120.93733978271484,
"logps/rejected": -139.880615234375,
"loss": 0.8714,
"margin_dpo/margin_mean": 14.018546104431152,
"margin_dpo/margin_std": 12.86544418334961,
"step": 158
},
{
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.05200641602277756,
"fcm_dpo/delta": -0.09326906502246857,
"fcm_dpo/margin": 13.204719543457031,
"fcm_dpo/q_t": 0.35249608755111694,
"grad_norm": 14.883583068847656,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -1.7073493003845215,
"logits/rejected": -1.2342015504837036,
"logps/chosen": -112.4176025390625,
"logps/ref_chosen": -108.2182388305664,
"logps/ref_rejected": -94.83901977539062,
"logps/rejected": -112.24309539794922,
"loss": 0.9328,
"margin_dpo/margin_mean": 13.204719543457031,
"margin_dpo/margin_std": 14.517641067504883,
"step": 159
},
{
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.05130424350500107,
"fcm_dpo/delta": -0.05173896625638008,
"fcm_dpo/margin": 12.633888244628906,
"fcm_dpo/q_t": 0.3558708131313324,
"grad_norm": 15.668502807617188,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -1.4224272966384888,
"logits/rejected": -0.8817993402481079,
"logps/chosen": -101.59320068359375,
"logps/ref_chosen": -98.16340637207031,
"logps/ref_rejected": -102.28097534179688,
"logps/rejected": -118.34466552734375,
"loss": 0.9332,
"margin_dpo/margin_mean": 12.633889198303223,
"margin_dpo/margin_std": 12.74842643737793,
"step": 160
},
{
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.05100458487868309,
"fcm_dpo/delta": -0.0010594986379146576,
"fcm_dpo/margin": 11.772229194641113,
"fcm_dpo/q_t": 0.3696848750114441,
"grad_norm": 16.675426483154297,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -1.661262035369873,
"logits/rejected": -1.1878881454467773,
"logps/chosen": -92.0523681640625,
"logps/ref_chosen": -86.59187316894531,
"logps/ref_rejected": -89.01643371582031,
"logps/rejected": -106.24916076660156,
"loss": 0.9897,
"margin_dpo/margin_mean": 11.772229194641113,
"margin_dpo/margin_std": 14.080129623413086,
"step": 161
},
{
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.050782568752765656,
"fcm_dpo/delta": -0.046759188175201416,
"fcm_dpo/margin": 12.671504974365234,
"fcm_dpo/q_t": 0.357089638710022,
"grad_norm": 14.808147430419922,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.9641022682189941,
"logits/rejected": -0.5625172257423401,
"logps/chosen": -102.07867431640625,
"logps/ref_chosen": -97.40288543701172,
"logps/ref_rejected": -95.86923217773438,
"logps/rejected": -113.2165298461914,
"loss": 0.9463,
"margin_dpo/margin_mean": 12.671504974365234,
"margin_dpo/margin_std": 13.506097793579102,
"step": 162
},
{
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.04923873394727707,
"fcm_dpo/delta": -0.19184833765029907,
"fcm_dpo/margin": 15.766294479370117,
"fcm_dpo/q_t": 0.3363766372203827,
"grad_norm": 13.101168632507324,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -1.5308029651641846,
"logits/rejected": -1.145806908607483,
"logps/chosen": -86.65454864501953,
"logps/ref_chosen": -82.1753921508789,
"logps/ref_rejected": -82.86473846435547,
"logps/rejected": -103.11018371582031,
"loss": 0.8757,
"margin_dpo/margin_mean": 15.766292572021484,
"margin_dpo/margin_std": 16.105995178222656,
"step": 163
},
{
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.04908021539449692,
"fcm_dpo/delta": 0.08986516296863556,
"fcm_dpo/margin": 10.468819618225098,
"fcm_dpo/q_t": 0.38547319173812866,
"grad_norm": 15.708305358886719,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -1.2432619333267212,
"logits/rejected": -0.7992422580718994,
"logps/chosen": -98.1744613647461,
"logps/ref_chosen": -90.47427368164062,
"logps/ref_rejected": -87.02809143066406,
"logps/rejected": -105.19710540771484,
"loss": 1.031,
"margin_dpo/margin_mean": 10.468820571899414,
"margin_dpo/margin_std": 12.44760799407959,
"step": 164
},
{
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.04876025393605232,
"fcm_dpo/delta": -0.1661471575498581,
"fcm_dpo/margin": 15.461935997009277,
"fcm_dpo/q_t": 0.3421006202697754,
"grad_norm": 13.452424049377441,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -1.3514103889465332,
"logits/rejected": -0.941323459148407,
"logps/chosen": -92.18637084960938,
"logps/ref_chosen": -87.313232421875,
"logps/ref_rejected": -81.902587890625,
"logps/rejected": -102.23765563964844,
"loss": 0.8884,
"margin_dpo/margin_mean": 15.461935997009277,
"margin_dpo/margin_std": 16.554237365722656,
"step": 165
},
{
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.04737546294927597,
"fcm_dpo/delta": -0.10841958224773407,
"fcm_dpo/margin": 14.792806625366211,
"fcm_dpo/q_t": 0.35638824105262756,
"grad_norm": 14.667271614074707,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -1.409278154373169,
"logits/rejected": -1.1956782341003418,
"logps/chosen": -90.57990264892578,
"logps/ref_chosen": -81.58606719970703,
"logps/ref_rejected": -103.16517639160156,
"logps/rejected": -126.95182800292969,
"loss": 0.9348,
"margin_dpo/margin_mean": 14.792806625366211,
"margin_dpo/margin_std": 17.73269271850586,
"step": 166
},
{
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.0457189604640007,
"fcm_dpo/delta": -0.15542298555374146,
"fcm_dpo/margin": 16.228336334228516,
"fcm_dpo/q_t": 0.33740636706352234,
"grad_norm": 14.068784713745117,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.9149848222732544,
"logits/rejected": -0.7234656810760498,
"logps/chosen": -91.24684143066406,
"logps/ref_chosen": -83.56167602539062,
"logps/ref_rejected": -99.05305480957031,
"logps/rejected": -122.966552734375,
"loss": 0.8664,
"margin_dpo/margin_mean": 16.228334426879883,
"margin_dpo/margin_std": 14.279183387756348,
"step": 167
},
{
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.04559946060180664,
"fcm_dpo/delta": -0.028263315558433533,
"fcm_dpo/margin": 13.73760986328125,
"fcm_dpo/q_t": 0.3622729182243347,
"grad_norm": 15.556920051574707,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -1.2671973705291748,
"logits/rejected": -0.9255896210670471,
"logps/chosen": -87.52298736572266,
"logps/ref_chosen": -79.93123626708984,
"logps/ref_rejected": -80.10216522216797,
"logps/rejected": -101.43153381347656,
"loss": 0.9486,
"margin_dpo/margin_mean": 13.73760986328125,
"margin_dpo/margin_std": 14.369284629821777,
"step": 168
},
{
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.0447533056139946,
"fcm_dpo/delta": -0.10509349405765533,
"fcm_dpo/margin": 15.592907905578613,
"fcm_dpo/q_t": 0.3495541214942932,
"grad_norm": 14.77795124053955,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -1.474048137664795,
"logits/rejected": -1.2569692134857178,
"logps/chosen": -89.0520248413086,
"logps/ref_chosen": -82.90370178222656,
"logps/ref_rejected": -90.12191009521484,
"logps/rejected": -111.8631362915039,
"loss": 0.9092,
"margin_dpo/margin_mean": 15.592906951904297,
"margin_dpo/margin_std": 15.848993301391602,
"step": 169
},
{
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.044094495475292206,
"fcm_dpo/delta": -0.01727929711341858,
"fcm_dpo/margin": 13.957527160644531,
"fcm_dpo/q_t": 0.3613913655281067,
"grad_norm": 15.097552299499512,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -1.403516173362732,
"logits/rejected": -1.1804269552230835,
"logps/chosen": -82.96324157714844,
"logps/ref_chosen": -74.98197174072266,
"logps/ref_rejected": -82.16560363769531,
"logps/rejected": -104.10440063476562,
"loss": 0.9393,
"margin_dpo/margin_mean": 13.957527160644531,
"margin_dpo/margin_std": 13.041166305541992,
"step": 170
},
{
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.043724410235881805,
"fcm_dpo/delta": -0.0946701318025589,
"fcm_dpo/margin": 15.739529609680176,
"fcm_dpo/q_t": 0.36089038848876953,
"grad_norm": 16.572017669677734,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -1.3717052936553955,
"logits/rejected": -0.8343698978424072,
"logps/chosen": -128.6714630126953,
"logps/ref_chosen": -118.97853088378906,
"logps/ref_rejected": -115.76930236816406,
"logps/rejected": -141.2017822265625,
"loss": 0.9651,
"margin_dpo/margin_mean": 15.73952865600586,
"margin_dpo/margin_std": 20.676483154296875,
"step": 171
},
{
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.043144889175891876,
"fcm_dpo/delta": -0.026401542127132416,
"fcm_dpo/margin": 14.471285820007324,
"fcm_dpo/q_t": 0.35602670907974243,
"grad_norm": 12.957140922546387,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -1.9741286039352417,
"logits/rejected": -1.2486335039138794,
"logps/chosen": -111.53092956542969,
"logps/ref_chosen": -101.15379333496094,
"logps/ref_rejected": -91.00568389892578,
"logps/rejected": -115.85411071777344,
"loss": 0.9095,
"margin_dpo/margin_mean": 14.471284866333008,
"margin_dpo/margin_std": 10.961095809936523,
"step": 172
},
{
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.04372722655534744,
"fcm_dpo/delta": 0.06630315631628036,
"fcm_dpo/margin": 12.284669876098633,
"fcm_dpo/q_t": 0.37949445843696594,
"grad_norm": 13.340476989746094,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -1.3271675109863281,
"logits/rejected": -0.8779016733169556,
"logps/chosen": -108.86444091796875,
"logps/ref_chosen": -99.05668640136719,
"logps/ref_rejected": -83.8507308959961,
"logps/rejected": -105.94315338134766,
"loss": 0.9963,
"margin_dpo/margin_mean": 12.284669876098633,
"margin_dpo/margin_std": 13.309188842773438,
"step": 173
},
{
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.04317288100719452,
"fcm_dpo/delta": -0.11161762475967407,
"fcm_dpo/margin": 16.303455352783203,
"fcm_dpo/q_t": 0.34579095244407654,
"grad_norm": 15.587017059326172,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -1.7453947067260742,
"logits/rejected": -1.6798264980316162,
"logps/chosen": -89.8681869506836,
"logps/ref_chosen": -80.40890502929688,
"logps/ref_rejected": -108.20171356201172,
"logps/rejected": -133.96444702148438,
"loss": 0.904,
"margin_dpo/margin_mean": 16.303455352783203,
"margin_dpo/margin_std": 16.183778762817383,
"step": 174
},
{
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.04247898980975151,
"fcm_dpo/delta": -0.09163874387741089,
"fcm_dpo/margin": 16.133861541748047,
"fcm_dpo/q_t": 0.3471967577934265,
"grad_norm": 15.230539321899414,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.9117776155471802,
"logits/rejected": -0.6414705514907837,
"logps/chosen": -95.48237609863281,
"logps/ref_chosen": -82.71649169921875,
"logps/ref_rejected": -96.66712951660156,
"logps/rejected": -125.56687927246094,
"loss": 0.8972,
"margin_dpo/margin_mean": 16.133861541748047,
"margin_dpo/margin_std": 14.272943496704102,
"step": 175
},
{
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.041571564972400665,
"fcm_dpo/delta": -0.08201269805431366,
"fcm_dpo/margin": 16.272031784057617,
"fcm_dpo/q_t": 0.3564577102661133,
"grad_norm": 13.990058898925781,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -1.4704281091690063,
"logits/rejected": -1.2904351949691772,
"logps/chosen": -90.04341888427734,
"logps/ref_chosen": -79.62930297851562,
"logps/ref_rejected": -102.6885986328125,
"logps/rejected": -129.374755859375,
"loss": 0.9461,
"margin_dpo/margin_mean": 16.272031784057617,
"margin_dpo/margin_std": 18.543813705444336,
"step": 176
},
{
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.04052150249481201,
"fcm_dpo/delta": -0.1876201629638672,
"fcm_dpo/margin": 19.08730125427246,
"fcm_dpo/q_t": 0.33940553665161133,
"grad_norm": 12.637894630432129,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -1.425307273864746,
"logits/rejected": -1.0769648551940918,
"logps/chosen": -103.93244171142578,
"logps/ref_chosen": -92.56109619140625,
"logps/ref_rejected": -103.33839416503906,
"logps/rejected": -133.79702758789062,
"loss": 0.876,
"margin_dpo/margin_mean": 19.08730125427246,
"margin_dpo/margin_std": 19.590530395507812,
"step": 177
},
{
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.03819538652896881,
"fcm_dpo/delta": -0.28372639417648315,
"fcm_dpo/margin": 22.48978614807129,
"fcm_dpo/q_t": 0.31603994965553284,
"grad_norm": 14.502700805664062,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -1.4522943496704102,
"logits/rejected": -1.1993482112884521,
"logps/chosen": -94.813720703125,
"logps/ref_chosen": -84.26579284667969,
"logps/ref_rejected": -110.79730987548828,
"logps/rejected": -143.83502197265625,
"loss": 0.8098,
"margin_dpo/margin_mean": 22.489788055419922,
"margin_dpo/margin_std": 19.596847534179688,
"step": 178
},
{
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.03770379722118378,
"fcm_dpo/delta": 0.03393785282969475,
"fcm_dpo/margin": 15.058501243591309,
"fcm_dpo/q_t": 0.3792954683303833,
"grad_norm": 14.292525291442871,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -1.4518247842788696,
"logits/rejected": -1.0252907276153564,
"logps/chosen": -107.04389953613281,
"logps/ref_chosen": -93.4479751586914,
"logps/ref_rejected": -94.25541687011719,
"logps/rejected": -122.90983581542969,
"loss": 1.0249,
"margin_dpo/margin_mean": 15.058500289916992,
"margin_dpo/margin_std": 20.123884201049805,
"step": 179
},
{
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.037198685109615326,
"fcm_dpo/delta": -0.1041245386004448,
"fcm_dpo/margin": 18.714359283447266,
"fcm_dpo/q_t": 0.34685850143432617,
"grad_norm": 16.52766227722168,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -1.3025885820388794,
"logits/rejected": -1.0561097860336304,
"logps/chosen": -100.08863830566406,
"logps/ref_chosen": -88.83554077148438,
"logps/ref_rejected": -103.90696716308594,
"logps/rejected": -133.87442016601562,
"loss": 0.894,
"margin_dpo/margin_mean": 18.714359283447266,
"margin_dpo/margin_std": 17.619686126708984,
"step": 180
},
{
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.037504084408283234,
"fcm_dpo/delta": 0.05481020361185074,
"fcm_dpo/margin": 14.614221572875977,
"fcm_dpo/q_t": 0.3806135952472687,
"grad_norm": 13.32461166381836,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -1.3358888626098633,
"logits/rejected": -0.8776803612709045,
"logps/chosen": -90.83370208740234,
"logps/ref_chosen": -79.23396301269531,
"logps/ref_rejected": -69.17442321777344,
"logps/rejected": -95.38838195800781,
"loss": 1.0004,
"margin_dpo/margin_mean": 14.614219665527344,
"margin_dpo/margin_std": 17.222267150878906,
"step": 181
},
{
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.036797843873500824,
"fcm_dpo/delta": -0.10142774879932404,
"fcm_dpo/margin": 18.840681076049805,
"fcm_dpo/q_t": 0.3518170118331909,
"grad_norm": 13.206196784973145,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.9532027244567871,
"logits/rejected": -0.7038797736167908,
"logps/chosen": -90.80711364746094,
"logps/ref_chosen": -76.08218383789062,
"logps/ref_rejected": -94.9686050415039,
"logps/rejected": -128.53421020507812,
"loss": 0.9321,
"margin_dpo/margin_mean": 18.840679168701172,
"margin_dpo/margin_std": 20.858131408691406,
"step": 182
},
{
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.03612806648015976,
"fcm_dpo/delta": -0.09574753046035767,
"fcm_dpo/margin": 19.054080963134766,
"fcm_dpo/q_t": 0.3482775390148163,
"grad_norm": 13.609108924865723,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -1.2038804292678833,
"logits/rejected": -0.5809817314147949,
"logps/chosen": -108.66018676757812,
"logps/ref_chosen": -93.90790557861328,
"logps/ref_rejected": -83.56061553955078,
"logps/rejected": -117.36697387695312,
"loss": 0.9029,
"margin_dpo/margin_mean": 19.054080963134766,
"margin_dpo/margin_std": 17.8356990814209,
"step": 183
},
{
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.03689391911029816,
"fcm_dpo/delta": 0.14321771264076233,
"fcm_dpo/margin": 12.56014633178711,
"fcm_dpo/q_t": 0.3973623514175415,
"grad_norm": 13.919024467468262,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -1.4166245460510254,
"logits/rejected": -1.154975175857544,
"logps/chosen": -100.36515045166016,
"logps/ref_chosen": -86.86790466308594,
"logps/ref_rejected": -84.28456115722656,
"logps/rejected": -110.34194946289062,
"loss": 1.0914,
"margin_dpo/margin_mean": 12.56014633178711,
"margin_dpo/margin_std": 19.446250915527344,
"step": 184
},
{
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.03748057410120964,
"fcm_dpo/delta": 0.027648886665701866,
"fcm_dpo/margin": 15.299049377441406,
"fcm_dpo/q_t": 0.3777276873588562,
"grad_norm": 15.781747817993164,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -1.9094171524047852,
"logits/rejected": -1.3005274534225464,
"logps/chosen": -108.41390991210938,
"logps/ref_chosen": -94.02142333984375,
"logps/ref_rejected": -81.45668029785156,
"logps/rejected": -111.1482162475586,
"loss": 1.0174,
"margin_dpo/margin_mean": 15.299049377441406,
"margin_dpo/margin_std": 20.224950790405273,
"step": 185
},
{
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.03701728954911232,
"fcm_dpo/delta": -0.03318379819393158,
"fcm_dpo/margin": 17.034767150878906,
"fcm_dpo/q_t": 0.3616868853569031,
"grad_norm": 15.530001640319824,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -2.2073938846588135,
"logits/rejected": -1.3490548133850098,
"logps/chosen": -131.6390380859375,
"logps/ref_chosen": -114.19168090820312,
"logps/ref_rejected": -93.33912658691406,
"logps/rejected": -127.82122802734375,
"loss": 0.9536,
"margin_dpo/margin_mean": 17.03476905822754,
"margin_dpo/margin_std": 18.32429313659668,
"step": 186
},
{
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.03734853118658066,
"fcm_dpo/delta": 0.018614832311868668,
"fcm_dpo/margin": 15.587629318237305,
"fcm_dpo/q_t": 0.37575846910476685,
"grad_norm": 13.691910743713379,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -1.4853415489196777,
"logits/rejected": -1.2423460483551025,
"logps/chosen": -92.11011505126953,
"logps/ref_chosen": -76.86955261230469,
"logps/ref_rejected": -88.16494750976562,
"logps/rejected": -118.99314880371094,
"loss": 1.013,
"margin_dpo/margin_mean": 15.587630271911621,
"margin_dpo/margin_std": 20.420238494873047,
"step": 187
},
{
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.0366520918905735,
"fcm_dpo/delta": -0.07102610170841217,
"fcm_dpo/margin": 18.150962829589844,
"fcm_dpo/q_t": 0.35110145807266235,
"grad_norm": 13.728299140930176,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -1.6838798522949219,
"logits/rejected": -1.4296061992645264,
"logps/chosen": -90.44649505615234,
"logps/ref_chosen": -79.1948013305664,
"logps/ref_rejected": -96.82889556884766,
"logps/rejected": -126.23155212402344,
"loss": 0.9272,
"margin_dpo/margin_mean": 18.150962829589844,
"margin_dpo/margin_std": 17.76366424560547,
"step": 188
},
{
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.0365501269698143,
"fcm_dpo/delta": -0.09013816714286804,
"fcm_dpo/margin": 18.695587158203125,
"fcm_dpo/q_t": 0.3573761284351349,
"grad_norm": 12.32153606414795,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -1.5201849937438965,
"logits/rejected": -0.8749784231185913,
"logps/chosen": -99.00448608398438,
"logps/ref_chosen": -83.8927993774414,
"logps/ref_rejected": -79.46589660644531,
"logps/rejected": -113.2731704711914,
"loss": 0.9442,
"margin_dpo/margin_mean": 18.695587158203125,
"margin_dpo/margin_std": 21.5902042388916,
"step": 189
},
{
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.03506568446755409,
"fcm_dpo/delta": -0.180100679397583,
"fcm_dpo/margin": 21.843788146972656,
"fcm_dpo/q_t": 0.331352174282074,
"grad_norm": 13.670723915100098,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -1.5921630859375,
"logits/rejected": -0.9873641133308411,
"logps/chosen": -115.09228515625,
"logps/ref_chosen": -103.44146728515625,
"logps/ref_rejected": -109.21502685546875,
"logps/rejected": -142.70962524414062,
"loss": 0.8555,
"margin_dpo/margin_mean": 21.843786239624023,
"margin_dpo/margin_std": 19.022666931152344,
"step": 190
},
{
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.03367793187499046,
"fcm_dpo/delta": -0.23534753918647766,
"fcm_dpo/margin": 24.244056701660156,
"fcm_dpo/q_t": 0.3260498046875,
"grad_norm": 14.045138359069824,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -1.439011812210083,
"logits/rejected": -1.1144874095916748,
"logps/chosen": -110.62017059326172,
"logps/ref_chosen": -99.78551483154297,
"logps/ref_rejected": -116.00028228759766,
"logps/rejected": -151.0789794921875,
"loss": 0.8445,
"margin_dpo/margin_mean": 24.244056701660156,
"margin_dpo/margin_std": 22.547290802001953,
"step": 191
},
{
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.03333042562007904,
"fcm_dpo/delta": 0.03673925995826721,
"fcm_dpo/margin": 16.96189308166504,
"fcm_dpo/q_t": 0.37764817476272583,
"grad_norm": 15.383365631103516,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -1.293067455291748,
"logits/rejected": -1.030430793762207,
"logps/chosen": -120.11058807373047,
"logps/ref_chosen": -100.86032104492188,
"logps/ref_rejected": -109.44500732421875,
"logps/rejected": -145.6571502685547,
"loss": 1.0054,
"margin_dpo/margin_mean": 16.961894989013672,
"margin_dpo/margin_std": 21.036739349365234,
"step": 192
},
{
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.0341544970870018,
"fcm_dpo/delta": 0.1115766242146492,
"fcm_dpo/margin": 14.43603515625,
"fcm_dpo/q_t": 0.3931388258934021,
"grad_norm": 14.872236251831055,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -1.807457685470581,
"logits/rejected": -1.221724271774292,
"logps/chosen": -108.34063720703125,
"logps/ref_chosen": -92.93831634521484,
"logps/ref_rejected": -83.10917663574219,
"logps/rejected": -112.94754028320312,
"loss": 1.0871,
"margin_dpo/margin_mean": 14.43603515625,
"margin_dpo/margin_std": 22.31793975830078,
"step": 193
},
{
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.0344829261302948,
"fcm_dpo/delta": 0.020925231277942657,
"fcm_dpo/margin": 16.809911727905273,
"fcm_dpo/q_t": 0.3748980760574341,
"grad_norm": 12.2451810836792,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -2.212796211242676,
"logits/rejected": -1.8392068147659302,
"logps/chosen": -79.75379180908203,
"logps/ref_chosen": -63.79157257080078,
"logps/ref_rejected": -68.72665405273438,
"logps/rejected": -101.498779296875,
"loss": 1.0019,
"margin_dpo/margin_mean": 16.809911727905273,
"margin_dpo/margin_std": 20.774627685546875,
"step": 194
},
{
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.03373473137617111,
"fcm_dpo/delta": -0.14405208826065063,
"fcm_dpo/margin": 21.749523162841797,
"fcm_dpo/q_t": 0.3414178490638733,
"grad_norm": 12.763561248779297,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -1.0410107374191284,
"logits/rejected": -0.9035718441009521,
"logps/chosen": -99.03205871582031,
"logps/ref_chosen": -83.88545989990234,
"logps/ref_rejected": -104.00125122070312,
"logps/rejected": -140.89736938476562,
"loss": 0.9105,
"margin_dpo/margin_mean": 21.749523162841797,
"margin_dpo/margin_std": 22.94123649597168,
"step": 195
},
{
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.032888732850551605,
"fcm_dpo/delta": -0.09442660957574844,
"fcm_dpo/margin": 20.915809631347656,
"fcm_dpo/q_t": 0.34680312871932983,
"grad_norm": 11.398536682128906,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -1.0971794128417969,
"logits/rejected": -0.6393330693244934,
"logps/chosen": -106.94159698486328,
"logps/ref_chosen": -89.4013671875,
"logps/ref_rejected": -83.34735107421875,
"logps/rejected": -121.80339050292969,
"loss": 0.9011,
"margin_dpo/margin_mean": 20.915809631347656,
"margin_dpo/margin_std": 19.22466278076172,
"step": 196
},
{
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.03359599411487579,
"fcm_dpo/delta": 0.17620685696601868,
"fcm_dpo/margin": 12.843008041381836,
"fcm_dpo/q_t": 0.40246933698654175,
"grad_norm": 13.048999786376953,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -1.3644063472747803,
"logits/rejected": -1.0448070764541626,
"logps/chosen": -105.15078735351562,
"logps/ref_chosen": -86.84070587158203,
"logps/ref_rejected": -82.79289245605469,
"logps/rejected": -113.94599914550781,
"loss": 1.0851,
"margin_dpo/margin_mean": 12.843008995056152,
"margin_dpo/margin_std": 18.110435485839844,
"step": 197
},
{
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.03354540839791298,
"fcm_dpo/delta": -0.08072011172771454,
"fcm_dpo/margin": 20.13166046142578,
"fcm_dpo/q_t": 0.3533306121826172,
"grad_norm": 15.39021110534668,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -1.7479376792907715,
"logits/rejected": -1.5593535900115967,
"logps/chosen": -106.19425964355469,
"logps/ref_chosen": -85.04824829101562,
"logps/ref_rejected": -104.10365295410156,
"logps/rejected": -145.38133239746094,
"loss": 0.9229,
"margin_dpo/margin_mean": 20.13166046142578,
"margin_dpo/margin_std": 20.80800437927246,
"step": 198
},
{
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.033529043197631836,
"fcm_dpo/delta": 0.020058371126651764,
"fcm_dpo/margin": 17.328767776489258,
"fcm_dpo/q_t": 0.373092383146286,
"grad_norm": 13.264050483703613,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -1.4798874855041504,
"logits/rejected": -1.2172993421554565,
"logps/chosen": -102.33380889892578,
"logps/ref_chosen": -81.78669738769531,
"logps/ref_rejected": -90.790771484375,
"logps/rejected": -128.66665649414062,
"loss": 0.992,
"margin_dpo/margin_mean": 17.328765869140625,
"margin_dpo/margin_std": 20.446475982666016,
"step": 199
},
{
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.03291964530944824,
"fcm_dpo/delta": -0.0771205723285675,
"fcm_dpo/margin": 20.37543296813965,
"fcm_dpo/q_t": 0.35671910643577576,
"grad_norm": 12.404348373413086,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -1.604175090789795,
"logits/rejected": -1.086242437362671,
"logps/chosen": -104.21473693847656,
"logps/ref_chosen": -84.26809692382812,
"logps/ref_rejected": -86.96786499023438,
"logps/rejected": -127.28994750976562,
"loss": 0.9603,
"margin_dpo/margin_mean": 20.37543296813965,
"margin_dpo/margin_std": 23.788284301757812,
"step": 200
},
{
"epoch": 0.2936857562408223,
"eval_fcm_dpo/beta": 0.0329931266605854,
"eval_logits/chosen": -1.439759373664856,
"eval_logits/rejected": -1.1262362003326416,
"eval_logps/chosen": -123.58830261230469,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -132.22947692871094,
"eval_loss": 0.534487247467041,
"eval_margin_dpo/margin_mean": 15.06699275970459,
"eval_margin_dpo/margin_std": 21.761512756347656,
"eval_runtime": 43.741,
"eval_samples_per_second": 53.474,
"eval_steps_per_second": 1.692,
"step": 200
},
{
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.03285796195268631,
"fcm_dpo/delta": -0.0432654544711113,
"fcm_dpo/margin": 19.492534637451172,
"fcm_dpo/q_t": 0.36308830976486206,
"grad_norm": 13.604676246643066,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -1.3040488958358765,
"logits/rejected": -0.9086974263191223,
"logps/chosen": -129.45172119140625,
"logps/ref_chosen": -107.77249145507812,
"logps/ref_rejected": -119.79248046875,
"logps/rejected": -160.9642333984375,
"loss": 0.9856,
"margin_dpo/margin_mean": 19.492534637451172,
"margin_dpo/margin_std": 24.657806396484375,
"step": 201
},
{
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.03282172232866287,
"fcm_dpo/delta": 0.012272818945348263,
"fcm_dpo/margin": 17.926525115966797,
"fcm_dpo/q_t": 0.3727894723415375,
"grad_norm": 13.259603500366211,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -1.6802592277526855,
"logits/rejected": -1.547353744506836,
"logps/chosen": -96.56066131591797,
"logps/ref_chosen": -75.97245788574219,
"logps/ref_rejected": -94.4599838256836,
"logps/rejected": -132.9747314453125,
"loss": 0.9903,
"margin_dpo/margin_mean": 17.926525115966797,
"margin_dpo/margin_std": 21.052555084228516,
"step": 202
},
{
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.032457903027534485,
"fcm_dpo/delta": -0.07600473612546921,
"fcm_dpo/margin": 20.671689987182617,
"fcm_dpo/q_t": 0.3520776033401489,
"grad_norm": 11.885881423950195,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -1.69942307472229,
"logits/rejected": -1.2441532611846924,
"logps/chosen": -112.8557357788086,
"logps/ref_chosen": -94.96715545654297,
"logps/ref_rejected": -92.8876724243164,
"logps/rejected": -131.44793701171875,
"loss": 0.9224,
"margin_dpo/margin_mean": 20.671688079833984,
"margin_dpo/margin_std": 20.679279327392578,
"step": 203
},
{
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.032535798847675323,
"fcm_dpo/delta": 0.02650514431297779,
"fcm_dpo/margin": 17.66338348388672,
"fcm_dpo/q_t": 0.3760603666305542,
"grad_norm": 14.426183700561523,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -1.7683607339859009,
"logits/rejected": -1.129054069519043,
"logps/chosen": -139.35574340820312,
"logps/ref_chosen": -116.35719299316406,
"logps/ref_rejected": -93.39759063720703,
"logps/rejected": -134.05953979492188,
"loss": 1.0127,
"margin_dpo/margin_mean": 17.66338348388672,
"margin_dpo/margin_std": 22.486337661743164,
"step": 204
},
{
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.031746186316013336,
"fcm_dpo/delta": -0.1341342329978943,
"fcm_dpo/margin": 22.80514907836914,
"fcm_dpo/q_t": 0.3369899392127991,
"grad_norm": 11.1696195602417,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -2.104504108428955,
"logits/rejected": -1.677614688873291,
"logps/chosen": -109.97764587402344,
"logps/ref_chosen": -88.85934448242188,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -135.77793884277344,
"loss": 0.8669,
"margin_dpo/margin_mean": 22.80514907836914,
"margin_dpo/margin_std": 18.854307174682617,
"step": 205
},
{
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.031177151948213577,
"fcm_dpo/delta": -0.0601668581366539,
"fcm_dpo/margin": 21.0255069732666,
"fcm_dpo/q_t": 0.3517179489135742,
"grad_norm": 14.36100959777832,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -1.566483497619629,
"logits/rejected": -1.3959765434265137,
"logps/chosen": -98.07730102539062,
"logps/ref_chosen": -76.54634857177734,
"logps/ref_rejected": -102.95314025878906,
"logps/rejected": -145.50961303710938,
"loss": 0.9067,
"margin_dpo/margin_mean": 21.0255069732666,
"margin_dpo/margin_std": 18.027606964111328,
"step": 206
},
{
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.030731763690710068,
"fcm_dpo/delta": -0.11052694916725159,
"fcm_dpo/margin": 22.86851692199707,
"fcm_dpo/q_t": 0.35563924908638,
"grad_norm": 14.994292259216309,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -1.886859655380249,
"logits/rejected": -1.45271635055542,
"logps/chosen": -109.18792724609375,
"logps/ref_chosen": -86.23164367675781,
"logps/ref_rejected": -90.65512084960938,
"logps/rejected": -136.47991943359375,
"loss": 0.9606,
"margin_dpo/margin_mean": 22.86851692199707,
"margin_dpo/margin_std": 28.827228546142578,
"step": 207
},
{
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.03015347383916378,
"fcm_dpo/delta": -0.12842844426631927,
"fcm_dpo/margin": 23.85171890258789,
"fcm_dpo/q_t": 0.3396170735359192,
"grad_norm": 13.180238723754883,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -1.829242467880249,
"logits/rejected": -1.4700978994369507,
"logps/chosen": -117.53231811523438,
"logps/ref_chosen": -94.1595458984375,
"logps/ref_rejected": -100.96233367919922,
"logps/rejected": -148.18682861328125,
"loss": 0.8877,
"margin_dpo/margin_mean": 23.85171890258789,
"margin_dpo/margin_std": 21.651004791259766,
"step": 208
},
{
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.029522810131311417,
"fcm_dpo/delta": -0.018623225390911102,
"fcm_dpo/margin": 20.891002655029297,
"fcm_dpo/q_t": 0.36556458473205566,
"grad_norm": 11.94274616241455,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -1.8460993766784668,
"logits/rejected": -1.3643391132354736,
"logps/chosen": -104.04109954833984,
"logps/ref_chosen": -84.17056274414062,
"logps/ref_rejected": -87.61955261230469,
"logps/rejected": -128.38108825683594,
"loss": 0.9693,
"margin_dpo/margin_mean": 20.891002655029297,
"margin_dpo/margin_std": 23.366682052612305,
"step": 209
},
{
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.029583394527435303,
"fcm_dpo/delta": 0.005795907229185104,
"fcm_dpo/margin": 20.090076446533203,
"fcm_dpo/q_t": 0.37067821621894836,
"grad_norm": 11.102954864501953,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -1.9762842655181885,
"logits/rejected": -1.5929546356201172,
"logps/chosen": -103.7457504272461,
"logps/ref_chosen": -81.17117309570312,
"logps/ref_rejected": -84.17478942871094,
"logps/rejected": -126.83943939208984,
"loss": 0.9765,
"margin_dpo/margin_mean": 20.090076446533203,
"margin_dpo/margin_std": 22.627273559570312,
"step": 210
},
{
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.029787559062242508,
"fcm_dpo/delta": 0.0069667138159275055,
"fcm_dpo/margin": 19.919384002685547,
"fcm_dpo/q_t": 0.37133318185806274,
"grad_norm": 13.3599214553833,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -1.7666494846343994,
"logits/rejected": -1.1323686838150024,
"logps/chosen": -133.63140869140625,
"logps/ref_chosen": -108.71270751953125,
"logps/ref_rejected": -93.55564880371094,
"logps/rejected": -138.39373779296875,
"loss": 0.9858,
"margin_dpo/margin_mean": 19.919384002685547,
"margin_dpo/margin_std": 23.80620574951172,
"step": 211
},
{
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.028646890074014664,
"fcm_dpo/delta": -0.22884619235992432,
"fcm_dpo/margin": 28.252504348754883,
"fcm_dpo/q_t": 0.32535678148269653,
"grad_norm": 11.055304527282715,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -1.8360583782196045,
"logits/rejected": -1.422875165939331,
"logps/chosen": -114.99956512451172,
"logps/ref_chosen": -98.36195373535156,
"logps/ref_rejected": -109.88999938964844,
"logps/rejected": -154.78012084960938,
"loss": 0.8428,
"margin_dpo/margin_mean": 28.252504348754883,
"margin_dpo/margin_std": 25.65581703186035,
"step": 212
},
{
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.028224751353263855,
"fcm_dpo/delta": -0.024367500096559525,
"fcm_dpo/margin": 22.059900283813477,
"fcm_dpo/q_t": 0.3628522753715515,
"grad_norm": 14.968526840209961,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -1.8746771812438965,
"logits/rejected": -1.1175596714019775,
"logps/chosen": -128.48854064941406,
"logps/ref_chosen": -108.05531311035156,
"logps/ref_rejected": -100.14414978027344,
"logps/rejected": -142.63729858398438,
"loss": 0.9689,
"margin_dpo/margin_mean": 22.059898376464844,
"margin_dpo/margin_std": 25.182662963867188,
"step": 213
},
{
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.02846229076385498,
"fcm_dpo/delta": 0.03237991780042648,
"fcm_dpo/margin": 20.002635955810547,
"fcm_dpo/q_t": 0.38030585646629333,
"grad_norm": 11.99617862701416,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -1.8247478008270264,
"logits/rejected": -1.70417320728302,
"logps/chosen": -99.7642593383789,
"logps/ref_chosen": -77.80473327636719,
"logps/ref_rejected": -84.27578735351562,
"logps/rejected": -126.2379379272461,
"loss": 1.0348,
"margin_dpo/margin_mean": 20.002635955810547,
"margin_dpo/margin_std": 28.244205474853516,
"step": 214
},
{
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.027820151299238205,
"fcm_dpo/delta": -0.1480301022529602,
"fcm_dpo/margin": 26.492956161499023,
"fcm_dpo/q_t": 0.3393728733062744,
"grad_norm": 11.24252986907959,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -1.7213172912597656,
"logits/rejected": -1.2412421703338623,
"logps/chosen": -103.63861083984375,
"logps/ref_chosen": -85.1138916015625,
"logps/ref_rejected": -96.86151123046875,
"logps/rejected": -141.87918090820312,
"loss": 0.8754,
"margin_dpo/margin_mean": 26.492958068847656,
"margin_dpo/margin_std": 24.533878326416016,
"step": 215
},
{
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.027603790163993835,
"fcm_dpo/delta": -0.03746962174773216,
"fcm_dpo/margin": 22.996862411499023,
"fcm_dpo/q_t": 0.3604411780834198,
"grad_norm": 10.716496467590332,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -1.9421966075897217,
"logits/rejected": -1.5539422035217285,
"logps/chosen": -101.21246337890625,
"logps/ref_chosen": -80.59230041503906,
"logps/ref_rejected": -81.41983795166016,
"logps/rejected": -125.036865234375,
"loss": 0.9377,
"margin_dpo/margin_mean": 22.996862411499023,
"margin_dpo/margin_std": 23.105133056640625,
"step": 216
},
{
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.027701357379555702,
"fcm_dpo/delta": 0.04847228527069092,
"fcm_dpo/margin": 19.994403839111328,
"fcm_dpo/q_t": 0.38077569007873535,
"grad_norm": 13.545554161071777,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -1.772913932800293,
"logits/rejected": -1.4578328132629395,
"logps/chosen": -117.97203063964844,
"logps/ref_chosen": -93.47257995605469,
"logps/ref_rejected": -103.488525390625,
"logps/rejected": -147.9823760986328,
"loss": 1.0186,
"margin_dpo/margin_mean": 19.994403839111328,
"margin_dpo/margin_std": 25.860273361206055,
"step": 217
},
{
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.02740955352783203,
"fcm_dpo/delta": -0.046444494277238846,
"fcm_dpo/margin": 23.46440887451172,
"fcm_dpo/q_t": 0.36053454875946045,
"grad_norm": 11.716761589050293,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -1.467943787574768,
"logits/rejected": -1.0809111595153809,
"logps/chosen": -112.5135498046875,
"logps/ref_chosen": -89.05883026123047,
"logps/ref_rejected": -94.30680847167969,
"logps/rejected": -141.22592163085938,
"loss": 0.9696,
"margin_dpo/margin_mean": 23.46440887451172,
"margin_dpo/margin_std": 27.73025894165039,
"step": 218
},
{
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.0276676956564188,
"fcm_dpo/delta": 0.009598739445209503,
"fcm_dpo/margin": 21.319698333740234,
"fcm_dpo/q_t": 0.37368935346603394,
"grad_norm": 14.510382652282715,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -1.6932134628295898,
"logits/rejected": -1.259705662727356,
"logps/chosen": -122.24336242675781,
"logps/ref_chosen": -101.00733947753906,
"logps/ref_rejected": -92.46794128417969,
"logps/rejected": -135.02365112304688,
"loss": 1.0173,
"margin_dpo/margin_mean": 21.319698333740234,
"margin_dpo/margin_std": 28.504629135131836,
"step": 219
},
{
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.027722153812646866,
"fcm_dpo/delta": 0.056987300515174866,
"fcm_dpo/margin": 19.699235916137695,
"fcm_dpo/q_t": 0.3782733082771301,
"grad_norm": 12.883475303649902,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -1.2188053131103516,
"logits/rejected": -0.8566290140151978,
"logps/chosen": -117.09075164794922,
"logps/ref_chosen": -95.53721618652344,
"logps/ref_rejected": -94.30703735351562,
"logps/rejected": -135.559814453125,
"loss": 0.9964,
"margin_dpo/margin_mean": 19.699234008789062,
"margin_dpo/margin_std": 21.820940017700195,
"step": 220
},
{
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.02716447226703167,
"fcm_dpo/delta": -0.2148474156856537,
"fcm_dpo/margin": 29.37576675415039,
"fcm_dpo/q_t": 0.3273373246192932,
"grad_norm": 12.420080184936523,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -1.346145749092102,
"logits/rejected": -1.196437120437622,
"logps/chosen": -103.59893035888672,
"logps/ref_chosen": -82.84486389160156,
"logps/ref_rejected": -110.81179809570312,
"logps/rejected": -160.94163513183594,
"loss": 0.8462,
"margin_dpo/margin_mean": 29.375770568847656,
"margin_dpo/margin_std": 25.051647186279297,
"step": 221
},
{
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.026609212160110474,
"fcm_dpo/delta": -0.045089878141880035,
"fcm_dpo/margin": 24.11734962463379,
"fcm_dpo/q_t": 0.36285531520843506,
"grad_norm": 14.774637222290039,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -1.9378188848495483,
"logits/rejected": -1.510035514831543,
"logps/chosen": -121.65461730957031,
"logps/ref_chosen": -95.14198303222656,
"logps/ref_rejected": -106.80441284179688,
"logps/rejected": -157.43441772460938,
"loss": 0.9999,
"margin_dpo/margin_mean": 24.117347717285156,
"margin_dpo/margin_std": 31.577239990234375,
"step": 222
},
{
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.026801597326993942,
"fcm_dpo/delta": 0.11458225548267365,
"fcm_dpo/margin": 18.32587432861328,
"fcm_dpo/q_t": 0.3927801251411438,
"grad_norm": 13.29928970336914,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -2.038062572479248,
"logits/rejected": -1.6366686820983887,
"logps/chosen": -110.8859634399414,
"logps/ref_chosen": -85.57511138916016,
"logps/ref_rejected": -86.45237731933594,
"logps/rejected": -130.089111328125,
"loss": 1.0622,
"margin_dpo/margin_mean": 18.32587432861328,
"margin_dpo/margin_std": 25.809633255004883,
"step": 223
},
{
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.026815548539161682,
"fcm_dpo/delta": -0.049758054316043854,
"fcm_dpo/margin": 24.111448287963867,
"fcm_dpo/q_t": 0.35970795154571533,
"grad_norm": 12.745670318603516,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -1.7115750312805176,
"logits/rejected": -1.2611249685287476,
"logps/chosen": -106.57003784179688,
"logps/ref_chosen": -82.72380065917969,
"logps/ref_rejected": -82.59538269042969,
"logps/rejected": -130.55307006835938,
"loss": 0.9542,
"margin_dpo/margin_mean": 24.111448287963867,
"margin_dpo/margin_std": 27.108388900756836,
"step": 224
},
{
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.026444047689437866,
"fcm_dpo/delta": -0.10222186148166656,
"fcm_dpo/margin": 26.288196563720703,
"fcm_dpo/q_t": 0.35237205028533936,
"grad_norm": 13.087396621704102,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -1.6562142372131348,
"logits/rejected": -1.1619675159454346,
"logps/chosen": -122.09429931640625,
"logps/ref_chosen": -95.67768859863281,
"logps/ref_rejected": -105.09687805175781,
"logps/rejected": -157.80169677734375,
"loss": 0.9433,
"margin_dpo/margin_mean": 26.288192749023438,
"margin_dpo/margin_std": 30.676586151123047,
"step": 225
},
{
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.02590518817305565,
"fcm_dpo/delta": -0.03390205651521683,
"fcm_dpo/margin": 24.35424041748047,
"fcm_dpo/q_t": 0.3604390621185303,
"grad_norm": 10.917281150817871,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -1.681097388267517,
"logits/rejected": -1.1455821990966797,
"logps/chosen": -115.38078308105469,
"logps/ref_chosen": -93.46092224121094,
"logps/ref_rejected": -86.7017593383789,
"logps/rejected": -132.97586059570312,
"loss": 0.9398,
"margin_dpo/margin_mean": 24.35424041748047,
"margin_dpo/margin_std": 24.179988861083984,
"step": 226
},
{
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.02585548348724842,
"fcm_dpo/delta": -0.045401234179735184,
"fcm_dpo/margin": 24.849811553955078,
"fcm_dpo/q_t": 0.362407386302948,
"grad_norm": 12.172209739685059,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -1.2314541339874268,
"logits/rejected": -0.8789317607879639,
"logps/chosen": -123.27052307128906,
"logps/ref_chosen": -92.81427001953125,
"logps/ref_rejected": -104.73693084716797,
"logps/rejected": -160.04299926757812,
"loss": 0.9737,
"margin_dpo/margin_mean": 24.849811553955078,
"margin_dpo/margin_std": 29.725021362304688,
"step": 227
},
{
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.026136575266718864,
"fcm_dpo/delta": 0.09344188123941422,
"fcm_dpo/margin": 19.564260482788086,
"fcm_dpo/q_t": 0.3862323760986328,
"grad_norm": 12.771748542785645,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -1.6004002094268799,
"logits/rejected": -1.1517033576965332,
"logps/chosen": -122.49995422363281,
"logps/ref_chosen": -94.03712463378906,
"logps/ref_rejected": -96.02151489257812,
"logps/rejected": -144.04861450195312,
"loss": 1.0429,
"margin_dpo/margin_mean": 19.56426239013672,
"margin_dpo/margin_std": 25.722578048706055,
"step": 228
},
{
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.025405660271644592,
"fcm_dpo/delta": -0.2284693717956543,
"fcm_dpo/margin": 31.885055541992188,
"fcm_dpo/q_t": 0.3289991319179535,
"grad_norm": 11.33955192565918,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -1.9561712741851807,
"logits/rejected": -1.348691701889038,
"logps/chosen": -120.61212921142578,
"logps/ref_chosen": -95.57766723632812,
"logps/ref_rejected": -89.17379760742188,
"logps/rejected": -146.0933074951172,
"loss": 0.8603,
"margin_dpo/margin_mean": 31.885055541992188,
"margin_dpo/margin_std": 31.453580856323242,
"step": 229
},
{
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.02440287545323372,
"fcm_dpo/delta": -0.1861468255519867,
"fcm_dpo/margin": 31.624526977539062,
"fcm_dpo/q_t": 0.3298434615135193,
"grad_norm": 12.601603507995605,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -1.638891339302063,
"logits/rejected": -1.555408239364624,
"logps/chosen": -90.70736694335938,
"logps/ref_chosen": -63.713626861572266,
"logps/ref_rejected": -91.90870666503906,
"logps/rejected": -150.5269775390625,
"loss": 0.8553,
"margin_dpo/margin_mean": 31.624528884887695,
"margin_dpo/margin_std": 27.797245025634766,
"step": 230
},
{
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.023886041715741158,
"fcm_dpo/delta": -0.04298706725239754,
"fcm_dpo/margin": 26.76896858215332,
"fcm_dpo/q_t": 0.3564389944076538,
"grad_norm": 10.671087265014648,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -1.6697638034820557,
"logits/rejected": -1.1297423839569092,
"logps/chosen": -125.79000091552734,
"logps/ref_chosen": -95.45567321777344,
"logps/ref_rejected": -80.95568084716797,
"logps/rejected": -138.05897521972656,
"loss": 0.9207,
"margin_dpo/margin_mean": 26.76896858215332,
"margin_dpo/margin_std": 24.022785186767578,
"step": 231
},
{
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.02407444454729557,
"fcm_dpo/delta": 0.04270622134208679,
"fcm_dpo/margin": 23.248355865478516,
"fcm_dpo/q_t": 0.37612390518188477,
"grad_norm": 11.53981876373291,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -1.4995675086975098,
"logits/rejected": -1.0280941724777222,
"logps/chosen": -128.28302001953125,
"logps/ref_chosen": -96.89726257324219,
"logps/ref_rejected": -89.76461791992188,
"logps/rejected": -144.39874267578125,
"loss": 0.9973,
"margin_dpo/margin_mean": 23.248355865478516,
"margin_dpo/margin_std": 26.718120574951172,
"step": 232
},
{
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.02357470616698265,
"fcm_dpo/delta": -0.1532088667154312,
"fcm_dpo/margin": 31.44921112060547,
"fcm_dpo/q_t": 0.3387308120727539,
"grad_norm": 11.674040794372559,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -1.721343994140625,
"logits/rejected": -1.4938092231750488,
"logps/chosen": -122.76866149902344,
"logps/ref_chosen": -89.05032348632812,
"logps/ref_rejected": -112.75917053222656,
"logps/rejected": -177.92672729492188,
"loss": 0.8881,
"margin_dpo/margin_mean": 31.44921112060547,
"margin_dpo/margin_std": 30.564104080200195,
"step": 233
},
{
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.022600866854190826,
"fcm_dpo/delta": -0.17494013905525208,
"fcm_dpo/margin": 33.52787780761719,
"fcm_dpo/q_t": 0.3380278944969177,
"grad_norm": 11.083850860595703,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -1.2067891359329224,
"logits/rejected": -1.1254383325576782,
"logps/chosen": -106.61786651611328,
"logps/ref_chosen": -74.318359375,
"logps/ref_rejected": -106.38758850097656,
"logps/rejected": -172.2149658203125,
"loss": 0.8916,
"margin_dpo/margin_mean": 33.52787780761719,
"margin_dpo/margin_std": 33.51675796508789,
"step": 234
},
{
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.022734005004167557,
"fcm_dpo/delta": 0.01751571148633957,
"fcm_dpo/margin": 25.663055419921875,
"fcm_dpo/q_t": 0.36923179030418396,
"grad_norm": 10.767046928405762,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -1.3730831146240234,
"logits/rejected": -0.8100583553314209,
"logps/chosen": -136.84141540527344,
"logps/ref_chosen": -98.217041015625,
"logps/ref_rejected": -97.24677276611328,
"logps/rejected": -161.53419494628906,
"loss": 0.9624,
"margin_dpo/margin_mean": 25.663055419921875,
"margin_dpo/margin_std": 25.689247131347656,
"step": 235
},
{
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.022359684109687805,
"fcm_dpo/delta": -0.12736767530441284,
"fcm_dpo/margin": 32.12920379638672,
"fcm_dpo/q_t": 0.35286009311676025,
"grad_norm": 11.318601608276367,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -1.2687420845031738,
"logits/rejected": -1.2513408660888672,
"logps/chosen": -111.73262023925781,
"logps/ref_chosen": -78.83773040771484,
"logps/ref_rejected": -109.06343078613281,
"logps/rejected": -174.0875244140625,
"loss": 0.924,
"margin_dpo/margin_mean": 32.12920379638672,
"margin_dpo/margin_std": 37.443138122558594,
"step": 236
},
{
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.02228570729494095,
"fcm_dpo/delta": 0.0468742772936821,
"fcm_dpo/margin": 24.937557220458984,
"fcm_dpo/q_t": 0.3862718939781189,
"grad_norm": 11.445574760437012,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -1.4098989963531494,
"logits/rejected": -1.0293853282928467,
"logps/chosen": -123.49139404296875,
"logps/ref_chosen": -85.21128845214844,
"logps/ref_rejected": -99.90999603271484,
"logps/rejected": -163.12767028808594,
"loss": 1.065,
"margin_dpo/margin_mean": 24.937557220458984,
"margin_dpo/margin_std": 39.00885009765625,
"step": 237
},
{
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.022245164960622787,
"fcm_dpo/delta": -0.023596640676259995,
"fcm_dpo/margin": 27.963947296142578,
"fcm_dpo/q_t": 0.36633193492889404,
"grad_norm": 10.700308799743652,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -1.0491883754730225,
"logits/rejected": -0.8851519227027893,
"logps/chosen": -120.75160217285156,
"logps/ref_chosen": -78.73123168945312,
"logps/ref_rejected": -95.41840362548828,
"logps/rejected": -165.4027099609375,
"loss": 0.9704,
"margin_dpo/margin_mean": 27.963947296142578,
"margin_dpo/margin_std": 32.5536003112793,
"step": 238
},
{
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.022143233567476273,
"fcm_dpo/delta": 0.018351107835769653,
"fcm_dpo/margin": 26.272315979003906,
"fcm_dpo/q_t": 0.37351781129837036,
"grad_norm": 11.168644905090332,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -1.7788722515106201,
"logits/rejected": -1.3021533489227295,
"logps/chosen": -131.14578247070312,
"logps/ref_chosen": -92.60093688964844,
"logps/ref_rejected": -100.51769256591797,
"logps/rejected": -165.3348388671875,
"loss": 0.9967,
"margin_dpo/margin_mean": 26.272315979003906,
"margin_dpo/margin_std": 30.892166137695312,
"step": 239
},
{
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.022500187158584595,
"fcm_dpo/delta": 0.07234937697649002,
"fcm_dpo/margin": 23.62274932861328,
"fcm_dpo/q_t": 0.3824573755264282,
"grad_norm": 12.867546081542969,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -1.9507150650024414,
"logits/rejected": -1.6044831275939941,
"logps/chosen": -125.81748962402344,
"logps/ref_chosen": -86.15977478027344,
"logps/ref_rejected": -80.45567321777344,
"logps/rejected": -143.7361297607422,
"loss": 1.0213,
"margin_dpo/margin_mean": 23.62274932861328,
"margin_dpo/margin_std": 28.90222930908203,
"step": 240
},
{
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0217452310025692,
"fcm_dpo/delta": -0.3266553282737732,
"fcm_dpo/margin": 41.317291259765625,
"fcm_dpo/q_t": 0.30971965193748474,
"grad_norm": 10.297433853149414,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -1.5102795362472534,
"logits/rejected": -0.8343143463134766,
"logps/chosen": -136.01959228515625,
"logps/ref_chosen": -101.04594421386719,
"logps/ref_rejected": -94.04934692382812,
"logps/rejected": -170.34030151367188,
"loss": 0.8042,
"margin_dpo/margin_mean": 41.317291259765625,
"margin_dpo/margin_std": 36.07255554199219,
"step": 241
},
{
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.021258074790239334,
"fcm_dpo/delta": -0.021016836166381836,
"fcm_dpo/margin": 29.1357479095459,
"fcm_dpo/q_t": 0.36231163144111633,
"grad_norm": 11.23208999633789,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -1.8308205604553223,
"logits/rejected": -1.1828442811965942,
"logps/chosen": -138.07565307617188,
"logps/ref_chosen": -95.67266845703125,
"logps/ref_rejected": -90.65454864501953,
"logps/rejected": -162.19329833984375,
"loss": 0.9561,
"margin_dpo/margin_mean": 29.135746002197266,
"margin_dpo/margin_std": 30.477272033691406,
"step": 242
},
{
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.021303560584783554,
"fcm_dpo/delta": -0.015803933143615723,
"fcm_dpo/margin": 28.77163314819336,
"fcm_dpo/q_t": 0.36690443754196167,
"grad_norm": 11.050640106201172,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -1.8111510276794434,
"logits/rejected": -1.0000836849212646,
"logps/chosen": -139.14649963378906,
"logps/ref_chosen": -98.03140258789062,
"logps/ref_rejected": -83.18806457519531,
"logps/rejected": -153.07479858398438,
"loss": 0.9904,
"margin_dpo/margin_mean": 28.77163314819336,
"margin_dpo/margin_std": 34.02631378173828,
"step": 243
},
{
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.020892877131700516,
"fcm_dpo/delta": -0.03988625854253769,
"fcm_dpo/margin": 30.495805740356445,
"fcm_dpo/q_t": 0.36164575815200806,
"grad_norm": 11.020362854003906,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -2.001028060913086,
"logits/rejected": -1.656126856803894,
"logps/chosen": -131.77944946289062,
"logps/ref_chosen": -88.8939208984375,
"logps/ref_rejected": -102.57278442382812,
"logps/rejected": -175.95413208007812,
"loss": 0.9556,
"margin_dpo/margin_mean": 30.495803833007812,
"margin_dpo/margin_std": 33.84397888183594,
"step": 244
},
{
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.02063831314444542,
"fcm_dpo/delta": -0.0902470201253891,
"fcm_dpo/margin": 33.148372650146484,
"fcm_dpo/q_t": 0.34877562522888184,
"grad_norm": 10.457184791564941,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -2.292057514190674,
"logits/rejected": -1.4950196743011475,
"logps/chosen": -140.51174926757812,
"logps/ref_chosen": -104.19400024414062,
"logps/ref_rejected": -92.65645599365234,
"logps/rejected": -162.12255859375,
"loss": 0.9075,
"margin_dpo/margin_mean": 33.14836883544922,
"margin_dpo/margin_std": 31.503116607666016,
"step": 245
},
{
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.020679624751210213,
"fcm_dpo/delta": 0.07523977756500244,
"fcm_dpo/margin": 25.567752838134766,
"fcm_dpo/q_t": 0.3833276629447937,
"grad_norm": 11.005114555358887,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -1.9735589027404785,
"logits/rejected": -1.4642459154129028,
"logps/chosen": -149.89358520507812,
"logps/ref_chosen": -103.21904754638672,
"logps/ref_rejected": -90.99221801757812,
"logps/rejected": -163.2344970703125,
"loss": 1.021,
"margin_dpo/margin_mean": 25.567752838134766,
"margin_dpo/margin_std": 31.323787689208984,
"step": 246
},
{
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.020045291632413864,
"fcm_dpo/delta": -0.22309784591197968,
"fcm_dpo/margin": 40.07710647583008,
"fcm_dpo/q_t": 0.328274130821228,
"grad_norm": 10.08717155456543,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -2.199232578277588,
"logits/rejected": -1.9928696155548096,
"logps/chosen": -129.20089721679688,
"logps/ref_chosen": -86.99437713623047,
"logps/ref_rejected": -111.33802795410156,
"logps/rejected": -193.62164306640625,
"loss": 0.8532,
"margin_dpo/margin_mean": 40.07710647583008,
"margin_dpo/margin_std": 37.57669448852539,
"step": 247
},
{
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.020135609433054924,
"fcm_dpo/delta": 0.0492338202893734,
"fcm_dpo/margin": 27.472003936767578,
"fcm_dpo/q_t": 0.3776289224624634,
"grad_norm": 11.691912651062012,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -1.5299792289733887,
"logits/rejected": -1.482997179031372,
"logps/chosen": -123.07820129394531,
"logps/ref_chosen": -74.7855224609375,
"logps/ref_rejected": -98.27689361572266,
"logps/rejected": -174.0415802001953,
"loss": 0.9963,
"margin_dpo/margin_mean": 27.472003936767578,
"margin_dpo/margin_std": 31.208698272705078,
"step": 248
},
{
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.02031414583325386,
"fcm_dpo/delta": 0.08933912217617035,
"fcm_dpo/margin": 25.36626434326172,
"fcm_dpo/q_t": 0.38820499181747437,
"grad_norm": 12.994771957397461,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -2.2492823600769043,
"logits/rejected": -1.739271640777588,
"logps/chosen": -148.12538146972656,
"logps/ref_chosen": -95.70379638671875,
"logps/ref_rejected": -87.14646911621094,
"logps/rejected": -164.934326171875,
"loss": 1.0546,
"margin_dpo/margin_mean": 25.36626434326172,
"margin_dpo/margin_std": 35.47053527832031,
"step": 249
},
{
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.020489081740379333,
"fcm_dpo/delta": 0.04790526628494263,
"fcm_dpo/margin": 27.043155670166016,
"fcm_dpo/q_t": 0.37720349431037903,
"grad_norm": 13.011922836303711,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -1.7810206413269043,
"logits/rejected": -0.8627009391784668,
"logps/chosen": -161.8379364013672,
"logps/ref_chosen": -114.05221557617188,
"logps/ref_rejected": -81.08767700195312,
"logps/rejected": -155.91656494140625,
"loss": 1.0209,
"margin_dpo/margin_mean": 27.043155670166016,
"margin_dpo/margin_std": 33.19232940673828,
"step": 250
},
{
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.020762145519256592,
"fcm_dpo/delta": 0.07583161443471909,
"fcm_dpo/margin": 25.410945892333984,
"fcm_dpo/q_t": 0.3874654173851013,
"grad_norm": 12.704300880432129,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -1.9245502948760986,
"logits/rejected": -1.244246006011963,
"logps/chosen": -144.52914428710938,
"logps/ref_chosen": -97.71128845214844,
"logps/ref_rejected": -83.52742004394531,
"logps/rejected": -155.7562255859375,
"loss": 1.0538,
"margin_dpo/margin_mean": 25.410945892333984,
"margin_dpo/margin_std": 35.388427734375,
"step": 251
},
{
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.02117466926574707,
"fcm_dpo/delta": 0.05208258330821991,
"fcm_dpo/margin": 26.01020050048828,
"fcm_dpo/q_t": 0.37829625606536865,
"grad_norm": 13.06750774383545,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -1.8967103958129883,
"logits/rejected": -1.3722825050354004,
"logps/chosen": -155.25262451171875,
"logps/ref_chosen": -108.9861068725586,
"logps/ref_rejected": -91.56423950195312,
"logps/rejected": -163.84097290039062,
"loss": 1.0169,
"margin_dpo/margin_mean": 26.01020050048828,
"margin_dpo/margin_std": 32.340423583984375,
"step": 252
},
{
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.021657634526491165,
"fcm_dpo/delta": 0.07324262708425522,
"fcm_dpo/margin": 24.389892578125,
"fcm_dpo/q_t": 0.3817564845085144,
"grad_norm": 12.639901161193848,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -2.0871338844299316,
"logits/rejected": -1.880314588546753,
"logps/chosen": -150.90634155273438,
"logps/ref_chosen": -100.21630096435547,
"logps/ref_rejected": -105.67670440673828,
"logps/rejected": -180.7566375732422,
"loss": 1.0268,
"margin_dpo/margin_mean": 24.389892578125,
"margin_dpo/margin_std": 29.48230743408203,
"step": 253
},
{
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.021658308804035187,
"fcm_dpo/delta": 0.0799470990896225,
"fcm_dpo/margin": 24.19228744506836,
"fcm_dpo/q_t": 0.38415348529815674,
"grad_norm": 12.239904403686523,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -1.8840199708938599,
"logits/rejected": -1.340732455253601,
"logps/chosen": -148.34524536132812,
"logps/ref_chosen": -98.73518371582031,
"logps/ref_rejected": -93.73825073242188,
"logps/rejected": -167.5406036376953,
"loss": 1.0444,
"margin_dpo/margin_mean": 24.19228744506836,
"margin_dpo/margin_std": 32.14631652832031,
"step": 254
},
{
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.021404996514320374,
"fcm_dpo/delta": -0.153507798910141,
"fcm_dpo/margin": 34.67428207397461,
"fcm_dpo/q_t": 0.33902448415756226,
"grad_norm": 11.967893600463867,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -1.910501480102539,
"logits/rejected": -1.5700030326843262,
"logps/chosen": -141.0276641845703,
"logps/ref_chosen": -90.382568359375,
"logps/ref_rejected": -97.07624816894531,
"logps/rejected": -182.3956298828125,
"loss": 0.8959,
"margin_dpo/margin_mean": 34.674278259277344,
"margin_dpo/margin_std": 34.8538818359375,
"step": 255
},
{
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.02097788080573082,
"fcm_dpo/delta": -0.10094420611858368,
"fcm_dpo/margin": 33.08401107788086,
"fcm_dpo/q_t": 0.3532879948616028,
"grad_norm": 12.47603702545166,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -1.874246597290039,
"logits/rejected": -1.5234124660491943,
"logps/chosen": -136.12049865722656,
"logps/ref_chosen": -88.7528076171875,
"logps/ref_rejected": -98.49383544921875,
"logps/rejected": -178.94552612304688,
"loss": 0.9704,
"margin_dpo/margin_mean": 33.08401107788086,
"margin_dpo/margin_std": 41.36225891113281,
"step": 256
},
{
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.020358510315418243,
"fcm_dpo/delta": -0.06614407151937485,
"fcm_dpo/margin": 32.34702682495117,
"fcm_dpo/q_t": 0.35797828435897827,
"grad_norm": 11.72060775756836,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -1.9517452716827393,
"logits/rejected": -1.572911262512207,
"logps/chosen": -132.3583984375,
"logps/ref_chosen": -82.80352020263672,
"logps/ref_rejected": -85.8677978515625,
"logps/rejected": -167.7696990966797,
"loss": 0.9599,
"margin_dpo/margin_mean": 32.34702682495117,
"margin_dpo/margin_std": 36.041168212890625,
"step": 257
},
{
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.02057628706097603,
"fcm_dpo/delta": -0.028431270271539688,
"fcm_dpo/margin": 30.42020034790039,
"fcm_dpo/q_t": 0.36126869916915894,
"grad_norm": 12.226306915283203,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -2.099985122680664,
"logits/rejected": -1.7437987327575684,
"logps/chosen": -140.68870544433594,
"logps/ref_chosen": -90.187744140625,
"logps/ref_rejected": -103.47068786621094,
"logps/rejected": -184.39186096191406,
"loss": 0.9628,
"margin_dpo/margin_mean": 30.42020034790039,
"margin_dpo/margin_std": 32.89296340942383,
"step": 258
},
{
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.020515816286206245,
"fcm_dpo/delta": 0.08078932762145996,
"fcm_dpo/margin": 25.49618148803711,
"fcm_dpo/q_t": 0.3858429789543152,
"grad_norm": 12.687917709350586,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -1.4180266857147217,
"logits/rejected": -1.1736819744110107,
"logps/chosen": -146.1452178955078,
"logps/ref_chosen": -90.77254486083984,
"logps/ref_rejected": -94.58816528320312,
"logps/rejected": -175.45701599121094,
"loss": 1.0682,
"margin_dpo/margin_mean": 25.49618148803711,
"margin_dpo/margin_std": 38.16563415527344,
"step": 259
},
{
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.021066930145025253,
"fcm_dpo/delta": 0.11459525674581528,
"fcm_dpo/margin": 23.313533782958984,
"fcm_dpo/q_t": 0.39374488592147827,
"grad_norm": 14.11849594116211,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -1.6266734600067139,
"logits/rejected": -1.6325747966766357,
"logps/chosen": -131.62429809570312,
"logps/ref_chosen": -75.59269714355469,
"logps/ref_rejected": -100.84554290771484,
"logps/rejected": -180.190673828125,
"loss": 1.113,
"margin_dpo/margin_mean": 23.313533782958984,
"margin_dpo/margin_std": 39.60096740722656,
"step": 260
},
{
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.02059752494096756,
"fcm_dpo/delta": -0.1640065312385559,
"fcm_dpo/margin": 36.42674255371094,
"fcm_dpo/q_t": 0.33720719814300537,
"grad_norm": 11.14237117767334,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -1.8282723426818848,
"logits/rejected": -1.453067421913147,
"logps/chosen": -138.38714599609375,
"logps/ref_chosen": -94.25491333007812,
"logps/ref_rejected": -93.65699005126953,
"logps/rejected": -174.21595764160156,
"loss": 0.8778,
"margin_dpo/margin_mean": 36.42674255371094,
"margin_dpo/margin_std": 34.84246826171875,
"step": 261
},
{
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.020204119384288788,
"fcm_dpo/delta": -0.09102602303028107,
"fcm_dpo/margin": 33.87533950805664,
"fcm_dpo/q_t": 0.3513264060020447,
"grad_norm": 11.49547290802002,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -1.7930582761764526,
"logits/rejected": -1.2549785375595093,
"logps/chosen": -133.25894165039062,
"logps/ref_chosen": -85.26730346679688,
"logps/ref_rejected": -90.82609558105469,
"logps/rejected": -172.69308471679688,
"loss": 0.9266,
"margin_dpo/margin_mean": 33.87533950805664,
"margin_dpo/margin_std": 35.84063720703125,
"step": 262
},
{
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.019306186586618423,
"fcm_dpo/delta": -0.29907822608947754,
"fcm_dpo/margin": 45.224769592285156,
"fcm_dpo/q_t": 0.3136481046676636,
"grad_norm": 10.701422691345215,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -1.9696038961410522,
"logits/rejected": -1.4115229845046997,
"logps/chosen": -134.94964599609375,
"logps/ref_chosen": -95.75790405273438,
"logps/ref_rejected": -101.83377075195312,
"logps/rejected": -186.2502899169922,
"loss": 0.83,
"margin_dpo/margin_mean": 45.224769592285156,
"margin_dpo/margin_std": 42.148963928222656,
"step": 263
},
{
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.01862514019012451,
"fcm_dpo/delta": -0.11325480043888092,
"fcm_dpo/margin": 37.84281921386719,
"fcm_dpo/q_t": 0.34645789861679077,
"grad_norm": 10.080230712890625,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -2.0475120544433594,
"logits/rejected": -1.7664682865142822,
"logps/chosen": -134.43109130859375,
"logps/ref_chosen": -90.6226577758789,
"logps/ref_rejected": -100.32554626464844,
"logps/rejected": -181.976806640625,
"loss": 0.9163,
"margin_dpo/margin_mean": 37.84281921386719,
"margin_dpo/margin_std": 38.84701919555664,
"step": 264
},
{
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.018290959298610687,
"fcm_dpo/delta": -0.06356701254844666,
"fcm_dpo/margin": 35.99824523925781,
"fcm_dpo/q_t": 0.3593207895755768,
"grad_norm": 10.072009086608887,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -1.925950050354004,
"logits/rejected": -1.5249955654144287,
"logps/chosen": -138.44578552246094,
"logps/ref_chosen": -92.37232971191406,
"logps/ref_rejected": -94.62757873535156,
"logps/rejected": -176.69927978515625,
"loss": 0.9603,
"margin_dpo/margin_mean": 35.99824905395508,
"margin_dpo/margin_std": 41.63231658935547,
"step": 265
},
{
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.018374420702457428,
"fcm_dpo/delta": 0.031407758593559265,
"fcm_dpo/margin": 31.039928436279297,
"fcm_dpo/q_t": 0.3749583959579468,
"grad_norm": 12.884138107299805,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -2.2668302059173584,
"logits/rejected": -1.4866418838500977,
"logps/chosen": -155.75039672851562,
"logps/ref_chosen": -102.20002746582031,
"logps/ref_rejected": -74.36642456054688,
"logps/rejected": -158.9567108154297,
"loss": 1.002,
"margin_dpo/margin_mean": 31.039928436279297,
"margin_dpo/margin_std": 37.078948974609375,
"step": 266
},
{
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.018142802640795708,
"fcm_dpo/delta": -0.07580061256885529,
"fcm_dpo/margin": 36.915672302246094,
"fcm_dpo/q_t": 0.35151273012161255,
"grad_norm": 10.919096946716309,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -1.7907602787017822,
"logits/rejected": -1.1931796073913574,
"logps/chosen": -150.03753662109375,
"logps/ref_chosen": -104.28599548339844,
"logps/ref_rejected": -95.98719024658203,
"logps/rejected": -178.6544189453125,
"loss": 0.9123,
"margin_dpo/margin_mean": 36.915672302246094,
"margin_dpo/margin_std": 34.50056076049805,
"step": 267
},
{
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.017767982557415962,
"fcm_dpo/delta": -0.12636685371398926,
"fcm_dpo/margin": 40.33136749267578,
"fcm_dpo/q_t": 0.34127211570739746,
"grad_norm": 10.990910530090332,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -1.6720902919769287,
"logits/rejected": -1.495853304862976,
"logps/chosen": -132.07077026367188,
"logps/ref_chosen": -86.69622039794922,
"logps/ref_rejected": -109.19184112548828,
"logps/rejected": -194.89776611328125,
"loss": 0.8877,
"margin_dpo/margin_mean": 40.33136749267578,
"margin_dpo/margin_std": 36.7901611328125,
"step": 268
},
{
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.017438966780900955,
"fcm_dpo/delta": -0.1256072223186493,
"fcm_dpo/margin": 41.10211944580078,
"fcm_dpo/q_t": 0.3434303104877472,
"grad_norm": 10.632892608642578,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -1.8700554370880127,
"logits/rejected": -1.408653974533081,
"logps/chosen": -152.26226806640625,
"logps/ref_chosen": -104.97181701660156,
"logps/ref_rejected": -112.47647094726562,
"logps/rejected": -200.86904907226562,
"loss": 0.8995,
"margin_dpo/margin_mean": 41.10211944580078,
"margin_dpo/margin_std": 40.0078125,
"step": 269
},
{
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.017290018498897552,
"fcm_dpo/delta": -0.01898367889225483,
"fcm_dpo/margin": 35.722171783447266,
"fcm_dpo/q_t": 0.3673768937587738,
"grad_norm": 10.380319595336914,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -2.5234313011169434,
"logits/rejected": -2.1158437728881836,
"logps/chosen": -153.2642822265625,
"logps/ref_chosen": -101.07383728027344,
"logps/ref_rejected": -117.75289154052734,
"logps/rejected": -205.66552734375,
"loss": 1.0017,
"margin_dpo/margin_mean": 35.722171783447266,
"margin_dpo/margin_std": 45.860267639160156,
"step": 270
},
{
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.0168063472956419,
"fcm_dpo/delta": -0.1483364999294281,
"fcm_dpo/margin": 43.856292724609375,
"fcm_dpo/q_t": 0.3364192843437195,
"grad_norm": 10.36974811553955,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -1.4026761054992676,
"logits/rejected": -0.9475973844528198,
"logps/chosen": -140.18299865722656,
"logps/ref_chosen": -96.72459411621094,
"logps/ref_rejected": -98.5244140625,
"logps/rejected": -185.839111328125,
"loss": 0.9012,
"margin_dpo/margin_mean": 43.856292724609375,
"margin_dpo/margin_std": 43.59748840332031,
"step": 271
},
{
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.016488002613186836,
"fcm_dpo/delta": -0.08753229677677155,
"fcm_dpo/margin": 41.335079193115234,
"fcm_dpo/q_t": 0.35309892892837524,
"grad_norm": 10.801867485046387,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -1.7714588642120361,
"logits/rejected": -1.6390788555145264,
"logps/chosen": -117.77945709228516,
"logps/ref_chosen": -80.97721862792969,
"logps/ref_rejected": -108.55535888671875,
"logps/rejected": -186.69268798828125,
"loss": 0.9354,
"margin_dpo/margin_mean": 41.3350830078125,
"margin_dpo/margin_std": 44.84138107299805,
"step": 272
},
{
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.016001848503947258,
"fcm_dpo/delta": -0.10035118460655212,
"fcm_dpo/margin": 43.12675857543945,
"fcm_dpo/q_t": 0.350558340549469,
"grad_norm": 9.880205154418945,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -1.484649419784546,
"logits/rejected": -1.0760293006896973,
"logps/chosen": -142.65541076660156,
"logps/ref_chosen": -92.22460174560547,
"logps/ref_rejected": -97.3630599975586,
"logps/rejected": -190.9206085205078,
"loss": 0.9532,
"margin_dpo/margin_mean": 43.12675857543945,
"margin_dpo/margin_std": 49.11865234375,
"step": 273
},
{
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.016166478395462036,
"fcm_dpo/delta": 0.05010783299803734,
"fcm_dpo/margin": 34.18390655517578,
"fcm_dpo/q_t": 0.37679600715637207,
"grad_norm": 12.417606353759766,
"learning_rate": 3.75e-07,
"logits/chosen": -1.9811569452285767,
"logits/rejected": -1.5822125673294067,
"logps/chosen": -133.99008178710938,
"logps/ref_chosen": -82.26089477539062,
"logps/ref_rejected": -83.87699890136719,
"logps/rejected": -169.79010009765625,
"loss": 0.9924,
"margin_dpo/margin_mean": 34.18390655517578,
"margin_dpo/margin_std": 37.70836639404297,
"step": 274
},
{
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.016330569982528687,
"fcm_dpo/delta": -0.013110160827636719,
"fcm_dpo/margin": 37.44886016845703,
"fcm_dpo/q_t": 0.3667539358139038,
"grad_norm": 11.84593391418457,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -2.191549301147461,
"logits/rejected": -1.8682620525360107,
"logps/chosen": -123.36338806152344,
"logps/ref_chosen": -79.68696594238281,
"logps/ref_rejected": -98.7509765625,
"logps/rejected": -179.87625122070312,
"loss": 1.0023,
"margin_dpo/margin_mean": 37.44886016845703,
"margin_dpo/margin_std": 47.065589904785156,
"step": 275
},
{
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.016055840998888016,
"fcm_dpo/delta": -0.008165836334228516,
"fcm_dpo/margin": 37.754920959472656,
"fcm_dpo/q_t": 0.3685857951641083,
"grad_norm": 12.060040473937988,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -2.0615105628967285,
"logits/rejected": -1.5026049613952637,
"logps/chosen": -140.79737854003906,
"logps/ref_chosen": -86.53970336914062,
"logps/ref_rejected": -77.85394287109375,
"logps/rejected": -169.86654663085938,
"loss": 0.9909,
"margin_dpo/margin_mean": 37.754920959472656,
"margin_dpo/margin_std": 45.30341339111328,
"step": 276
},
{
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.016346724703907967,
"fcm_dpo/delta": 0.08950374275445938,
"fcm_dpo/margin": 31.514963150024414,
"fcm_dpo/q_t": 0.38510024547576904,
"grad_norm": 13.390870094299316,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -2.061767101287842,
"logits/rejected": -1.646317481994629,
"logps/chosen": -141.05987548828125,
"logps/ref_chosen": -84.24411010742188,
"logps/ref_rejected": -98.83421325683594,
"logps/rejected": -187.1649169921875,
"loss": 1.0305,
"margin_dpo/margin_mean": 31.514963150024414,
"margin_dpo/margin_std": 38.913230895996094,
"step": 277
},
{
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0163891538977623,
"fcm_dpo/delta": -0.006686456501483917,
"fcm_dpo/margin": 36.96751403808594,
"fcm_dpo/q_t": 0.36568570137023926,
"grad_norm": 12.524548530578613,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -2.1564512252807617,
"logits/rejected": -1.948979139328003,
"logps/chosen": -136.97509765625,
"logps/ref_chosen": -82.431884765625,
"logps/ref_rejected": -97.85691833496094,
"logps/rejected": -189.3676300048828,
"loss": 0.9865,
"margin_dpo/margin_mean": 36.9675178527832,
"margin_dpo/margin_std": 43.24678039550781,
"step": 278
},
{
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.01617228053510189,
"fcm_dpo/delta": -0.08998230844736099,
"fcm_dpo/margin": 42.24311447143555,
"fcm_dpo/q_t": 0.35100412368774414,
"grad_norm": 11.406697273254395,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -1.8248711824417114,
"logits/rejected": -1.607770562171936,
"logps/chosen": -143.13755798339844,
"logps/ref_chosen": -85.16799926757812,
"logps/ref_rejected": -94.12664794921875,
"logps/rejected": -194.33932495117188,
"loss": 0.9281,
"margin_dpo/margin_mean": 42.24311828613281,
"margin_dpo/margin_std": 44.3424072265625,
"step": 279
},
{
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.01589279994368553,
"fcm_dpo/delta": -0.12768404185771942,
"fcm_dpo/margin": 45.2197265625,
"fcm_dpo/q_t": 0.34247255325317383,
"grad_norm": 11.279327392578125,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -1.7638534307479858,
"logits/rejected": -1.555558204650879,
"logps/chosen": -143.5164337158203,
"logps/ref_chosen": -85.85641479492188,
"logps/ref_rejected": -104.11859130859375,
"logps/rejected": -206.9983367919922,
"loss": 0.8992,
"margin_dpo/margin_mean": 45.2197265625,
"margin_dpo/margin_std": 43.852996826171875,
"step": 280
},
{
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.015449896454811096,
"fcm_dpo/delta": -0.1223934069275856,
"fcm_dpo/margin": 46.17839050292969,
"fcm_dpo/q_t": 0.3489765524864197,
"grad_norm": 9.723531723022461,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -2.1463782787323,
"logits/rejected": -1.422343373298645,
"logps/chosen": -147.69259643554688,
"logps/ref_chosen": -86.55082702636719,
"logps/ref_rejected": -88.62866973876953,
"logps/rejected": -195.94882202148438,
"loss": 0.9317,
"margin_dpo/margin_mean": 46.17839050292969,
"margin_dpo/margin_std": 52.58583068847656,
"step": 281
},
{
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.014763940125703812,
"fcm_dpo/delta": -0.25571346282958984,
"fcm_dpo/margin": 56.46161651611328,
"fcm_dpo/q_t": 0.3185679316520691,
"grad_norm": 9.334823608398438,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -1.9072599411010742,
"logits/rejected": -1.542052984237671,
"logps/chosen": -164.61256408691406,
"logps/ref_chosen": -105.10511016845703,
"logps/ref_rejected": -102.85336303710938,
"logps/rejected": -218.82244873046875,
"loss": 0.8212,
"margin_dpo/margin_mean": 56.46161651611328,
"margin_dpo/margin_std": 48.17137908935547,
"step": 282
},
{
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.014660017564892769,
"fcm_dpo/delta": 0.026379965245723724,
"fcm_dpo/margin": 39.232139587402344,
"fcm_dpo/q_t": 0.3760044276714325,
"grad_norm": 12.626619338989258,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -2.1855897903442383,
"logits/rejected": -1.8464176654815674,
"logps/chosen": -164.91322326660156,
"logps/ref_chosen": -95.05259704589844,
"logps/ref_rejected": -103.54454803466797,
"logps/rejected": -212.63731384277344,
"loss": 1.0276,
"margin_dpo/margin_mean": 39.232139587402344,
"margin_dpo/margin_std": 53.09544372558594,
"step": 283
},
{
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.01422587689012289,
"fcm_dpo/delta": -0.17442496120929718,
"fcm_dpo/margin": 53.390506744384766,
"fcm_dpo/q_t": 0.33068281412124634,
"grad_norm": 9.979324340820312,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -1.7961182594299316,
"logits/rejected": -1.4766089916229248,
"logps/chosen": -148.95257568359375,
"logps/ref_chosen": -87.66647338867188,
"logps/ref_rejected": -98.75103759765625,
"logps/rejected": -213.42764282226562,
"loss": 0.8488,
"margin_dpo/margin_mean": 53.3905029296875,
"margin_dpo/margin_std": 44.12660217285156,
"step": 284
},
{
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.01436292938888073,
"fcm_dpo/delta": 0.06020905822515488,
"fcm_dpo/margin": 37.7760009765625,
"fcm_dpo/q_t": 0.3767179250717163,
"grad_norm": 14.075101852416992,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -2.3716185092926025,
"logits/rejected": -2.0925536155700684,
"logps/chosen": -164.59915161132812,
"logps/ref_chosen": -94.43303680419922,
"logps/ref_rejected": -104.07194519042969,
"logps/rejected": -212.01405334472656,
"loss": 0.9954,
"margin_dpo/margin_mean": 37.7760009765625,
"margin_dpo/margin_std": 40.709510803222656,
"step": 285
},
{
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.014110833406448364,
"fcm_dpo/delta": -0.10124547779560089,
"fcm_dpo/margin": 49.17796325683594,
"fcm_dpo/q_t": 0.3466648757457733,
"grad_norm": 10.761466026306152,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -1.7981414794921875,
"logits/rejected": -1.32814621925354,
"logps/chosen": -178.29949951171875,
"logps/ref_chosen": -103.72039794921875,
"logps/ref_rejected": -96.25775909423828,
"logps/rejected": -220.01480102539062,
"loss": 0.9137,
"margin_dpo/margin_mean": 49.17796325683594,
"margin_dpo/margin_std": 48.17053985595703,
"step": 286
},
{
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.014512348920106888,
"fcm_dpo/delta": 0.16535604000091553,
"fcm_dpo/margin": 30.320154190063477,
"fcm_dpo/q_t": 0.4061054587364197,
"grad_norm": 11.057209968566895,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -2.0745434761047363,
"logits/rejected": -1.4789698123931885,
"logps/chosen": -181.6774444580078,
"logps/ref_chosen": -93.88988494873047,
"logps/ref_rejected": -83.33365631103516,
"logps/rejected": -201.44137573242188,
"loss": 1.1162,
"margin_dpo/margin_mean": 30.320154190063477,
"margin_dpo/margin_std": 49.62172317504883,
"step": 287
},
{
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.014212418347597122,
"fcm_dpo/delta": -0.13853885233402252,
"fcm_dpo/margin": 51.22802734375,
"fcm_dpo/q_t": 0.34565305709838867,
"grad_norm": 10.420333862304688,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -1.952336311340332,
"logits/rejected": -1.555095911026001,
"logps/chosen": -172.57025146484375,
"logps/ref_chosen": -88.15602111816406,
"logps/ref_rejected": -93.28195190429688,
"logps/rejected": -228.92422485351562,
"loss": 0.9128,
"margin_dpo/margin_mean": 51.22802734375,
"margin_dpo/margin_std": 54.86450958251953,
"step": 288
},
{
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.013875823467969894,
"fcm_dpo/delta": -0.14725634455680847,
"fcm_dpo/margin": 53.087974548339844,
"fcm_dpo/q_t": 0.34943529963493347,
"grad_norm": 9.893397331237793,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -1.9582253694534302,
"logits/rejected": -1.757028341293335,
"logps/chosen": -158.78115844726562,
"logps/ref_chosen": -75.39292907714844,
"logps/ref_rejected": -93.15428161621094,
"logps/rejected": -229.6304931640625,
"loss": 0.9302,
"margin_dpo/margin_mean": 53.087974548339844,
"margin_dpo/margin_std": 61.871124267578125,
"step": 289
},
{
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.013547876849770546,
"fcm_dpo/delta": -0.14979226887226105,
"fcm_dpo/margin": 54.51033020019531,
"fcm_dpo/q_t": 0.3426121473312378,
"grad_norm": 10.341540336608887,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -1.157566785812378,
"logits/rejected": -1.1871843338012695,
"logps/chosen": -179.93722534179688,
"logps/ref_chosen": -88.04197692871094,
"logps/ref_rejected": -123.212158203125,
"logps/rejected": -269.61773681640625,
"loss": 0.9071,
"margin_dpo/margin_mean": 54.51033020019531,
"margin_dpo/margin_std": 57.139404296875,
"step": 290
},
{
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.013345437124371529,
"fcm_dpo/delta": -0.0021981801837682724,
"fcm_dpo/margin": 45.109642028808594,
"fcm_dpo/q_t": 0.3656271994113922,
"grad_norm": 10.851905822753906,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -2.3467676639556885,
"logits/rejected": -1.8936057090759277,
"logps/chosen": -181.8697967529297,
"logps/ref_chosen": -94.09524536132812,
"logps/ref_rejected": -96.05006408691406,
"logps/rejected": -228.93426513671875,
"loss": 0.9784,
"margin_dpo/margin_mean": 45.109642028808594,
"margin_dpo/margin_std": 51.156219482421875,
"step": 291
},
{
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.013189633376896381,
"fcm_dpo/delta": -0.062153078615665436,
"fcm_dpo/margin": 49.88909912109375,
"fcm_dpo/q_t": 0.3522568643093109,
"grad_norm": 10.579051971435547,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -2.1592330932617188,
"logits/rejected": -1.730700969696045,
"logps/chosen": -178.15887451171875,
"logps/ref_chosen": -88.25041198730469,
"logps/ref_rejected": -96.41764068603516,
"logps/rejected": -236.21517944335938,
"loss": 0.9094,
"margin_dpo/margin_mean": 49.889095306396484,
"margin_dpo/margin_std": 44.906211853027344,
"step": 292
},
{
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.013408501632511616,
"fcm_dpo/delta": 0.0868854746222496,
"fcm_dpo/margin": 38.51031494140625,
"fcm_dpo/q_t": 0.3900435268878937,
"grad_norm": 9.791333198547363,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -2.625321388244629,
"logits/rejected": -2.0894412994384766,
"logps/chosen": -180.33511352539062,
"logps/ref_chosen": -87.37654113769531,
"logps/ref_rejected": -85.75579833984375,
"logps/rejected": -217.22467041015625,
"loss": 1.0576,
"margin_dpo/margin_mean": 38.51031494140625,
"margin_dpo/margin_std": 54.91911315917969,
"step": 293
},
{
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.013757754117250443,
"fcm_dpo/delta": 0.16101771593093872,
"fcm_dpo/margin": 32.374855041503906,
"fcm_dpo/q_t": 0.4055211544036865,
"grad_norm": 11.367213249206543,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -1.6977550983428955,
"logits/rejected": -1.5189887285232544,
"logps/chosen": -170.7028045654297,
"logps/ref_chosen": -73.50794982910156,
"logps/ref_rejected": -88.08877563476562,
"logps/rejected": -217.65847778320312,
"loss": 1.1326,
"margin_dpo/margin_mean": 32.374855041503906,
"margin_dpo/margin_std": 57.856483459472656,
"step": 294
},
{
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.013606134802103043,
"fcm_dpo/delta": -0.06477653235197067,
"fcm_dpo/margin": 48.46016311645508,
"fcm_dpo/q_t": 0.3579823672771454,
"grad_norm": 10.51223373413086,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -1.6513359546661377,
"logits/rejected": -1.5038331747055054,
"logps/chosen": -172.31985473632812,
"logps/ref_chosen": -77.76548767089844,
"logps/ref_rejected": -94.24726867675781,
"logps/rejected": -237.2617950439453,
"loss": 0.9532,
"margin_dpo/margin_mean": 48.46016311645508,
"margin_dpo/margin_std": 54.77019500732422,
"step": 295
},
{
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.01337181031703949,
"fcm_dpo/delta": -0.10661374777555466,
"fcm_dpo/margin": 52.22368240356445,
"fcm_dpo/q_t": 0.34752362966537476,
"grad_norm": 11.95097827911377,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -2.5504400730133057,
"logits/rejected": -2.0862655639648438,
"logps/chosen": -198.88778686523438,
"logps/ref_chosen": -97.85641479492188,
"logps/ref_rejected": -100.81631469726562,
"logps/rejected": -254.07135009765625,
"loss": 0.9174,
"margin_dpo/margin_mean": 52.22368240356445,
"margin_dpo/margin_std": 53.6197509765625,
"step": 296
},
{
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.013420840725302696,
"fcm_dpo/delta": 0.018867414444684982,
"fcm_dpo/margin": 43.375144958496094,
"fcm_dpo/q_t": 0.37541282176971436,
"grad_norm": 13.033909797668457,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -1.2705745697021484,
"logits/rejected": -0.9568313360214233,
"logps/chosen": -192.59295654296875,
"logps/ref_chosen": -88.56583404541016,
"logps/ref_rejected": -101.55656433105469,
"logps/rejected": -248.9588165283203,
"loss": 1.0032,
"margin_dpo/margin_mean": 43.375144958496094,
"margin_dpo/margin_std": 54.68861389160156,
"step": 297
},
{
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.013230125419795513,
"fcm_dpo/delta": -0.11564385145902634,
"fcm_dpo/margin": 53.48369216918945,
"fcm_dpo/q_t": 0.35127192735671997,
"grad_norm": 16.36045265197754,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -2.012399911880493,
"logits/rejected": -1.9581105709075928,
"logps/chosen": -186.6326904296875,
"logps/ref_chosen": -94.88043975830078,
"logps/ref_rejected": -122.31101989746094,
"logps/rejected": -267.54693603515625,
"loss": 0.9249,
"margin_dpo/margin_mean": 53.48369216918945,
"margin_dpo/margin_std": 59.249088287353516,
"step": 298
},
{
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.012806476093828678,
"fcm_dpo/delta": -0.13323679566383362,
"fcm_dpo/margin": 56.45828628540039,
"fcm_dpo/q_t": 0.34347003698349,
"grad_norm": 10.563586235046387,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.9475471377372742,
"logits/rejected": -0.7138333320617676,
"logps/chosen": -175.18414306640625,
"logps/ref_chosen": -80.40835571289062,
"logps/ref_rejected": -89.53717041015625,
"logps/rejected": -240.771240234375,
"loss": 0.9033,
"margin_dpo/margin_mean": 56.458290100097656,
"margin_dpo/margin_std": 57.19697570800781,
"step": 299
},
{
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.012716519646346569,
"fcm_dpo/delta": 0.005831874907016754,
"fcm_dpo/margin": 46.72724151611328,
"fcm_dpo/q_t": 0.37154459953308105,
"grad_norm": 11.477516174316406,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -1.3495773077011108,
"logits/rejected": -1.1283777952194214,
"logps/chosen": -187.920654296875,
"logps/ref_chosen": -88.15890502929688,
"logps/ref_rejected": -100.93919372558594,
"logps/rejected": -247.4281768798828,
"loss": 1.0079,
"margin_dpo/margin_mean": 46.727237701416016,
"margin_dpo/margin_std": 59.988731384277344,
"step": 300
},
{
"epoch": 0.44052863436123346,
"eval_fcm_dpo/beta": 0.01277273241430521,
"eval_logits/chosen": -1.7159112691879272,
"eval_logits/rejected": -1.3590905666351318,
"eval_logps/chosen": -199.09510803222656,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -235.35121154785156,
"eval_loss": 0.5174583792686462,
"eval_margin_dpo/margin_mean": 42.681915283203125,
"eval_margin_dpo/margin_std": 56.65989685058594,
"eval_runtime": 43.743,
"eval_samples_per_second": 53.471,
"eval_steps_per_second": 1.692,
"step": 300
},
{
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.012361442670226097,
"fcm_dpo/delta": -0.16100239753723145,
"fcm_dpo/margin": 60.36054992675781,
"fcm_dpo/q_t": 0.3411176800727844,
"grad_norm": 10.854103088378906,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -1.7507340908050537,
"logits/rejected": -1.315748691558838,
"logps/chosen": -178.2888946533203,
"logps/ref_chosen": -85.50491333007812,
"logps/ref_rejected": -98.91683959960938,
"logps/rejected": -252.06137084960938,
"loss": 0.9009,
"margin_dpo/margin_mean": 60.36054992675781,
"margin_dpo/margin_std": 62.385398864746094,
"step": 301
},
{
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.012308208271861076,
"fcm_dpo/delta": -0.004877906292676926,
"fcm_dpo/margin": 49.0919189453125,
"fcm_dpo/q_t": 0.3680340647697449,
"grad_norm": 11.78134536743164,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -1.391321063041687,
"logits/rejected": -1.2118873596191406,
"logps/chosen": -172.2723388671875,
"logps/ref_chosen": -79.45040893554688,
"logps/ref_rejected": -103.0591049194336,
"logps/rejected": -244.97293090820312,
"loss": 0.9875,
"margin_dpo/margin_mean": 49.091915130615234,
"margin_dpo/margin_std": 58.554237365722656,
"step": 302
},
{
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.01226123422384262,
"fcm_dpo/delta": -0.08874759823083878,
"fcm_dpo/margin": 55.66910934448242,
"fcm_dpo/q_t": 0.3502175211906433,
"grad_norm": 10.105868339538574,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -1.8867647647857666,
"logits/rejected": -1.4471575021743774,
"logps/chosen": -179.00445556640625,
"logps/ref_chosen": -95.59829711914062,
"logps/ref_rejected": -92.0645751953125,
"logps/rejected": -231.1398468017578,
"loss": 0.9267,
"margin_dpo/margin_mean": 55.669105529785156,
"margin_dpo/margin_std": 56.76445007324219,
"step": 303
},
{
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.011957298964262009,
"fcm_dpo/delta": -0.11238184571266174,
"fcm_dpo/margin": 58.92472839355469,
"fcm_dpo/q_t": 0.3453929126262665,
"grad_norm": 11.284574508666992,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -2.2299141883850098,
"logits/rejected": -1.8196072578430176,
"logps/chosen": -183.23292541503906,
"logps/ref_chosen": -99.4531478881836,
"logps/ref_rejected": -98.08226013183594,
"logps/rejected": -240.78677368164062,
"loss": 0.9148,
"margin_dpo/margin_mean": 58.92472839355469,
"margin_dpo/margin_std": 60.10872268676758,
"step": 304
},
{
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.011641704477369785,
"fcm_dpo/delta": -0.06990103423595428,
"fcm_dpo/margin": 56.961299896240234,
"fcm_dpo/q_t": 0.35451453924179077,
"grad_norm": 10.1016263961792,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -2.020512104034424,
"logits/rejected": -1.5305230617523193,
"logps/chosen": -190.84634399414062,
"logps/ref_chosen": -103.26339721679688,
"logps/ref_rejected": -97.82897186279297,
"logps/rejected": -242.3732147216797,
"loss": 0.9438,
"margin_dpo/margin_mean": 56.96129608154297,
"margin_dpo/margin_std": 59.46485137939453,
"step": 305
},
{
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.011314811185002327,
"fcm_dpo/delta": -0.20089520514011383,
"fcm_dpo/margin": 69.29483032226562,
"fcm_dpo/q_t": 0.3232324719429016,
"grad_norm": 10.051947593688965,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -2.1980226039886475,
"logits/rejected": -1.6485023498535156,
"logps/chosen": -177.880615234375,
"logps/ref_chosen": -100.3038330078125,
"logps/ref_rejected": -92.23086547851562,
"logps/rejected": -239.10247802734375,
"loss": 0.8261,
"margin_dpo/margin_mean": 69.29483032226562,
"margin_dpo/margin_std": 51.61286926269531,
"step": 306
},
{
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.011268500238656998,
"fcm_dpo/delta": 0.04018982872366905,
"fcm_dpo/margin": 49.86820983886719,
"fcm_dpo/q_t": 0.37846463918685913,
"grad_norm": 10.908080101013184,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -2.024630308151245,
"logits/rejected": -1.6950145959854126,
"logps/chosen": -181.67286682128906,
"logps/ref_chosen": -96.64976501464844,
"logps/ref_rejected": -93.34860229492188,
"logps/rejected": -228.23992919921875,
"loss": 1.026,
"margin_dpo/margin_mean": 49.86820602416992,
"margin_dpo/margin_std": 66.2600326538086,
"step": 307
},
{
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.011317985132336617,
"fcm_dpo/delta": 0.04552755504846573,
"fcm_dpo/margin": 49.12928009033203,
"fcm_dpo/q_t": 0.38196253776550293,
"grad_norm": 11.59121322631836,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -1.7888622283935547,
"logits/rejected": -1.5279297828674316,
"logps/chosen": -173.8710479736328,
"logps/ref_chosen": -85.72467041015625,
"logps/ref_rejected": -86.42317962646484,
"logps/rejected": -223.69882202148438,
"loss": 1.0387,
"margin_dpo/margin_mean": 49.12928009033203,
"margin_dpo/margin_std": 66.89312744140625,
"step": 308
},
{
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.011532912030816078,
"fcm_dpo/delta": 0.044411551207304,
"fcm_dpo/margin": 48.37786102294922,
"fcm_dpo/q_t": 0.3778493404388428,
"grad_norm": 14.045127868652344,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -1.8127508163452148,
"logits/rejected": -1.389906883239746,
"logps/chosen": -179.91854858398438,
"logps/ref_chosen": -83.15145874023438,
"logps/ref_rejected": -86.58602142333984,
"logps/rejected": -231.73095703125,
"loss": 1.0173,
"margin_dpo/margin_mean": 48.37785720825195,
"margin_dpo/margin_std": 60.719688415527344,
"step": 309
},
{
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.011562330648303032,
"fcm_dpo/delta": 0.013482505455613136,
"fcm_dpo/margin": 50.79509735107422,
"fcm_dpo/q_t": 0.3710099160671234,
"grad_norm": 10.965914726257324,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -2.2912349700927734,
"logits/rejected": -1.8876441717147827,
"logps/chosen": -175.0748291015625,
"logps/ref_chosen": -89.73799133300781,
"logps/ref_rejected": -87.07083129882812,
"logps/rejected": -223.2027587890625,
"loss": 0.9925,
"margin_dpo/margin_mean": 50.79509353637695,
"margin_dpo/margin_std": 60.45803451538086,
"step": 310
},
{
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.011323259212076664,
"fcm_dpo/delta": -0.14433330297470093,
"fcm_dpo/margin": 64.79643249511719,
"fcm_dpo/q_t": 0.33913880586624146,
"grad_norm": 10.768723487854004,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -1.8347514867782593,
"logits/rejected": -1.1266942024230957,
"logps/chosen": -192.47508239746094,
"logps/ref_chosen": -88.52644348144531,
"logps/ref_rejected": -82.26608276367188,
"logps/rejected": -251.0111541748047,
"loss": 0.875,
"margin_dpo/margin_mean": 64.79643249511719,
"margin_dpo/margin_std": 58.919742584228516,
"step": 311
},
{
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.011255646124482155,
"fcm_dpo/delta": -0.024957779794931412,
"fcm_dpo/margin": 55.35394287109375,
"fcm_dpo/q_t": 0.36366888880729675,
"grad_norm": 10.373575210571289,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -1.5337181091308594,
"logits/rejected": -1.003326654434204,
"logps/chosen": -179.2374267578125,
"logps/ref_chosen": -80.40069580078125,
"logps/ref_rejected": -82.4471435546875,
"logps/rejected": -236.63783264160156,
"loss": 0.9707,
"margin_dpo/margin_mean": 55.35394287109375,
"margin_dpo/margin_std": 63.06084442138672,
"step": 312
},
{
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.01121255662292242,
"fcm_dpo/delta": 0.0006225090473890305,
"fcm_dpo/margin": 53.44978332519531,
"fcm_dpo/q_t": 0.3703840374946594,
"grad_norm": 11.122648239135742,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -2.4546828269958496,
"logits/rejected": -2.0480782985687256,
"logps/chosen": -176.66587829589844,
"logps/ref_chosen": -84.14190673828125,
"logps/ref_rejected": -93.42684173583984,
"logps/rejected": -239.40060424804688,
"loss": 1.008,
"margin_dpo/margin_mean": 53.44977569580078,
"margin_dpo/margin_std": 69.2515869140625,
"step": 313
},
{
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.011162392795085907,
"fcm_dpo/delta": 0.010066971182823181,
"fcm_dpo/margin": 52.88270568847656,
"fcm_dpo/q_t": 0.3719868063926697,
"grad_norm": 11.244734764099121,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -1.9083623886108398,
"logits/rejected": -1.6891626119613647,
"logps/chosen": -181.61160278320312,
"logps/ref_chosen": -88.80972290039062,
"logps/ref_rejected": -104.15892028808594,
"logps/rejected": -249.843505859375,
"loss": 0.9914,
"margin_dpo/margin_mean": 52.88270568847656,
"margin_dpo/margin_std": 63.093101501464844,
"step": 314
},
{
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.011188652366399765,
"fcm_dpo/delta": -0.006126678083091974,
"fcm_dpo/margin": 54.138145446777344,
"fcm_dpo/q_t": 0.37136292457580566,
"grad_norm": 12.068286895751953,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -1.4232127666473389,
"logits/rejected": -1.2410612106323242,
"logps/chosen": -171.7213134765625,
"logps/ref_chosen": -80.793701171875,
"logps/ref_rejected": -92.13711547851562,
"logps/rejected": -237.202880859375,
"loss": 1.0249,
"margin_dpo/margin_mean": 54.138145446777344,
"margin_dpo/margin_std": 74.93022155761719,
"step": 315
},
{
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.011153796687722206,
"fcm_dpo/delta": -0.032783642411231995,
"fcm_dpo/margin": 56.54811096191406,
"fcm_dpo/q_t": 0.364329993724823,
"grad_norm": 10.284096717834473,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -1.860671043395996,
"logits/rejected": -1.8614685535430908,
"logps/chosen": -179.6904296875,
"logps/ref_chosen": -84.18424987792969,
"logps/ref_rejected": -105.6908187866211,
"logps/rejected": -257.7451171875,
"loss": 0.9706,
"margin_dpo/margin_mean": 56.54811096191406,
"margin_dpo/margin_std": 67.02822875976562,
"step": 316
},
{
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.010856937617063522,
"fcm_dpo/delta": -0.19055171310901642,
"fcm_dpo/margin": 71.48945617675781,
"fcm_dpo/q_t": 0.3373391628265381,
"grad_norm": 12.972799301147461,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -1.9898185729980469,
"logits/rejected": -1.512327790260315,
"logps/chosen": -185.63662719726562,
"logps/ref_chosen": -87.87348937988281,
"logps/ref_rejected": -108.26519775390625,
"logps/rejected": -277.5177917480469,
"loss": 0.9042,
"margin_dpo/margin_mean": 71.48945617675781,
"margin_dpo/margin_std": 77.33946228027344,
"step": 317
},
{
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.010556013323366642,
"fcm_dpo/delta": -0.08519938588142395,
"fcm_dpo/margin": 64.34666442871094,
"fcm_dpo/q_t": 0.351060688495636,
"grad_norm": 10.77450180053711,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -2.1584458351135254,
"logits/rejected": -1.4637277126312256,
"logps/chosen": -193.25021362304688,
"logps/ref_chosen": -98.20553588867188,
"logps/ref_rejected": -88.13629150390625,
"logps/rejected": -247.52764892578125,
"loss": 0.925,
"margin_dpo/margin_mean": 64.34667205810547,
"margin_dpo/margin_std": 66.24368286132812,
"step": 318
},
{
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.010318400338292122,
"fcm_dpo/delta": -0.1146102324128151,
"fcm_dpo/margin": 68.42571258544922,
"fcm_dpo/q_t": 0.3458014726638794,
"grad_norm": 10.647491455078125,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -1.9602619409561157,
"logits/rejected": -1.8778321743011475,
"logps/chosen": -164.64324951171875,
"logps/ref_chosen": -82.50337219238281,
"logps/ref_rejected": -118.65068054199219,
"logps/rejected": -269.21624755859375,
"loss": 0.9143,
"margin_dpo/margin_mean": 68.42571258544922,
"margin_dpo/margin_std": 70.17927551269531,
"step": 319
},
{
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.010274864733219147,
"fcm_dpo/delta": 0.008025115355849266,
"fcm_dpo/margin": 57.651084899902344,
"fcm_dpo/q_t": 0.369371235370636,
"grad_norm": 10.72866439819336,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -2.084186553955078,
"logits/rejected": -1.8974592685699463,
"logps/chosen": -172.93923950195312,
"logps/ref_chosen": -87.13722229003906,
"logps/ref_rejected": -103.22412109375,
"logps/rejected": -246.67721557617188,
"loss": 0.974,
"margin_dpo/margin_mean": 57.651084899902344,
"margin_dpo/margin_std": 64.01679992675781,
"step": 320
},
{
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.010156042873859406,
"fcm_dpo/delta": -0.06489168107509613,
"fcm_dpo/margin": 64.97313690185547,
"fcm_dpo/q_t": 0.3531816005706787,
"grad_norm": 10.770258903503418,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -2.1117072105407715,
"logits/rejected": -1.7369625568389893,
"logps/chosen": -188.44654846191406,
"logps/ref_chosen": -93.10466766357422,
"logps/ref_rejected": -97.4681396484375,
"logps/rejected": -257.78314208984375,
"loss": 0.9341,
"margin_dpo/margin_mean": 64.97313690185547,
"margin_dpo/margin_std": 66.21844482421875,
"step": 321
},
{
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.009905429556965828,
"fcm_dpo/delta": -0.12228944152593613,
"fcm_dpo/margin": 71.8099594116211,
"fcm_dpo/q_t": 0.3459140360355377,
"grad_norm": 13.35485553741455,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -1.7865667343139648,
"logits/rejected": -1.4356210231781006,
"logps/chosen": -182.97547912597656,
"logps/ref_chosen": -92.19291687011719,
"logps/ref_rejected": -100.89321899414062,
"logps/rejected": -263.4857177734375,
"loss": 0.9237,
"margin_dpo/margin_mean": 71.8099594116211,
"margin_dpo/margin_std": 75.52076721191406,
"step": 322
},
{
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.009861964732408524,
"fcm_dpo/delta": -0.021682463586330414,
"fcm_dpo/margin": 62.875831604003906,
"fcm_dpo/q_t": 0.3650767207145691,
"grad_norm": 10.933591842651367,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -2.0441133975982666,
"logits/rejected": -1.6695351600646973,
"logps/chosen": -184.3953857421875,
"logps/ref_chosen": -80.27210235595703,
"logps/ref_rejected": -80.3033447265625,
"logps/rejected": -247.302490234375,
"loss": 0.9606,
"margin_dpo/margin_mean": 62.875831604003906,
"margin_dpo/margin_std": 68.70439147949219,
"step": 323
},
{
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.009814316406846046,
"fcm_dpo/delta": -0.05098802596330643,
"fcm_dpo/margin": 65.9963150024414,
"fcm_dpo/q_t": 0.3542330861091614,
"grad_norm": 10.099251747131348,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -1.5629843473434448,
"logits/rejected": -0.8973981142044067,
"logps/chosen": -200.52267456054688,
"logps/ref_chosen": -95.72935485839844,
"logps/ref_rejected": -94.16924285888672,
"logps/rejected": -264.9588623046875,
"loss": 0.9235,
"margin_dpo/margin_mean": 65.9963150024414,
"margin_dpo/margin_std": 62.014060974121094,
"step": 324
},
{
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.009698811918497086,
"fcm_dpo/delta": -0.06353026628494263,
"fcm_dpo/margin": 67.9854965209961,
"fcm_dpo/q_t": 0.3524365723133087,
"grad_norm": 8.840824127197266,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -1.7049955129623413,
"logits/rejected": -0.977316677570343,
"logps/chosen": -196.92950439453125,
"logps/ref_chosen": -98.70686340332031,
"logps/ref_rejected": -76.40809631347656,
"logps/rejected": -242.61624145507812,
"loss": 0.9246,
"margin_dpo/margin_mean": 67.9854965209961,
"margin_dpo/margin_std": 65.83146667480469,
"step": 325
},
{
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.009657101705670357,
"fcm_dpo/delta": 0.0018943261820822954,
"fcm_dpo/margin": 61.94574737548828,
"fcm_dpo/q_t": 0.36700719594955444,
"grad_norm": 10.065813064575195,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -2.4169249534606934,
"logits/rejected": -1.762148380279541,
"logps/chosen": -214.70034790039062,
"logps/ref_chosen": -104.66130828857422,
"logps/ref_rejected": -86.82537841796875,
"logps/rejected": -258.8101806640625,
"loss": 0.9749,
"margin_dpo/margin_mean": 61.94574737548828,
"margin_dpo/margin_std": 68.8028793334961,
"step": 326
},
{
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.00965862162411213,
"fcm_dpo/delta": 0.015111330896615982,
"fcm_dpo/margin": 60.6379508972168,
"fcm_dpo/q_t": 0.3714819848537445,
"grad_norm": 10.074041366577148,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -1.8287179470062256,
"logits/rejected": -1.238955020904541,
"logps/chosen": -213.91966247558594,
"logps/ref_chosen": -107.44023895263672,
"logps/ref_rejected": -117.05323028564453,
"logps/rejected": -284.17059326171875,
"loss": 0.9792,
"margin_dpo/margin_mean": 60.6379508972168,
"margin_dpo/margin_std": 67.89471435546875,
"step": 327
},
{
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0094821248203516,
"fcm_dpo/delta": -0.14909884333610535,
"fcm_dpo/margin": 77.84373474121094,
"fcm_dpo/q_t": 0.33735549449920654,
"grad_norm": 12.075971603393555,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -1.6724544763565063,
"logits/rejected": -1.3806228637695312,
"logps/chosen": -192.94517517089844,
"logps/ref_chosen": -86.6106948852539,
"logps/ref_rejected": -97.40188598632812,
"logps/rejected": -281.5801086425781,
"loss": 0.8778,
"margin_dpo/margin_mean": 77.84373474121094,
"margin_dpo/margin_std": 71.33271789550781,
"step": 328
},
{
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0093916654586792,
"fcm_dpo/delta": 0.019437361508607864,
"fcm_dpo/margin": 61.90422058105469,
"fcm_dpo/q_t": 0.371404230594635,
"grad_norm": 11.608654975891113,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -1.5943433046340942,
"logits/rejected": -1.0977287292480469,
"logps/chosen": -212.25148010253906,
"logps/ref_chosen": -100.61308288574219,
"logps/ref_rejected": -104.95370483398438,
"logps/rejected": -278.496337890625,
"loss": 0.9812,
"margin_dpo/margin_mean": 61.90422058105469,
"margin_dpo/margin_std": 68.27637481689453,
"step": 329
},
{
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.00939023308455944,
"fcm_dpo/delta": -0.020257247611880302,
"fcm_dpo/margin": 65.90046691894531,
"fcm_dpo/q_t": 0.3579813539981842,
"grad_norm": 9.650596618652344,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -2.022364377975464,
"logits/rejected": -1.9466552734375,
"logps/chosen": -175.4729766845703,
"logps/ref_chosen": -76.95843505859375,
"logps/ref_rejected": -92.81494140625,
"logps/rejected": -257.22991943359375,
"loss": 0.9311,
"margin_dpo/margin_mean": 65.90046691894531,
"margin_dpo/margin_std": 57.93687438964844,
"step": 330
},
{
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.009334873408079147,
"fcm_dpo/delta": -0.08815296739339828,
"fcm_dpo/margin": 73.05988311767578,
"fcm_dpo/q_t": 0.34697699546813965,
"grad_norm": 12.887969970703125,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -1.7896149158477783,
"logits/rejected": -1.4816310405731201,
"logps/chosen": -188.48745727539062,
"logps/ref_chosen": -83.05941772460938,
"logps/ref_rejected": -94.13113403320312,
"logps/rejected": -272.6190490722656,
"loss": 0.8953,
"margin_dpo/margin_mean": 73.05988311767578,
"margin_dpo/margin_std": 62.86804962158203,
"step": 331
},
{
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.00939233973622322,
"fcm_dpo/delta": 0.1038806289434433,
"fcm_dpo/margin": 53.36603927612305,
"fcm_dpo/q_t": 0.3885546326637268,
"grad_norm": 9.494643211364746,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -2.1970367431640625,
"logits/rejected": -1.778074026107788,
"logps/chosen": -191.7408447265625,
"logps/ref_chosen": -81.93089294433594,
"logps/ref_rejected": -82.84590148925781,
"logps/rejected": -246.02191162109375,
"loss": 1.0397,
"margin_dpo/margin_mean": 53.36603546142578,
"margin_dpo/margin_std": 68.24900817871094,
"step": 332
},
{
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.009271856397390366,
"fcm_dpo/delta": -0.06427404284477234,
"fcm_dpo/margin": 71.00698852539062,
"fcm_dpo/q_t": 0.35842740535736084,
"grad_norm": 10.024383544921875,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -1.5461020469665527,
"logits/rejected": -1.212287425994873,
"logps/chosen": -216.07131958007812,
"logps/ref_chosen": -88.53257751464844,
"logps/ref_rejected": -102.46601867675781,
"logps/rejected": -301.01171875,
"loss": 0.968,
"margin_dpo/margin_mean": 71.00698852539062,
"margin_dpo/margin_std": 83.59139251708984,
"step": 333
},
{
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.009264590218663216,
"fcm_dpo/delta": 0.004638519138097763,
"fcm_dpo/margin": 64.21815490722656,
"fcm_dpo/q_t": 0.3684922456741333,
"grad_norm": 11.035482406616211,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -1.366419792175293,
"logits/rejected": -1.0714776515960693,
"logps/chosen": -213.7954864501953,
"logps/ref_chosen": -97.75778198242188,
"logps/ref_rejected": -100.0058364868164,
"logps/rejected": -280.2616882324219,
"loss": 1.0037,
"margin_dpo/margin_mean": 64.21814727783203,
"margin_dpo/margin_std": 79.95184326171875,
"step": 334
},
{
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.009400870651006699,
"fcm_dpo/delta": 0.05613856390118599,
"fcm_dpo/margin": 58.17888641357422,
"fcm_dpo/q_t": 0.38059741258621216,
"grad_norm": 11.443718910217285,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -2.1963934898376465,
"logits/rejected": -1.7753444910049438,
"logps/chosen": -212.1436767578125,
"logps/ref_chosen": -96.37603759765625,
"logps/ref_rejected": -94.92401123046875,
"logps/rejected": -268.87054443359375,
"loss": 1.0258,
"margin_dpo/margin_mean": 58.17888641357422,
"margin_dpo/margin_std": 75.49417114257812,
"step": 335
},
{
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.009332764893770218,
"fcm_dpo/delta": -0.08782423287630081,
"fcm_dpo/margin": 73.06648254394531,
"fcm_dpo/q_t": 0.3505851626396179,
"grad_norm": 11.681970596313477,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -1.6232037544250488,
"logits/rejected": -1.4630122184753418,
"logps/chosen": -203.955078125,
"logps/ref_chosen": -88.52952575683594,
"logps/ref_rejected": -109.5557861328125,
"logps/rejected": -298.04779052734375,
"loss": 0.9236,
"margin_dpo/margin_mean": 73.06648254394531,
"margin_dpo/margin_std": 76.55876159667969,
"step": 336
},
{
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.009262502193450928,
"fcm_dpo/delta": -0.02163381688296795,
"fcm_dpo/margin": 66.955322265625,
"fcm_dpo/q_t": 0.36714088916778564,
"grad_norm": 12.159340858459473,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -2.4223062992095947,
"logits/rejected": -2.1396877765655518,
"logps/chosen": -213.5079345703125,
"logps/ref_chosen": -97.3123779296875,
"logps/ref_rejected": -110.13134765625,
"logps/rejected": -293.2822265625,
"loss": 0.9977,
"margin_dpo/margin_mean": 66.955322265625,
"margin_dpo/margin_std": 85.39122772216797,
"step": 337
},
{
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.00921336468309164,
"fcm_dpo/delta": 0.004760187119245529,
"fcm_dpo/margin": 64.63055419921875,
"fcm_dpo/q_t": 0.36690112948417664,
"grad_norm": 10.842123985290527,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -1.7986483573913574,
"logits/rejected": -1.6366183757781982,
"logps/chosen": -202.72119140625,
"logps/ref_chosen": -83.63011169433594,
"logps/ref_rejected": -93.69235229492188,
"logps/rejected": -277.41400146484375,
"loss": 0.9782,
"margin_dpo/margin_mean": 64.63055419921875,
"margin_dpo/margin_std": 72.1498794555664,
"step": 338
},
{
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.009260453283786774,
"fcm_dpo/delta": -0.030668731778860092,
"fcm_dpo/margin": 67.80081939697266,
"fcm_dpo/q_t": 0.3631947636604309,
"grad_norm": 10.40760612487793,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -3.007730484008789,
"logits/rejected": -2.590148448944092,
"logps/chosen": -223.9995880126953,
"logps/ref_chosen": -99.85182189941406,
"logps/ref_rejected": -109.0743637084961,
"logps/rejected": -301.02294921875,
"loss": 0.9867,
"margin_dpo/margin_mean": 67.80081939697266,
"margin_dpo/margin_std": 81.80381774902344,
"step": 339
},
{
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.009243492037057877,
"fcm_dpo/delta": 0.04011049121618271,
"fcm_dpo/margin": 60.805809020996094,
"fcm_dpo/q_t": 0.37543347477912903,
"grad_norm": 12.52755355834961,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -1.4719023704528809,
"logits/rejected": -0.9998151659965515,
"logps/chosen": -218.23899841308594,
"logps/ref_chosen": -84.97396850585938,
"logps/ref_rejected": -78.80033874511719,
"logps/rejected": -272.87115478515625,
"loss": 1.0069,
"margin_dpo/margin_mean": 60.805809020996094,
"margin_dpo/margin_std": 75.30453491210938,
"step": 340
},
{
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.009315932169556618,
"fcm_dpo/delta": 0.06518840789794922,
"fcm_dpo/margin": 57.787986755371094,
"fcm_dpo/q_t": 0.383889377117157,
"grad_norm": 12.319915771484375,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -1.5798420906066895,
"logits/rejected": -1.3175228834152222,
"logps/chosen": -224.97544860839844,
"logps/ref_chosen": -85.26399230957031,
"logps/ref_rejected": -106.08595275878906,
"logps/rejected": -303.58538818359375,
"loss": 1.0502,
"margin_dpo/margin_mean": 57.787986755371094,
"margin_dpo/margin_std": 81.41255187988281,
"step": 341
},
{
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.009050893597304821,
"fcm_dpo/delta": -0.1713675707578659,
"fcm_dpo/margin": 83.44084167480469,
"fcm_dpo/q_t": 0.33266481757164,
"grad_norm": 10.313692092895508,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -1.3843135833740234,
"logits/rejected": -0.633280336856842,
"logps/chosen": -212.62014770507812,
"logps/ref_chosen": -101.03860473632812,
"logps/ref_rejected": -98.89851379394531,
"logps/rejected": -293.9208984375,
"loss": 0.8903,
"margin_dpo/margin_mean": 83.44084167480469,
"margin_dpo/margin_std": 78.70523834228516,
"step": 342
},
{
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.009080128744244576,
"fcm_dpo/delta": -0.036977700889110565,
"fcm_dpo/margin": 69.78305053710938,
"fcm_dpo/q_t": 0.3612483739852905,
"grad_norm": 12.353619575500488,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -2.0943527221679688,
"logits/rejected": -1.9702728986740112,
"logps/chosen": -203.23928833007812,
"logps/ref_chosen": -83.74736022949219,
"logps/ref_rejected": -116.57908630371094,
"logps/rejected": -305.85406494140625,
"loss": 0.9632,
"margin_dpo/margin_mean": 69.78305053710938,
"margin_dpo/margin_std": 76.31233215332031,
"step": 343
},
{
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.008997892960906029,
"fcm_dpo/delta": -0.030076302587985992,
"fcm_dpo/margin": 69.76876068115234,
"fcm_dpo/q_t": 0.3652355372905731,
"grad_norm": 10.976330757141113,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -1.680814504623413,
"logits/rejected": -1.425823450088501,
"logps/chosen": -202.2406768798828,
"logps/ref_chosen": -78.25396728515625,
"logps/ref_rejected": -95.42870330810547,
"logps/rejected": -289.1842041015625,
"loss": 0.9842,
"margin_dpo/margin_mean": 69.76876068115234,
"margin_dpo/margin_std": 85.37368774414062,
"step": 344
},
{
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.008805211633443832,
"fcm_dpo/delta": -0.08041529357433319,
"fcm_dpo/margin": 76.61846160888672,
"fcm_dpo/q_t": 0.3534577190876007,
"grad_norm": 11.980306625366211,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -1.5205774307250977,
"logits/rejected": -1.2206867933273315,
"logps/chosen": -221.85317993164062,
"logps/ref_chosen": -98.0794906616211,
"logps/ref_rejected": -104.03477478027344,
"logps/rejected": -304.42694091796875,
"loss": 0.9355,
"margin_dpo/margin_mean": 76.61846923828125,
"margin_dpo/margin_std": 82.15892028808594,
"step": 345
},
{
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.008871208876371384,
"fcm_dpo/delta": 0.0582590326666832,
"fcm_dpo/margin": 61.423545837402344,
"fcm_dpo/q_t": 0.3825676441192627,
"grad_norm": 10.243391036987305,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -1.9583630561828613,
"logits/rejected": -1.651193618774414,
"logps/chosen": -219.33963012695312,
"logps/ref_chosen": -80.68348693847656,
"logps/ref_rejected": -85.89260864257812,
"logps/rejected": -285.9722900390625,
"loss": 1.0424,
"margin_dpo/margin_mean": 61.423545837402344,
"margin_dpo/margin_std": 86.02749633789062,
"step": 346
},
{
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.009024476632475853,
"fcm_dpo/delta": 0.0737469345331192,
"fcm_dpo/margin": 58.69518280029297,
"fcm_dpo/q_t": 0.385664701461792,
"grad_norm": 11.432808876037598,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -1.6128592491149902,
"logits/rejected": -1.2691833972930908,
"logps/chosen": -223.65142822265625,
"logps/ref_chosen": -93.67938232421875,
"logps/ref_rejected": -97.54251098632812,
"logps/rejected": -286.209716796875,
"loss": 1.047,
"margin_dpo/margin_mean": 58.69518280029297,
"margin_dpo/margin_std": 81.67510986328125,
"step": 347
},
{
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.008929546922445297,
"fcm_dpo/delta": -0.0036687031388282776,
"fcm_dpo/margin": 67.33739471435547,
"fcm_dpo/q_t": 0.3671216368675232,
"grad_norm": 9.5287446975708,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -1.8974089622497559,
"logits/rejected": -1.6134772300720215,
"logps/chosen": -202.92189025878906,
"logps/ref_chosen": -90.25224304199219,
"logps/ref_rejected": -95.77027893066406,
"logps/rejected": -275.7773132324219,
"loss": 0.9868,
"margin_dpo/margin_mean": 67.33739471435547,
"margin_dpo/margin_std": 76.04698181152344,
"step": 348
},
{
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.009099038317799568,
"fcm_dpo/delta": 0.019386742264032364,
"fcm_dpo/margin": 63.853363037109375,
"fcm_dpo/q_t": 0.3738081157207489,
"grad_norm": 11.725251197814941,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -1.996931552886963,
"logits/rejected": -1.7708909511566162,
"logps/chosen": -214.01986694335938,
"logps/ref_chosen": -92.05809020996094,
"logps/ref_rejected": -103.96527099609375,
"logps/rejected": -289.7804260253906,
"loss": 1.0031,
"margin_dpo/margin_mean": 63.853363037109375,
"margin_dpo/margin_std": 78.11573791503906,
"step": 349
},
{
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.009226538240909576,
"fcm_dpo/delta": 0.1344294250011444,
"fcm_dpo/margin": 51.163047790527344,
"fcm_dpo/q_t": 0.39684417843818665,
"grad_norm": 12.850371360778809,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -1.785254955291748,
"logits/rejected": -1.556738018989563,
"logps/chosen": -219.18960571289062,
"logps/ref_chosen": -87.05419921875,
"logps/ref_rejected": -95.72193145751953,
"logps/rejected": -279.0203857421875,
"loss": 1.0736,
"margin_dpo/margin_mean": 51.16304397583008,
"margin_dpo/margin_std": 73.25152587890625,
"step": 350
},
{
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.009196523576974869,
"fcm_dpo/delta": -0.1071363314986229,
"fcm_dpo/margin": 76.08428192138672,
"fcm_dpo/q_t": 0.34812554717063904,
"grad_norm": 11.473617553710938,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -1.5400876998901367,
"logits/rejected": -1.2160134315490723,
"logps/chosen": -218.47886657714844,
"logps/ref_chosen": -89.16903686523438,
"logps/ref_rejected": -108.29801940917969,
"logps/rejected": -313.692138671875,
"loss": 0.9389,
"margin_dpo/margin_mean": 76.08428192138672,
"margin_dpo/margin_std": 84.34982299804688,
"step": 351
},
{
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.009029903449118137,
"fcm_dpo/delta": -0.0580860860645771,
"fcm_dpo/margin": 72.45895385742188,
"fcm_dpo/q_t": 0.3607955574989319,
"grad_norm": 11.246063232421875,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -1.9249012470245361,
"logits/rejected": -1.3872631788253784,
"logps/chosen": -215.51181030273438,
"logps/ref_chosen": -96.81147766113281,
"logps/ref_rejected": -84.40483093261719,
"logps/rejected": -275.5640869140625,
"loss": 0.9591,
"margin_dpo/margin_mean": 72.45895385742188,
"margin_dpo/margin_std": 83.17478942871094,
"step": 352
},
{
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.009232399985194206,
"fcm_dpo/delta": 0.1866103559732437,
"fcm_dpo/margin": 45.66265869140625,
"fcm_dpo/q_t": 0.40671753883361816,
"grad_norm": 12.68930721282959,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -1.9192724227905273,
"logits/rejected": -1.527640700340271,
"logps/chosen": -221.19326782226562,
"logps/ref_chosen": -85.36248779296875,
"logps/ref_rejected": -87.36732482910156,
"logps/rejected": -268.86077880859375,
"loss": 1.1274,
"margin_dpo/margin_mean": 45.66265869140625,
"margin_dpo/margin_std": 76.734375,
"step": 353
},
{
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.009293053299188614,
"fcm_dpo/delta": -0.0588521845638752,
"fcm_dpo/margin": 70.47891235351562,
"fcm_dpo/q_t": 0.35432109236717224,
"grad_norm": 10.813716888427734,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -1.8877973556518555,
"logits/rejected": -1.4194233417510986,
"logps/chosen": -215.95494079589844,
"logps/ref_chosen": -95.8011703491211,
"logps/ref_rejected": -94.86614227294922,
"logps/rejected": -285.49884033203125,
"loss": 0.9172,
"margin_dpo/margin_mean": 70.47891235351562,
"margin_dpo/margin_std": 65.81661987304688,
"step": 354
},
{
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.009149078279733658,
"fcm_dpo/delta": -0.0197719968855381,
"fcm_dpo/margin": 67.51188659667969,
"fcm_dpo/q_t": 0.3642021119594574,
"grad_norm": 11.531266212463379,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -2.4402689933776855,
"logits/rejected": -1.9639763832092285,
"logps/chosen": -205.29983520507812,
"logps/ref_chosen": -86.185546875,
"logps/ref_rejected": -88.4615707397461,
"logps/rejected": -275.0877685546875,
"loss": 0.9754,
"margin_dpo/margin_mean": 67.51187896728516,
"margin_dpo/margin_std": 75.14073181152344,
"step": 355
},
{
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.009418745525181293,
"fcm_dpo/delta": 0.13494420051574707,
"fcm_dpo/margin": 49.970237731933594,
"fcm_dpo/q_t": 0.3972604274749756,
"grad_norm": 13.176185607910156,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -1.8549089431762695,
"logits/rejected": -1.394954800605774,
"logps/chosen": -231.3189697265625,
"logps/ref_chosen": -90.818115234375,
"logps/ref_rejected": -82.1273193359375,
"logps/rejected": -272.5984191894531,
"loss": 1.1045,
"margin_dpo/margin_mean": 49.970237731933594,
"margin_dpo/margin_std": 81.81539916992188,
"step": 356
},
{
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.009089802391827106,
"fcm_dpo/delta": -0.18602949380874634,
"fcm_dpo/margin": 84.42666625976562,
"fcm_dpo/q_t": 0.3301994502544403,
"grad_norm": 9.98563003540039,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -2.099191188812256,
"logits/rejected": -1.7402398586273193,
"logps/chosen": -202.91748046875,
"logps/ref_chosen": -87.5047607421875,
"logps/ref_rejected": -92.24937438964844,
"logps/rejected": -292.0887756347656,
"loss": 0.861,
"margin_dpo/margin_mean": 84.42666625976562,
"margin_dpo/margin_std": 71.71357727050781,
"step": 357
},
{
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.009189213626086712,
"fcm_dpo/delta": 0.059452179819345474,
"fcm_dpo/margin": 59.17223358154297,
"fcm_dpo/q_t": 0.3846597373485565,
"grad_norm": 11.328146934509277,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -1.47278892993927,
"logits/rejected": -0.9826061725616455,
"logps/chosen": -225.228271484375,
"logps/ref_chosen": -95.05546569824219,
"logps/ref_rejected": -96.47611999511719,
"logps/rejected": -285.8211669921875,
"loss": 1.0297,
"margin_dpo/margin_mean": 59.17223358154297,
"margin_dpo/margin_std": 79.3198471069336,
"step": 358
},
{
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.009268088266253471,
"fcm_dpo/delta": 0.09260153770446777,
"fcm_dpo/margin": 55.203125,
"fcm_dpo/q_t": 0.39258408546447754,
"grad_norm": 11.787094116210938,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -1.5292563438415527,
"logits/rejected": -0.9679604768753052,
"logps/chosen": -230.56655883789062,
"logps/ref_chosen": -87.37332916259766,
"logps/ref_rejected": -88.77188110351562,
"logps/rejected": -287.1682434082031,
"loss": 1.0731,
"margin_dpo/margin_mean": 55.203128814697266,
"margin_dpo/margin_std": 82.28927612304688,
"step": 359
},
{
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.009283397346735,
"fcm_dpo/delta": -0.06985802948474884,
"fcm_dpo/margin": 71.65193176269531,
"fcm_dpo/q_t": 0.35196417570114136,
"grad_norm": 14.178664207458496,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -1.6124732494354248,
"logits/rejected": -1.2661912441253662,
"logps/chosen": -209.75942993164062,
"logps/ref_chosen": -75.22344970703125,
"logps/ref_rejected": -76.09400177001953,
"logps/rejected": -282.28192138671875,
"loss": 0.9315,
"margin_dpo/margin_mean": 71.65193176269531,
"margin_dpo/margin_std": 73.52704620361328,
"step": 360
},
{
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.009296550415456295,
"fcm_dpo/delta": 0.019596338272094727,
"fcm_dpo/margin": 62.54603958129883,
"fcm_dpo/q_t": 0.37713533639907837,
"grad_norm": 12.717004776000977,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -1.715510606765747,
"logits/rejected": -1.4632185697555542,
"logps/chosen": -218.74740600585938,
"logps/ref_chosen": -81.95333099365234,
"logps/ref_rejected": -96.67919921875,
"logps/rejected": -296.0193176269531,
"loss": 1.0228,
"margin_dpo/margin_mean": 62.54604721069336,
"margin_dpo/margin_std": 84.71768188476562,
"step": 361
},
{
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.009247587993741035,
"fcm_dpo/delta": -0.015775587409734726,
"fcm_dpo/margin": 66.46771240234375,
"fcm_dpo/q_t": 0.3636864423751831,
"grad_norm": 11.942319869995117,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -1.6847763061523438,
"logits/rejected": -1.467708706855774,
"logps/chosen": -230.143310546875,
"logps/ref_chosen": -89.33971405029297,
"logps/ref_rejected": -110.50196838378906,
"logps/rejected": -317.7733154296875,
"loss": 0.9741,
"margin_dpo/margin_mean": 66.46771240234375,
"margin_dpo/margin_std": 74.62181854248047,
"step": 362
},
{
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.009058544412255287,
"fcm_dpo/delta": -0.18251146376132965,
"fcm_dpo/margin": 84.8752670288086,
"fcm_dpo/q_t": 0.33618301153182983,
"grad_norm": 12.65180778503418,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -1.7909729480743408,
"logits/rejected": -1.3521623611450195,
"logps/chosen": -214.4105987548828,
"logps/ref_chosen": -80.77499389648438,
"logps/ref_rejected": -79.43267822265625,
"logps/rejected": -297.94354248046875,
"loss": 0.8884,
"margin_dpo/margin_mean": 84.87525939941406,
"margin_dpo/margin_std": 86.01889038085938,
"step": 363
},
{
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.008983662351965904,
"fcm_dpo/delta": -0.018669523298740387,
"fcm_dpo/margin": 68.58975219726562,
"fcm_dpo/q_t": 0.36190158128738403,
"grad_norm": 13.727071762084961,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -2.0300331115722656,
"logits/rejected": -1.877135992050171,
"logps/chosen": -209.77499389648438,
"logps/ref_chosen": -76.52960205078125,
"logps/ref_rejected": -97.56173706054688,
"logps/rejected": -299.39691162109375,
"loss": 0.9511,
"margin_dpo/margin_mean": 68.58975219726562,
"margin_dpo/margin_std": 66.92192840576172,
"step": 364
},
{
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.008843779563903809,
"fcm_dpo/delta": -0.06726966798305511,
"fcm_dpo/margin": 74.93986511230469,
"fcm_dpo/q_t": 0.35855019092559814,
"grad_norm": 10.768917083740234,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -1.5261728763580322,
"logits/rejected": -1.3805526494979858,
"logps/chosen": -210.90951538085938,
"logps/ref_chosen": -80.54489135742188,
"logps/ref_rejected": -99.32316589355469,
"logps/rejected": -304.62762451171875,
"loss": 0.9621,
"margin_dpo/margin_mean": 74.93986511230469,
"margin_dpo/margin_std": 88.24283599853516,
"step": 365
},
{
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.008807620033621788,
"fcm_dpo/delta": 0.024917863309383392,
"fcm_dpo/margin": 65.45736694335938,
"fcm_dpo/q_t": 0.37715762853622437,
"grad_norm": 10.28955364227295,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -1.706976294517517,
"logits/rejected": -1.7277071475982666,
"logps/chosen": -201.939697265625,
"logps/ref_chosen": -72.23806762695312,
"logps/ref_rejected": -91.67182922363281,
"logps/rejected": -286.830810546875,
"loss": 0.9911,
"margin_dpo/margin_mean": 65.45736694335938,
"margin_dpo/margin_std": 78.65293884277344,
"step": 366
},
{
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.008751322515308857,
"fcm_dpo/delta": -0.05687132105231285,
"fcm_dpo/margin": 74.63752746582031,
"fcm_dpo/q_t": 0.3563315272331238,
"grad_norm": 10.187382698059082,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -1.7047667503356934,
"logits/rejected": -1.3686751127243042,
"logps/chosen": -207.76351928710938,
"logps/ref_chosen": -85.42201232910156,
"logps/ref_rejected": -83.51779174804688,
"logps/rejected": -280.496826171875,
"loss": 0.9312,
"margin_dpo/margin_mean": 74.63752746582031,
"margin_dpo/margin_std": 76.12283325195312,
"step": 367
},
{
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.008638489991426468,
"fcm_dpo/delta": -0.10402532666921616,
"fcm_dpo/margin": 80.64979553222656,
"fcm_dpo/q_t": 0.34605395793914795,
"grad_norm": 10.933233261108398,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -1.6838431358337402,
"logits/rejected": -1.6408352851867676,
"logps/chosen": -189.81283569335938,
"logps/ref_chosen": -73.84112548828125,
"logps/ref_rejected": -104.69528198242188,
"logps/rejected": -301.3167724609375,
"loss": 0.9216,
"margin_dpo/margin_mean": 80.64979553222656,
"margin_dpo/margin_std": 81.0087661743164,
"step": 368
},
{
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.008491117507219315,
"fcm_dpo/delta": -0.05438908189535141,
"fcm_dpo/margin": 76.65316772460938,
"fcm_dpo/q_t": 0.35426294803619385,
"grad_norm": 11.064504623413086,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -1.6709437370300293,
"logits/rejected": -1.30281400680542,
"logps/chosen": -224.9449920654297,
"logps/ref_chosen": -98.46195983886719,
"logps/ref_rejected": -99.15672302246094,
"logps/rejected": -302.29290771484375,
"loss": 0.915,
"margin_dpo/margin_mean": 76.65316772460938,
"margin_dpo/margin_std": 69.63180541992188,
"step": 369
},
{
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.008652754127979279,
"fcm_dpo/delta": 0.14852631092071533,
"fcm_dpo/margin": 52.930091857910156,
"fcm_dpo/q_t": 0.3984871506690979,
"grad_norm": 11.095954895019531,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -1.8884553909301758,
"logits/rejected": -1.686102032661438,
"logps/chosen": -236.5087432861328,
"logps/ref_chosen": -96.77574157714844,
"logps/ref_rejected": -102.74411010742188,
"logps/rejected": -295.4071960449219,
"loss": 1.0924,
"margin_dpo/margin_mean": 52.930091857910156,
"margin_dpo/margin_std": 80.57627868652344,
"step": 370
},
{
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.008863227441906929,
"fcm_dpo/delta": 0.09689568728208542,
"fcm_dpo/margin": 57.24983215332031,
"fcm_dpo/q_t": 0.38820239901542664,
"grad_norm": 12.403904914855957,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -1.7923095226287842,
"logits/rejected": -1.426107406616211,
"logps/chosen": -235.0094451904297,
"logps/ref_chosen": -102.99020385742188,
"logps/ref_rejected": -105.35062408447266,
"logps/rejected": -294.61968994140625,
"loss": 1.0522,
"margin_dpo/margin_mean": 57.24983215332031,
"margin_dpo/margin_std": 77.89881134033203,
"step": 371
},
{
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.00865775067359209,
"fcm_dpo/delta": -0.1739071160554886,
"fcm_dpo/margin": 87.87632751464844,
"fcm_dpo/q_t": 0.3345174789428711,
"grad_norm": 9.950636863708496,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -1.8596062660217285,
"logits/rejected": -1.2332671880722046,
"logps/chosen": -204.2379608154297,
"logps/ref_chosen": -91.02623748779297,
"logps/ref_rejected": -85.6102294921875,
"logps/rejected": -286.69830322265625,
"loss": 0.8762,
"margin_dpo/margin_mean": 87.87632751464844,
"margin_dpo/margin_std": 83.01412963867188,
"step": 372
},
{
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.008587082847952843,
"fcm_dpo/delta": 0.02046929858624935,
"fcm_dpo/margin": 67.62651824951172,
"fcm_dpo/q_t": 0.3730928897857666,
"grad_norm": 13.208666801452637,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -2.059213161468506,
"logits/rejected": -1.5383615493774414,
"logps/chosen": -229.08303833007812,
"logps/ref_chosen": -100.07307434082031,
"logps/ref_rejected": -91.11634826660156,
"logps/rejected": -287.7528076171875,
"loss": 1.0004,
"margin_dpo/margin_mean": 67.62651062011719,
"margin_dpo/margin_std": 82.54972839355469,
"step": 373
},
{
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.008557072840631008,
"fcm_dpo/delta": -0.01339799165725708,
"fcm_dpo/margin": 71.54954528808594,
"fcm_dpo/q_t": 0.36381733417510986,
"grad_norm": 10.388188362121582,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -2.204723834991455,
"logits/rejected": -1.814666986465454,
"logps/chosen": -211.86734008789062,
"logps/ref_chosen": -98.01234436035156,
"logps/ref_rejected": -100.75224304199219,
"logps/rejected": -286.1567687988281,
"loss": 0.9477,
"margin_dpo/margin_mean": 71.54954528808594,
"margin_dpo/margin_std": 71.1581802368164,
"step": 374
},
{
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.00840807519853115,
"fcm_dpo/delta": -0.07696233689785004,
"fcm_dpo/margin": 79.625244140625,
"fcm_dpo/q_t": 0.3517950177192688,
"grad_norm": 10.864441871643066,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -1.7122812271118164,
"logits/rejected": -1.4867812395095825,
"logps/chosen": -202.69105529785156,
"logps/ref_chosen": -84.39627838134766,
"logps/ref_rejected": -95.7188491821289,
"logps/rejected": -293.63885498046875,
"loss": 0.9258,
"margin_dpo/margin_mean": 79.62525177001953,
"margin_dpo/margin_std": 77.73374938964844,
"step": 375
},
{
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.00827401876449585,
"fcm_dpo/delta": -0.13464483618736267,
"fcm_dpo/margin": 87.5845718383789,
"fcm_dpo/q_t": 0.3414178490638733,
"grad_norm": 11.585356712341309,
"learning_rate": 2.5e-07,
"logits/chosen": -1.7216506004333496,
"logits/rejected": -1.4160125255584717,
"logps/chosen": -206.31048583984375,
"logps/ref_chosen": -95.73770141601562,
"logps/ref_rejected": -111.97331237792969,
"logps/rejected": -310.13067626953125,
"loss": 0.8974,
"margin_dpo/margin_mean": 87.5845718383789,
"margin_dpo/margin_std": 86.00065612792969,
"step": 376
},
{
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.008009790442883968,
"fcm_dpo/delta": -0.14207610487937927,
"fcm_dpo/margin": 91.17427062988281,
"fcm_dpo/q_t": 0.339458703994751,
"grad_norm": 10.219127655029297,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -1.5154132843017578,
"logits/rejected": -1.391338586807251,
"logps/chosen": -199.823486328125,
"logps/ref_chosen": -78.92694854736328,
"logps/ref_rejected": -105.89892578125,
"logps/rejected": -317.9697265625,
"loss": 0.8889,
"margin_dpo/margin_mean": 91.17427062988281,
"margin_dpo/margin_std": 85.8505859375,
"step": 377
},
{
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.008050184696912766,
"fcm_dpo/delta": 0.0010249018669128418,
"fcm_dpo/margin": 74.31245422363281,
"fcm_dpo/q_t": 0.36496371030807495,
"grad_norm": 10.996868133544922,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -2.0298149585723877,
"logits/rejected": -1.7492952346801758,
"logps/chosen": -192.58639526367188,
"logps/ref_chosen": -75.37240600585938,
"logps/ref_rejected": -93.70409393310547,
"logps/rejected": -285.2305603027344,
"loss": 0.9746,
"margin_dpo/margin_mean": 74.31245422363281,
"margin_dpo/margin_std": 80.56228637695312,
"step": 378
},
{
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.008063051849603653,
"fcm_dpo/delta": 0.05794180929660797,
"fcm_dpo/margin": 67.62254333496094,
"fcm_dpo/q_t": 0.38331836462020874,
"grad_norm": 10.951027870178223,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -1.8049771785736084,
"logits/rejected": -1.708174467086792,
"logps/chosen": -221.08468627929688,
"logps/ref_chosen": -88.85568237304688,
"logps/ref_rejected": -121.38114166259766,
"logps/rejected": -321.2326965332031,
"loss": 1.0446,
"margin_dpo/margin_mean": 67.62254333496094,
"margin_dpo/margin_std": 94.80967712402344,
"step": 379
},
{
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.008038859814405441,
"fcm_dpo/delta": -0.04042629525065422,
"fcm_dpo/margin": 79.3418197631836,
"fcm_dpo/q_t": 0.364249050617218,
"grad_norm": 13.981470108032227,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -1.3323712348937988,
"logits/rejected": -1.299829125404358,
"logps/chosen": -227.69309997558594,
"logps/ref_chosen": -79.83321380615234,
"logps/ref_rejected": -110.7447509765625,
"logps/rejected": -337.94647216796875,
"loss": 0.9705,
"margin_dpo/margin_mean": 79.3418197631836,
"margin_dpo/margin_std": 95.55670928955078,
"step": 380
},
{
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.00806824117898941,
"fcm_dpo/delta": 0.03291797637939453,
"fcm_dpo/margin": 70.51676940917969,
"fcm_dpo/q_t": 0.37344565987586975,
"grad_norm": 12.015775680541992,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -1.5715270042419434,
"logits/rejected": -1.0036929845809937,
"logps/chosen": -214.4466552734375,
"logps/ref_chosen": -86.56683349609375,
"logps/ref_rejected": -84.416748046875,
"logps/rejected": -282.8133544921875,
"loss": 0.9954,
"margin_dpo/margin_mean": 70.51676940917969,
"margin_dpo/margin_std": 82.17634582519531,
"step": 381
},
{
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.007982393726706505,
"fcm_dpo/delta": -0.02618768811225891,
"fcm_dpo/margin": 78.04951477050781,
"fcm_dpo/q_t": 0.36266398429870605,
"grad_norm": 10.413619041442871,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -1.9479053020477295,
"logits/rejected": -1.6788208484649658,
"logps/chosen": -233.09474182128906,
"logps/ref_chosen": -104.46180725097656,
"logps/ref_rejected": -114.60223388671875,
"logps/rejected": -321.28466796875,
"loss": 0.9563,
"margin_dpo/margin_mean": 78.04950714111328,
"margin_dpo/margin_std": 81.13053894042969,
"step": 382
},
{
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.008084449917078018,
"fcm_dpo/delta": 0.05731251835823059,
"fcm_dpo/margin": 67.51455688476562,
"fcm_dpo/q_t": 0.3798357844352722,
"grad_norm": 11.326423645019531,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -1.5943937301635742,
"logits/rejected": -1.4752601385116577,
"logps/chosen": -219.62017822265625,
"logps/ref_chosen": -82.11607360839844,
"logps/ref_rejected": -98.77716064453125,
"logps/rejected": -303.79583740234375,
"loss": 1.0111,
"margin_dpo/margin_mean": 67.51455688476562,
"margin_dpo/margin_std": 80.03602600097656,
"step": 383
},
{
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.008265078067779541,
"fcm_dpo/delta": 0.04529654234647751,
"fcm_dpo/margin": 67.14823913574219,
"fcm_dpo/q_t": 0.37842482328414917,
"grad_norm": 13.126636505126953,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -1.7413570880889893,
"logits/rejected": -1.5120550394058228,
"logps/chosen": -228.9114990234375,
"logps/ref_chosen": -80.58525085449219,
"logps/ref_rejected": -91.44789123535156,
"logps/rejected": -306.92236328125,
"loss": 1.0505,
"margin_dpo/margin_mean": 67.14823913574219,
"margin_dpo/margin_std": 92.4796371459961,
"step": 384
},
{
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.008029671385884285,
"fcm_dpo/delta": -0.09009828418493271,
"fcm_dpo/margin": 84.93191528320312,
"fcm_dpo/q_t": 0.35304510593414307,
"grad_norm": 12.234119415283203,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -2.2260751724243164,
"logits/rejected": -1.6197378635406494,
"logps/chosen": -217.18515014648438,
"logps/ref_chosen": -92.76016235351562,
"logps/ref_rejected": -84.45997619628906,
"logps/rejected": -293.81689453125,
"loss": 0.9373,
"margin_dpo/margin_mean": 84.93191528320312,
"margin_dpo/margin_std": 90.8809814453125,
"step": 385
},
{
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.007991664111614227,
"fcm_dpo/delta": -0.05802081525325775,
"fcm_dpo/margin": 81.86547088623047,
"fcm_dpo/q_t": 0.35287386178970337,
"grad_norm": 9.855354309082031,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -1.597925066947937,
"logits/rejected": -1.3119235038757324,
"logps/chosen": -215.71453857421875,
"logps/ref_chosen": -85.35896301269531,
"logps/ref_rejected": -96.99065399169922,
"logps/rejected": -309.211669921875,
"loss": 0.9444,
"margin_dpo/margin_mean": 81.86546325683594,
"margin_dpo/margin_std": 86.34203338623047,
"step": 386
},
{
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.007988503202795982,
"fcm_dpo/delta": 0.00018469803035259247,
"fcm_dpo/margin": 75.06538391113281,
"fcm_dpo/q_t": 0.36637941002845764,
"grad_norm": 12.132233619689941,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -2.5154049396514893,
"logits/rejected": -1.8449430465698242,
"logps/chosen": -253.63768005371094,
"logps/ref_chosen": -102.68814086914062,
"logps/ref_rejected": -99.42858123779297,
"logps/rejected": -325.4435119628906,
"loss": 0.9641,
"margin_dpo/margin_mean": 75.06538391113281,
"margin_dpo/margin_std": 79.062255859375,
"step": 387
},
{
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.00789298489689827,
"fcm_dpo/delta": -0.05572465807199478,
"fcm_dpo/margin": 82.60919952392578,
"fcm_dpo/q_t": 0.35591650009155273,
"grad_norm": 10.612791061401367,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -1.7140839099884033,
"logits/rejected": -1.5836071968078613,
"logps/chosen": -234.08363342285156,
"logps/ref_chosen": -83.58523559570312,
"logps/ref_rejected": -110.10037231445312,
"logps/rejected": -343.2079772949219,
"loss": 0.9393,
"margin_dpo/margin_mean": 82.60919189453125,
"margin_dpo/margin_std": 85.55877685546875,
"step": 388
},
{
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.007903015241026878,
"fcm_dpo/delta": 0.023557592183351517,
"fcm_dpo/margin": 73.11224365234375,
"fcm_dpo/q_t": 0.3752698302268982,
"grad_norm": 12.45739459991455,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -2.490940570831299,
"logits/rejected": -2.0545895099639893,
"logps/chosen": -244.65711975097656,
"logps/ref_chosen": -89.822265625,
"logps/ref_rejected": -84.85376739501953,
"logps/rejected": -312.80084228515625,
"loss": 1.0175,
"margin_dpo/margin_mean": 73.11224365234375,
"margin_dpo/margin_std": 96.38984680175781,
"step": 389
},
{
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.007932371459901333,
"fcm_dpo/delta": 0.060371283441782,
"fcm_dpo/margin": 68.37907409667969,
"fcm_dpo/q_t": 0.3861868381500244,
"grad_norm": 10.967287063598633,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -1.6026420593261719,
"logits/rejected": -1.389652132987976,
"logps/chosen": -234.77609252929688,
"logps/ref_chosen": -77.53244018554688,
"logps/ref_rejected": -92.58392333984375,
"logps/rejected": -318.2066650390625,
"loss": 1.0473,
"margin_dpo/margin_mean": 68.37908172607422,
"margin_dpo/margin_std": 97.29962158203125,
"step": 390
},
{
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.007975282147526741,
"fcm_dpo/delta": -0.00302848219871521,
"fcm_dpo/margin": 75.55020141601562,
"fcm_dpo/q_t": 0.3686513304710388,
"grad_norm": 11.11927604675293,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -1.8575489521026611,
"logits/rejected": -1.4008793830871582,
"logps/chosen": -229.98764038085938,
"logps/ref_chosen": -88.2329330444336,
"logps/ref_rejected": -93.00756072998047,
"logps/rejected": -310.3124694824219,
"loss": 0.9856,
"margin_dpo/margin_mean": 75.55020141601562,
"margin_dpo/margin_std": 88.94224548339844,
"step": 391
},
{
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.007913609966635704,
"fcm_dpo/delta": -0.104721799492836,
"fcm_dpo/margin": 88.1395263671875,
"fcm_dpo/q_t": 0.3488277792930603,
"grad_norm": 13.28966999053955,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -1.2604981660842896,
"logits/rejected": -1.1343798637390137,
"logps/chosen": -231.86595153808594,
"logps/ref_chosen": -87.74442291259766,
"logps/ref_rejected": -108.57071685791016,
"logps/rejected": -340.831787109375,
"loss": 0.9373,
"margin_dpo/margin_mean": 88.1395263671875,
"margin_dpo/margin_std": 97.56378173828125,
"step": 392
},
{
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0078022656962275505,
"fcm_dpo/delta": -0.027617137879133224,
"fcm_dpo/margin": 80.21553802490234,
"fcm_dpo/q_t": 0.3618813455104828,
"grad_norm": 10.270767211914062,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -1.5781824588775635,
"logits/rejected": -1.2512006759643555,
"logps/chosen": -205.15457153320312,
"logps/ref_chosen": -76.56381225585938,
"logps/ref_rejected": -79.72618103027344,
"logps/rejected": -288.5325012207031,
"loss": 0.9551,
"margin_dpo/margin_mean": 80.21553802490234,
"margin_dpo/margin_std": 85.68644714355469,
"step": 393
},
{
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.007711788639426231,
"fcm_dpo/delta": -0.06800325214862823,
"fcm_dpo/margin": 86.02757263183594,
"fcm_dpo/q_t": 0.3508681356906891,
"grad_norm": 10.564422607421875,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -1.5955944061279297,
"logits/rejected": -1.3630306720733643,
"logps/chosen": -219.39657592773438,
"logps/ref_chosen": -84.69866180419922,
"logps/ref_rejected": -97.457763671875,
"logps/rejected": -318.1832275390625,
"loss": 0.9127,
"margin_dpo/margin_mean": 86.02757263183594,
"margin_dpo/margin_std": 78.89590454101562,
"step": 394
},
{
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.00774542149156332,
"fcm_dpo/delta": 0.08366027474403381,
"fcm_dpo/margin": 67.20747375488281,
"fcm_dpo/q_t": 0.3928409814834595,
"grad_norm": 11.868879318237305,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -2.6224658489227295,
"logits/rejected": -2.231447219848633,
"logps/chosen": -229.33065795898438,
"logps/ref_chosen": -88.88763427734375,
"logps/ref_rejected": -91.0455322265625,
"logps/rejected": -298.696044921875,
"loss": 1.1162,
"margin_dpo/margin_mean": 67.20747375488281,
"margin_dpo/margin_std": 117.15966796875,
"step": 395
},
{
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.007993882521986961,
"fcm_dpo/delta": 0.14788684248924255,
"fcm_dpo/margin": 57.40891647338867,
"fcm_dpo/q_t": 0.3995135426521301,
"grad_norm": 15.381148338317871,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -1.979184627532959,
"logits/rejected": -1.4956262111663818,
"logps/chosen": -252.1164093017578,
"logps/ref_chosen": -102.47132873535156,
"logps/ref_rejected": -93.23478698730469,
"logps/rejected": -300.28875732421875,
"loss": 1.0905,
"margin_dpo/margin_mean": 57.40891647338867,
"margin_dpo/margin_std": 87.18148803710938,
"step": 396
},
{
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0077405949123203754,
"fcm_dpo/delta": -0.2481895387172699,
"fcm_dpo/margin": 106.80502319335938,
"fcm_dpo/q_t": 0.3245747983455658,
"grad_norm": 11.806785583496094,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -2.333134889602661,
"logits/rejected": -1.7398998737335205,
"logps/chosen": -234.05245971679688,
"logps/ref_chosen": -101.04441833496094,
"logps/ref_rejected": -108.8511734008789,
"logps/rejected": -348.66424560546875,
"loss": 0.8711,
"margin_dpo/margin_mean": 106.80502319335938,
"margin_dpo/margin_std": 107.98983001708984,
"step": 397
},
{
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.00766055379062891,
"fcm_dpo/delta": 0.006077233701944351,
"fcm_dpo/margin": 77.572265625,
"fcm_dpo/q_t": 0.3723902702331543,
"grad_norm": 11.528839111328125,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -2.045728921890259,
"logits/rejected": -1.755936861038208,
"logps/chosen": -235.32550048828125,
"logps/ref_chosen": -95.04922485351562,
"logps/ref_rejected": -114.00563049316406,
"logps/rejected": -331.85418701171875,
"loss": 1.0144,
"margin_dpo/margin_mean": 77.572265625,
"margin_dpo/margin_std": 103.19320678710938,
"step": 398
},
{
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0075286137871444225,
"fcm_dpo/delta": -0.15626700222492218,
"fcm_dpo/margin": 98.94232940673828,
"fcm_dpo/q_t": 0.3343579173088074,
"grad_norm": 10.617576599121094,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -2.003676414489746,
"logits/rejected": -1.1966228485107422,
"logps/chosen": -222.55133056640625,
"logps/ref_chosen": -96.63854217529297,
"logps/ref_rejected": -83.53645324707031,
"logps/rejected": -308.3916015625,
"loss": 0.8622,
"margin_dpo/margin_mean": 98.94233703613281,
"margin_dpo/margin_std": 83.51806640625,
"step": 399
},
{
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.007274920120835304,
"fcm_dpo/delta": -0.14528706669807434,
"fcm_dpo/margin": 100.96365356445312,
"fcm_dpo/q_t": 0.3401756286621094,
"grad_norm": 14.20820426940918,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -2.292538642883301,
"logits/rejected": -1.8154387474060059,
"logps/chosen": -234.29095458984375,
"logps/ref_chosen": -105.30314636230469,
"logps/ref_rejected": -108.6555404663086,
"logps/rejected": -338.60699462890625,
"loss": 0.9031,
"margin_dpo/margin_mean": 100.96365356445312,
"margin_dpo/margin_std": 101.42951965332031,
"step": 400
},
{
"epoch": 0.5873715124816447,
"eval_fcm_dpo/beta": 0.0072255851700901985,
"eval_logits/chosen": -1.9563562870025635,
"eval_logits/rejected": -1.5862782001495361,
"eval_logps/chosen": -255.01856994628906,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -314.7674865722656,
"eval_loss": 0.5320852398872375,
"eval_margin_dpo/margin_mean": 66.17474365234375,
"eval_margin_dpo/margin_std": 89.82697296142578,
"eval_runtime": 43.7256,
"eval_samples_per_second": 53.493,
"eval_steps_per_second": 1.692,
"step": 400
},
{
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.007348657585680485,
"fcm_dpo/delta": 0.13697752356529236,
"fcm_dpo/margin": 63.91277313232422,
"fcm_dpo/q_t": 0.40259644389152527,
"grad_norm": 14.34986686706543,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -1.7043498754501343,
"logits/rejected": -1.4674514532089233,
"logps/chosen": -227.89923095703125,
"logps/ref_chosen": -84.28916931152344,
"logps/ref_rejected": -90.943115234375,
"logps/rejected": -298.4659423828125,
"loss": 1.1053,
"margin_dpo/margin_mean": 63.91277313232422,
"margin_dpo/margin_std": 107.23680114746094,
"step": 401
},
{
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.007161559071391821,
"fcm_dpo/delta": -0.25808143615722656,
"fcm_dpo/margin": 116.8514404296875,
"fcm_dpo/q_t": 0.3132210671901703,
"grad_norm": 11.211475372314453,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -1.7466460466384888,
"logits/rejected": -1.2662107944488525,
"logps/chosen": -213.04330444335938,
"logps/ref_chosen": -83.59312438964844,
"logps/ref_rejected": -87.81027221679688,
"logps/rejected": -334.11187744140625,
"loss": 0.8053,
"margin_dpo/margin_mean": 116.8514404296875,
"margin_dpo/margin_std": 88.61773681640625,
"step": 402
},
{
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.007018306292593479,
"fcm_dpo/delta": -0.011818572878837585,
"fcm_dpo/margin": 87.03970336914062,
"fcm_dpo/q_t": 0.3649277985095978,
"grad_norm": 10.342928886413574,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -2.358241081237793,
"logits/rejected": -1.9733853340148926,
"logps/chosen": -205.4605712890625,
"logps/ref_chosen": -73.75308227539062,
"logps/ref_rejected": -83.92012786865234,
"logps/rejected": -302.6673278808594,
"loss": 0.9675,
"margin_dpo/margin_mean": 87.03970336914062,
"margin_dpo/margin_std": 95.28384399414062,
"step": 403
},
{
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.007098081987351179,
"fcm_dpo/delta": 0.07683449983596802,
"fcm_dpo/margin": 74.28379821777344,
"fcm_dpo/q_t": 0.38582703471183777,
"grad_norm": 15.550983428955078,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -2.0557281970977783,
"logits/rejected": -1.7230305671691895,
"logps/chosen": -237.1645050048828,
"logps/ref_chosen": -79.67617797851562,
"logps/ref_rejected": -84.280517578125,
"logps/rejected": -316.0526428222656,
"loss": 1.0557,
"margin_dpo/margin_mean": 74.28379821777344,
"margin_dpo/margin_std": 107.16883087158203,
"step": 404
},
{
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.007242183201014996,
"fcm_dpo/delta": 0.046799372881650925,
"fcm_dpo/margin": 76.58912658691406,
"fcm_dpo/q_t": 0.3775564432144165,
"grad_norm": 9.922403335571289,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -1.4873124361038208,
"logits/rejected": -1.0152088403701782,
"logps/chosen": -255.55612182617188,
"logps/ref_chosen": -96.46195220947266,
"logps/ref_rejected": -92.87071228027344,
"logps/rejected": -328.55401611328125,
"loss": 0.9945,
"margin_dpo/margin_mean": 76.58912658691406,
"margin_dpo/margin_std": 85.40648651123047,
"step": 405
},
{
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.007322330959141254,
"fcm_dpo/delta": 0.09602566063404083,
"fcm_dpo/margin": 69.48136138916016,
"fcm_dpo/q_t": 0.3899584114551544,
"grad_norm": 17.966711044311523,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -2.0658977031707764,
"logits/rejected": -1.8964214324951172,
"logps/chosen": -228.33712768554688,
"logps/ref_chosen": -78.13396453857422,
"logps/ref_rejected": -98.28359985351562,
"logps/rejected": -317.9681396484375,
"loss": 1.0736,
"margin_dpo/margin_mean": 69.48136138916016,
"margin_dpo/margin_std": 104.44835662841797,
"step": 406
},
{
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.007518821395933628,
"fcm_dpo/delta": 0.14913788437843323,
"fcm_dpo/margin": 60.868133544921875,
"fcm_dpo/q_t": 0.39967426657676697,
"grad_norm": 13.635825157165527,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -2.2239134311676025,
"logits/rejected": -1.7089815139770508,
"logps/chosen": -243.0015869140625,
"logps/ref_chosen": -84.01283264160156,
"logps/ref_rejected": -82.78103637695312,
"logps/rejected": -302.6379089355469,
"loss": 1.0895,
"margin_dpo/margin_mean": 60.868133544921875,
"margin_dpo/margin_std": 93.1767578125,
"step": 407
},
{
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.007688861805945635,
"fcm_dpo/delta": 0.11370319128036499,
"fcm_dpo/margin": 63.99238204956055,
"fcm_dpo/q_t": 0.39551639556884766,
"grad_norm": 13.247913360595703,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -2.340407133102417,
"logits/rejected": -1.736910343170166,
"logps/chosen": -254.63917541503906,
"logps/ref_chosen": -104.46174621582031,
"logps/ref_rejected": -96.37217712402344,
"logps/rejected": -310.5419921875,
"loss": 1.0843,
"margin_dpo/margin_mean": 63.99237823486328,
"margin_dpo/margin_std": 99.77020263671875,
"step": 408
},
{
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.007855923846364021,
"fcm_dpo/delta": 0.098321832716465,
"fcm_dpo/margin": 64.50284576416016,
"fcm_dpo/q_t": 0.3866105079650879,
"grad_norm": 11.660709381103516,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -1.5960277318954468,
"logits/rejected": -1.1638531684875488,
"logps/chosen": -259.2583923339844,
"logps/ref_chosen": -97.66830444335938,
"logps/ref_rejected": -90.04583740234375,
"logps/rejected": -316.13873291015625,
"loss": 1.0741,
"margin_dpo/margin_mean": 64.50284576416016,
"margin_dpo/margin_std": 95.21458435058594,
"step": 409
},
{
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.007921447977423668,
"fcm_dpo/delta": 0.01372436247766018,
"fcm_dpo/margin": 74.1120834350586,
"fcm_dpo/q_t": 0.3695540428161621,
"grad_norm": 11.11070442199707,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -1.7564365863800049,
"logits/rejected": -1.6230534315109253,
"logps/chosen": -235.60093688964844,
"logps/ref_chosen": -76.46923828125,
"logps/ref_rejected": -88.64064025878906,
"logps/rejected": -321.8843994140625,
"loss": 0.9988,
"margin_dpo/margin_mean": 74.11209106445312,
"margin_dpo/margin_std": 90.84548950195312,
"step": 410
},
{
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.007924282923340797,
"fcm_dpo/delta": 0.025149427354335785,
"fcm_dpo/margin": 72.70220184326172,
"fcm_dpo/q_t": 0.37517866492271423,
"grad_norm": 11.753535270690918,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -2.482978343963623,
"logits/rejected": -2.093186140060425,
"logps/chosen": -254.4764862060547,
"logps/ref_chosen": -87.16630554199219,
"logps/ref_rejected": -87.09603118896484,
"logps/rejected": -327.1083984375,
"loss": 0.9966,
"margin_dpo/margin_mean": 72.70220184326172,
"margin_dpo/margin_std": 86.63870239257812,
"step": 411
},
{
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.007970331236720085,
"fcm_dpo/delta": 0.03290639817714691,
"fcm_dpo/margin": 71.35936737060547,
"fcm_dpo/q_t": 0.38153523206710815,
"grad_norm": 12.409209251403809,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -2.412627696990967,
"logits/rejected": -2.191441059112549,
"logps/chosen": -236.4732208251953,
"logps/ref_chosen": -78.94734191894531,
"logps/ref_rejected": -106.10554504394531,
"logps/rejected": -334.99078369140625,
"loss": 1.0581,
"margin_dpo/margin_mean": 71.35936737060547,
"margin_dpo/margin_std": 107.4913330078125,
"step": 412
},
{
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.0079441387206316,
"fcm_dpo/delta": -0.033406391739845276,
"fcm_dpo/margin": 79.40829467773438,
"fcm_dpo/q_t": 0.3635585904121399,
"grad_norm": 11.994926452636719,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -1.2549076080322266,
"logits/rejected": -0.9982696771621704,
"logps/chosen": -265.3006591796875,
"logps/ref_chosen": -95.69470977783203,
"logps/ref_rejected": -107.96085357666016,
"logps/rejected": -356.97509765625,
"loss": 0.9942,
"margin_dpo/margin_mean": 79.40829467773438,
"margin_dpo/margin_std": 102.14022827148438,
"step": 413
},
{
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.007994389161467552,
"fcm_dpo/delta": 0.021429970860481262,
"fcm_dpo/margin": 72.52461242675781,
"fcm_dpo/q_t": 0.37816792726516724,
"grad_norm": 16.40660858154297,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -1.648315668106079,
"logits/rejected": -1.2972140312194824,
"logps/chosen": -245.08993530273438,
"logps/ref_chosen": -88.27667236328125,
"logps/ref_rejected": -92.87004089355469,
"logps/rejected": -322.2078857421875,
"loss": 1.0534,
"margin_dpo/margin_mean": 72.52461242675781,
"margin_dpo/margin_std": 109.23208618164062,
"step": 414
},
{
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.008126772940158844,
"fcm_dpo/delta": 0.09716188907623291,
"fcm_dpo/margin": 62.45792007446289,
"fcm_dpo/q_t": 0.39152491092681885,
"grad_norm": 13.653444290161133,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -1.602043867111206,
"logits/rejected": -1.2246946096420288,
"logps/chosen": -278.5191955566406,
"logps/ref_chosen": -108.91590118408203,
"logps/ref_rejected": -107.47135925292969,
"logps/rejected": -339.5325622558594,
"loss": 1.0767,
"margin_dpo/margin_mean": 62.45792007446289,
"margin_dpo/margin_std": 95.18862915039062,
"step": 415
},
{
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.008111116476356983,
"fcm_dpo/delta": -0.1124371886253357,
"fcm_dpo/margin": 86.70777893066406,
"fcm_dpo/q_t": 0.3507331311702728,
"grad_norm": 11.193100929260254,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -1.8388745784759521,
"logits/rejected": -1.336613655090332,
"logps/chosen": -259.5759582519531,
"logps/ref_chosen": -93.39888000488281,
"logps/ref_rejected": -97.6729736328125,
"logps/rejected": -350.5578308105469,
"loss": 0.9505,
"margin_dpo/margin_mean": 86.70777893066406,
"margin_dpo/margin_std": 98.13165283203125,
"step": 416
},
{
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.007943946868181229,
"fcm_dpo/delta": -0.04033021628856659,
"fcm_dpo/margin": 80.280517578125,
"fcm_dpo/q_t": 0.36008456349372864,
"grad_norm": 11.344613075256348,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -2.1126794815063477,
"logits/rejected": -1.1838053464889526,
"logps/chosen": -239.34365844726562,
"logps/ref_chosen": -83.53533172607422,
"logps/ref_rejected": -74.44184875488281,
"logps/rejected": -310.53070068359375,
"loss": 0.9601,
"margin_dpo/margin_mean": 80.280517578125,
"margin_dpo/margin_std": 89.32632446289062,
"step": 417
},
{
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.007776966318488121,
"fcm_dpo/delta": -0.20287159085273743,
"fcm_dpo/margin": 101.0872802734375,
"fcm_dpo/q_t": 0.3297736644744873,
"grad_norm": 14.166253089904785,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -2.374035358428955,
"logits/rejected": -1.742133378982544,
"logps/chosen": -255.0244598388672,
"logps/ref_chosen": -108.22152709960938,
"logps/ref_rejected": -111.8646469116211,
"logps/rejected": -359.7548828125,
"loss": 0.8604,
"margin_dpo/margin_mean": 101.0872802734375,
"margin_dpo/margin_std": 87.68946838378906,
"step": 418
},
{
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.007567869499325752,
"fcm_dpo/delta": -0.06575581431388855,
"fcm_dpo/margin": 87.35041809082031,
"fcm_dpo/q_t": 0.3550986051559448,
"grad_norm": 11.697413444519043,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -1.4137977361679077,
"logits/rejected": -0.960525631904602,
"logps/chosen": -251.655517578125,
"logps/ref_chosen": -97.18328094482422,
"logps/ref_rejected": -98.18531799316406,
"logps/rejected": -340.0079345703125,
"loss": 0.9383,
"margin_dpo/margin_mean": 87.35041809082031,
"margin_dpo/margin_std": 90.81410217285156,
"step": 419
},
{
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.007550552487373352,
"fcm_dpo/delta": 0.013763319700956345,
"fcm_dpo/margin": 77.66873931884766,
"fcm_dpo/q_t": 0.37317919731140137,
"grad_norm": 16.0394344329834,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -2.1781749725341797,
"logits/rejected": -1.0760204792022705,
"logps/chosen": -272.736572265625,
"logps/ref_chosen": -114.30848693847656,
"logps/ref_rejected": -75.68355560302734,
"logps/rejected": -311.7803955078125,
"loss": 1.0063,
"margin_dpo/margin_mean": 77.66874694824219,
"margin_dpo/margin_std": 97.64486694335938,
"step": 420
},
{
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.007453433237969875,
"fcm_dpo/delta": -0.04194428399205208,
"fcm_dpo/margin": 85.74388885498047,
"fcm_dpo/q_t": 0.35998445749282837,
"grad_norm": 10.710517883300781,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -1.8413752317428589,
"logits/rejected": -1.4720008373260498,
"logps/chosen": -239.76809692382812,
"logps/ref_chosen": -85.87985229492188,
"logps/ref_rejected": -96.33648681640625,
"logps/rejected": -335.9686279296875,
"loss": 0.9597,
"margin_dpo/margin_mean": 85.743896484375,
"margin_dpo/margin_std": 95.09271240234375,
"step": 421
},
{
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.007458934560418129,
"fcm_dpo/delta": -0.008748026564717293,
"fcm_dpo/margin": 81.53353881835938,
"fcm_dpo/q_t": 0.3671715557575226,
"grad_norm": 12.771909713745117,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -2.399049758911133,
"logits/rejected": -2.0303146839141846,
"logps/chosen": -251.56912231445312,
"logps/ref_chosen": -100.48060607910156,
"logps/ref_rejected": -94.40821838378906,
"logps/rejected": -327.0303039550781,
"loss": 0.9712,
"margin_dpo/margin_mean": 81.53353881835938,
"margin_dpo/margin_std": 92.20245361328125,
"step": 422
},
{
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.00747031532227993,
"fcm_dpo/delta": -0.004844721406698227,
"fcm_dpo/margin": 80.87257385253906,
"fcm_dpo/q_t": 0.3732362687587738,
"grad_norm": 12.241605758666992,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -1.9506278038024902,
"logits/rejected": -1.833554744720459,
"logps/chosen": -219.573486328125,
"logps/ref_chosen": -78.44993591308594,
"logps/ref_rejected": -92.04652404785156,
"logps/rejected": -314.04266357421875,
"loss": 1.0157,
"margin_dpo/margin_mean": 80.8725814819336,
"margin_dpo/margin_std": 108.63528442382812,
"step": 423
},
{
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.007418894208967686,
"fcm_dpo/delta": -0.012280648574233055,
"fcm_dpo/margin": 82.428466796875,
"fcm_dpo/q_t": 0.3642910420894623,
"grad_norm": 10.28632640838623,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -1.6936588287353516,
"logits/rejected": -1.089149832725525,
"logps/chosen": -245.31788635253906,
"logps/ref_chosen": -87.64236450195312,
"logps/ref_rejected": -72.36566162109375,
"logps/rejected": -312.46966552734375,
"loss": 0.973,
"margin_dpo/margin_mean": 82.428466796875,
"margin_dpo/margin_std": 93.11031341552734,
"step": 424
},
{
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.007599799428135157,
"fcm_dpo/delta": 0.1809379905462265,
"fcm_dpo/margin": 56.19786071777344,
"fcm_dpo/q_t": 0.41294217109680176,
"grad_norm": 13.262221336364746,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -1.6487901210784912,
"logits/rejected": -1.5741477012634277,
"logps/chosen": -241.32943725585938,
"logps/ref_chosen": -78.51979064941406,
"logps/ref_rejected": -102.74864196777344,
"logps/rejected": -321.75616455078125,
"loss": 1.1604,
"margin_dpo/margin_mean": 56.19786071777344,
"margin_dpo/margin_std": 110.29562377929688,
"step": 425
},
{
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.007676470093429089,
"fcm_dpo/delta": -0.026276282966136932,
"fcm_dpo/margin": 81.36367797851562,
"fcm_dpo/q_t": 0.36279982328414917,
"grad_norm": 14.7294282913208,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -1.8185904026031494,
"logits/rejected": -1.1083682775497437,
"logps/chosen": -247.37643432617188,
"logps/ref_chosen": -108.50582885742188,
"logps/ref_rejected": -88.300048828125,
"logps/rejected": -308.53436279296875,
"loss": 0.9699,
"margin_dpo/margin_mean": 81.36367797851562,
"margin_dpo/margin_std": 92.48495483398438,
"step": 426
},
{
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.007682330906391144,
"fcm_dpo/delta": 0.018056850880384445,
"fcm_dpo/margin": 75.87571716308594,
"fcm_dpo/q_t": 0.3702627420425415,
"grad_norm": 10.52995491027832,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -2.527616262435913,
"logits/rejected": -1.911233901977539,
"logps/chosen": -213.69146728515625,
"logps/ref_chosen": -99.12046813964844,
"logps/ref_rejected": -85.724609375,
"logps/rejected": -276.17132568359375,
"loss": 0.9796,
"margin_dpo/margin_mean": 75.87571716308594,
"margin_dpo/margin_std": 83.9215316772461,
"step": 427
},
{
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.007750866934657097,
"fcm_dpo/delta": 0.1085626631975174,
"fcm_dpo/margin": 64.08061981201172,
"fcm_dpo/q_t": 0.3931909203529358,
"grad_norm": 11.369186401367188,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -1.6850427389144897,
"logits/rejected": -1.4209712743759155,
"logps/chosen": -267.068115234375,
"logps/ref_chosen": -105.96925354003906,
"logps/ref_rejected": -109.10218048095703,
"logps/rejected": -334.28167724609375,
"loss": 1.0875,
"margin_dpo/margin_mean": 64.08061981201172,
"margin_dpo/margin_std": 99.4907455444336,
"step": 428
},
{
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0078573664650321,
"fcm_dpo/delta": -0.057729579508304596,
"fcm_dpo/margin": 83.04490661621094,
"fcm_dpo/q_t": 0.3629707396030426,
"grad_norm": 11.991065979003906,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -2.019653797149658,
"logits/rejected": -1.3182734251022339,
"logps/chosen": -259.6905517578125,
"logps/ref_chosen": -113.54487609863281,
"logps/ref_rejected": -98.24201965332031,
"logps/rejected": -327.4326171875,
"loss": 0.9858,
"margin_dpo/margin_mean": 83.04490661621094,
"margin_dpo/margin_std": 102.54899597167969,
"step": 429
},
{
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.00791855901479721,
"fcm_dpo/delta": 0.11582081019878387,
"fcm_dpo/margin": 61.77325439453125,
"fcm_dpo/q_t": 0.39517396688461304,
"grad_norm": 11.80427074432373,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -1.3399078845977783,
"logits/rejected": -1.298473596572876,
"logps/chosen": -235.0428009033203,
"logps/ref_chosen": -91.31936645507812,
"logps/ref_rejected": -110.1096420288086,
"logps/rejected": -315.6063232421875,
"loss": 1.0881,
"margin_dpo/margin_mean": 61.773258209228516,
"margin_dpo/margin_std": 96.4741439819336,
"step": 430
},
{
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.007927455008029938,
"fcm_dpo/delta": 0.0042446693405508995,
"fcm_dpo/margin": 75.17669677734375,
"fcm_dpo/q_t": 0.371150940656662,
"grad_norm": 12.48121166229248,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -1.9271609783172607,
"logits/rejected": -1.5382022857666016,
"logps/chosen": -235.30166625976562,
"logps/ref_chosen": -93.18122100830078,
"logps/ref_rejected": -98.13226318359375,
"logps/rejected": -315.4294128417969,
"loss": 1.0076,
"margin_dpo/margin_mean": 75.17669677734375,
"margin_dpo/margin_std": 96.68666076660156,
"step": 431
},
{
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.007823891006410122,
"fcm_dpo/delta": -0.13029845058918,
"fcm_dpo/margin": 92.16277313232422,
"fcm_dpo/q_t": 0.34236711263656616,
"grad_norm": 10.779977798461914,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -1.6222333908081055,
"logits/rejected": -1.0763202905654907,
"logps/chosen": -229.423828125,
"logps/ref_chosen": -104.43478393554688,
"logps/ref_rejected": -105.08955383300781,
"logps/rejected": -322.24139404296875,
"loss": 0.9015,
"margin_dpo/margin_mean": 92.16278076171875,
"margin_dpo/margin_std": 89.09420776367188,
"step": 432
},
{
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.007831670343875885,
"fcm_dpo/delta": 0.033429112285375595,
"fcm_dpo/margin": 72.43510437011719,
"fcm_dpo/q_t": 0.3780878186225891,
"grad_norm": 11.7706880569458,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -1.7953574657440186,
"logits/rejected": -1.4741289615631104,
"logps/chosen": -237.90676879882812,
"logps/ref_chosen": -89.84322357177734,
"logps/ref_rejected": -101.73345947265625,
"logps/rejected": -322.23211669921875,
"loss": 1.0302,
"margin_dpo/margin_mean": 72.43511199951172,
"margin_dpo/margin_std": 97.10591125488281,
"step": 433
},
{
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.007778637111186981,
"fcm_dpo/delta": -0.019643208011984825,
"fcm_dpo/margin": 79.50172424316406,
"fcm_dpo/q_t": 0.37190669775009155,
"grad_norm": 11.87337875366211,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -2.532245635986328,
"logits/rejected": -1.888089656829834,
"logps/chosen": -229.92483520507812,
"logps/ref_chosen": -97.6925277709961,
"logps/ref_rejected": -84.09129333496094,
"logps/rejected": -295.8253173828125,
"loss": 1.0079,
"margin_dpo/margin_mean": 79.50172424316406,
"margin_dpo/margin_std": 108.16815185546875,
"step": 434
},
{
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.007717899512499571,
"fcm_dpo/delta": -0.0468808189034462,
"fcm_dpo/margin": 83.42731475830078,
"fcm_dpo/q_t": 0.36141249537467957,
"grad_norm": 12.820802688598633,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -2.351439952850342,
"logits/rejected": -1.8964674472808838,
"logps/chosen": -214.12286376953125,
"logps/ref_chosen": -86.17192077636719,
"logps/ref_rejected": -93.751708984375,
"logps/rejected": -305.12994384765625,
"loss": 0.9617,
"margin_dpo/margin_mean": 83.42730712890625,
"margin_dpo/margin_std": 96.24213409423828,
"step": 435
},
{
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.00758952833712101,
"fcm_dpo/delta": -0.08244708925485611,
"fcm_dpo/margin": 89.17771911621094,
"fcm_dpo/q_t": 0.3484029769897461,
"grad_norm": 11.64830493927002,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -1.9208111763000488,
"logits/rejected": -1.0720462799072266,
"logps/chosen": -211.42588806152344,
"logps/ref_chosen": -105.78710174560547,
"logps/ref_rejected": -88.62471008300781,
"logps/rejected": -283.44122314453125,
"loss": 0.9044,
"margin_dpo/margin_mean": 89.17771911621094,
"margin_dpo/margin_std": 80.94717407226562,
"step": 436
},
{
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.007655961439013481,
"fcm_dpo/delta": 0.07603086531162262,
"fcm_dpo/margin": 68.94998168945312,
"fcm_dpo/q_t": 0.38024958968162537,
"grad_norm": 11.85803508758545,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -1.9982421398162842,
"logits/rejected": -1.3477120399475098,
"logps/chosen": -220.46435546875,
"logps/ref_chosen": -96.06204223632812,
"logps/ref_rejected": -89.01220703125,
"logps/rejected": -282.364501953125,
"loss": 1.0144,
"margin_dpo/margin_mean": 68.94998168945312,
"margin_dpo/margin_std": 80.42849731445312,
"step": 437
},
{
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.00769776338711381,
"fcm_dpo/delta": 0.05381428450345993,
"fcm_dpo/margin": 71.3000717163086,
"fcm_dpo/q_t": 0.3784894645214081,
"grad_norm": 12.725931167602539,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -2.1903576850891113,
"logits/rejected": -1.3835668563842773,
"logps/chosen": -227.7496337890625,
"logps/ref_chosen": -101.85537719726562,
"logps/ref_rejected": -89.44766235351562,
"logps/rejected": -286.6419677734375,
"loss": 1.0157,
"margin_dpo/margin_mean": 71.3000717163086,
"margin_dpo/margin_std": 86.569091796875,
"step": 438
},
{
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.007735989056527615,
"fcm_dpo/delta": 0.037646569311618805,
"fcm_dpo/margin": 72.81299591064453,
"fcm_dpo/q_t": 0.3774023652076721,
"grad_norm": 13.056401252746582,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -1.570444941520691,
"logits/rejected": -1.3217337131500244,
"logps/chosen": -219.38943481445312,
"logps/ref_chosen": -81.75563049316406,
"logps/ref_rejected": -90.58636474609375,
"logps/rejected": -301.03314208984375,
"loss": 1.0128,
"margin_dpo/margin_mean": 72.81299591064453,
"margin_dpo/margin_std": 88.81855773925781,
"step": 439
},
{
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.007678865920752287,
"fcm_dpo/delta": -0.07341113686561584,
"fcm_dpo/margin": 86.8636703491211,
"fcm_dpo/q_t": 0.35673287510871887,
"grad_norm": 10.554864883422852,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -2.4091434478759766,
"logits/rejected": -1.9146919250488281,
"logps/chosen": -229.0126190185547,
"logps/ref_chosen": -105.64108276367188,
"logps/ref_rejected": -103.4010009765625,
"logps/rejected": -313.63623046875,
"loss": 0.9517,
"margin_dpo/margin_mean": 86.86366271972656,
"margin_dpo/margin_std": 97.84516143798828,
"step": 440
},
{
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.007652001921087503,
"fcm_dpo/delta": -0.02822795696556568,
"fcm_dpo/margin": 81.84053039550781,
"fcm_dpo/q_t": 0.36398565769195557,
"grad_norm": 11.060223579406738,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -2.021010398864746,
"logits/rejected": -1.6652534008026123,
"logps/chosen": -243.90728759765625,
"logps/ref_chosen": -91.529541015625,
"logps/ref_rejected": -103.619384765625,
"logps/rejected": -337.8376770019531,
"loss": 0.9485,
"margin_dpo/margin_mean": 81.84053039550781,
"margin_dpo/margin_std": 87.4005126953125,
"step": 441
},
{
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.007508729584515095,
"fcm_dpo/delta": -0.13639724254608154,
"fcm_dpo/margin": 96.75260162353516,
"fcm_dpo/q_t": 0.33930784463882446,
"grad_norm": 11.458242416381836,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -1.8102037906646729,
"logits/rejected": -1.3520574569702148,
"logps/chosen": -236.66015625,
"logps/ref_chosen": -84.77755737304688,
"logps/ref_rejected": -83.82415771484375,
"logps/rejected": -332.4593505859375,
"loss": 0.8827,
"margin_dpo/margin_mean": 96.75260162353516,
"margin_dpo/margin_std": 88.31156158447266,
"step": 442
},
{
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.007541971281170845,
"fcm_dpo/delta": 0.12367384880781174,
"fcm_dpo/margin": 63.93056869506836,
"fcm_dpo/q_t": 0.3926345407962799,
"grad_norm": 11.647642135620117,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -2.023066997528076,
"logits/rejected": -1.4911339282989502,
"logps/chosen": -249.26025390625,
"logps/ref_chosen": -102.64927673339844,
"logps/ref_rejected": -93.03807067871094,
"logps/rejected": -303.5796203613281,
"loss": 1.057,
"margin_dpo/margin_mean": 63.930564880371094,
"margin_dpo/margin_std": 82.83998107910156,
"step": 443
},
{
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0077362204901874065,
"fcm_dpo/delta": 0.07435087114572525,
"fcm_dpo/margin": 68.44133758544922,
"fcm_dpo/q_t": 0.3861270546913147,
"grad_norm": 12.801765441894531,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -1.6765544414520264,
"logits/rejected": -1.5086448192596436,
"logps/chosen": -242.2406463623047,
"logps/ref_chosen": -87.91971588134766,
"logps/ref_rejected": -103.32345581054688,
"logps/rejected": -326.08575439453125,
"loss": 1.0539,
"margin_dpo/margin_mean": 68.44134521484375,
"margin_dpo/margin_std": 97.85394287109375,
"step": 444
},
{
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.007698110304772854,
"fcm_dpo/delta": -0.021563369780778885,
"fcm_dpo/margin": 80.50859832763672,
"fcm_dpo/q_t": 0.3627839982509613,
"grad_norm": 11.485919952392578,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -1.9858006238937378,
"logits/rejected": -1.5744715929031372,
"logps/chosen": -236.77423095703125,
"logps/ref_chosen": -101.40086364746094,
"logps/ref_rejected": -99.03790283203125,
"logps/rejected": -314.91986083984375,
"loss": 0.9579,
"margin_dpo/margin_mean": 80.50859832763672,
"margin_dpo/margin_std": 85.82842254638672,
"step": 445
},
{
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.007764734327793121,
"fcm_dpo/delta": 0.028329739347100258,
"fcm_dpo/margin": 73.82595825195312,
"fcm_dpo/q_t": 0.3735341429710388,
"grad_norm": 14.396162986755371,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -1.9688878059387207,
"logits/rejected": -1.821672797203064,
"logps/chosen": -217.6196746826172,
"logps/ref_chosen": -87.4223403930664,
"logps/ref_rejected": -106.70075988769531,
"logps/rejected": -310.72406005859375,
"loss": 0.9884,
"margin_dpo/margin_mean": 73.82595825195312,
"margin_dpo/margin_std": 83.8984603881836,
"step": 446
},
{
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.007610865402966738,
"fcm_dpo/delta": -0.12789002060890198,
"fcm_dpo/margin": 94.40399169921875,
"fcm_dpo/q_t": 0.34479495882987976,
"grad_norm": 11.825496673583984,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -2.1955559253692627,
"logits/rejected": -2.148773670196533,
"logps/chosen": -214.28981018066406,
"logps/ref_chosen": -83.6152572631836,
"logps/ref_rejected": -104.91239929199219,
"logps/rejected": -329.990966796875,
"loss": 0.9191,
"margin_dpo/margin_mean": 94.40399169921875,
"margin_dpo/margin_std": 100.11077880859375,
"step": 447
},
{
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.007517419755458832,
"fcm_dpo/delta": -0.09112964570522308,
"fcm_dpo/margin": 91.07563781738281,
"fcm_dpo/q_t": 0.35592204332351685,
"grad_norm": 12.4849271774292,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -1.9084062576293945,
"logits/rejected": -1.6441185474395752,
"logps/chosen": -232.83050537109375,
"logps/ref_chosen": -92.5757827758789,
"logps/ref_rejected": -107.68977355957031,
"logps/rejected": -339.0201110839844,
"loss": 0.9746,
"margin_dpo/margin_mean": 91.07563781738281,
"margin_dpo/margin_std": 112.18666076660156,
"step": 448
},
{
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.007335230708122253,
"fcm_dpo/delta": -0.09747156500816345,
"fcm_dpo/margin": 94.17698669433594,
"fcm_dpo/q_t": 0.34920936822891235,
"grad_norm": 10.423608779907227,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -1.7010339498519897,
"logits/rejected": -1.2772597074508667,
"logps/chosen": -240.6586151123047,
"logps/ref_chosen": -102.39893341064453,
"logps/ref_rejected": -95.14886474609375,
"logps/rejected": -327.58551025390625,
"loss": 0.9312,
"margin_dpo/margin_mean": 94.17698669433594,
"margin_dpo/margin_std": 100.80087280273438,
"step": 449
},
{
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.007234559860080481,
"fcm_dpo/delta": -0.046480074524879456,
"fcm_dpo/margin": 88.93949127197266,
"fcm_dpo/q_t": 0.35815033316612244,
"grad_norm": 11.052569389343262,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -2.46598482131958,
"logits/rejected": -2.271839141845703,
"logps/chosen": -210.14553833007812,
"logps/ref_chosen": -86.99285888671875,
"logps/ref_rejected": -108.53203582763672,
"logps/rejected": -320.62420654296875,
"loss": 0.9593,
"margin_dpo/margin_mean": 88.93949890136719,
"margin_dpo/margin_std": 99.60723876953125,
"step": 450
},
{
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.007310773245990276,
"fcm_dpo/delta": 0.062415413558483124,
"fcm_dpo/margin": 73.95240020751953,
"fcm_dpo/q_t": 0.3855394721031189,
"grad_norm": 19.515153884887695,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -2.4347171783447266,
"logits/rejected": -1.9352984428405762,
"logps/chosen": -226.25865173339844,
"logps/ref_chosen": -86.811279296875,
"logps/ref_rejected": -79.8555908203125,
"logps/rejected": -293.25537109375,
"loss": 1.0629,
"margin_dpo/margin_mean": 73.952392578125,
"margin_dpo/margin_std": 110.59036254882812,
"step": 451
},
{
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.007255923468619585,
"fcm_dpo/delta": 0.012697272002696991,
"fcm_dpo/margin": 80.79637145996094,
"fcm_dpo/q_t": 0.36846262216567993,
"grad_norm": 10.28548812866211,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -1.8156119585037231,
"logits/rejected": -1.7515356540679932,
"logps/chosen": -201.019287109375,
"logps/ref_chosen": -79.379638671875,
"logps/ref_rejected": -103.71539306640625,
"logps/rejected": -306.15142822265625,
"loss": 0.9648,
"margin_dpo/margin_mean": 80.79637145996094,
"margin_dpo/margin_std": 77.09156799316406,
"step": 452
},
{
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.007229423150420189,
"fcm_dpo/delta": -0.10436852276325226,
"fcm_dpo/margin": 96.43700408935547,
"fcm_dpo/q_t": 0.34717243909835815,
"grad_norm": 14.366576194763184,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -2.0531060695648193,
"logits/rejected": -1.5909814834594727,
"logps/chosen": -234.8137664794922,
"logps/ref_chosen": -87.6951904296875,
"logps/ref_rejected": -90.0582275390625,
"logps/rejected": -333.6138000488281,
"loss": 0.9015,
"margin_dpo/margin_mean": 96.43701171875,
"margin_dpo/margin_std": 90.89103698730469,
"step": 453
},
{
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.007256688550114632,
"fcm_dpo/delta": 0.03500938415527344,
"fcm_dpo/margin": 78.03349304199219,
"fcm_dpo/q_t": 0.3794366121292114,
"grad_norm": 12.678661346435547,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -1.8184224367141724,
"logits/rejected": -1.3561699390411377,
"logps/chosen": -246.05104064941406,
"logps/ref_chosen": -89.56623840332031,
"logps/ref_rejected": -92.92105102539062,
"logps/rejected": -327.4393310546875,
"loss": 1.0581,
"margin_dpo/margin_mean": 78.03349304199219,
"margin_dpo/margin_std": 115.84363555908203,
"step": 454
},
{
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0073183816857635975,
"fcm_dpo/delta": 0.04829338192939758,
"fcm_dpo/margin": 75.66259765625,
"fcm_dpo/q_t": 0.37961339950561523,
"grad_norm": 11.24074935913086,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -2.125541925430298,
"logits/rejected": -2.0756983757019043,
"logps/chosen": -222.00599670410156,
"logps/ref_chosen": -77.62998962402344,
"logps/ref_rejected": -118.97795104980469,
"logps/rejected": -339.0165710449219,
"loss": 1.0307,
"margin_dpo/margin_mean": 75.66259765625,
"margin_dpo/margin_std": 100.04164123535156,
"step": 455
},
{
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.00738774798810482,
"fcm_dpo/delta": 0.056851405650377274,
"fcm_dpo/margin": 73.88893127441406,
"fcm_dpo/q_t": 0.37978994846343994,
"grad_norm": 13.592082023620605,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -1.9894077777862549,
"logits/rejected": -1.8558869361877441,
"logps/chosen": -236.3739013671875,
"logps/ref_chosen": -89.61686706542969,
"logps/ref_rejected": -109.5597152709961,
"logps/rejected": -330.2056884765625,
"loss": 1.0154,
"margin_dpo/margin_mean": 73.88893127441406,
"margin_dpo/margin_std": 90.75523376464844,
"step": 456
},
{
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.007179590407758951,
"fcm_dpo/delta": -0.12014248967170715,
"fcm_dpo/margin": 98.66180419921875,
"fcm_dpo/q_t": 0.33989059925079346,
"grad_norm": 12.305672645568848,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -2.244877338409424,
"logits/rejected": -1.9393136501312256,
"logps/chosen": -209.8995819091797,
"logps/ref_chosen": -87.32168579101562,
"logps/ref_rejected": -90.76660919189453,
"logps/rejected": -312.0063171386719,
"loss": 0.8772,
"margin_dpo/margin_mean": 98.66180419921875,
"margin_dpo/margin_std": 78.503662109375,
"step": 457
},
{
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.00729702552780509,
"fcm_dpo/delta": 0.06375245004892349,
"fcm_dpo/margin": 73.92202758789062,
"fcm_dpo/q_t": 0.38597458600997925,
"grad_norm": 12.738073348999023,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -1.7707667350769043,
"logits/rejected": -1.5331363677978516,
"logps/chosen": -233.474365234375,
"logps/ref_chosen": -93.52044677734375,
"logps/ref_rejected": -103.36898803710938,
"logps/rejected": -317.24493408203125,
"loss": 1.0455,
"margin_dpo/margin_mean": 73.92202758789062,
"margin_dpo/margin_std": 105.11477661132812,
"step": 458
},
{
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.007197215687483549,
"fcm_dpo/delta": -0.05717407539486885,
"fcm_dpo/margin": 90.64775085449219,
"fcm_dpo/q_t": 0.35962265729904175,
"grad_norm": 9.387142181396484,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -1.5463656187057495,
"logits/rejected": -1.285736322402954,
"logps/chosen": -208.82803344726562,
"logps/ref_chosen": -75.68820190429688,
"logps/ref_rejected": -92.17048645019531,
"logps/rejected": -315.95806884765625,
"loss": 0.9552,
"margin_dpo/margin_mean": 90.64775085449219,
"margin_dpo/margin_std": 101.39945983886719,
"step": 459
},
{
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.007193270605057478,
"fcm_dpo/delta": 0.00503632053732872,
"fcm_dpo/margin": 82.70344543457031,
"fcm_dpo/q_t": 0.37113451957702637,
"grad_norm": 13.572725296020508,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -1.9946367740631104,
"logits/rejected": -2.009538412094116,
"logps/chosen": -219.1749267578125,
"logps/ref_chosen": -81.11788940429688,
"logps/ref_rejected": -110.31238555908203,
"logps/rejected": -331.0728759765625,
"loss": 0.9981,
"margin_dpo/margin_mean": 82.70344543457031,
"margin_dpo/margin_std": 101.98342895507812,
"step": 460
},
{
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.007337551563978195,
"fcm_dpo/delta": 0.12816324830055237,
"fcm_dpo/margin": 65.16386413574219,
"fcm_dpo/q_t": 0.3958069682121277,
"grad_norm": 10.882566452026367,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -1.553586483001709,
"logits/rejected": -1.35811448097229,
"logps/chosen": -250.552734375,
"logps/ref_chosen": -81.58352661132812,
"logps/ref_rejected": -93.87710571289062,
"logps/rejected": -328.01019287109375,
"loss": 1.0685,
"margin_dpo/margin_mean": 65.16386413574219,
"margin_dpo/margin_std": 90.7364501953125,
"step": 461
},
{
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.007375137880444527,
"fcm_dpo/delta": -0.05297492444515228,
"fcm_dpo/margin": 88.07096862792969,
"fcm_dpo/q_t": 0.3572496771812439,
"grad_norm": 10.971365928649902,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -2.2870004177093506,
"logits/rejected": -1.787416696548462,
"logps/chosen": -221.18679809570312,
"logps/ref_chosen": -94.19854736328125,
"logps/ref_rejected": -85.63162994384766,
"logps/rejected": -300.69085693359375,
"loss": 0.9407,
"margin_dpo/margin_mean": 88.07096862792969,
"margin_dpo/margin_std": 91.79960632324219,
"step": 462
},
{
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.007232788018882275,
"fcm_dpo/delta": -0.1597738415002823,
"fcm_dpo/margin": 103.35470581054688,
"fcm_dpo/q_t": 0.33559200167655945,
"grad_norm": 14.430492401123047,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -1.9360418319702148,
"logits/rejected": -1.5115666389465332,
"logps/chosen": -210.00100708007812,
"logps/ref_chosen": -85.92474365234375,
"logps/ref_rejected": -96.90184020996094,
"logps/rejected": -324.33282470703125,
"loss": 0.8987,
"margin_dpo/margin_mean": 103.35470581054688,
"margin_dpo/margin_std": 98.86663818359375,
"step": 463
},
{
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.007100261747837067,
"fcm_dpo/delta": 0.008855840191245079,
"fcm_dpo/margin": 83.32122802734375,
"fcm_dpo/q_t": 0.37016385793685913,
"grad_norm": 14.36965274810791,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -2.0437088012695312,
"logits/rejected": -2.1165390014648438,
"logps/chosen": -232.46543884277344,
"logps/ref_chosen": -79.68919372558594,
"logps/ref_rejected": -107.29232025146484,
"logps/rejected": -343.3898010253906,
"loss": 0.9944,
"margin_dpo/margin_mean": 83.32122802734375,
"margin_dpo/margin_std": 101.45448303222656,
"step": 464
},
{
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.006998377852141857,
"fcm_dpo/delta": -0.09421905875205994,
"fcm_dpo/margin": 98.20915222167969,
"fcm_dpo/q_t": 0.3482474386692047,
"grad_norm": 12.179363250732422,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -1.6612213850021362,
"logits/rejected": -1.3575751781463623,
"logps/chosen": -235.50955200195312,
"logps/ref_chosen": -91.8602294921875,
"logps/ref_rejected": -118.71001434326172,
"logps/rejected": -360.5684814453125,
"loss": 0.9084,
"margin_dpo/margin_mean": 98.20915222167969,
"margin_dpo/margin_std": 94.13998413085938,
"step": 465
},
{
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.006858200766146183,
"fcm_dpo/delta": -0.10749548673629761,
"fcm_dpo/margin": 102.01173400878906,
"fcm_dpo/q_t": 0.3455418348312378,
"grad_norm": 8.609355926513672,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -1.733262300491333,
"logits/rejected": -1.417356252670288,
"logps/chosen": -213.08628845214844,
"logps/ref_chosen": -84.70140838623047,
"logps/ref_rejected": -96.05084228515625,
"logps/rejected": -326.44744873046875,
"loss": 0.9046,
"margin_dpo/margin_mean": 102.01173400878906,
"margin_dpo/margin_std": 98.38445281982422,
"step": 466
},
{
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.006808539852499962,
"fcm_dpo/delta": 0.01279231533408165,
"fcm_dpo/margin": 86.26240539550781,
"fcm_dpo/q_t": 0.3721468448638916,
"grad_norm": 13.353436470031738,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -2.1667706966400146,
"logits/rejected": -1.576673984527588,
"logps/chosen": -269.39898681640625,
"logps/ref_chosen": -109.29832458496094,
"logps/ref_rejected": -108.8436508178711,
"logps/rejected": -355.20672607421875,
"loss": 1.0157,
"margin_dpo/margin_mean": 86.26240539550781,
"margin_dpo/margin_std": 111.6978759765625,
"step": 467
},
{
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.006919668987393379,
"fcm_dpo/delta": 0.07496071606874466,
"fcm_dpo/margin": 76.45877075195312,
"fcm_dpo/q_t": 0.38796383142471313,
"grad_norm": 11.61962604522705,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -1.8730924129486084,
"logits/rejected": -1.6779046058654785,
"logps/chosen": -226.63394165039062,
"logps/ref_chosen": -98.32164764404297,
"logps/ref_rejected": -106.68048095703125,
"logps/rejected": -311.4515380859375,
"loss": 1.0509,
"margin_dpo/margin_mean": 76.45877075195312,
"margin_dpo/margin_std": 109.24627685546875,
"step": 468
},
{
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.006858726963400841,
"fcm_dpo/delta": -0.05379898101091385,
"fcm_dpo/margin": 94.6535873413086,
"fcm_dpo/q_t": 0.35775983333587646,
"grad_norm": 13.751016616821289,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -2.398892879486084,
"logits/rejected": -1.9867908954620361,
"logps/chosen": -252.68528747558594,
"logps/ref_chosen": -96.76420593261719,
"logps/ref_rejected": -109.59500885009766,
"logps/rejected": -360.169677734375,
"loss": 0.9539,
"margin_dpo/margin_mean": 94.6535873413086,
"margin_dpo/margin_std": 103.79216003417969,
"step": 469
},
{
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.00688820518553257,
"fcm_dpo/delta": 0.0024176451843231916,
"fcm_dpo/margin": 86.77494812011719,
"fcm_dpo/q_t": 0.3669547140598297,
"grad_norm": 10.803339004516602,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -1.6873784065246582,
"logits/rejected": -1.2658147811889648,
"logps/chosen": -228.20758056640625,
"logps/ref_chosen": -83.49665832519531,
"logps/ref_rejected": -88.48578643798828,
"logps/rejected": -319.9716796875,
"loss": 0.962,
"margin_dpo/margin_mean": 86.77494812011719,
"margin_dpo/margin_std": 90.58299255371094,
"step": 470
},
{
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.00690112542361021,
"fcm_dpo/delta": 0.02034003473818302,
"fcm_dpo/margin": 84.1663589477539,
"fcm_dpo/q_t": 0.36976000666618347,
"grad_norm": 11.01422119140625,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -1.8118414878845215,
"logits/rejected": -1.4235670566558838,
"logps/chosen": -255.23658752441406,
"logps/ref_chosen": -88.47430419921875,
"logps/ref_rejected": -90.48171997070312,
"logps/rejected": -341.41033935546875,
"loss": 0.973,
"margin_dpo/margin_mean": 84.16635131835938,
"margin_dpo/margin_std": 89.08399963378906,
"step": 471
},
{
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.006775230169296265,
"fcm_dpo/delta": -0.1039826050400734,
"fcm_dpo/margin": 102.69633483886719,
"fcm_dpo/q_t": 0.3456118404865265,
"grad_norm": 9.285819053649902,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -2.095188617706299,
"logits/rejected": -1.393007516860962,
"logps/chosen": -276.9514465332031,
"logps/ref_chosen": -125.23370361328125,
"logps/ref_rejected": -121.05349731445312,
"logps/rejected": -375.46759033203125,
"loss": 0.8906,
"margin_dpo/margin_mean": 102.69633483886719,
"margin_dpo/margin_std": 90.67658233642578,
"step": 472
},
{
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0069533102214336395,
"fcm_dpo/delta": 0.16166609525680542,
"fcm_dpo/margin": 64.00907897949219,
"fcm_dpo/q_t": 0.4053763747215271,
"grad_norm": 12.549628257751465,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -1.8435192108154297,
"logits/rejected": -1.3511978387832642,
"logps/chosen": -268.58892822265625,
"logps/ref_chosen": -95.61137390136719,
"logps/ref_rejected": -88.15115356445312,
"logps/rejected": -325.1377868652344,
"loss": 1.1087,
"margin_dpo/margin_mean": 64.00907897949219,
"margin_dpo/margin_std": 105.01445770263672,
"step": 473
},
{
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.007215796038508415,
"fcm_dpo/delta": 0.14754922688007355,
"fcm_dpo/margin": 63.232147216796875,
"fcm_dpo/q_t": 0.4007920026779175,
"grad_norm": 14.762158393859863,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -1.478015422821045,
"logits/rejected": -1.284541368484497,
"logps/chosen": -245.74856567382812,
"logps/ref_chosen": -81.47975158691406,
"logps/ref_rejected": -96.46562957763672,
"logps/rejected": -323.9665832519531,
"loss": 1.112,
"margin_dpo/margin_mean": 63.23215103149414,
"margin_dpo/margin_std": 100.12031555175781,
"step": 474
},
{
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.007110551930963993,
"fcm_dpo/delta": -0.09296688437461853,
"fcm_dpo/margin": 96.56515502929688,
"fcm_dpo/q_t": 0.35164913535118103,
"grad_norm": 10.715070724487305,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -2.3761649131774902,
"logits/rejected": -2.3209524154663086,
"logps/chosen": -197.6427001953125,
"logps/ref_chosen": -74.19598388671875,
"logps/ref_rejected": -94.69242095947266,
"logps/rejected": -314.70428466796875,
"loss": 0.9244,
"margin_dpo/margin_mean": 96.56515502929688,
"margin_dpo/margin_std": 101.15115356445312,
"step": 475
},
{
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.007045960985124111,
"fcm_dpo/delta": 0.005972862243652344,
"fcm_dpo/margin": 84.30451965332031,
"fcm_dpo/q_t": 0.3728053569793701,
"grad_norm": 14.778757095336914,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -2.1326091289520264,
"logits/rejected": -1.7579033374786377,
"logps/chosen": -218.2198944091797,
"logps/ref_chosen": -71.97109985351562,
"logps/ref_rejected": -80.26224517822266,
"logps/rejected": -310.8155517578125,
"loss": 0.9929,
"margin_dpo/margin_mean": 84.30451965332031,
"margin_dpo/margin_std": 102.45613098144531,
"step": 476
},
{
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.007064092438668013,
"fcm_dpo/delta": -0.026253901422023773,
"fcm_dpo/margin": 88.41944885253906,
"fcm_dpo/q_t": 0.3624919652938843,
"grad_norm": 11.6608304977417,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -2.2218666076660156,
"logits/rejected": -1.326930284500122,
"logps/chosen": -249.8019561767578,
"logps/ref_chosen": -105.00555419921875,
"logps/ref_rejected": -81.87843322753906,
"logps/rejected": -315.09429931640625,
"loss": 0.9793,
"margin_dpo/margin_mean": 88.41944885253906,
"margin_dpo/margin_std": 104.81526947021484,
"step": 477
},
{
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.007006206549704075,
"fcm_dpo/delta": -0.0406154990196228,
"fcm_dpo/margin": 91.0680923461914,
"fcm_dpo/q_t": 0.3596052825450897,
"grad_norm": 10.416852951049805,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -1.7539746761322021,
"logits/rejected": -1.58416748046875,
"logps/chosen": -215.23123168945312,
"logps/ref_chosen": -76.7882080078125,
"logps/ref_rejected": -90.43994140625,
"logps/rejected": -319.9510498046875,
"loss": 0.9565,
"margin_dpo/margin_mean": 91.06809997558594,
"margin_dpo/margin_std": 100.75262451171875,
"step": 478
},
{
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.007025442086160183,
"fcm_dpo/delta": 0.00436464324593544,
"fcm_dpo/margin": 84.7488784790039,
"fcm_dpo/q_t": 0.36682647466659546,
"grad_norm": 13.093260765075684,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -1.6674690246582031,
"logits/rejected": -1.5295199155807495,
"logps/chosen": -226.80955505371094,
"logps/ref_chosen": -81.3623046875,
"logps/ref_rejected": -101.09114074707031,
"logps/rejected": -331.2872619628906,
"loss": 0.9753,
"margin_dpo/margin_mean": 84.7488784790039,
"margin_dpo/margin_std": 92.19709014892578,
"step": 479
},
{
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.007094934582710266,
"fcm_dpo/delta": 0.09958771616220474,
"fcm_dpo/margin": 71.2434310913086,
"fcm_dpo/q_t": 0.3897866904735565,
"grad_norm": 13.344147682189941,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -1.7132782936096191,
"logits/rejected": -1.0284159183502197,
"logps/chosen": -262.8040771484375,
"logps/ref_chosen": -96.7739028930664,
"logps/ref_rejected": -86.40473937988281,
"logps/rejected": -323.6783447265625,
"loss": 1.0558,
"margin_dpo/margin_mean": 71.2434310913086,
"margin_dpo/margin_std": 99.02870178222656,
"step": 480
},
{
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.007131170481443405,
"fcm_dpo/delta": 0.0027018017135560513,
"fcm_dpo/margin": 83.7805404663086,
"fcm_dpo/q_t": 0.3683137893676758,
"grad_norm": 14.994175910949707,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -2.007399559020996,
"logits/rejected": -1.5706861019134521,
"logps/chosen": -255.43331909179688,
"logps/ref_chosen": -91.670166015625,
"logps/ref_rejected": -98.69490051269531,
"logps/rejected": -346.2386169433594,
"loss": 0.9915,
"margin_dpo/margin_mean": 83.7805404663086,
"margin_dpo/margin_std": 100.96444702148438,
"step": 481
},
{
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.00711690541356802,
"fcm_dpo/delta": 0.00016288645565509796,
"fcm_dpo/margin": 84.26334381103516,
"fcm_dpo/q_t": 0.3666335344314575,
"grad_norm": 14.931113243103027,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -1.6981770992279053,
"logits/rejected": -1.342320442199707,
"logps/chosen": -254.57460021972656,
"logps/ref_chosen": -98.52011108398438,
"logps/ref_rejected": -94.82945251464844,
"logps/rejected": -335.14727783203125,
"loss": 0.9789,
"margin_dpo/margin_mean": 84.26334381103516,
"margin_dpo/margin_std": 95.54747009277344,
"step": 482
},
{
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.007158739026635885,
"fcm_dpo/delta": -0.03249619901180267,
"fcm_dpo/margin": 87.9349365234375,
"fcm_dpo/q_t": 0.3601437211036682,
"grad_norm": 9.465551376342773,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -1.9062191247940063,
"logits/rejected": -1.2519001960754395,
"logps/chosen": -269.0156555175781,
"logps/ref_chosen": -107.11860656738281,
"logps/ref_rejected": -101.11499786376953,
"logps/rejected": -350.9469909667969,
"loss": 0.9553,
"margin_dpo/margin_mean": 87.9349365234375,
"margin_dpo/margin_std": 92.60682678222656,
"step": 483
},
{
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.007042921148240566,
"fcm_dpo/delta": -0.07625816017389297,
"fcm_dpo/margin": 95.28274536132812,
"fcm_dpo/q_t": 0.3532974123954773,
"grad_norm": 10.351212501525879,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -2.5401973724365234,
"logits/rejected": -2.068960189819336,
"logps/chosen": -238.0645751953125,
"logps/ref_chosen": -86.97991943359375,
"logps/ref_rejected": -90.72367095947266,
"logps/rejected": -337.091064453125,
"loss": 0.9457,
"margin_dpo/margin_mean": 95.28274536132812,
"margin_dpo/margin_std": 104.58644104003906,
"step": 484
},
{
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.006840351969003677,
"fcm_dpo/delta": -0.14380024373531342,
"fcm_dpo/margin": 107.2056884765625,
"fcm_dpo/q_t": 0.3391224145889282,
"grad_norm": 10.769954681396484,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -1.80538809299469,
"logits/rejected": -1.378275990486145,
"logps/chosen": -233.96734619140625,
"logps/ref_chosen": -94.05874633789062,
"logps/ref_rejected": -108.56298828125,
"logps/rejected": -355.67724609375,
"loss": 0.881,
"margin_dpo/margin_mean": 107.2056884765625,
"margin_dpo/margin_std": 98.861328125,
"step": 485
},
{
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.006765207275748253,
"fcm_dpo/delta": -0.04138827323913574,
"fcm_dpo/margin": 94.40589904785156,
"fcm_dpo/q_t": 0.3596769869327545,
"grad_norm": 12.390087127685547,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -1.9631407260894775,
"logits/rejected": -1.2379772663116455,
"logps/chosen": -255.2804718017578,
"logps/ref_chosen": -93.74588012695312,
"logps/ref_rejected": -98.07064819335938,
"logps/rejected": -354.0111389160156,
"loss": 0.9607,
"margin_dpo/margin_mean": 94.40589904785156,
"margin_dpo/margin_std": 104.61402893066406,
"step": 486
},
{
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.006735553964972496,
"fcm_dpo/delta": 0.016399431973695755,
"fcm_dpo/margin": 86.7838134765625,
"fcm_dpo/q_t": 0.3709105849266052,
"grad_norm": 11.133604049682617,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -1.875716209411621,
"logits/rejected": -1.5496511459350586,
"logps/chosen": -243.07168579101562,
"logps/ref_chosen": -88.0335693359375,
"logps/ref_rejected": -98.47209930419922,
"logps/rejected": -340.2940673828125,
"loss": 0.9787,
"margin_dpo/margin_mean": 86.78382110595703,
"margin_dpo/margin_std": 96.44297790527344,
"step": 487
},
{
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.006739957258105278,
"fcm_dpo/delta": -0.01644238643348217,
"fcm_dpo/margin": 91.31090545654297,
"fcm_dpo/q_t": 0.3608691692352295,
"grad_norm": 12.61165714263916,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -1.6958119869232178,
"logits/rejected": -1.4399720430374146,
"logps/chosen": -229.36561584472656,
"logps/ref_chosen": -84.78964233398438,
"logps/ref_rejected": -90.2734603881836,
"logps/rejected": -326.16033935546875,
"loss": 0.9391,
"margin_dpo/margin_mean": 91.3109130859375,
"margin_dpo/margin_std": 84.56864929199219,
"step": 488
},
{
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.006757946684956551,
"fcm_dpo/delta": 0.025971170514822006,
"fcm_dpo/margin": 85.16073608398438,
"fcm_dpo/q_t": 0.3735937476158142,
"grad_norm": 11.42379093170166,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -1.6805338859558105,
"logits/rejected": -1.1919324398040771,
"logps/chosen": -250.7588653564453,
"logps/ref_chosen": -90.46929931640625,
"logps/ref_rejected": -86.39761352539062,
"logps/rejected": -331.847900390625,
"loss": 0.9907,
"margin_dpo/margin_mean": 85.16073608398438,
"margin_dpo/margin_std": 98.77789306640625,
"step": 489
},
{
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.006898378022015095,
"fcm_dpo/delta": 0.06882701814174652,
"fcm_dpo/margin": 77.1955795288086,
"fcm_dpo/q_t": 0.38805437088012695,
"grad_norm": 12.629081726074219,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -2.1147122383117676,
"logits/rejected": -2.184906482696533,
"logps/chosen": -248.49630737304688,
"logps/ref_chosen": -85.32012939453125,
"logps/ref_rejected": -115.99385070800781,
"logps/rejected": -356.3656311035156,
"loss": 1.091,
"margin_dpo/margin_mean": 77.1955795288086,
"margin_dpo/margin_std": 123.41070556640625,
"step": 490
},
{
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.006943402346223593,
"fcm_dpo/delta": 0.0842859223484993,
"fcm_dpo/margin": 74.90750122070312,
"fcm_dpo/q_t": 0.38507401943206787,
"grad_norm": 11.258004188537598,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -1.9617629051208496,
"logits/rejected": -1.4733606576919556,
"logps/chosen": -249.405517578125,
"logps/ref_chosen": -86.14352416992188,
"logps/ref_rejected": -80.67945861816406,
"logps/rejected": -318.8489685058594,
"loss": 1.0404,
"margin_dpo/margin_mean": 74.9074935913086,
"margin_dpo/margin_std": 99.26988220214844,
"step": 491
},
{
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0071650012396276,
"fcm_dpo/delta": 0.1844560205936432,
"fcm_dpo/margin": 59.10064697265625,
"fcm_dpo/q_t": 0.40985268354415894,
"grad_norm": 16.328256607055664,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -2.2341504096984863,
"logits/rejected": -1.856196641921997,
"logps/chosen": -232.42469787597656,
"logps/ref_chosen": -81.10757446289062,
"logps/ref_rejected": -80.75199890136719,
"logps/rejected": -291.1697692871094,
"loss": 1.1711,
"margin_dpo/margin_mean": 59.10065460205078,
"margin_dpo/margin_std": 119.12300872802734,
"step": 492
},
{
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.007225475274026394,
"fcm_dpo/delta": 0.03164070099592209,
"fcm_dpo/margin": 78.82869720458984,
"fcm_dpo/q_t": 0.37277752161026,
"grad_norm": 15.412538528442383,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -2.5492968559265137,
"logits/rejected": -1.8315892219543457,
"logps/chosen": -262.19671630859375,
"logps/ref_chosen": -112.20733642578125,
"logps/ref_rejected": -93.60719299316406,
"logps/rejected": -322.42523193359375,
"loss": 0.9933,
"margin_dpo/margin_mean": 78.82870483398438,
"margin_dpo/margin_std": 88.707275390625,
"step": 493
},
{
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.007072390988469124,
"fcm_dpo/delta": -0.1455509066581726,
"fcm_dpo/margin": 103.59664916992188,
"fcm_dpo/q_t": 0.3390905261039734,
"grad_norm": 12.173338890075684,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -1.4517457485198975,
"logits/rejected": -1.3680830001831055,
"logps/chosen": -218.484375,
"logps/ref_chosen": -73.11489868164062,
"logps/ref_rejected": -92.16300201416016,
"logps/rejected": -341.1291198730469,
"loss": 0.8945,
"margin_dpo/margin_mean": 103.59664916992188,
"margin_dpo/margin_std": 99.80902862548828,
"step": 494
},
{
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0070437416434288025,
"fcm_dpo/delta": -0.02011135406792164,
"fcm_dpo/margin": 87.85247802734375,
"fcm_dpo/q_t": 0.36548200249671936,
"grad_norm": 12.353002548217773,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -2.263014316558838,
"logits/rejected": -1.8048030138015747,
"logps/chosen": -248.50387573242188,
"logps/ref_chosen": -99.52031707763672,
"logps/ref_rejected": -97.93089294433594,
"logps/rejected": -334.76690673828125,
"loss": 0.9874,
"margin_dpo/margin_mean": 87.85247802734375,
"margin_dpo/margin_std": 108.39643859863281,
"step": 495
},
{
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.007146789226680994,
"fcm_dpo/delta": 0.0695204809308052,
"fcm_dpo/margin": 74.6668701171875,
"fcm_dpo/q_t": 0.3810140788555145,
"grad_norm": 11.570090293884277,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -2.214664936065674,
"logits/rejected": -1.5754952430725098,
"logps/chosen": -272.01220703125,
"logps/ref_chosen": -107.85675048828125,
"logps/ref_rejected": -92.77056121826172,
"logps/rejected": -331.5928955078125,
"loss": 1.0234,
"margin_dpo/margin_mean": 74.6668701171875,
"margin_dpo/margin_std": 91.9163818359375,
"step": 496
},
{
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.007371243089437485,
"fcm_dpo/delta": 0.22922512888908386,
"fcm_dpo/margin": 51.54722595214844,
"fcm_dpo/q_t": 0.4190720319747925,
"grad_norm": 11.735376358032227,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -1.9056165218353271,
"logits/rejected": -1.4678566455841064,
"logps/chosen": -271.3009033203125,
"logps/ref_chosen": -92.08322143554688,
"logps/ref_rejected": -81.79503631591797,
"logps/rejected": -312.5599365234375,
"loss": 1.1688,
"margin_dpo/margin_mean": 51.54722213745117,
"margin_dpo/margin_std": 99.53231048583984,
"step": 497
},
{
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.007443058304488659,
"fcm_dpo/delta": -0.029292959719896317,
"fcm_dpo/margin": 84.29811096191406,
"fcm_dpo/q_t": 0.3616979122161865,
"grad_norm": 13.830591201782227,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -1.9414262771606445,
"logits/rejected": -1.4592478275299072,
"logps/chosen": -244.99317932128906,
"logps/ref_chosen": -98.19437408447266,
"logps/ref_rejected": -90.68745422363281,
"logps/rejected": -321.7843933105469,
"loss": 0.9608,
"margin_dpo/margin_mean": 84.29811096191406,
"margin_dpo/margin_std": 93.23235321044922,
"step": 498
},
{
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0074178483337163925,
"fcm_dpo/delta": -0.029103467240929604,
"fcm_dpo/margin": 84.56077575683594,
"fcm_dpo/q_t": 0.36542099714279175,
"grad_norm": 15.131891250610352,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -1.9305214881896973,
"logits/rejected": -1.8768131732940674,
"logps/chosen": -239.75906372070312,
"logps/ref_chosen": -81.0399169921875,
"logps/ref_rejected": -106.92170715332031,
"logps/rejected": -350.2016296386719,
"loss": 0.992,
"margin_dpo/margin_mean": 84.56077575683594,
"margin_dpo/margin_std": 106.96080017089844,
"step": 499
},
{
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.007414363324642181,
"fcm_dpo/delta": 0.03089684061706066,
"fcm_dpo/margin": 76.994140625,
"fcm_dpo/q_t": 0.381319522857666,
"grad_norm": 14.555001258850098,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -1.7026747465133667,
"logits/rejected": -1.5297117233276367,
"logps/chosen": -260.76824951171875,
"logps/ref_chosen": -89.248046875,
"logps/ref_rejected": -100.41021728515625,
"logps/rejected": -348.9245300292969,
"loss": 1.0372,
"margin_dpo/margin_mean": 76.994140625,
"margin_dpo/margin_std": 109.05560302734375,
"step": 500
},
{
"epoch": 0.7342143906020558,
"eval_fcm_dpo/beta": 0.0074382987804710865,
"eval_logits/chosen": -1.973371148109436,
"eval_logits/rejected": -1.5978766679763794,
"eval_logps/chosen": -264.56976318359375,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -330.363525390625,
"eval_loss": 0.5229191184043884,
"eval_margin_dpo/margin_mean": 72.21954345703125,
"eval_margin_dpo/margin_std": 99.1610107421875,
"eval_runtime": 43.7228,
"eval_samples_per_second": 53.496,
"eval_steps_per_second": 1.692,
"step": 500
},
{
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.007509919814765453,
"fcm_dpo/delta": -0.011263996362686157,
"fcm_dpo/margin": 80.9957046508789,
"fcm_dpo/q_t": 0.3736144006252289,
"grad_norm": 14.008468627929688,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -1.6845519542694092,
"logits/rejected": -1.2528064250946045,
"logps/chosen": -241.5672607421875,
"logps/ref_chosen": -90.596923828125,
"logps/ref_rejected": -85.2972640991211,
"logps/rejected": -317.2633361816406,
"loss": 1.0472,
"margin_dpo/margin_mean": 80.99571228027344,
"margin_dpo/margin_std": 117.20719909667969,
"step": 501
},
{
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.007532845251262188,
"fcm_dpo/delta": 0.05232547968626022,
"fcm_dpo/margin": 72.90962982177734,
"fcm_dpo/q_t": 0.38146716356277466,
"grad_norm": 12.48260498046875,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -1.9113637208938599,
"logits/rejected": -1.1727447509765625,
"logps/chosen": -257.06707763671875,
"logps/ref_chosen": -103.47073364257812,
"logps/ref_rejected": -89.04415893554688,
"logps/rejected": -315.55010986328125,
"loss": 1.0651,
"margin_dpo/margin_mean": 72.90962982177734,
"margin_dpo/margin_std": 107.84392547607422,
"step": 502
},
{
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.007508490234613419,
"fcm_dpo/delta": -0.042639248073101044,
"fcm_dpo/margin": 85.12161254882812,
"fcm_dpo/q_t": 0.36175140738487244,
"grad_norm": 11.972295761108398,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -1.7123284339904785,
"logits/rejected": -1.2041406631469727,
"logps/chosen": -243.31890869140625,
"logps/ref_chosen": -93.72998809814453,
"logps/ref_rejected": -87.60896301269531,
"logps/rejected": -322.3194885253906,
"loss": 0.9774,
"margin_dpo/margin_mean": 85.12161254882812,
"margin_dpo/margin_std": 100.05804443359375,
"step": 503
},
{
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.007257069461047649,
"fcm_dpo/delta": -0.17726479470729828,
"fcm_dpo/margin": 105.27082824707031,
"fcm_dpo/q_t": 0.3317434787750244,
"grad_norm": 15.06264877319336,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -2.0711169242858887,
"logits/rejected": -1.7322639226913452,
"logps/chosen": -242.6743927001953,
"logps/ref_chosen": -92.2918701171875,
"logps/ref_rejected": -109.03765106201172,
"logps/rejected": -364.6910095214844,
"loss": 0.8702,
"margin_dpo/margin_mean": 105.27082824707031,
"margin_dpo/margin_std": 96.85734558105469,
"step": 504
},
{
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.007209095172584057,
"fcm_dpo/delta": 0.027257125824689865,
"fcm_dpo/margin": 79.66241455078125,
"fcm_dpo/q_t": 0.3737871050834656,
"grad_norm": 11.139007568359375,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -2.458643674850464,
"logits/rejected": -1.9072757959365845,
"logps/chosen": -274.4703063964844,
"logps/ref_chosen": -106.27693176269531,
"logps/ref_rejected": -94.20611572265625,
"logps/rejected": -342.0618896484375,
"loss": 0.9951,
"margin_dpo/margin_mean": 79.66241455078125,
"margin_dpo/margin_std": 93.06168365478516,
"step": 505
},
{
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.007173755206167698,
"fcm_dpo/delta": -0.01147904247045517,
"fcm_dpo/margin": 85.08673095703125,
"fcm_dpo/q_t": 0.3622966408729553,
"grad_norm": 11.042329788208008,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -2.1348423957824707,
"logits/rejected": -1.7820146083831787,
"logps/chosen": -245.61795043945312,
"logps/ref_chosen": -81.44086456298828,
"logps/ref_rejected": -85.66439819335938,
"logps/rejected": -334.92822265625,
"loss": 0.9651,
"margin_dpo/margin_mean": 85.08673095703125,
"margin_dpo/margin_std": 90.8485107421875,
"step": 506
},
{
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.007161298766732216,
"fcm_dpo/delta": -0.0466289147734642,
"fcm_dpo/margin": 89.87873077392578,
"fcm_dpo/q_t": 0.3620367646217346,
"grad_norm": 11.227087020874023,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -2.110520362854004,
"logits/rejected": -1.6617255210876465,
"logps/chosen": -269.304931640625,
"logps/ref_chosen": -99.42694854736328,
"logps/ref_rejected": -102.25296020507812,
"logps/rejected": -362.00970458984375,
"loss": 0.9832,
"margin_dpo/margin_mean": 89.87872314453125,
"margin_dpo/margin_std": 110.67439270019531,
"step": 507
},
{
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.007206980139017105,
"fcm_dpo/delta": 0.07583308964967728,
"fcm_dpo/margin": 73.293212890625,
"fcm_dpo/q_t": 0.3894771933555603,
"grad_norm": 15.699549674987793,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -2.079139232635498,
"logits/rejected": -1.599036455154419,
"logps/chosen": -264.4894714355469,
"logps/ref_chosen": -97.9156494140625,
"logps/ref_rejected": -103.79782104492188,
"logps/rejected": -343.66485595703125,
"loss": 1.0835,
"margin_dpo/margin_mean": 73.293212890625,
"margin_dpo/margin_std": 114.59175872802734,
"step": 508
},
{
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.007344736717641354,
"fcm_dpo/delta": 0.07915890216827393,
"fcm_dpo/margin": 71.44049072265625,
"fcm_dpo/q_t": 0.3829602003097534,
"grad_norm": 17.265066146850586,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -2.11910080909729,
"logits/rejected": -1.4653677940368652,
"logps/chosen": -257.30712890625,
"logps/ref_chosen": -100.72984313964844,
"logps/ref_rejected": -86.943359375,
"logps/rejected": -314.961181640625,
"loss": 1.0586,
"margin_dpo/margin_mean": 71.44050598144531,
"margin_dpo/margin_std": 101.78379821777344,
"step": 509
},
{
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0073591480031609535,
"fcm_dpo/delta": -0.015946993604302406,
"fcm_dpo/margin": 83.5523452758789,
"fcm_dpo/q_t": 0.36579430103302,
"grad_norm": 14.097626686096191,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -1.7049915790557861,
"logits/rejected": -1.5141171216964722,
"logps/chosen": -272.86090087890625,
"logps/ref_chosen": -89.16938781738281,
"logps/ref_rejected": -107.91939544677734,
"logps/rejected": -375.1632385253906,
"loss": 0.9857,
"margin_dpo/margin_mean": 83.55235290527344,
"margin_dpo/margin_std": 98.11227416992188,
"step": 510
},
{
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.007278499659150839,
"fcm_dpo/delta": -0.07066594809293747,
"fcm_dpo/margin": 91.5009765625,
"fcm_dpo/q_t": 0.3567890226840973,
"grad_norm": 12.136595726013184,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -2.2216238975524902,
"logits/rejected": -1.6600124835968018,
"logps/chosen": -255.50558471679688,
"logps/ref_chosen": -89.74979400634766,
"logps/ref_rejected": -97.73127746582031,
"logps/rejected": -354.988037109375,
"loss": 0.9465,
"margin_dpo/margin_mean": 91.5009765625,
"margin_dpo/margin_std": 102.25430297851562,
"step": 511
},
{
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0072515252977609634,
"fcm_dpo/delta": -0.02720758691430092,
"fcm_dpo/margin": 86.15798950195312,
"fcm_dpo/q_t": 0.3647354543209076,
"grad_norm": 16.666259765625,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -1.8731253147125244,
"logits/rejected": -1.4305176734924316,
"logps/chosen": -244.8845977783203,
"logps/ref_chosen": -85.73515319824219,
"logps/ref_rejected": -87.6248779296875,
"logps/rejected": -332.93231201171875,
"loss": 0.9965,
"margin_dpo/margin_mean": 86.1579818725586,
"margin_dpo/margin_std": 108.06047058105469,
"step": 512
},
{
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.007211950141936541,
"fcm_dpo/delta": 0.06090360879898071,
"fcm_dpo/margin": 75.15151977539062,
"fcm_dpo/q_t": 0.37852686643600464,
"grad_norm": 12.995072364807129,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -2.311227798461914,
"logits/rejected": -1.6142702102661133,
"logps/chosen": -264.4798583984375,
"logps/ref_chosen": -96.19050598144531,
"logps/ref_rejected": -82.91773986816406,
"logps/rejected": -326.3586120605469,
"loss": 1.0292,
"margin_dpo/margin_mean": 75.15151977539062,
"margin_dpo/margin_std": 95.23574829101562,
"step": 513
},
{
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.007290856912732124,
"fcm_dpo/delta": 0.017535630613565445,
"fcm_dpo/margin": 80.029296875,
"fcm_dpo/q_t": 0.37469518184661865,
"grad_norm": 14.053443908691406,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -1.6758110523223877,
"logits/rejected": -1.0973663330078125,
"logps/chosen": -259.1905822753906,
"logps/ref_chosen": -97.82363891601562,
"logps/ref_rejected": -85.36982727050781,
"logps/rejected": -326.76605224609375,
"loss": 1.0028,
"margin_dpo/margin_mean": 80.029296875,
"margin_dpo/margin_std": 99.76383972167969,
"step": 514
},
{
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.00737146707251668,
"fcm_dpo/delta": 0.06299495697021484,
"fcm_dpo/margin": 73.31365203857422,
"fcm_dpo/q_t": 0.381864994764328,
"grad_norm": 17.193218231201172,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -1.7850375175476074,
"logits/rejected": -1.5526132583618164,
"logps/chosen": -240.5751953125,
"logps/ref_chosen": -81.93355560302734,
"logps/ref_rejected": -93.57656860351562,
"logps/rejected": -325.5318603515625,
"loss": 1.0369,
"margin_dpo/margin_mean": 73.31365203857422,
"margin_dpo/margin_std": 97.8846206665039,
"step": 515
},
{
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.00745687261223793,
"fcm_dpo/delta": 0.0405082106590271,
"fcm_dpo/margin": 75.3163833618164,
"fcm_dpo/q_t": 0.3833583891391754,
"grad_norm": 13.243304252624512,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -2.4003708362579346,
"logits/rejected": -2.211030960083008,
"logps/chosen": -257.3096923828125,
"logps/ref_chosen": -97.09166717529297,
"logps/ref_rejected": -118.55667114257812,
"logps/rejected": -354.0910949707031,
"loss": 1.0819,
"margin_dpo/margin_mean": 75.3163833618164,
"margin_dpo/margin_std": 121.37615966796875,
"step": 516
},
{
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.007305598817765713,
"fcm_dpo/delta": -0.07763826102018356,
"fcm_dpo/margin": 91.7294921875,
"fcm_dpo/q_t": 0.3567124009132385,
"grad_norm": 12.040420532226562,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -1.9732433557510376,
"logits/rejected": -1.5516693592071533,
"logps/chosen": -264.7029113769531,
"logps/ref_chosen": -93.18656921386719,
"logps/ref_rejected": -95.5927963256836,
"logps/rejected": -358.838623046875,
"loss": 0.9666,
"margin_dpo/margin_mean": 91.7294921875,
"margin_dpo/margin_std": 106.93476104736328,
"step": 517
},
{
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.007446288131177425,
"fcm_dpo/delta": 0.09759702533483505,
"fcm_dpo/margin": 68.14192199707031,
"fcm_dpo/q_t": 0.3923153877258301,
"grad_norm": 15.57699966430664,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -2.0085532665252686,
"logits/rejected": -1.6788051128387451,
"logps/chosen": -266.467041015625,
"logps/ref_chosen": -87.0213623046875,
"logps/ref_rejected": -91.24456024169922,
"logps/rejected": -338.8321533203125,
"loss": 1.0852,
"margin_dpo/margin_mean": 68.14192199707031,
"margin_dpo/margin_std": 108.119873046875,
"step": 518
},
{
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.007489525713026524,
"fcm_dpo/delta": 0.032412007451057434,
"fcm_dpo/margin": 75.99348449707031,
"fcm_dpo/q_t": 0.37510281801223755,
"grad_norm": 12.590337753295898,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -1.9630098342895508,
"logits/rejected": -1.5859870910644531,
"logps/chosen": -266.3951416015625,
"logps/ref_chosen": -100.68755340576172,
"logps/ref_rejected": -109.03042602539062,
"logps/rejected": -350.73150634765625,
"loss": 0.9882,
"margin_dpo/margin_mean": 75.99348449707031,
"margin_dpo/margin_std": 85.33174133300781,
"step": 519
},
{
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.007443825248628855,
"fcm_dpo/delta": -0.06601864099502563,
"fcm_dpo/margin": 88.84642028808594,
"fcm_dpo/q_t": 0.3531188368797302,
"grad_norm": 11.373416900634766,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -1.54036545753479,
"logits/rejected": -1.0939218997955322,
"logps/chosen": -250.6941375732422,
"logps/ref_chosen": -98.77762603759766,
"logps/ref_rejected": -106.9457015991211,
"logps/rejected": -347.7086486816406,
"loss": 0.9295,
"margin_dpo/margin_mean": 88.84642028808594,
"margin_dpo/margin_std": 88.97753143310547,
"step": 520
},
{
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.0074075451120734215,
"fcm_dpo/delta": -0.06705917418003082,
"fcm_dpo/margin": 89.41288757324219,
"fcm_dpo/q_t": 0.35395270586013794,
"grad_norm": 12.826030731201172,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -2.1257705688476562,
"logits/rejected": -1.7991995811462402,
"logps/chosen": -244.10650634765625,
"logps/ref_chosen": -86.35108947753906,
"logps/ref_rejected": -90.39317321777344,
"logps/rejected": -337.56146240234375,
"loss": 0.9291,
"margin_dpo/margin_mean": 89.41287994384766,
"margin_dpo/margin_std": 89.05442810058594,
"step": 521
},
{
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.007335927337408066,
"fcm_dpo/delta": -0.02064390294253826,
"fcm_dpo/margin": 84.41744232177734,
"fcm_dpo/q_t": 0.36679452657699585,
"grad_norm": 12.274141311645508,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -1.7425016164779663,
"logits/rejected": -1.3717331886291504,
"logps/chosen": -236.16275024414062,
"logps/ref_chosen": -76.008056640625,
"logps/ref_rejected": -74.44125366210938,
"logps/rejected": -319.0133972167969,
"loss": 0.9823,
"margin_dpo/margin_mean": 84.41744995117188,
"margin_dpo/margin_std": 101.350341796875,
"step": 522
},
{
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.007474465761333704,
"fcm_dpo/delta": 0.16874706745147705,
"fcm_dpo/margin": 58.72737121582031,
"fcm_dpo/q_t": 0.40729784965515137,
"grad_norm": 14.717813491821289,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -1.9797134399414062,
"logits/rejected": -1.2677710056304932,
"logps/chosen": -276.0887451171875,
"logps/ref_chosen": -96.49627685546875,
"logps/ref_rejected": -91.0152816772461,
"logps/rejected": -329.3351135253906,
"loss": 1.1446,
"margin_dpo/margin_mean": 58.72737503051758,
"margin_dpo/margin_std": 108.98553466796875,
"step": 523
},
{
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.007531831972301006,
"fcm_dpo/delta": -0.0187044907361269,
"fcm_dpo/margin": 81.98780822753906,
"fcm_dpo/q_t": 0.36835557222366333,
"grad_norm": 16.366233825683594,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -2.228161334991455,
"logits/rejected": -1.5479540824890137,
"logps/chosen": -268.5673828125,
"logps/ref_chosen": -103.40550994873047,
"logps/ref_rejected": -87.91015625,
"logps/rejected": -335.059814453125,
"loss": 0.9845,
"margin_dpo/margin_mean": 81.98780822753906,
"margin_dpo/margin_std": 100.14673614501953,
"step": 524
},
{
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.007332179695367813,
"fcm_dpo/delta": -0.08620290458202362,
"fcm_dpo/margin": 92.11900329589844,
"fcm_dpo/q_t": 0.3595668077468872,
"grad_norm": 12.162001609802246,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -1.6582214832305908,
"logits/rejected": -1.572461724281311,
"logps/chosen": -239.363525390625,
"logps/ref_chosen": -84.20648193359375,
"logps/ref_rejected": -106.26158905029297,
"logps/rejected": -353.53765869140625,
"loss": 0.9656,
"margin_dpo/margin_mean": 92.11900329589844,
"margin_dpo/margin_std": 107.05292510986328,
"step": 525
},
{
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.007367183454334736,
"fcm_dpo/delta": -0.03810001537203789,
"fcm_dpo/margin": 86.28730773925781,
"fcm_dpo/q_t": 0.36337071657180786,
"grad_norm": 10.43152141571045,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -1.8134328126907349,
"logits/rejected": -1.1642718315124512,
"logps/chosen": -264.8533935546875,
"logps/ref_chosen": -102.85411071777344,
"logps/ref_rejected": -93.65145874023438,
"logps/rejected": -341.9380187988281,
"loss": 0.9824,
"margin_dpo/margin_mean": 86.28730773925781,
"margin_dpo/margin_std": 105.58222961425781,
"step": 526
},
{
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.007302050478756428,
"fcm_dpo/delta": -0.04052596166729927,
"fcm_dpo/margin": 87.3643798828125,
"fcm_dpo/q_t": 0.3573962152004242,
"grad_norm": 12.692599296569824,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -3.0793709754943848,
"logits/rejected": -2.2557575702667236,
"logps/chosen": -257.8551025390625,
"logps/ref_chosen": -96.76543426513672,
"logps/ref_rejected": -85.39518737792969,
"logps/rejected": -333.8492431640625,
"loss": 0.9283,
"margin_dpo/margin_mean": 87.36438751220703,
"margin_dpo/margin_std": 82.9266357421875,
"step": 527
},
{
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0072532519698143005,
"fcm_dpo/delta": -0.04537132754921913,
"fcm_dpo/margin": 88.57562255859375,
"fcm_dpo/q_t": 0.3604862689971924,
"grad_norm": 12.81235122680664,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -2.018289089202881,
"logits/rejected": -1.5019323825836182,
"logps/chosen": -244.18003845214844,
"logps/ref_chosen": -94.1649169921875,
"logps/ref_rejected": -100.372314453125,
"logps/rejected": -338.96307373046875,
"loss": 0.9672,
"margin_dpo/margin_mean": 88.57562255859375,
"margin_dpo/margin_std": 102.981201171875,
"step": 528
},
{
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.007266518659889698,
"fcm_dpo/delta": 0.034581415355205536,
"fcm_dpo/margin": 78.07312774658203,
"fcm_dpo/q_t": 0.3804980516433716,
"grad_norm": 12.89798641204834,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.6607023477554321,
"logits/rejected": -1.4771153926849365,
"logps/chosen": -255.24560546875,
"logps/ref_chosen": -80.77680206298828,
"logps/ref_rejected": -97.191162109375,
"logps/rejected": -349.73309326171875,
"loss": 1.0404,
"margin_dpo/margin_mean": 78.0731201171875,
"margin_dpo/margin_std": 111.84912109375,
"step": 529
},
{
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.007262610364705324,
"fcm_dpo/delta": 0.023171119391918182,
"fcm_dpo/margin": 79.56475830078125,
"fcm_dpo/q_t": 0.37278905510902405,
"grad_norm": 11.019624710083008,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -1.5359987020492554,
"logits/rejected": -1.2066240310668945,
"logps/chosen": -239.765869140625,
"logps/ref_chosen": -79.7757339477539,
"logps/ref_rejected": -92.05390167236328,
"logps/rejected": -331.6087646484375,
"loss": 0.977,
"margin_dpo/margin_mean": 79.56475830078125,
"margin_dpo/margin_std": 86.05628967285156,
"step": 530
},
{
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.007175838574767113,
"fcm_dpo/delta": -0.11706428229808807,
"fcm_dpo/margin": 98.76272583007812,
"fcm_dpo/q_t": 0.3418416380882263,
"grad_norm": 11.516433715820312,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.8055527210235596,
"logits/rejected": -1.1960854530334473,
"logps/chosen": -235.0717315673828,
"logps/ref_chosen": -79.31919860839844,
"logps/ref_rejected": -78.10832214355469,
"logps/rejected": -332.62359619140625,
"loss": 0.8993,
"margin_dpo/margin_mean": 98.76272583007812,
"margin_dpo/margin_std": 93.84896850585938,
"step": 531
},
{
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.006985160522162914,
"fcm_dpo/delta": -0.16617754101753235,
"fcm_dpo/margin": 107.93487548828125,
"fcm_dpo/q_t": 0.334781676530838,
"grad_norm": 11.529740333557129,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -1.7615710496902466,
"logits/rejected": -1.1453497409820557,
"logps/chosen": -266.70928955078125,
"logps/ref_chosen": -101.63691711425781,
"logps/ref_rejected": -93.34539031982422,
"logps/rejected": -366.3526611328125,
"loss": 0.8779,
"margin_dpo/margin_mean": 107.93487548828125,
"margin_dpo/margin_std": 99.59811401367188,
"step": 532
},
{
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.007072297390550375,
"fcm_dpo/delta": 0.15511363744735718,
"fcm_dpo/margin": 63.86847686767578,
"fcm_dpo/q_t": 0.4039548635482788,
"grad_norm": 12.298042297363281,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -2.4961955547332764,
"logits/rejected": -1.911888837814331,
"logps/chosen": -251.64068603515625,
"logps/ref_chosen": -90.74664306640625,
"logps/ref_rejected": -82.88848876953125,
"logps/rejected": -307.6510009765625,
"loss": 1.1014,
"margin_dpo/margin_mean": 63.86847686767578,
"margin_dpo/margin_std": 102.30418395996094,
"step": 533
},
{
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.007072807289659977,
"fcm_dpo/delta": -0.053309373557567596,
"fcm_dpo/margin": 91.88386535644531,
"fcm_dpo/q_t": 0.3540825843811035,
"grad_norm": 10.521297454833984,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -2.0540428161621094,
"logits/rejected": -1.6685407161712646,
"logps/chosen": -231.44000244140625,
"logps/ref_chosen": -86.6308364868164,
"logps/ref_rejected": -84.87196350097656,
"logps/rejected": -321.56500244140625,
"loss": 0.9216,
"margin_dpo/margin_mean": 91.88386535644531,
"margin_dpo/margin_std": 85.76850128173828,
"step": 534
},
{
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.0070062074810266495,
"fcm_dpo/delta": -0.07865796238183975,
"fcm_dpo/margin": 96.07039642333984,
"fcm_dpo/q_t": 0.3533538579940796,
"grad_norm": 10.109777450561523,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -1.8793187141418457,
"logits/rejected": -1.6919231414794922,
"logps/chosen": -236.0832061767578,
"logps/ref_chosen": -90.34539031982422,
"logps/ref_rejected": -100.7779769897461,
"logps/rejected": -342.586181640625,
"loss": 0.939,
"margin_dpo/margin_mean": 96.07039642333984,
"margin_dpo/margin_std": 102.51802062988281,
"step": 535
},
{
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.006879711989313364,
"fcm_dpo/delta": -0.037070631980895996,
"fcm_dpo/margin": 92.2296371459961,
"fcm_dpo/q_t": 0.3588649034500122,
"grad_norm": 10.414005279541016,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -1.927714228630066,
"logits/rejected": -1.678464651107788,
"logps/chosen": -227.02041625976562,
"logps/ref_chosen": -78.95956420898438,
"logps/ref_rejected": -93.08779907226562,
"logps/rejected": -333.3782958984375,
"loss": 0.9498,
"margin_dpo/margin_mean": 92.22964477539062,
"margin_dpo/margin_std": 97.07234954833984,
"step": 536
},
{
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.006824993994086981,
"fcm_dpo/delta": -0.11446214467287064,
"fcm_dpo/margin": 103.43373107910156,
"fcm_dpo/q_t": 0.34388312697410583,
"grad_norm": 9.952900886535645,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -1.647277593612671,
"logits/rejected": -0.9549525380134583,
"logps/chosen": -214.62451171875,
"logps/ref_chosen": -86.70939636230469,
"logps/ref_rejected": -78.90766906738281,
"logps/rejected": -310.25653076171875,
"loss": 0.8996,
"margin_dpo/margin_mean": 103.43373107910156,
"margin_dpo/margin_std": 94.62214660644531,
"step": 537
},
{
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.006767315790057182,
"fcm_dpo/delta": 0.02974122203886509,
"fcm_dpo/margin": 84.50604248046875,
"fcm_dpo/q_t": 0.37143373489379883,
"grad_norm": 11.350445747375488,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -1.9193155765533447,
"logits/rejected": -1.3912053108215332,
"logps/chosen": -269.3667297363281,
"logps/ref_chosen": -98.44214630126953,
"logps/ref_rejected": -91.09465026855469,
"logps/rejected": -346.5252990722656,
"loss": 0.9744,
"margin_dpo/margin_mean": 84.50605010986328,
"margin_dpo/margin_std": 87.87483215332031,
"step": 538
},
{
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.006792771629989147,
"fcm_dpo/delta": 0.016240041702985764,
"fcm_dpo/margin": 86.07090759277344,
"fcm_dpo/q_t": 0.3723304867744446,
"grad_norm": 10.20103645324707,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -2.425452709197998,
"logits/rejected": -1.947152018547058,
"logps/chosen": -242.98489379882812,
"logps/ref_chosen": -90.93211364746094,
"logps/ref_rejected": -97.44546508789062,
"logps/rejected": -335.56915283203125,
"loss": 1.005,
"margin_dpo/margin_mean": 86.07090759277344,
"margin_dpo/margin_std": 107.96048736572266,
"step": 539
},
{
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.006723560392856598,
"fcm_dpo/delta": -0.10000798851251602,
"fcm_dpo/margin": 103.08011627197266,
"fcm_dpo/q_t": 0.3510266840457916,
"grad_norm": 12.132619857788086,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -2.134742498397827,
"logits/rejected": -1.8956007957458496,
"logps/chosen": -265.84393310546875,
"logps/ref_chosen": -107.86990356445312,
"logps/ref_rejected": -121.39350128173828,
"logps/rejected": -382.4476318359375,
"loss": 0.9372,
"margin_dpo/margin_mean": 103.08011627197266,
"margin_dpo/margin_std": 114.15684509277344,
"step": 540
},
{
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.00676216883584857,
"fcm_dpo/delta": 0.15339398384094238,
"fcm_dpo/margin": 67.08287048339844,
"fcm_dpo/q_t": 0.4000394940376282,
"grad_norm": 11.626500129699707,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -2.422119140625,
"logits/rejected": -1.958017349243164,
"logps/chosen": -283.84246826171875,
"logps/ref_chosen": -106.56748962402344,
"logps/ref_rejected": -109.06633758544922,
"logps/rejected": -353.4242248535156,
"loss": 1.0758,
"margin_dpo/margin_mean": 67.08287048339844,
"margin_dpo/margin_std": 90.2316665649414,
"step": 541
},
{
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.006774522364139557,
"fcm_dpo/delta": -0.09346829354763031,
"fcm_dpo/margin": 101.42529296875,
"fcm_dpo/q_t": 0.3474395275115967,
"grad_norm": 11.156708717346191,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -2.3203277587890625,
"logits/rejected": -2.1964328289031982,
"logps/chosen": -237.198486328125,
"logps/ref_chosen": -87.36929321289062,
"logps/ref_rejected": -102.82034301757812,
"logps/rejected": -354.0748291015625,
"loss": 0.9173,
"margin_dpo/margin_mean": 101.42529296875,
"margin_dpo/margin_std": 100.42752838134766,
"step": 542
},
{
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.006940866820514202,
"fcm_dpo/delta": 0.23537762463092804,
"fcm_dpo/margin": 53.89453887939453,
"fcm_dpo/q_t": 0.422155499458313,
"grad_norm": 14.22153377532959,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -1.7961983680725098,
"logits/rejected": -1.828367829322815,
"logps/chosen": -261.8563232421875,
"logps/ref_chosen": -87.152587890625,
"logps/ref_rejected": -111.53172302246094,
"logps/rejected": -340.1300048828125,
"loss": 1.1992,
"margin_dpo/margin_mean": 53.89453887939453,
"margin_dpo/margin_std": 116.86991882324219,
"step": 543
},
{
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.007195335812866688,
"fcm_dpo/delta": 0.13186031579971313,
"fcm_dpo/margin": 65.94219207763672,
"fcm_dpo/q_t": 0.3974402844905853,
"grad_norm": 13.812807083129883,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -2.0716145038604736,
"logits/rejected": -1.9726595878601074,
"logps/chosen": -265.1961669921875,
"logps/ref_chosen": -86.14691162109375,
"logps/ref_rejected": -106.3939208984375,
"logps/rejected": -351.3853759765625,
"loss": 1.0679,
"margin_dpo/margin_mean": 65.94218444824219,
"margin_dpo/margin_std": 94.23214721679688,
"step": 544
},
{
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.007269757799804211,
"fcm_dpo/delta": 0.005659213289618492,
"fcm_dpo/margin": 81.78932189941406,
"fcm_dpo/q_t": 0.37121522426605225,
"grad_norm": 12.46208667755127,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -1.712825059890747,
"logits/rejected": -1.2983132600784302,
"logps/chosen": -258.543701171875,
"logps/ref_chosen": -96.42424011230469,
"logps/ref_rejected": -100.73405456542969,
"logps/rejected": -344.642822265625,
"loss": 1.0252,
"margin_dpo/margin_mean": 81.7893295288086,
"margin_dpo/margin_std": 111.28877258300781,
"step": 545
},
{
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.007279456127434969,
"fcm_dpo/delta": 0.041319385170936584,
"fcm_dpo/margin": 77.05686950683594,
"fcm_dpo/q_t": 0.37605637311935425,
"grad_norm": 12.261453628540039,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -1.5237065553665161,
"logits/rejected": -1.1611510515213013,
"logps/chosen": -252.166259765625,
"logps/ref_chosen": -86.02112579345703,
"logps/ref_rejected": -87.37263488769531,
"logps/rejected": -330.5746154785156,
"loss": 1.0274,
"margin_dpo/margin_mean": 77.05686950683594,
"margin_dpo/margin_std": 102.12185668945312,
"step": 546
},
{
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.007295276969671249,
"fcm_dpo/delta": -0.05277668312191963,
"fcm_dpo/margin": 88.98194885253906,
"fcm_dpo/q_t": 0.3542296886444092,
"grad_norm": 11.10513973236084,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -1.8882625102996826,
"logits/rejected": -1.4319205284118652,
"logps/chosen": -260.6878662109375,
"logps/ref_chosen": -100.6510009765625,
"logps/ref_rejected": -97.9117431640625,
"logps/rejected": -346.9305419921875,
"loss": 0.9216,
"margin_dpo/margin_mean": 88.98194885253906,
"margin_dpo/margin_std": 82.17151641845703,
"step": 547
},
{
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.00735054537653923,
"fcm_dpo/delta": 0.093236543238163,
"fcm_dpo/margin": 69.57292175292969,
"fcm_dpo/q_t": 0.39055800437927246,
"grad_norm": 13.908289909362793,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -2.291468858718872,
"logits/rejected": -1.938688039779663,
"logps/chosen": -255.26414489746094,
"logps/ref_chosen": -96.03010559082031,
"logps/ref_rejected": -98.41322326660156,
"logps/rejected": -327.2201843261719,
"loss": 1.1036,
"margin_dpo/margin_mean": 69.57292938232422,
"margin_dpo/margin_std": 116.0625,
"step": 548
},
{
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.007444350048899651,
"fcm_dpo/delta": 0.020497113466262817,
"fcm_dpo/margin": 77.89366149902344,
"fcm_dpo/q_t": 0.3747206926345825,
"grad_norm": 14.169367790222168,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -1.9140429496765137,
"logits/rejected": -1.494286060333252,
"logps/chosen": -231.77978515625,
"logps/ref_chosen": -83.45095825195312,
"logps/ref_rejected": -81.38023376464844,
"logps/rejected": -307.60272216796875,
"loss": 1.0385,
"margin_dpo/margin_mean": 77.89366149902344,
"margin_dpo/margin_std": 108.47966003417969,
"step": 549
},
{
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.007483618333935738,
"fcm_dpo/delta": 0.04548526555299759,
"fcm_dpo/margin": 74.3836898803711,
"fcm_dpo/q_t": 0.3776836395263672,
"grad_norm": 12.159344673156738,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -1.6402368545532227,
"logits/rejected": -1.4186840057373047,
"logps/chosen": -255.8831329345703,
"logps/ref_chosen": -89.59434509277344,
"logps/ref_rejected": -101.31549835205078,
"logps/rejected": -341.98797607421875,
"loss": 1.0083,
"margin_dpo/margin_mean": 74.38369750976562,
"margin_dpo/margin_std": 88.22019958496094,
"step": 550
},
{
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.007342686876654625,
"fcm_dpo/delta": -0.06708859652280807,
"fcm_dpo/margin": 90.0142593383789,
"fcm_dpo/q_t": 0.3585403561592102,
"grad_norm": 14.69068717956543,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -1.7167030572891235,
"logits/rejected": -1.5703403949737549,
"logps/chosen": -241.60235595703125,
"logps/ref_chosen": -95.08216094970703,
"logps/ref_rejected": -117.7192611694336,
"logps/rejected": -354.25372314453125,
"loss": 0.942,
"margin_dpo/margin_mean": 90.0142593383789,
"margin_dpo/margin_std": 97.10916137695312,
"step": 551
},
{
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0069796196185052395,
"fcm_dpo/delta": -0.29933837056159973,
"fcm_dpo/margin": 124.37358093261719,
"fcm_dpo/q_t": 0.31151920557022095,
"grad_norm": 10.166218757629395,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -2.0635464191436768,
"logits/rejected": -1.6969006061553955,
"logps/chosen": -239.97976684570312,
"logps/ref_chosen": -101.40918731689453,
"logps/ref_rejected": -110.83843994140625,
"logps/rejected": -373.7825927734375,
"loss": 0.8214,
"margin_dpo/margin_mean": 124.37358093261719,
"margin_dpo/margin_std": 107.26200866699219,
"step": 552
},
{
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.007029990665614605,
"fcm_dpo/delta": 0.06483317166566849,
"fcm_dpo/margin": 76.58863830566406,
"fcm_dpo/q_t": 0.3810678720474243,
"grad_norm": 11.677241325378418,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -1.5280718803405762,
"logits/rejected": -1.200793981552124,
"logps/chosen": -251.30337524414062,
"logps/ref_chosen": -97.96388244628906,
"logps/ref_rejected": -103.57719421386719,
"logps/rejected": -333.50531005859375,
"loss": 1.0384,
"margin_dpo/margin_mean": 76.58863830566406,
"margin_dpo/margin_std": 102.57685089111328,
"step": 553
},
{
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.007085290737450123,
"fcm_dpo/delta": 0.005978398025035858,
"fcm_dpo/margin": 83.78956604003906,
"fcm_dpo/q_t": 0.37405747175216675,
"grad_norm": 11.701247215270996,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -1.9315733909606934,
"logits/rejected": -1.1649553775787354,
"logps/chosen": -265.73052978515625,
"logps/ref_chosen": -103.68680572509766,
"logps/ref_rejected": -92.99210357666016,
"logps/rejected": -338.82537841796875,
"loss": 1.0497,
"margin_dpo/margin_mean": 83.78956604003906,
"margin_dpo/margin_std": 123.03929138183594,
"step": 554
},
{
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.007088308688253164,
"fcm_dpo/delta": 0.04228687286376953,
"fcm_dpo/margin": 79.01618957519531,
"fcm_dpo/q_t": 0.38176795840263367,
"grad_norm": 12.377523422241211,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -1.7843549251556396,
"logits/rejected": -1.688063144683838,
"logps/chosen": -255.608154296875,
"logps/ref_chosen": -86.20362854003906,
"logps/ref_rejected": -114.54719543457031,
"logps/rejected": -362.9678955078125,
"loss": 1.0493,
"margin_dpo/margin_mean": 79.01618957519531,
"margin_dpo/margin_std": 114.8555908203125,
"step": 555
},
{
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.007074237335473299,
"fcm_dpo/delta": -0.01300017535686493,
"fcm_dpo/margin": 86.51844024658203,
"fcm_dpo/q_t": 0.36618727445602417,
"grad_norm": 12.069602012634277,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -1.5851771831512451,
"logits/rejected": -1.2715528011322021,
"logps/chosen": -238.14895629882812,
"logps/ref_chosen": -88.40348815917969,
"logps/ref_rejected": -106.63911437988281,
"logps/rejected": -342.90301513671875,
"loss": 0.9853,
"margin_dpo/margin_mean": 86.5184326171875,
"margin_dpo/margin_std": 104.031494140625,
"step": 556
},
{
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.006955698132514954,
"fcm_dpo/delta": -0.08430520445108414,
"fcm_dpo/margin": 97.38102722167969,
"fcm_dpo/q_t": 0.3552389144897461,
"grad_norm": 10.077157974243164,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -1.8743904829025269,
"logits/rejected": -1.6990039348602295,
"logps/chosen": -259.66729736328125,
"logps/ref_chosen": -98.83523559570312,
"logps/ref_rejected": -115.64888763427734,
"logps/rejected": -373.8619689941406,
"loss": 0.9496,
"margin_dpo/margin_mean": 97.38102722167969,
"margin_dpo/margin_std": 110.20993041992188,
"step": 557
},
{
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.006852718070149422,
"fcm_dpo/delta": -0.09594091773033142,
"fcm_dpo/margin": 100.5401840209961,
"fcm_dpo/q_t": 0.346677303314209,
"grad_norm": 12.043631553649902,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -2.3513083457946777,
"logits/rejected": -1.7356767654418945,
"logps/chosen": -267.2415771484375,
"logps/ref_chosen": -106.01183319091797,
"logps/ref_rejected": -108.39131927490234,
"logps/rejected": -370.1612548828125,
"loss": 0.9025,
"margin_dpo/margin_mean": 100.5401840209961,
"margin_dpo/margin_std": 92.63386535644531,
"step": 558
},
{
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.006862045265734196,
"fcm_dpo/delta": 0.020558176562190056,
"fcm_dpo/margin": 84.61210632324219,
"fcm_dpo/q_t": 0.3711848556995392,
"grad_norm": 13.065723419189453,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -2.5563955307006836,
"logits/rejected": -2.0908899307250977,
"logps/chosen": -271.6568908691406,
"logps/ref_chosen": -100.42474365234375,
"logps/ref_rejected": -100.41270446777344,
"logps/rejected": -356.2569580078125,
"loss": 1.0026,
"margin_dpo/margin_mean": 84.61210632324219,
"margin_dpo/margin_std": 103.49603271484375,
"step": 559
},
{
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.006905478425323963,
"fcm_dpo/delta": 0.04596704989671707,
"fcm_dpo/margin": 80.60478973388672,
"fcm_dpo/q_t": 0.37765079736709595,
"grad_norm": 13.970322608947754,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -1.1586146354675293,
"logits/rejected": -0.6924932599067688,
"logps/chosen": -254.54705810546875,
"logps/ref_chosen": -87.63551330566406,
"logps/ref_rejected": -87.37985229492188,
"logps/rejected": -334.89617919921875,
"loss": 1.0214,
"margin_dpo/margin_mean": 80.60478210449219,
"margin_dpo/margin_std": 103.96781921386719,
"step": 560
},
{
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0070221684873104095,
"fcm_dpo/delta": 0.06730255484580994,
"fcm_dpo/margin": 76.30671691894531,
"fcm_dpo/q_t": 0.38102492690086365,
"grad_norm": 10.298686981201172,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.9619371891021729,
"logits/rejected": -1.9289535284042358,
"logps/chosen": -249.20779418945312,
"logps/ref_chosen": -74.88140106201172,
"logps/ref_rejected": -106.04796600341797,
"logps/rejected": -356.68109130859375,
"loss": 1.0044,
"margin_dpo/margin_mean": 76.30671691894531,
"margin_dpo/margin_std": 86.78927612304688,
"step": 561
},
{
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0070929634384810925,
"fcm_dpo/delta": 0.0866638720035553,
"fcm_dpo/margin": 73.01620483398438,
"fcm_dpo/q_t": 0.38909482955932617,
"grad_norm": 15.064519882202148,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -2.21871280670166,
"logits/rejected": -1.770498514175415,
"logps/chosen": -259.5347900390625,
"logps/ref_chosen": -100.66322326660156,
"logps/ref_rejected": -102.24087524414062,
"logps/rejected": -334.1286315917969,
"loss": 1.0786,
"margin_dpo/margin_mean": 73.01620483398438,
"margin_dpo/margin_std": 114.18824768066406,
"step": 562
},
{
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.007309372071176767,
"fcm_dpo/delta": 0.12443551421165466,
"fcm_dpo/margin": 65.76155090332031,
"fcm_dpo/q_t": 0.3946084678173065,
"grad_norm": 12.58653736114502,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -1.7142508029937744,
"logits/rejected": -1.3739852905273438,
"logps/chosen": -265.404296875,
"logps/ref_chosen": -96.25921630859375,
"logps/ref_rejected": -101.20503997802734,
"logps/rejected": -336.1116943359375,
"loss": 1.0736,
"margin_dpo/margin_mean": 65.76155090332031,
"margin_dpo/margin_std": 95.51123046875,
"step": 563
},
{
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.007428737357258797,
"fcm_dpo/delta": 0.06579644232988358,
"fcm_dpo/margin": 72.30021667480469,
"fcm_dpo/q_t": 0.3801688849925995,
"grad_norm": 13.077413558959961,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -2.345478057861328,
"logits/rejected": -1.7775757312774658,
"logps/chosen": -277.65167236328125,
"logps/ref_chosen": -112.96040344238281,
"logps/ref_rejected": -99.52803802490234,
"logps/rejected": -336.51953125,
"loss": 1.0098,
"margin_dpo/margin_mean": 72.30021667480469,
"margin_dpo/margin_std": 83.55976867675781,
"step": 564
},
{
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.007322209887206554,
"fcm_dpo/delta": -0.07626369595527649,
"fcm_dpo/margin": 91.62553405761719,
"fcm_dpo/q_t": 0.3552612066268921,
"grad_norm": 11.750452995300293,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -1.7991347312927246,
"logits/rejected": -1.296679139137268,
"logps/chosen": -253.31402587890625,
"logps/ref_chosen": -95.36558532714844,
"logps/ref_rejected": -92.21737670898438,
"logps/rejected": -341.7913513183594,
"loss": 0.9521,
"margin_dpo/margin_mean": 91.62553405761719,
"margin_dpo/margin_std": 103.9658203125,
"step": 565
},
{
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.007290005683898926,
"fcm_dpo/delta": -0.03795613348484039,
"fcm_dpo/margin": 87.17108154296875,
"fcm_dpo/q_t": 0.36021772027015686,
"grad_norm": 10.284100532531738,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -2.10237455368042,
"logits/rejected": -1.6776249408721924,
"logps/chosen": -264.3515625,
"logps/ref_chosen": -98.97320556640625,
"logps/ref_rejected": -110.82090759277344,
"logps/rejected": -363.3703308105469,
"loss": 0.9705,
"margin_dpo/margin_mean": 87.17108154296875,
"margin_dpo/margin_std": 100.97877502441406,
"step": 566
},
{
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.007268569432199001,
"fcm_dpo/delta": 0.050699688494205475,
"fcm_dpo/margin": 75.91093444824219,
"fcm_dpo/q_t": 0.3769448399543762,
"grad_norm": 13.014694213867188,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -2.2701256275177,
"logits/rejected": -1.6002724170684814,
"logps/chosen": -259.28155517578125,
"logps/ref_chosen": -92.7035903930664,
"logps/ref_rejected": -91.22431945800781,
"logps/rejected": -333.7132568359375,
"loss": 1.0209,
"margin_dpo/margin_mean": 75.91093444824219,
"margin_dpo/margin_std": 94.99116516113281,
"step": 567
},
{
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.007200797088444233,
"fcm_dpo/delta": -0.09029556810855865,
"fcm_dpo/margin": 94.91342163085938,
"fcm_dpo/q_t": 0.35067832469940186,
"grad_norm": 10.619403839111328,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -1.4319794178009033,
"logits/rejected": -1.2937116622924805,
"logps/chosen": -230.70367431640625,
"logps/ref_chosen": -73.13226318359375,
"logps/ref_rejected": -94.95265197753906,
"logps/rejected": -347.4375,
"loss": 0.9189,
"margin_dpo/margin_mean": 94.91342163085938,
"margin_dpo/margin_std": 95.35987091064453,
"step": 568
},
{
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.007249847054481506,
"fcm_dpo/delta": 0.02976367063820362,
"fcm_dpo/margin": 78.85527801513672,
"fcm_dpo/q_t": 0.37523531913757324,
"grad_norm": 17.318199157714844,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -2.0498688220977783,
"logits/rejected": -1.6809823513031006,
"logps/chosen": -256.4047546386719,
"logps/ref_chosen": -89.21708679199219,
"logps/ref_rejected": -86.41105651855469,
"logps/rejected": -332.4539794921875,
"loss": 1.0482,
"margin_dpo/margin_mean": 78.85527801513672,
"margin_dpo/margin_std": 112.83515930175781,
"step": 569
},
{
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.00708900997415185,
"fcm_dpo/delta": -0.13322904706001282,
"fcm_dpo/margin": 102.04464721679688,
"fcm_dpo/q_t": 0.3389042615890503,
"grad_norm": 11.147782325744629,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -2.2344322204589844,
"logits/rejected": -1.5383317470550537,
"logps/chosen": -243.58880615234375,
"logps/ref_chosen": -94.76277160644531,
"logps/ref_rejected": -88.846923828125,
"logps/rejected": -339.71759033203125,
"loss": 0.8871,
"margin_dpo/margin_mean": 102.04464721679688,
"margin_dpo/margin_std": 94.04344177246094,
"step": 570
},
{
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.00701398029923439,
"fcm_dpo/delta": -0.053493283689022064,
"fcm_dpo/margin": 92.67195129394531,
"fcm_dpo/q_t": 0.3594912588596344,
"grad_norm": 14.784183502197266,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -1.6442519426345825,
"logits/rejected": -1.4821221828460693,
"logps/chosen": -234.88565063476562,
"logps/ref_chosen": -80.8212890625,
"logps/ref_rejected": -97.01528930664062,
"logps/rejected": -343.7516174316406,
"loss": 0.9712,
"margin_dpo/margin_mean": 92.67195129394531,
"margin_dpo/margin_std": 109.70027160644531,
"step": 571
},
{
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.00692150229588151,
"fcm_dpo/delta": -0.035205624997615814,
"fcm_dpo/margin": 91.42375183105469,
"fcm_dpo/q_t": 0.36460578441619873,
"grad_norm": 11.84706974029541,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.739823341369629,
"logits/rejected": -1.5698119401931763,
"logps/chosen": -235.00161743164062,
"logps/ref_chosen": -77.37992858886719,
"logps/ref_rejected": -90.56597900390625,
"logps/rejected": -339.6114196777344,
"loss": 0.9675,
"margin_dpo/margin_mean": 91.42375183105469,
"margin_dpo/margin_std": 106.2401123046875,
"step": 572
},
{
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.006989034358412027,
"fcm_dpo/delta": -0.0046057626605033875,
"fcm_dpo/margin": 86.20860290527344,
"fcm_dpo/q_t": 0.3665117621421814,
"grad_norm": 10.864632606506348,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -2.1315410137176514,
"logits/rejected": -1.9030604362487793,
"logps/chosen": -236.8482208251953,
"logps/ref_chosen": -83.7060775756836,
"logps/ref_rejected": -90.69746398925781,
"logps/rejected": -330.0482177734375,
"loss": 0.9925,
"margin_dpo/margin_mean": 86.20860290527344,
"margin_dpo/margin_std": 100.88923645019531,
"step": 573
},
{
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.006914999336004257,
"fcm_dpo/delta": -0.02837441675364971,
"fcm_dpo/margin": 90.59143829345703,
"fcm_dpo/q_t": 0.36350974440574646,
"grad_norm": 13.734319686889648,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -2.0994553565979004,
"logits/rejected": -1.7679946422576904,
"logps/chosen": -246.83309936523438,
"logps/ref_chosen": -85.12948608398438,
"logps/ref_rejected": -99.05403137207031,
"logps/rejected": -351.3490905761719,
"loss": 0.9786,
"margin_dpo/margin_mean": 90.59143829345703,
"margin_dpo/margin_std": 107.298828125,
"step": 574
},
{
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.006864962168037891,
"fcm_dpo/delta": -0.005945529788732529,
"fcm_dpo/margin": 88.20824432373047,
"fcm_dpo/q_t": 0.3688689172267914,
"grad_norm": 15.33983039855957,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -1.398423433303833,
"logits/rejected": -0.9362001419067383,
"logps/chosen": -272.59100341796875,
"logps/ref_chosen": -105.77084350585938,
"logps/ref_rejected": -100.94944763183594,
"logps/rejected": -355.9778747558594,
"loss": 1.0027,
"margin_dpo/margin_mean": 88.20824432373047,
"margin_dpo/margin_std": 113.10353088378906,
"step": 575
},
{
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.006903272587805986,
"fcm_dpo/delta": 0.013031164184212685,
"fcm_dpo/margin": 85.11408996582031,
"fcm_dpo/q_t": 0.37200257182121277,
"grad_norm": 12.17516803741455,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -2.0569982528686523,
"logits/rejected": -1.9845412969589233,
"logps/chosen": -248.39340209960938,
"logps/ref_chosen": -81.20452880859375,
"logps/ref_rejected": -105.40940856933594,
"logps/rejected": -357.7123718261719,
"loss": 1.0148,
"margin_dpo/margin_mean": 85.11408996582031,
"margin_dpo/margin_std": 110.67799377441406,
"step": 576
},
{
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.00673964899033308,
"fcm_dpo/delta": -0.11396480351686478,
"fcm_dpo/margin": 104.58753204345703,
"fcm_dpo/q_t": 0.34595972299575806,
"grad_norm": 9.555338859558105,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.9512568712234497,
"logits/rejected": -1.2471516132354736,
"logps/chosen": -249.70469665527344,
"logps/ref_chosen": -101.46419525146484,
"logps/ref_rejected": -89.32637023925781,
"logps/rejected": -342.1544189453125,
"loss": 0.9266,
"margin_dpo/margin_mean": 104.58753204345703,
"margin_dpo/margin_std": 111.21625518798828,
"step": 577
},
{
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.006825780496001244,
"fcm_dpo/delta": 0.08758461475372314,
"fcm_dpo/margin": 75.72772979736328,
"fcm_dpo/q_t": 0.38606128096580505,
"grad_norm": 16.20516586303711,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -1.3706796169281006,
"logits/rejected": -1.131410837173462,
"logps/chosen": -253.5721435546875,
"logps/ref_chosen": -77.29241943359375,
"logps/ref_rejected": -88.24701690673828,
"logps/rejected": -340.25445556640625,
"loss": 1.0516,
"margin_dpo/margin_mean": 75.72772979736328,
"margin_dpo/margin_std": 105.21282196044922,
"step": 578
},
{
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.006886948365718126,
"fcm_dpo/delta": 0.06752566248178482,
"fcm_dpo/margin": 77.81400299072266,
"fcm_dpo/q_t": 0.3831830620765686,
"grad_norm": 11.248205184936523,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -1.7072319984436035,
"logits/rejected": -1.3880934715270996,
"logps/chosen": -245.95306396484375,
"logps/ref_chosen": -87.34305572509766,
"logps/ref_rejected": -102.12100982666016,
"logps/rejected": -338.5450439453125,
"loss": 1.027,
"margin_dpo/margin_mean": 77.81400299072266,
"margin_dpo/margin_std": 96.24092102050781,
"step": 579
},
{
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.006863095331937075,
"fcm_dpo/delta": -0.040312498807907104,
"fcm_dpo/margin": 92.7848129272461,
"fcm_dpo/q_t": 0.3631054759025574,
"grad_norm": 11.509198188781738,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -1.6127009391784668,
"logits/rejected": -1.3059601783752441,
"logps/chosen": -242.31298828125,
"logps/ref_chosen": -88.92879486083984,
"logps/ref_rejected": -88.69661712646484,
"logps/rejected": -334.8656311035156,
"loss": 0.9711,
"margin_dpo/margin_mean": 92.7848129272461,
"margin_dpo/margin_std": 108.14720153808594,
"step": 580
},
{
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.006930704694241285,
"fcm_dpo/delta": 0.026802916079759598,
"fcm_dpo/margin": 82.91061401367188,
"fcm_dpo/q_t": 0.3752003312110901,
"grad_norm": 13.487149238586426,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -2.016202449798584,
"logits/rejected": -1.6698317527770996,
"logps/chosen": -254.0625762939453,
"logps/ref_chosen": -92.22323608398438,
"logps/ref_rejected": -99.38943481445312,
"logps/rejected": -344.139404296875,
"loss": 1.0399,
"margin_dpo/margin_mean": 82.91061401367188,
"margin_dpo/margin_std": 116.20519256591797,
"step": 581
},
{
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.007061149924993515,
"fcm_dpo/delta": 0.04960351437330246,
"fcm_dpo/margin": 77.88203430175781,
"fcm_dpo/q_t": 0.3824002146720886,
"grad_norm": 11.220653533935547,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -1.9758110046386719,
"logits/rejected": -1.9616727828979492,
"logps/chosen": -229.19464111328125,
"logps/ref_chosen": -87.33561706542969,
"logps/ref_rejected": -115.1544189453125,
"logps/rejected": -334.8954772949219,
"loss": 1.039,
"margin_dpo/margin_mean": 77.88203430175781,
"margin_dpo/margin_std": 104.54690551757812,
"step": 582
},
{
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.006990609224885702,
"fcm_dpo/delta": 0.02330578863620758,
"fcm_dpo/margin": 82.64878845214844,
"fcm_dpo/q_t": 0.37635916471481323,
"grad_norm": 12.386331558227539,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -1.5934879779815674,
"logits/rejected": -1.364257574081421,
"logps/chosen": -257.1393737792969,
"logps/ref_chosen": -83.71707153320312,
"logps/ref_rejected": -94.58465576171875,
"logps/rejected": -350.65576171875,
"loss": 1.0101,
"margin_dpo/margin_mean": 82.64878845214844,
"margin_dpo/margin_std": 105.64147186279297,
"step": 583
},
{
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.007066408172249794,
"fcm_dpo/delta": -0.031313344836235046,
"fcm_dpo/margin": 88.83890533447266,
"fcm_dpo/q_t": 0.3632817268371582,
"grad_norm": 10.694197654724121,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -2.124901294708252,
"logits/rejected": -1.6727513074874878,
"logps/chosen": -246.2718963623047,
"logps/ref_chosen": -94.25955963134766,
"logps/ref_rejected": -104.48269653320312,
"logps/rejected": -345.33392333984375,
"loss": 0.9905,
"margin_dpo/margin_mean": 88.83891296386719,
"margin_dpo/margin_std": 106.91671752929688,
"step": 584
},
{
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.00690193846821785,
"fcm_dpo/delta": -0.11694584041833878,
"fcm_dpo/margin": 102.68148040771484,
"fcm_dpo/q_t": 0.350594162940979,
"grad_norm": 11.292356491088867,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.7038193941116333,
"logits/rejected": -1.4119065999984741,
"logps/chosen": -247.07659912109375,
"logps/ref_chosen": -85.63676452636719,
"logps/ref_rejected": -90.28547668457031,
"logps/rejected": -354.40679931640625,
"loss": 0.955,
"margin_dpo/margin_mean": 102.68147277832031,
"margin_dpo/margin_std": 123.2242431640625,
"step": 585
},
{
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.006955640856176615,
"fcm_dpo/delta": 0.1244577169418335,
"fcm_dpo/margin": 69.21916198730469,
"fcm_dpo/q_t": 0.3929838240146637,
"grad_norm": 13.513263702392578,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.7758688926696777,
"logits/rejected": -1.507314682006836,
"logps/chosen": -263.46990966796875,
"logps/ref_chosen": -83.68114471435547,
"logps/ref_rejected": -98.35916137695312,
"logps/rejected": -347.3670654296875,
"loss": 1.0502,
"margin_dpo/margin_mean": 69.21916198730469,
"margin_dpo/margin_std": 89.19205474853516,
"step": 586
},
{
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0070916032418608665,
"fcm_dpo/delta": 0.06968455016613007,
"fcm_dpo/margin": 75.24751281738281,
"fcm_dpo/q_t": 0.3832997679710388,
"grad_norm": 11.80888557434082,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -2.346747636795044,
"logits/rejected": -2.054532527923584,
"logps/chosen": -222.94000244140625,
"logps/ref_chosen": -77.57750701904297,
"logps/ref_rejected": -89.32032775878906,
"logps/rejected": -309.93035888671875,
"loss": 1.0274,
"margin_dpo/margin_mean": 75.24751281738281,
"margin_dpo/margin_std": 96.26628875732422,
"step": 587
},
{
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.00717865489423275,
"fcm_dpo/delta": 0.10119879245758057,
"fcm_dpo/margin": 70.20915222167969,
"fcm_dpo/q_t": 0.39323359727859497,
"grad_norm": 14.625384330749512,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.971228003501892,
"logits/rejected": -1.9192923307418823,
"logps/chosen": -241.52024841308594,
"logps/ref_chosen": -83.80426025390625,
"logps/ref_rejected": -113.11943054199219,
"logps/rejected": -341.0445861816406,
"loss": 1.0898,
"margin_dpo/margin_mean": 70.20915222167969,
"margin_dpo/margin_std": 111.68272399902344,
"step": 588
},
{
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.007164877839386463,
"fcm_dpo/delta": -0.06353701651096344,
"fcm_dpo/margin": 92.01499938964844,
"fcm_dpo/q_t": 0.35298681259155273,
"grad_norm": 12.12359619140625,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -1.600557565689087,
"logits/rejected": -1.1946024894714355,
"logps/chosen": -266.53143310546875,
"logps/ref_chosen": -96.46501922607422,
"logps/ref_rejected": -116.03719329833984,
"logps/rejected": -378.11859130859375,
"loss": 0.9228,
"margin_dpo/margin_mean": 92.01498413085938,
"margin_dpo/margin_std": 88.87408447265625,
"step": 589
},
{
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.007136746309697628,
"fcm_dpo/delta": -0.05460678040981293,
"fcm_dpo/margin": 91.158203125,
"fcm_dpo/q_t": 0.3565479516983032,
"grad_norm": 13.865069389343262,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -1.906883716583252,
"logits/rejected": -1.7435002326965332,
"logps/chosen": -242.1221160888672,
"logps/ref_chosen": -88.4278564453125,
"logps/ref_rejected": -110.19876861572266,
"logps/rejected": -355.0512390136719,
"loss": 0.9316,
"margin_dpo/margin_mean": 91.158203125,
"margin_dpo/margin_std": 90.19902038574219,
"step": 590
},
{
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.00695719663053751,
"fcm_dpo/delta": -0.08778176456689835,
"fcm_dpo/margin": 97.93551635742188,
"fcm_dpo/q_t": 0.3495037257671356,
"grad_norm": 8.954543113708496,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -2.069058418273926,
"logits/rejected": -1.4569966793060303,
"logps/chosen": -236.16778564453125,
"logps/ref_chosen": -87.84968566894531,
"logps/ref_rejected": -89.57919311523438,
"logps/rejected": -335.83282470703125,
"loss": 0.9107,
"margin_dpo/margin_mean": 97.93551635742188,
"margin_dpo/margin_std": 93.53607177734375,
"step": 591
},
{
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.007020566612482071,
"fcm_dpo/delta": 0.06493528187274933,
"fcm_dpo/margin": 76.70272827148438,
"fcm_dpo/q_t": 0.3827671408653259,
"grad_norm": 13.41107177734375,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.726281762123108,
"logits/rejected": -1.3931875228881836,
"logps/chosen": -250.2864990234375,
"logps/ref_chosen": -74.97573852539062,
"logps/ref_rejected": -81.55262756347656,
"logps/rejected": -333.5661315917969,
"loss": 1.0491,
"margin_dpo/margin_mean": 76.70272827148438,
"margin_dpo/margin_std": 108.41436004638672,
"step": 592
},
{
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.007145709358155727,
"fcm_dpo/delta": 0.1387026607990265,
"fcm_dpo/margin": 65.48255157470703,
"fcm_dpo/q_t": 0.399899959564209,
"grad_norm": 17.311737060546875,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.939947247505188,
"logits/rejected": -0.6396098136901855,
"logps/chosen": -234.7220458984375,
"logps/ref_chosen": -72.23902893066406,
"logps/ref_rejected": -70.09652709960938,
"logps/rejected": -298.0621337890625,
"loss": 1.099,
"margin_dpo/margin_mean": 65.48255157470703,
"margin_dpo/margin_std": 103.57083129882812,
"step": 593
},
{
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0071470364928245544,
"fcm_dpo/delta": -0.11245208233594894,
"fcm_dpo/margin": 98.58993530273438,
"fcm_dpo/q_t": 0.3443548083305359,
"grad_norm": 10.641993522644043,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.9720404148101807,
"logits/rejected": -1.4504088163375854,
"logps/chosen": -236.50711059570312,
"logps/ref_chosen": -90.43771362304688,
"logps/ref_rejected": -92.267578125,
"logps/rejected": -336.92694091796875,
"loss": 0.8928,
"margin_dpo/margin_mean": 98.58993530273438,
"margin_dpo/margin_std": 89.36225891113281,
"step": 594
},
{
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.007122883573174477,
"fcm_dpo/delta": 0.020702892914414406,
"fcm_dpo/margin": 81.45152282714844,
"fcm_dpo/q_t": 0.3758925795555115,
"grad_norm": 12.234829902648926,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -1.6966605186462402,
"logits/rejected": -1.2726609706878662,
"logps/chosen": -236.58111572265625,
"logps/ref_chosen": -81.51480102539062,
"logps/ref_rejected": -86.52047729492188,
"logps/rejected": -323.038330078125,
"loss": 1.0138,
"margin_dpo/margin_mean": 81.45152282714844,
"margin_dpo/margin_std": 105.23150634765625,
"step": 595
},
{
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.00720050185918808,
"fcm_dpo/delta": 0.08340231329202652,
"fcm_dpo/margin": 72.33407592773438,
"fcm_dpo/q_t": 0.3886866271495819,
"grad_norm": 15.935720443725586,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -1.7379255294799805,
"logits/rejected": -1.1650152206420898,
"logps/chosen": -248.61962890625,
"logps/ref_chosen": -88.20805358886719,
"logps/ref_rejected": -86.09375,
"logps/rejected": -318.83941650390625,
"loss": 1.0738,
"margin_dpo/margin_mean": 72.33407592773438,
"margin_dpo/margin_std": 109.93353271484375,
"step": 596
},
{
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.007105865050107241,
"fcm_dpo/delta": -0.09072263538837433,
"fcm_dpo/margin": 96.26971435546875,
"fcm_dpo/q_t": 0.3528348505496979,
"grad_norm": 10.61738109588623,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -1.3981832265853882,
"logits/rejected": -1.2777040004730225,
"logps/chosen": -240.2840576171875,
"logps/ref_chosen": -83.58998107910156,
"logps/ref_rejected": -106.94439697265625,
"logps/rejected": -359.908203125,
"loss": 0.9408,
"margin_dpo/margin_mean": 96.26971435546875,
"margin_dpo/margin_std": 107.11369323730469,
"step": 597
},
{
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.007089652121067047,
"fcm_dpo/delta": 0.03473873436450958,
"fcm_dpo/margin": 79.942138671875,
"fcm_dpo/q_t": 0.37478604912757874,
"grad_norm": 11.30370807647705,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -2.0580103397369385,
"logits/rejected": -1.748920202255249,
"logps/chosen": -239.97232055664062,
"logps/ref_chosen": -90.18262481689453,
"logps/ref_rejected": -94.79856872558594,
"logps/rejected": -324.5303955078125,
"loss": 1.021,
"margin_dpo/margin_mean": 79.942138671875,
"margin_dpo/margin_std": 102.13404846191406,
"step": 598
},
{
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.007190364878624678,
"fcm_dpo/delta": 0.09109681099653244,
"fcm_dpo/margin": 71.37487030029297,
"fcm_dpo/q_t": 0.38869598507881165,
"grad_norm": 12.38364028930664,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -1.799156665802002,
"logits/rejected": -1.6733660697937012,
"logps/chosen": -232.6363067626953,
"logps/ref_chosen": -77.84226989746094,
"logps/ref_rejected": -89.1976318359375,
"logps/rejected": -315.36651611328125,
"loss": 1.0679,
"margin_dpo/margin_mean": 71.37487030029297,
"margin_dpo/margin_std": 102.9161148071289,
"step": 599
},
{
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.007273955270648003,
"fcm_dpo/delta": -0.006405937019735575,
"fcm_dpo/margin": 83.31228637695312,
"fcm_dpo/q_t": 0.3687729835510254,
"grad_norm": 10.347834587097168,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -2.4206528663635254,
"logits/rejected": -2.0758280754089355,
"logps/chosen": -257.964111328125,
"logps/ref_chosen": -92.18781280517578,
"logps/ref_rejected": -92.95429992675781,
"logps/rejected": -342.0428771972656,
"loss": 0.9868,
"margin_dpo/margin_mean": 83.31228637695312,
"margin_dpo/margin_std": 99.32923889160156,
"step": 600
},
{
"epoch": 0.8810572687224669,
"eval_fcm_dpo/beta": 0.0072656371630728245,
"eval_logits/chosen": -1.840718150138855,
"eval_logits/rejected": -1.4554615020751953,
"eval_logps/chosen": -265.1147155761719,
"eval_logps/ref_chosen": -100.49356842041016,
"eval_logps/ref_rejected": -94.06775665283203,
"eval_logps/rejected": -332.2911376953125,
"eval_loss": 0.5223754644393921,
"eval_margin_dpo/margin_mean": 73.60225677490234,
"eval_margin_dpo/margin_std": 100.21179962158203,
"eval_runtime": 43.7553,
"eval_samples_per_second": 53.456,
"eval_steps_per_second": 1.691,
"step": 600
},
{
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.00720963254570961,
"fcm_dpo/delta": -0.023707086220383644,
"fcm_dpo/margin": 86.24479675292969,
"fcm_dpo/q_t": 0.3629060983657837,
"grad_norm": 11.4024019241333,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -1.7290210723876953,
"logits/rejected": -1.4024295806884766,
"logps/chosen": -259.41070556640625,
"logps/ref_chosen": -89.21614074707031,
"logps/ref_rejected": -100.17054748535156,
"logps/rejected": -356.60992431640625,
"loss": 0.9523,
"margin_dpo/margin_mean": 86.24479675292969,
"margin_dpo/margin_std": 90.01136016845703,
"step": 601
},
{
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.007033023051917553,
"fcm_dpo/delta": -0.185970276594162,
"fcm_dpo/margin": 109.70604705810547,
"fcm_dpo/q_t": 0.3343247175216675,
"grad_norm": 10.687939643859863,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -2.0846588611602783,
"logits/rejected": -1.826683759689331,
"logps/chosen": -214.779052734375,
"logps/ref_chosen": -80.05760192871094,
"logps/ref_rejected": -93.197509765625,
"logps/rejected": -337.625,
"loss": 0.8887,
"margin_dpo/margin_mean": 109.70604705810547,
"margin_dpo/margin_std": 112.28135681152344,
"step": 602
},
{
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.006929402239620686,
"fcm_dpo/delta": -0.0485604926943779,
"fcm_dpo/margin": 93.14605712890625,
"fcm_dpo/q_t": 0.3590378165245056,
"grad_norm": 10.695706367492676,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -1.9837377071380615,
"logits/rejected": -1.3861087560653687,
"logps/chosen": -247.59893798828125,
"logps/ref_chosen": -102.30958557128906,
"logps/ref_rejected": -108.06884765625,
"logps/rejected": -346.5042724609375,
"loss": 0.9534,
"margin_dpo/margin_mean": 93.14605712890625,
"margin_dpo/margin_std": 104.23910522460938,
"step": 603
},
{
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.006826011463999748,
"fcm_dpo/delta": -0.04624610394239426,
"fcm_dpo/margin": 94.17044067382812,
"fcm_dpo/q_t": 0.35456353425979614,
"grad_norm": 10.141494750976562,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -1.782772183418274,
"logits/rejected": -1.626922845840454,
"logps/chosen": -228.49160766601562,
"logps/ref_chosen": -78.17408752441406,
"logps/ref_rejected": -99.49610900878906,
"logps/rejected": -343.98406982421875,
"loss": 0.9365,
"margin_dpo/margin_mean": 94.17044067382812,
"margin_dpo/margin_std": 92.8728256225586,
"step": 604
},
{
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.006894152611494064,
"fcm_dpo/delta": 0.06398770958185196,
"fcm_dpo/margin": 78.25582122802734,
"fcm_dpo/q_t": 0.38114598393440247,
"grad_norm": 10.904273986816406,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -1.7464425563812256,
"logits/rejected": -1.2376749515533447,
"logps/chosen": -258.21099853515625,
"logps/ref_chosen": -94.77332305908203,
"logps/ref_rejected": -107.30490112304688,
"logps/rejected": -348.9984130859375,
"loss": 1.0158,
"margin_dpo/margin_mean": 78.25582885742188,
"margin_dpo/margin_std": 95.33082580566406,
"step": 605
},
{
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.007000097073614597,
"fcm_dpo/delta": 0.02021496742963791,
"fcm_dpo/margin": 82.80071258544922,
"fcm_dpo/q_t": 0.3707614541053772,
"grad_norm": 11.527751922607422,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -1.9277280569076538,
"logits/rejected": -1.524000883102417,
"logps/chosen": -245.60330200195312,
"logps/ref_chosen": -87.7533950805664,
"logps/ref_rejected": -104.2422103881836,
"logps/rejected": -344.892822265625,
"loss": 0.9999,
"margin_dpo/margin_mean": 82.80071258544922,
"margin_dpo/margin_std": 97.99324035644531,
"step": 606
},
{
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0069069629535079,
"fcm_dpo/delta": -0.04535501077771187,
"fcm_dpo/margin": 93.01445007324219,
"fcm_dpo/q_t": 0.36220258474349976,
"grad_norm": 14.510759353637695,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -1.7667630910873413,
"logits/rejected": -1.2318663597106934,
"logps/chosen": -243.1822509765625,
"logps/ref_chosen": -88.32904052734375,
"logps/ref_rejected": -86.76810455322266,
"logps/rejected": -334.6357421875,
"loss": 1.0038,
"margin_dpo/margin_mean": 93.01445007324219,
"margin_dpo/margin_std": 122.94918823242188,
"step": 607
},
{
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.006772700231522322,
"fcm_dpo/delta": -0.05694134533405304,
"fcm_dpo/margin": 96.16629028320312,
"fcm_dpo/q_t": 0.35457879304885864,
"grad_norm": 12.842971801757812,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.7402276992797852,
"logits/rejected": -1.4407744407653809,
"logps/chosen": -226.5384521484375,
"logps/ref_chosen": -85.76937866210938,
"logps/ref_rejected": -100.23281860351562,
"logps/rejected": -337.1681823730469,
"loss": 0.9581,
"margin_dpo/margin_mean": 96.16629028320312,
"margin_dpo/margin_std": 104.13595581054688,
"step": 608
},
{
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.006742427125573158,
"fcm_dpo/delta": -0.037574149668216705,
"fcm_dpo/margin": 94.12151336669922,
"fcm_dpo/q_t": 0.36158236861228943,
"grad_norm": 13.74590015411377,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -2.4941606521606445,
"logits/rejected": -2.0870494842529297,
"logps/chosen": -246.841796875,
"logps/ref_chosen": -92.9665756225586,
"logps/ref_rejected": -95.91818237304688,
"logps/rejected": -343.9149169921875,
"loss": 0.9629,
"margin_dpo/margin_mean": 94.12152099609375,
"margin_dpo/margin_std": 102.90473937988281,
"step": 609
},
{
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.00668362807482481,
"fcm_dpo/delta": -0.060604121536016464,
"fcm_dpo/margin": 98.20524597167969,
"fcm_dpo/q_t": 0.3524959683418274,
"grad_norm": 10.179794311523438,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -1.8149501085281372,
"logits/rejected": -1.2743051052093506,
"logps/chosen": -239.31069946289062,
"logps/ref_chosen": -94.70028686523438,
"logps/ref_rejected": -89.68739318847656,
"logps/rejected": -332.5030517578125,
"loss": 0.934,
"margin_dpo/margin_mean": 98.20524597167969,
"margin_dpo/margin_std": 99.45082092285156,
"step": 610
},
{
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.00667162798345089,
"fcm_dpo/delta": -0.047587163746356964,
"fcm_dpo/margin": 96.54525756835938,
"fcm_dpo/q_t": 0.35953009128570557,
"grad_norm": 12.610694885253906,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -2.17702317237854,
"logits/rejected": -1.8332421779632568,
"logps/chosen": -237.6696319580078,
"logps/ref_chosen": -86.78334045410156,
"logps/ref_rejected": -89.84307861328125,
"logps/rejected": -337.274658203125,
"loss": 0.9441,
"margin_dpo/margin_mean": 96.54525756835938,
"margin_dpo/margin_std": 100.653076171875,
"step": 611
},
{
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.006550832651555538,
"fcm_dpo/delta": -0.049320608377456665,
"fcm_dpo/margin": 98.5962142944336,
"fcm_dpo/q_t": 0.35573434829711914,
"grad_norm": 10.573701858520508,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -2.050720453262329,
"logits/rejected": -1.197265625,
"logps/chosen": -265.29229736328125,
"logps/ref_chosen": -101.02015686035156,
"logps/ref_rejected": -93.78302764892578,
"logps/rejected": -356.6513671875,
"loss": 0.9267,
"margin_dpo/margin_mean": 98.5962142944336,
"margin_dpo/margin_std": 94.86685180664062,
"step": 612
},
{
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.00644554290920496,
"fcm_dpo/delta": -0.09590654820203781,
"fcm_dpo/margin": 106.90154266357422,
"fcm_dpo/q_t": 0.34533023834228516,
"grad_norm": 10.872828483581543,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -1.9289722442626953,
"logits/rejected": -1.728402853012085,
"logps/chosen": -224.5867919921875,
"logps/ref_chosen": -88.98861694335938,
"logps/ref_rejected": -109.99551391601562,
"logps/rejected": -352.4952392578125,
"loss": 0.8885,
"margin_dpo/margin_mean": 106.90153503417969,
"margin_dpo/margin_std": 91.88870239257812,
"step": 613
},
{
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0064603416249156,
"fcm_dpo/delta": 0.04875949025154114,
"fcm_dpo/margin": 85.74708557128906,
"fcm_dpo/q_t": 0.37675923109054565,
"grad_norm": 12.27489948272705,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -1.6303455829620361,
"logits/rejected": -1.6167235374450684,
"logps/chosen": -247.53366088867188,
"logps/ref_chosen": -80.20005798339844,
"logps/ref_rejected": -109.86239624023438,
"logps/rejected": -362.94305419921875,
"loss": 1.0172,
"margin_dpo/margin_mean": 85.74708557128906,
"margin_dpo/margin_std": 105.97724151611328,
"step": 614
},
{
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0066406624391674995,
"fcm_dpo/delta": 0.09489738941192627,
"fcm_dpo/margin": 76.43423461914062,
"fcm_dpo/q_t": 0.38481491804122925,
"grad_norm": 11.700821876525879,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -1.9045593738555908,
"logits/rejected": -1.5077332258224487,
"logps/chosen": -256.5118408203125,
"logps/ref_chosen": -100.43525695800781,
"logps/ref_rejected": -101.1800537109375,
"logps/rejected": -333.6908874511719,
"loss": 1.0219,
"margin_dpo/margin_mean": 76.43423461914062,
"margin_dpo/margin_std": 84.58399963378906,
"step": 615
},
{
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.006577151361852884,
"fcm_dpo/delta": -0.0695214495062828,
"fcm_dpo/margin": 101.07377624511719,
"fcm_dpo/q_t": 0.35348814725875854,
"grad_norm": 10.295190811157227,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.8900903463363647,
"logits/rejected": -1.4016035795211792,
"logps/chosen": -250.1123504638672,
"logps/ref_chosen": -92.49292755126953,
"logps/ref_rejected": -82.06065368652344,
"logps/rejected": -340.7538757324219,
"loss": 0.937,
"margin_dpo/margin_mean": 101.07378387451172,
"margin_dpo/margin_std": 106.29083251953125,
"step": 616
},
{
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.006596277467906475,
"fcm_dpo/delta": 0.07905448973178864,
"fcm_dpo/margin": 79.61018371582031,
"fcm_dpo/q_t": 0.3824957013130188,
"grad_norm": 10.861001014709473,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -1.6198773384094238,
"logits/rejected": -0.9793082475662231,
"logps/chosen": -255.09112548828125,
"logps/ref_chosen": -91.09699249267578,
"logps/ref_rejected": -70.41004943847656,
"logps/rejected": -314.01434326171875,
"loss": 1.0275,
"margin_dpo/margin_mean": 79.61019897460938,
"margin_dpo/margin_std": 98.98773193359375,
"step": 617
},
{
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.006635970436036587,
"fcm_dpo/delta": 0.026904039084911346,
"fcm_dpo/margin": 86.58537292480469,
"fcm_dpo/q_t": 0.37111544609069824,
"grad_norm": 11.58753776550293,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.829303503036499,
"logits/rejected": -1.4636094570159912,
"logps/chosen": -266.91119384765625,
"logps/ref_chosen": -102.02059936523438,
"logps/ref_rejected": -99.80119323730469,
"logps/rejected": -351.27716064453125,
"loss": 1.0048,
"margin_dpo/margin_mean": 86.58537292480469,
"margin_dpo/margin_std": 104.85649108886719,
"step": 618
},
{
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.006731549743562937,
"fcm_dpo/delta": 0.07085485756397247,
"fcm_dpo/margin": 79.16950988769531,
"fcm_dpo/q_t": 0.38630783557891846,
"grad_norm": 11.098875999450684,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.452412724494934,
"logits/rejected": -1.0741287469863892,
"logps/chosen": -248.23838806152344,
"logps/ref_chosen": -89.74136352539062,
"logps/ref_rejected": -99.90138244628906,
"logps/rejected": -337.56793212890625,
"loss": 1.0501,
"margin_dpo/margin_mean": 79.16950988769531,
"margin_dpo/margin_std": 113.035400390625,
"step": 619
},
{
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.006684327032417059,
"fcm_dpo/delta": 0.012951068580150604,
"fcm_dpo/margin": 87.47955322265625,
"fcm_dpo/q_t": 0.3729515075683594,
"grad_norm": 11.075997352600098,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -2.05159068107605,
"logits/rejected": -1.514744758605957,
"logps/chosen": -238.4269256591797,
"logps/ref_chosen": -85.12431335449219,
"logps/ref_rejected": -85.41253662109375,
"logps/rejected": -326.1947021484375,
"loss": 1.0182,
"margin_dpo/margin_mean": 87.47955322265625,
"margin_dpo/margin_std": 107.8419189453125,
"step": 620
},
{
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.006873922888189554,
"fcm_dpo/delta": 0.017858199775218964,
"fcm_dpo/margin": 84.41682434082031,
"fcm_dpo/q_t": 0.3753046989440918,
"grad_norm": 11.407662391662598,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.8650214672088623,
"logits/rejected": -1.598212480545044,
"logps/chosen": -245.00628662109375,
"logps/ref_chosen": -89.24842071533203,
"logps/ref_rejected": -95.46463775634766,
"logps/rejected": -335.6393127441406,
"loss": 1.0371,
"margin_dpo/margin_mean": 84.41682434082031,
"margin_dpo/margin_std": 113.9649658203125,
"step": 621
},
{
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0067746308632195,
"fcm_dpo/delta": -0.006477778777480125,
"fcm_dpo/margin": 89.44147491455078,
"fcm_dpo/q_t": 0.3655218780040741,
"grad_norm": 12.676264762878418,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -1.4255738258361816,
"logits/rejected": -0.7507696747779846,
"logps/chosen": -255.17373657226562,
"logps/ref_chosen": -99.79412841796875,
"logps/ref_rejected": -90.82821655273438,
"logps/rejected": -335.6492919921875,
"loss": 0.9635,
"margin_dpo/margin_mean": 89.44148254394531,
"margin_dpo/margin_std": 94.65943908691406,
"step": 622
},
{
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.006937139667570591,
"fcm_dpo/delta": 0.16734254360198975,
"fcm_dpo/margin": 63.47694396972656,
"fcm_dpo/q_t": 0.40689754486083984,
"grad_norm": 13.347342491149902,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -2.32456111907959,
"logits/rejected": -1.8910653591156006,
"logps/chosen": -251.2813262939453,
"logps/ref_chosen": -90.45555114746094,
"logps/ref_rejected": -91.32276153564453,
"logps/rejected": -315.62548828125,
"loss": 1.138,
"margin_dpo/margin_mean": 63.47693634033203,
"margin_dpo/margin_std": 114.36991882324219,
"step": 623
},
{
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.006922868080437183,
"fcm_dpo/delta": -0.09390503168106079,
"fcm_dpo/margin": 99.29826354980469,
"fcm_dpo/q_t": 0.35036206245422363,
"grad_norm": 12.173335075378418,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -2.5920774936676025,
"logits/rejected": -1.668988585472107,
"logps/chosen": -272.736083984375,
"logps/ref_chosen": -109.87522888183594,
"logps/ref_rejected": -104.77320861816406,
"logps/rejected": -366.93231201171875,
"loss": 0.9295,
"margin_dpo/margin_mean": 99.29826354980469,
"margin_dpo/margin_std": 105.40048217773438,
"step": 624
},
{
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.006994827184826136,
"fcm_dpo/delta": 0.08261488378047943,
"fcm_dpo/margin": 74.53196716308594,
"fcm_dpo/q_t": 0.38579094409942627,
"grad_norm": 10.665149688720703,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -1.9561140537261963,
"logits/rejected": -1.590733289718628,
"logps/chosen": -259.02740478515625,
"logps/ref_chosen": -87.16815948486328,
"logps/ref_rejected": -91.86148071289062,
"logps/rejected": -338.252685546875,
"loss": 1.0348,
"margin_dpo/margin_mean": 74.53196716308594,
"margin_dpo/margin_std": 97.0251693725586,
"step": 625
},
{
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.007120197638869286,
"fcm_dpo/delta": 0.08808340132236481,
"fcm_dpo/margin": 72.4712142944336,
"fcm_dpo/q_t": 0.39089781045913696,
"grad_norm": 13.34603214263916,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -1.5797321796417236,
"logits/rejected": -1.1557903289794922,
"logps/chosen": -267.87542724609375,
"logps/ref_chosen": -104.22421264648438,
"logps/ref_rejected": -104.1774673461914,
"logps/rejected": -340.29986572265625,
"loss": 1.0775,
"margin_dpo/margin_mean": 72.47120666503906,
"margin_dpo/margin_std": 112.5821533203125,
"step": 626
},
{
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.007080005016177893,
"fcm_dpo/delta": -0.04163035377860069,
"fcm_dpo/margin": 90.24308776855469,
"fcm_dpo/q_t": 0.36456674337387085,
"grad_norm": 9.200769424438477,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -1.5551083087921143,
"logits/rejected": -1.6972510814666748,
"logps/chosen": -221.11859130859375,
"logps/ref_chosen": -74.93461608886719,
"logps/ref_rejected": -112.57289123535156,
"logps/rejected": -348.9999694824219,
"loss": 0.9833,
"margin_dpo/margin_mean": 90.24308776855469,
"margin_dpo/margin_std": 113.31100463867188,
"step": 627
},
{
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.006989369168877602,
"fcm_dpo/delta": -0.0777251347899437,
"fcm_dpo/margin": 96.21630859375,
"fcm_dpo/q_t": 0.3529139757156372,
"grad_norm": 12.8143892288208,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -2.0264172554016113,
"logits/rejected": -1.7620849609375,
"logps/chosen": -246.24087524414062,
"logps/ref_chosen": -93.69107818603516,
"logps/ref_rejected": -107.34395599365234,
"logps/rejected": -356.11004638671875,
"loss": 0.9296,
"margin_dpo/margin_mean": 96.21630096435547,
"margin_dpo/margin_std": 98.98152923583984,
"step": 628
},
{
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0069330655969679356,
"fcm_dpo/delta": -0.0012096241116523743,
"fcm_dpo/margin": 86.66081237792969,
"fcm_dpo/q_t": 0.36651545763015747,
"grad_norm": 11.387245178222656,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -2.008543014526367,
"logits/rejected": -1.5725985765457153,
"logps/chosen": -266.6309814453125,
"logps/ref_chosen": -103.23037719726562,
"logps/ref_rejected": -97.16841888427734,
"logps/rejected": -347.2298278808594,
"loss": 0.9856,
"margin_dpo/margin_mean": 86.66081237792969,
"margin_dpo/margin_std": 100.66366577148438,
"step": 629
},
{
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.007189781405031681,
"fcm_dpo/delta": 0.1778494417667389,
"fcm_dpo/margin": 59.536293029785156,
"fcm_dpo/q_t": 0.40600132942199707,
"grad_norm": 15.609967231750488,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -2.0184245109558105,
"logits/rejected": -1.4799931049346924,
"logps/chosen": -244.27438354492188,
"logps/ref_chosen": -93.89755249023438,
"logps/ref_rejected": -89.3743896484375,
"logps/rejected": -299.28753662109375,
"loss": 1.1223,
"margin_dpo/margin_mean": 59.536293029785156,
"margin_dpo/margin_std": 98.0714111328125,
"step": 630
},
{
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.007201473228633404,
"fcm_dpo/delta": -0.001188160851597786,
"fcm_dpo/margin": 83.4664535522461,
"fcm_dpo/q_t": 0.36686837673187256,
"grad_norm": 11.061304092407227,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -2.5078396797180176,
"logits/rejected": -2.078655958175659,
"logps/chosen": -235.43612670898438,
"logps/ref_chosen": -87.77082061767578,
"logps/ref_rejected": -88.68241882324219,
"logps/rejected": -319.81414794921875,
"loss": 0.9793,
"margin_dpo/margin_mean": 83.4664535522461,
"margin_dpo/margin_std": 95.37765502929688,
"step": 631
},
{
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.007080842275172472,
"fcm_dpo/delta": -0.06554602086544037,
"fcm_dpo/margin": 93.11445617675781,
"fcm_dpo/q_t": 0.35878896713256836,
"grad_norm": 12.773686408996582,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.9766130447387695,
"logits/rejected": -1.5455785989761353,
"logps/chosen": -233.4774169921875,
"logps/ref_chosen": -88.62652587890625,
"logps/ref_rejected": -91.45091247558594,
"logps/rejected": -329.416259765625,
"loss": 0.9759,
"margin_dpo/margin_mean": 93.11445617675781,
"margin_dpo/margin_std": 112.72152709960938,
"step": 632
},
{
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.007372128311544657,
"fcm_dpo/delta": 0.18217022716999054,
"fcm_dpo/margin": 57.35297393798828,
"fcm_dpo/q_t": 0.41445010900497437,
"grad_norm": 14.506038665771484,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -2.240860939025879,
"logits/rejected": -2.0498335361480713,
"logps/chosen": -260.7783203125,
"logps/ref_chosen": -81.37442016601562,
"logps/ref_rejected": -98.62571716308594,
"logps/rejected": -335.3825988769531,
"loss": 1.1966,
"margin_dpo/margin_mean": 57.35297393798828,
"margin_dpo/margin_std": 124.81423950195312,
"step": 633
},
{
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.007527489215135574,
"fcm_dpo/delta": 0.04844316840171814,
"fcm_dpo/margin": 73.15084838867188,
"fcm_dpo/q_t": 0.3781312108039856,
"grad_norm": 13.961141586303711,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -1.4511160850524902,
"logits/rejected": -1.2699205875396729,
"logps/chosen": -263.0788879394531,
"logps/ref_chosen": -96.12284851074219,
"logps/ref_rejected": -112.84780883789062,
"logps/rejected": -352.9547119140625,
"loss": 1.0141,
"margin_dpo/margin_mean": 73.15084838867188,
"margin_dpo/margin_std": 84.39659881591797,
"step": 634
},
{
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.007684573996812105,
"fcm_dpo/delta": 0.16829924285411835,
"fcm_dpo/margin": 56.93890380859375,
"fcm_dpo/q_t": 0.40733087062835693,
"grad_norm": 16.105562210083008,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.7153419256210327,
"logits/rejected": -1.1771185398101807,
"logps/chosen": -269.976806640625,
"logps/ref_chosen": -98.68411254882812,
"logps/ref_rejected": -89.89920043945312,
"logps/rejected": -318.13079833984375,
"loss": 1.1449,
"margin_dpo/margin_mean": 56.93890380859375,
"margin_dpo/margin_std": 105.87216186523438,
"step": 635
},
{
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.007606941275298595,
"fcm_dpo/delta": -0.09959021955728531,
"fcm_dpo/margin": 91.07327270507812,
"fcm_dpo/q_t": 0.35048407316207886,
"grad_norm": 12.00423526763916,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -1.446263074874878,
"logits/rejected": -0.9623331427574158,
"logps/chosen": -246.98480224609375,
"logps/ref_chosen": -90.41796112060547,
"logps/ref_rejected": -87.70687866210938,
"logps/rejected": -335.34698486328125,
"loss": 0.9282,
"margin_dpo/margin_mean": 91.07327270507812,
"margin_dpo/margin_std": 97.26042175292969,
"step": 636
},
{
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0074293059296905994,
"fcm_dpo/delta": -0.1272699534893036,
"fcm_dpo/margin": 96.68272399902344,
"fcm_dpo/q_t": 0.34441307187080383,
"grad_norm": 10.729310989379883,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -2.0841870307922363,
"logits/rejected": -1.981583595275879,
"logps/chosen": -240.16139221191406,
"logps/ref_chosen": -82.44971466064453,
"logps/ref_rejected": -104.02860260009766,
"logps/rejected": -358.4230041503906,
"loss": 0.9195,
"margin_dpo/margin_mean": 96.68272399902344,
"margin_dpo/margin_std": 101.56159973144531,
"step": 637
},
{
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.007293237838894129,
"fcm_dpo/delta": -0.11687527596950531,
"fcm_dpo/margin": 97.12511444091797,
"fcm_dpo/q_t": 0.3418697118759155,
"grad_norm": 10.414422988891602,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -1.5123820304870605,
"logits/rejected": -0.8182043433189392,
"logps/chosen": -238.3617401123047,
"logps/ref_chosen": -91.92498779296875,
"logps/ref_rejected": -86.28703308105469,
"logps/rejected": -329.8489074707031,
"loss": 0.8832,
"margin_dpo/margin_mean": 97.12511444091797,
"margin_dpo/margin_std": 81.02644348144531,
"step": 638
},
{
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.007484002038836479,
"fcm_dpo/delta": 0.23280690610408783,
"fcm_dpo/margin": 50.03688430786133,
"fcm_dpo/q_t": 0.41923975944519043,
"grad_norm": 14.05245304107666,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -1.3220126628875732,
"logits/rejected": -0.5478463768959045,
"logps/chosen": -296.0784912109375,
"logps/ref_chosen": -104.52755737304688,
"logps/ref_rejected": -81.4803466796875,
"logps/rejected": -323.06817626953125,
"loss": 1.2022,
"margin_dpo/margin_mean": 50.036888122558594,
"margin_dpo/margin_std": 110.06419372558594,
"step": 639
},
{
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.00756090460345149,
"fcm_dpo/delta": -0.06717022508382797,
"fcm_dpo/margin": 87.34074401855469,
"fcm_dpo/q_t": 0.3554985821247101,
"grad_norm": 12.982050895690918,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -1.7683886289596558,
"logits/rejected": -1.2369413375854492,
"logps/chosen": -261.49896240234375,
"logps/ref_chosen": -97.88526916503906,
"logps/ref_rejected": -112.70502471923828,
"logps/rejected": -363.65948486328125,
"loss": 0.9568,
"margin_dpo/margin_mean": 87.34074401855469,
"margin_dpo/margin_std": 92.46745300292969,
"step": 640
},
{
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.007357077207416296,
"fcm_dpo/delta": -0.08074723184108734,
"fcm_dpo/margin": 91.79606628417969,
"fcm_dpo/q_t": 0.3599125146865845,
"grad_norm": 10.860076904296875,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -2.443859338760376,
"logits/rejected": -1.9180916547775269,
"logps/chosen": -245.45460510253906,
"logps/ref_chosen": -96.4456787109375,
"logps/ref_rejected": -95.13568878173828,
"logps/rejected": -335.940673828125,
"loss": 0.9735,
"margin_dpo/margin_mean": 91.79606628417969,
"margin_dpo/margin_std": 117.18472290039062,
"step": 641
},
{
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.007323989644646645,
"fcm_dpo/delta": -0.019782835617661476,
"fcm_dpo/margin": 84.43093872070312,
"fcm_dpo/q_t": 0.3701496422290802,
"grad_norm": 12.883811950683594,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -1.569532036781311,
"logits/rejected": -1.235536813735962,
"logps/chosen": -269.09429931640625,
"logps/ref_chosen": -100.75984954833984,
"logps/ref_rejected": -114.7076416015625,
"logps/rejected": -367.4729919433594,
"loss": 1.0067,
"margin_dpo/margin_mean": 84.43093872070312,
"margin_dpo/margin_std": 113.8532485961914,
"step": 642
},
{
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.007224624045193195,
"fcm_dpo/delta": 0.009218744933605194,
"fcm_dpo/margin": 81.56900787353516,
"fcm_dpo/q_t": 0.37049052119255066,
"grad_norm": 13.749687194824219,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -2.2235236167907715,
"logits/rejected": -1.8363418579101562,
"logps/chosen": -241.62869262695312,
"logps/ref_chosen": -84.74365997314453,
"logps/ref_rejected": -94.31842041015625,
"logps/rejected": -332.7724609375,
"loss": 1.0051,
"margin_dpo/margin_mean": 81.56900787353516,
"margin_dpo/margin_std": 98.04161071777344,
"step": 643
},
{
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0074260118417441845,
"fcm_dpo/delta": 0.10681871324777603,
"fcm_dpo/margin": 67.11778259277344,
"fcm_dpo/q_t": 0.38967859745025635,
"grad_norm": 14.644425392150879,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -1.818040370941162,
"logits/rejected": -1.5412225723266602,
"logps/chosen": -246.3582763671875,
"logps/ref_chosen": -85.32274627685547,
"logps/ref_rejected": -94.60861206054688,
"logps/rejected": -322.76190185546875,
"loss": 1.0502,
"margin_dpo/margin_mean": 67.11778259277344,
"margin_dpo/margin_std": 89.98631286621094,
"step": 644
},
{
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.007773152552545071,
"fcm_dpo/delta": 0.2508290708065033,
"fcm_dpo/margin": 45.93043518066406,
"fcm_dpo/q_t": 0.425441175699234,
"grad_norm": 14.240182876586914,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -1.446195363998413,
"logits/rejected": -1.2669878005981445,
"logps/chosen": -257.103515625,
"logps/ref_chosen": -82.59024047851562,
"logps/ref_rejected": -103.00375366210938,
"logps/rejected": -323.4474792480469,
"loss": 1.2211,
"margin_dpo/margin_mean": 45.93043518066406,
"margin_dpo/margin_std": 107.12353515625,
"step": 645
},
{
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.007703070994466543,
"fcm_dpo/delta": -0.08048854768276215,
"fcm_dpo/margin": 87.47425842285156,
"fcm_dpo/q_t": 0.3519340753555298,
"grad_norm": 11.884034156799316,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -1.8835480213165283,
"logits/rejected": -1.2794466018676758,
"logps/chosen": -241.96484375,
"logps/ref_chosen": -91.1920394897461,
"logps/ref_rejected": -83.77833557128906,
"logps/rejected": -322.025390625,
"loss": 0.9277,
"margin_dpo/margin_mean": 87.47425842285156,
"margin_dpo/margin_std": 88.36001586914062,
"step": 646
},
{
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0077326856553554535,
"fcm_dpo/delta": 0.018357396125793457,
"fcm_dpo/margin": 75.35665130615234,
"fcm_dpo/q_t": 0.37991610169410706,
"grad_norm": 12.295222282409668,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -1.3169585466384888,
"logits/rejected": -1.1735953092575073,
"logps/chosen": -254.99407958984375,
"logps/ref_chosen": -88.8444595336914,
"logps/ref_rejected": -99.49833679199219,
"logps/rejected": -341.00457763671875,
"loss": 1.0523,
"margin_dpo/margin_mean": 75.35665130615234,
"margin_dpo/margin_std": 113.26918029785156,
"step": 647
},
{
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.007531121838837862,
"fcm_dpo/delta": -0.2151474952697754,
"fcm_dpo/margin": 106.015625,
"fcm_dpo/q_t": 0.3290584087371826,
"grad_norm": 12.194585800170898,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.9958875179290771,
"logits/rejected": -1.9187660217285156,
"logps/chosen": -240.71719360351562,
"logps/ref_chosen": -87.84810638427734,
"logps/ref_rejected": -104.67005920410156,
"logps/rejected": -363.5547790527344,
"loss": 0.8804,
"margin_dpo/margin_mean": 106.015625,
"margin_dpo/margin_std": 108.19938659667969,
"step": 648
},
{
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.007324899546802044,
"fcm_dpo/delta": -0.042867887765169144,
"fcm_dpo/margin": 87.23211669921875,
"fcm_dpo/q_t": 0.35840868949890137,
"grad_norm": 13.264395713806152,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -2.3595848083496094,
"logits/rejected": -1.9544248580932617,
"logps/chosen": -229.560302734375,
"logps/ref_chosen": -89.6925048828125,
"logps/ref_rejected": -88.70658111572266,
"logps/rejected": -315.8064880371094,
"loss": 0.9392,
"margin_dpo/margin_mean": 87.23211669921875,
"margin_dpo/margin_std": 85.69928741455078,
"step": 649
},
{
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.007353082299232483,
"fcm_dpo/delta": -0.015879113227128983,
"fcm_dpo/margin": 83.62142181396484,
"fcm_dpo/q_t": 0.3669193983078003,
"grad_norm": 12.675481796264648,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -1.3255770206451416,
"logits/rejected": -1.12962007522583,
"logps/chosen": -255.35781860351562,
"logps/ref_chosen": -89.93060302734375,
"logps/ref_rejected": -102.61282348632812,
"logps/rejected": -351.66143798828125,
"loss": 1.0064,
"margin_dpo/margin_mean": 83.62141418457031,
"margin_dpo/margin_std": 110.35063934326172,
"step": 650
},
{
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.007182122208178043,
"fcm_dpo/delta": -0.1339079886674881,
"fcm_dpo/margin": 100.80975341796875,
"fcm_dpo/q_t": 0.3390667736530304,
"grad_norm": 11.261962890625,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -2.088024854660034,
"logits/rejected": -1.914245367050171,
"logps/chosen": -218.3494873046875,
"logps/ref_chosen": -79.18731689453125,
"logps/ref_rejected": -105.93333435058594,
"logps/rejected": -345.9052734375,
"loss": 0.8862,
"margin_dpo/margin_mean": 100.80976104736328,
"margin_dpo/margin_std": 92.29314422607422,
"step": 651
},
{
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.007138779386878014,
"fcm_dpo/delta": -0.021067023277282715,
"fcm_dpo/margin": 86.80287170410156,
"fcm_dpo/q_t": 0.36111682653427124,
"grad_norm": 10.429312705993652,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -2.4728636741638184,
"logits/rejected": -2.168954849243164,
"logps/chosen": -246.78160095214844,
"logps/ref_chosen": -101.79022216796875,
"logps/ref_rejected": -116.3245849609375,
"logps/rejected": -348.11883544921875,
"loss": 0.9352,
"margin_dpo/margin_mean": 86.80288696289062,
"margin_dpo/margin_std": 81.80032348632812,
"step": 652
},
{
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.007149814628064632,
"fcm_dpo/delta": 0.018516050651669502,
"fcm_dpo/margin": 81.45347595214844,
"fcm_dpo/q_t": 0.36881303787231445,
"grad_norm": 12.834737777709961,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -2.072822093963623,
"logits/rejected": -1.819368600845337,
"logps/chosen": -257.77667236328125,
"logps/ref_chosen": -93.35359191894531,
"logps/ref_rejected": -109.12324523925781,
"logps/rejected": -354.99981689453125,
"loss": 0.9894,
"margin_dpo/margin_mean": 81.45347595214844,
"margin_dpo/margin_std": 93.0560073852539,
"step": 653
},
{
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.007231119088828564,
"fcm_dpo/delta": 0.08581791818141937,
"fcm_dpo/margin": 71.71923828125,
"fcm_dpo/q_t": 0.3922192454338074,
"grad_norm": 15.544170379638672,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -2.3582053184509277,
"logits/rejected": -2.17484712600708,
"logps/chosen": -254.76153564453125,
"logps/ref_chosen": -89.11553955078125,
"logps/ref_rejected": -104.91995239257812,
"logps/rejected": -342.28515625,
"loss": 1.0873,
"margin_dpo/margin_mean": 71.71923828125,
"margin_dpo/margin_std": 116.78224182128906,
"step": 654
},
{
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.007248979061841965,
"fcm_dpo/delta": -0.02873397059738636,
"fcm_dpo/margin": 86.4774169921875,
"fcm_dpo/q_t": 0.362434983253479,
"grad_norm": 11.312616348266602,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -2.129537582397461,
"logits/rejected": -2.125460147857666,
"logps/chosen": -233.4827423095703,
"logps/ref_chosen": -81.03610229492188,
"logps/ref_rejected": -102.80233764648438,
"logps/rejected": -341.72637939453125,
"loss": 0.9844,
"margin_dpo/margin_mean": 86.4774169921875,
"margin_dpo/margin_std": 104.30917358398438,
"step": 655
},
{
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.007026261650025845,
"fcm_dpo/delta": -0.16489456593990326,
"fcm_dpo/margin": 107.01776123046875,
"fcm_dpo/q_t": 0.3365343511104584,
"grad_norm": 12.231975555419922,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -1.2732338905334473,
"logits/rejected": -1.3455628156661987,
"logps/chosen": -237.61703491210938,
"logps/ref_chosen": -85.32534790039062,
"logps/ref_rejected": -118.33866119384766,
"logps/rejected": -377.6481018066406,
"loss": 0.8735,
"margin_dpo/margin_mean": 107.01776123046875,
"margin_dpo/margin_std": 99.96194458007812,
"step": 656
},
{
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.006966191343963146,
"fcm_dpo/delta": 0.019626092165708542,
"fcm_dpo/margin": 83.40191650390625,
"fcm_dpo/q_t": 0.3764026165008545,
"grad_norm": 14.666167259216309,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.931036114692688,
"logits/rejected": -1.757138729095459,
"logps/chosen": -242.91915893554688,
"logps/ref_chosen": -80.19772338867188,
"logps/ref_rejected": -102.581298828125,
"logps/rejected": -348.70465087890625,
"loss": 1.0244,
"margin_dpo/margin_mean": 83.40192413330078,
"margin_dpo/margin_std": 111.77230834960938,
"step": 657
},
{
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.007131836377084255,
"fcm_dpo/delta": 0.08433780819177628,
"fcm_dpo/margin": 72.8197250366211,
"fcm_dpo/q_t": 0.38564208149909973,
"grad_norm": 11.62498664855957,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -2.0923075675964355,
"logits/rejected": -1.626826286315918,
"logps/chosen": -262.4381103515625,
"logps/ref_chosen": -94.634521484375,
"logps/ref_rejected": -101.6316146850586,
"logps/rejected": -342.25494384765625,
"loss": 1.0369,
"margin_dpo/margin_mean": 72.8197250366211,
"margin_dpo/margin_std": 94.01678466796875,
"step": 658
},
{
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.007036149501800537,
"fcm_dpo/delta": -0.056030645966529846,
"fcm_dpo/margin": 92.60414123535156,
"fcm_dpo/q_t": 0.36230194568634033,
"grad_norm": 11.66299819946289,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -2.0208144187927246,
"logits/rejected": -1.0372681617736816,
"logps/chosen": -259.408447265625,
"logps/ref_chosen": -112.55587005615234,
"logps/ref_rejected": -93.9216079711914,
"logps/rejected": -333.37835693359375,
"loss": 1.0067,
"margin_dpo/margin_mean": 92.6041488647461,
"margin_dpo/margin_std": 122.59965515136719,
"step": 659
},
{
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.006981555372476578,
"fcm_dpo/delta": -0.06499287486076355,
"fcm_dpo/margin": 94.63003540039062,
"fcm_dpo/q_t": 0.35565003752708435,
"grad_norm": 14.255919456481934,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.8492063283920288,
"logits/rejected": -1.2232909202575684,
"logps/chosen": -247.27349853515625,
"logps/ref_chosen": -93.37742614746094,
"logps/ref_rejected": -81.39482116699219,
"logps/rejected": -329.9209289550781,
"loss": 0.9439,
"margin_dpo/margin_mean": 94.63003540039062,
"margin_dpo/margin_std": 103.12388610839844,
"step": 660
},
{
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.006860889494419098,
"fcm_dpo/delta": -0.1109163761138916,
"fcm_dpo/margin": 102.49998474121094,
"fcm_dpo/q_t": 0.34864702820777893,
"grad_norm": 11.033123970031738,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.9187567234039307,
"logits/rejected": -1.4068787097930908,
"logps/chosen": -245.22006225585938,
"logps/ref_chosen": -87.85516357421875,
"logps/ref_rejected": -92.40330505371094,
"logps/rejected": -352.2681884765625,
"loss": 0.931,
"margin_dpo/margin_mean": 102.49998474121094,
"margin_dpo/margin_std": 111.98723602294922,
"step": 661
},
{
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.006795001216232777,
"fcm_dpo/delta": -0.02861689031124115,
"fcm_dpo/margin": 92.24244689941406,
"fcm_dpo/q_t": 0.3603462278842926,
"grad_norm": 10.01567554473877,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -1.5263056755065918,
"logits/rejected": -0.9408450722694397,
"logps/chosen": -255.88613891601562,
"logps/ref_chosen": -102.77980041503906,
"logps/ref_rejected": -95.22531127929688,
"logps/rejected": -340.5740966796875,
"loss": 0.942,
"margin_dpo/margin_mean": 92.24244689941406,
"margin_dpo/margin_std": 90.77671813964844,
"step": 662
},
{
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.006806028075516224,
"fcm_dpo/delta": 0.015626247972249985,
"fcm_dpo/margin": 85.96340942382812,
"fcm_dpo/q_t": 0.3726150095462799,
"grad_norm": 11.392067909240723,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -2.2485814094543457,
"logits/rejected": -2.024899959564209,
"logps/chosen": -249.08914184570312,
"logps/ref_chosen": -96.34658813476562,
"logps/ref_rejected": -120.52645874023438,
"logps/rejected": -359.232421875,
"loss": 1.0095,
"margin_dpo/margin_mean": 85.96340942382812,
"margin_dpo/margin_std": 108.69853210449219,
"step": 663
},
{
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.006872647441923618,
"fcm_dpo/delta": 0.062802754342556,
"fcm_dpo/margin": 78.6165771484375,
"fcm_dpo/q_t": 0.37933892011642456,
"grad_norm": 11.565393447875977,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -1.6666076183319092,
"logits/rejected": -1.4859673976898193,
"logps/chosen": -259.26226806640625,
"logps/ref_chosen": -91.84242248535156,
"logps/ref_rejected": -111.8366928100586,
"logps/rejected": -357.8731384277344,
"loss": 1.0112,
"margin_dpo/margin_mean": 78.6165771484375,
"margin_dpo/margin_std": 91.88618469238281,
"step": 664
},
{
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.006901168264448643,
"fcm_dpo/delta": 0.007144978269934654,
"fcm_dpo/margin": 85.94855499267578,
"fcm_dpo/q_t": 0.36612850427627563,
"grad_norm": 9.935822486877441,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -2.3310766220092773,
"logits/rejected": -1.89298677444458,
"logps/chosen": -232.2900848388672,
"logps/ref_chosen": -88.18618774414062,
"logps/ref_rejected": -91.9120101928711,
"logps/rejected": -321.9644775390625,
"loss": 0.9693,
"margin_dpo/margin_mean": 85.94855499267578,
"margin_dpo/margin_std": 91.62937927246094,
"step": 665
},
{
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.006875505670905113,
"fcm_dpo/delta": 0.01688658446073532,
"fcm_dpo/margin": 84.91444396972656,
"fcm_dpo/q_t": 0.37375444173812866,
"grad_norm": 14.775041580200195,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -1.6999645233154297,
"logits/rejected": -1.3265091180801392,
"logps/chosen": -261.86273193359375,
"logps/ref_chosen": -100.97459411621094,
"logps/ref_rejected": -101.24992370605469,
"logps/rejected": -347.052490234375,
"loss": 1.0104,
"margin_dpo/margin_mean": 84.91444396972656,
"margin_dpo/margin_std": 106.82335662841797,
"step": 666
},
{
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.006852601654827595,
"fcm_dpo/delta": -0.03697332367300987,
"fcm_dpo/margin": 92.57352447509766,
"fcm_dpo/q_t": 0.3577842712402344,
"grad_norm": 11.277308464050293,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -1.651948094367981,
"logits/rejected": -1.0065885782241821,
"logps/chosen": -270.1557922363281,
"logps/ref_chosen": -97.5711669921875,
"logps/ref_rejected": -93.58476257324219,
"logps/rejected": -358.742919921875,
"loss": 0.9343,
"margin_dpo/margin_mean": 92.57352447509766,
"margin_dpo/margin_std": 89.03514099121094,
"step": 667
},
{
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.006754533387720585,
"fcm_dpo/delta": -0.11870689690113068,
"fcm_dpo/margin": 105.17715454101562,
"fcm_dpo/q_t": 0.3479474186897278,
"grad_norm": 14.153767585754395,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -2.2566616535186768,
"logits/rejected": -1.7255539894104004,
"logps/chosen": -255.34854125976562,
"logps/ref_chosen": -102.5750503540039,
"logps/ref_rejected": -108.81768798828125,
"logps/rejected": -366.7683410644531,
"loss": 0.911,
"margin_dpo/margin_mean": 105.17715454101562,
"margin_dpo/margin_std": 108.63129425048828,
"step": 668
},
{
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.006622363813221455,
"fcm_dpo/delta": -0.0791027769446373,
"fcm_dpo/margin": 101.74537658691406,
"fcm_dpo/q_t": 0.350632905960083,
"grad_norm": 11.706164360046387,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -1.6500535011291504,
"logits/rejected": -1.2420462369918823,
"logps/chosen": -229.16888427734375,
"logps/ref_chosen": -89.49478149414062,
"logps/ref_rejected": -82.51950073242188,
"logps/rejected": -323.9389953613281,
"loss": 0.9178,
"margin_dpo/margin_mean": 101.74537658691406,
"margin_dpo/margin_std": 98.94613647460938,
"step": 669
},
{
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.006471183151006699,
"fcm_dpo/delta": -0.12788358330726624,
"fcm_dpo/margin": 111.06494140625,
"fcm_dpo/q_t": 0.3385842442512512,
"grad_norm": 9.694023132324219,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -1.7507426738739014,
"logits/rejected": -1.2104510068893433,
"logps/chosen": -234.94271850585938,
"logps/ref_chosen": -95.45459747314453,
"logps/ref_rejected": -101.53292846679688,
"logps/rejected": -352.08599853515625,
"loss": 0.8892,
"margin_dpo/margin_mean": 111.06494140625,
"margin_dpo/margin_std": 101.67810821533203,
"step": 670
},
{
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.006587146315723658,
"fcm_dpo/delta": 0.11642096936702728,
"fcm_dpo/margin": 73.98999786376953,
"fcm_dpo/q_t": 0.3927046060562134,
"grad_norm": 11.580408096313477,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.8810558319091797,
"logits/rejected": -1.6646201610565186,
"logps/chosen": -254.25238037109375,
"logps/ref_chosen": -82.12312316894531,
"logps/ref_rejected": -90.21969604492188,
"logps/rejected": -336.33892822265625,
"loss": 1.0615,
"margin_dpo/margin_mean": 73.98999786376953,
"margin_dpo/margin_std": 101.05743408203125,
"step": 671
},
{
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0067345621064305305,
"fcm_dpo/delta": 0.16344712674617767,
"fcm_dpo/margin": 65.91778564453125,
"fcm_dpo/q_t": 0.4018493890762329,
"grad_norm": 13.346258163452148,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.9560139179229736,
"logits/rejected": -1.6140336990356445,
"logps/chosen": -260.7767333984375,
"logps/ref_chosen": -90.0619125366211,
"logps/ref_rejected": -100.45323181152344,
"logps/rejected": -337.0858154296875,
"loss": 1.0877,
"margin_dpo/margin_mean": 65.91779327392578,
"margin_dpo/margin_std": 93.67379760742188,
"step": 672
},
{
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.006898353807628155,
"fcm_dpo/delta": 0.08046399801969528,
"fcm_dpo/margin": 75.86769104003906,
"fcm_dpo/q_t": 0.38369953632354736,
"grad_norm": 13.707597732543945,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -2.0346531867980957,
"logits/rejected": -1.667980432510376,
"logps/chosen": -230.58885192871094,
"logps/ref_chosen": -79.26315307617188,
"logps/ref_rejected": -91.34925079345703,
"logps/rejected": -318.5426330566406,
"loss": 1.0206,
"margin_dpo/margin_mean": 75.86769104003906,
"margin_dpo/margin_std": 91.65807342529297,
"step": 673
},
{
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.006784925237298012,
"fcm_dpo/delta": -0.10350759327411652,
"fcm_dpo/margin": 102.5726318359375,
"fcm_dpo/q_t": 0.3477708697319031,
"grad_norm": 10.508623123168945,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -1.6836049556732178,
"logits/rejected": -1.2868695259094238,
"logps/chosen": -219.0989990234375,
"logps/ref_chosen": -75.45831298828125,
"logps/ref_rejected": -76.20362854003906,
"logps/rejected": -322.41693115234375,
"loss": 0.9032,
"margin_dpo/margin_mean": 102.57262420654297,
"margin_dpo/margin_std": 99.2423095703125,
"step": 674
},
{
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.006704601459205151,
"fcm_dpo/delta": -0.0026360079646110535,
"fcm_dpo/margin": 89.63999938964844,
"fcm_dpo/q_t": 0.3685849606990814,
"grad_norm": 13.317923545837402,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.458611249923706,
"logits/rejected": -1.1178425550460815,
"logps/chosen": -239.28143310546875,
"logps/ref_chosen": -79.90953063964844,
"logps/ref_rejected": -81.21824645996094,
"logps/rejected": -330.23016357421875,
"loss": 1.0169,
"margin_dpo/margin_mean": 89.63999938964844,
"margin_dpo/margin_std": 117.47451782226562,
"step": 675
},
{
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.006719764322042465,
"fcm_dpo/delta": -0.015834422782063484,
"fcm_dpo/margin": 91.44828796386719,
"fcm_dpo/q_t": 0.36421921849250793,
"grad_norm": 11.21393871307373,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -1.9717350006103516,
"logits/rejected": -1.388458251953125,
"logps/chosen": -268.1275329589844,
"logps/ref_chosen": -98.17111206054688,
"logps/ref_rejected": -95.024658203125,
"logps/rejected": -356.42938232421875,
"loss": 0.9797,
"margin_dpo/margin_mean": 91.44828796386719,
"margin_dpo/margin_std": 105.38261413574219,
"step": 676
},
{
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.006692584604024887,
"fcm_dpo/delta": -0.030026260763406754,
"fcm_dpo/margin": 93.82452392578125,
"fcm_dpo/q_t": 0.35947829484939575,
"grad_norm": 10.283641815185547,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.81768798828125,
"logits/rejected": -1.1459150314331055,
"logps/chosen": -243.63877868652344,
"logps/ref_chosen": -91.37928771972656,
"logps/ref_rejected": -82.87776947021484,
"logps/rejected": -328.9617919921875,
"loss": 0.9365,
"margin_dpo/margin_mean": 93.82452392578125,
"margin_dpo/margin_std": 89.39640808105469,
"step": 677
},
{
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.00660483306273818,
"fcm_dpo/delta": -0.14169199764728546,
"fcm_dpo/margin": 110.73104858398438,
"fcm_dpo/q_t": 0.33739006519317627,
"grad_norm": 10.643210411071777,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -1.7930124998092651,
"logits/rejected": -1.4681642055511475,
"logps/chosen": -219.61293029785156,
"logps/ref_chosen": -75.6458740234375,
"logps/ref_rejected": -86.96611022949219,
"logps/rejected": -341.66424560546875,
"loss": 0.88,
"margin_dpo/margin_mean": 110.73104858398438,
"margin_dpo/margin_std": 97.85108947753906,
"step": 678
},
{
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.006523388437926769,
"fcm_dpo/delta": 0.013953792862594128,
"fcm_dpo/margin": 89.96353149414062,
"fcm_dpo/q_t": 0.36797723174095154,
"grad_norm": 10.297441482543945,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -2.299855947494507,
"logits/rejected": -1.7864303588867188,
"logps/chosen": -245.36917114257812,
"logps/ref_chosen": -80.77344512939453,
"logps/ref_rejected": -82.87850189208984,
"logps/rejected": -337.4377746582031,
"loss": 0.9515,
"margin_dpo/margin_mean": 89.96353912353516,
"margin_dpo/margin_std": 85.39958190917969,
"step": 679
},
{
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.006581049412488937,
"fcm_dpo/delta": -0.020787008106708527,
"fcm_dpo/margin": 93.83221435546875,
"fcm_dpo/q_t": 0.36515846848487854,
"grad_norm": 9.382950782775879,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -1.9891002178192139,
"logits/rejected": -1.6667735576629639,
"logps/chosen": -276.03668212890625,
"logps/ref_chosen": -107.68292999267578,
"logps/ref_rejected": -116.0948715209961,
"logps/rejected": -378.2808837890625,
"loss": 0.9739,
"margin_dpo/margin_mean": 93.83221435546875,
"margin_dpo/margin_std": 103.33714294433594,
"step": 680
},
{
"epoch": 1.0,
"fcm_dpo/beta": 0.006566454656422138,
"fcm_dpo/delta": 0.05062846094369888,
"fcm_dpo/margin": 84.06166076660156,
"fcm_dpo/q_t": 0.3779335916042328,
"grad_norm": 10.81689167022705,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -2.369196653366089,
"logits/rejected": -2.0650794506073,
"logps/chosen": -243.9755859375,
"logps/ref_chosen": -93.01106262207031,
"logps/ref_rejected": -94.8221664428711,
"logps/rejected": -329.84832763671875,
"loss": 1.0069,
"margin_dpo/margin_mean": 84.06166076660156,
"margin_dpo/margin_std": 100.67030334472656,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.0153061533114283,
"train_runtime": 2239.7507,
"train_samples_per_second": 19.466,
"train_steps_per_second": 0.304
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink