W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.05
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
aef2411924440c971e4c6d76120b7d778cd0d65d
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC aef2411924 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.05
Source: Original Platform
2026-05-18 22:13:03 +08:00

15707 lines
589 KiB
JSON
Raw Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.52085876464844,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.19752502441406,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": -0.017164230346679688,
"KL/mean": -0.008938148617744446,
"KL/rejected_KL_mean": -0.00070953369140625,
"KL/std": 0.25200146436691284,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.016454964876174927,
"fcm_dpo/q_t": 0.500411331653595,
"grad_norm": 70.90198516845703,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.4816335439682007,
"logits/rejected": -0.442240834236145,
"logps/chosen": -60.99876022338867,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.67330932617188,
"loss": 1.3883,
"margin_dpo/margin_mean": -0.016454339027404785,
"margin_dpo/margin_std": 0.3718428313732147,
"step": 3
},
{
"KL/chosen_KL_mean": 0.0011157989501953125,
"KL/mean": 0.030593425035476685,
"KL/rejected_KL_mean": 0.060070037841796875,
"KL/std": 0.2528386116027832,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.058952271938323975,
"fcm_dpo/q_t": 0.5014750957489014,
"grad_norm": 72.30359649658203,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.46882253885269165,
"logits/rejected": -0.44118714332580566,
"logps/chosen": -56.76659393310547,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.5870361328125,
"loss": 1.3926,
"margin_dpo/margin_mean": -0.058952778577804565,
"margin_dpo/margin_std": 0.3786373734474182,
"step": 4
},
{
"KL/chosen_KL_mean": 0.04130363464355469,
"KL/mean": 0.028823688626289368,
"KL/rejected_KL_mean": 0.016345977783203125,
"KL/std": 0.2790408730506897,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.024959057569503784,
"fcm_dpo/q_t": 0.4993762671947479,
"grad_norm": 89.41995239257812,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5145682096481323,
"logits/rejected": -0.47082412242889404,
"logps/chosen": -53.81806945800781,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.13284301757812,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.024959444999694824,
"margin_dpo/margin_std": 0.38526445627212524,
"step": 5
},
{
"KL/chosen_KL_mean": -0.04564094543457031,
"KL/mean": -0.020483508706092834,
"KL/rejected_KL_mean": 0.00467681884765625,
"KL/std": 0.26991650462150574,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0503142774105072,
"fcm_dpo/q_t": 0.50125652551651,
"grad_norm": 92.1827621459961,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5298000574111938,
"logits/rejected": -0.4896334409713745,
"logps/chosen": -63.053123474121094,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.64067077636719,
"loss": 1.3918,
"margin_dpo/margin_mean": -0.05031454563140869,
"margin_dpo/margin_std": 0.4164780378341675,
"step": 6
},
{
"KL/chosen_KL_mean": 0.05684852600097656,
"KL/mean": 0.05007433891296387,
"KL/rejected_KL_mean": 0.043300628662109375,
"KL/std": 0.2920554578304291,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.013551175594329834,
"fcm_dpo/q_t": 0.4996613562107086,
"grad_norm": 82.23982238769531,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5054950714111328,
"logits/rejected": -0.4716752767562866,
"logps/chosen": -57.71797180175781,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.87728881835938,
"loss": 1.3854,
"margin_dpo/margin_mean": 0.013551592826843262,
"margin_dpo/margin_std": 0.43186938762664795,
"step": 7
},
{
"KL/chosen_KL_mean": 0.017724990844726562,
"KL/mean": 0.02704182267189026,
"KL/rejected_KL_mean": 0.0363616943359375,
"KL/std": 0.3007723093032837,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.018639206886291504,
"fcm_dpo/q_t": 0.500464916229248,
"grad_norm": 78.59706115722656,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.5154826641082764,
"logits/rejected": -0.4903911352157593,
"logps/chosen": -58.69831085205078,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.27506256103516,
"loss": 1.3886,
"margin_dpo/margin_mean": -0.01863950490951538,
"margin_dpo/margin_std": 0.3895518183708191,
"step": 8
},
{
"KL/chosen_KL_mean": 0.00308990478515625,
"KL/mean": -0.0010503679513931274,
"KL/rejected_KL_mean": -0.00518798828125,
"KL/std": 0.28215163946151733,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.008274465799331665,
"fcm_dpo/q_t": 0.499793142080307,
"grad_norm": 85.33476257324219,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.5284153819084167,
"logits/rejected": -0.4838245213031769,
"logps/chosen": -69.86375427246094,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.60784912109375,
"loss": 1.3859,
"margin_dpo/margin_mean": 0.008274048566818237,
"margin_dpo/margin_std": 0.4120550751686096,
"step": 9
},
{
"KL/chosen_KL_mean": 0.007932662963867188,
"KL/mean": 0.01091562956571579,
"KL/rejected_KL_mean": 0.0139007568359375,
"KL/std": 0.2518649697303772,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.00596599280834198,
"fcm_dpo/q_t": 0.5001487135887146,
"grad_norm": 70.95951080322266,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.4845648407936096,
"logits/rejected": -0.4408423602581024,
"logps/chosen": -48.349754333496094,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.35816192626953,
"loss": 1.3872,
"margin_dpo/margin_mean": -0.00596579909324646,
"margin_dpo/margin_std": 0.3506859540939331,
"step": 10
},
{
"KL/chosen_KL_mean": 0.06185722351074219,
"KL/mean": 0.0011502206325531006,
"KL/rejected_KL_mean": -0.0595550537109375,
"KL/std": 0.27010950446128845,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.12140899896621704,
"fcm_dpo/q_t": 0.4969657063484192,
"grad_norm": 67.91629028320312,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4723697900772095,
"logits/rejected": -0.44793397188186646,
"logps/chosen": -52.95499801635742,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.83993530273438,
"loss": 1.3745,
"margin_dpo/margin_mean": 0.12140867114067078,
"margin_dpo/margin_std": 0.3221217393875122,
"step": 11
},
{
"KL/chosen_KL_mean": 0.0077342987060546875,
"KL/mean": -0.026624195277690887,
"KL/rejected_KL_mean": -0.060977935791015625,
"KL/std": 0.26408517360687256,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06871287524700165,
"fcm_dpo/q_t": 0.49828261137008667,
"grad_norm": 95.73152923583984,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5401338338851929,
"logits/rejected": -0.5040255784988403,
"logps/chosen": -61.79770278930664,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.91925048828125,
"loss": 1.3797,
"margin_dpo/margin_mean": 0.06871210038661957,
"margin_dpo/margin_std": 0.33830663561820984,
"step": 12
},
{
"KL/chosen_KL_mean": 0.0069599151611328125,
"KL/mean": -0.017511114478111267,
"KL/rejected_KL_mean": -0.04198455810546875,
"KL/std": 0.3143480122089386,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.048950523138046265,
"fcm_dpo/q_t": 0.4987792670726776,
"grad_norm": 79.25921630859375,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4748424291610718,
"logits/rejected": -0.4463379681110382,
"logps/chosen": -64.2533950805664,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.24507141113281,
"loss": 1.382,
"margin_dpo/margin_mean": 0.048950642347335815,
"margin_dpo/margin_std": 0.44883352518081665,
"step": 13
},
{
"KL/chosen_KL_mean": -0.021654129028320312,
"KL/mean": 0.0024417340755462646,
"KL/rejected_KL_mean": 0.0265350341796875,
"KL/std": 0.2979321777820587,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04818415641784668,
"fcm_dpo/q_t": 0.5012027025222778,
"grad_norm": 85.64781188964844,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.48348790407180786,
"logits/rejected": -0.4443298876285553,
"logps/chosen": -58.131866455078125,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.02054595947266,
"loss": 1.3916,
"margin_dpo/margin_mean": -0.048183709383010864,
"margin_dpo/margin_std": 0.44763094186782837,
"step": 14
},
{
"KL/chosen_KL_mean": -0.005901336669921875,
"KL/mean": -0.012522846460342407,
"KL/rejected_KL_mean": -0.019145965576171875,
"KL/std": 0.21723134815692902,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.013249039649963379,
"fcm_dpo/q_t": 0.49966850876808167,
"grad_norm": 64.4452133178711,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.498607337474823,
"logits/rejected": -0.48026347160339355,
"logps/chosen": -56.972808837890625,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.82778930664062,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.013249337673187256,
"margin_dpo/margin_std": 0.3437996804714203,
"step": 15
},
{
"KL/chosen_KL_mean": -0.027292251586914062,
"KL/mean": -0.009725570678710938,
"KL/rejected_KL_mean": 0.007839202880859375,
"KL/std": 0.2627600431442261,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.035129427909851074,
"fcm_dpo/q_t": 0.5008782148361206,
"grad_norm": 84.21294403076172,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5192116498947144,
"logits/rejected": -0.47780126333236694,
"logps/chosen": -61.76718521118164,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.36163330078125,
"loss": 1.3901,
"margin_dpo/margin_mean": -0.035130053758621216,
"margin_dpo/margin_std": 0.35660019516944885,
"step": 16
},
{
"KL/chosen_KL_mean": 0.0523681640625,
"KL/mean": 0.011688470840454102,
"KL/rejected_KL_mean": -0.02899169921875,
"KL/std": 0.2670317590236664,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08135926723480225,
"fcm_dpo/q_t": 0.49796566367149353,
"grad_norm": 78.71238708496094,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.48786813020706177,
"logits/rejected": -0.4491962194442749,
"logps/chosen": -67.65796661376953,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.40764617919922,
"loss": 1.3785,
"margin_dpo/margin_mean": 0.08135849237442017,
"margin_dpo/margin_std": 0.38243818283081055,
"step": 17
},
{
"KL/chosen_KL_mean": 0.039516448974609375,
"KL/mean": -0.002365991473197937,
"KL/rejected_KL_mean": -0.04425048828125,
"KL/std": 0.2152850478887558,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08376440405845642,
"fcm_dpo/q_t": 0.4979061186313629,
"grad_norm": 81.61917877197266,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.493377149105072,
"logits/rejected": -0.43791770935058594,
"logps/chosen": -47.699974060058594,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.51654052734375,
"loss": 1.3782,
"margin_dpo/margin_mean": 0.08376404643058777,
"margin_dpo/margin_std": 0.2884919345378876,
"step": 18
},
{
"KL/chosen_KL_mean": 0.016864776611328125,
"KL/mean": -0.012162566184997559,
"KL/rejected_KL_mean": -0.041187286376953125,
"KL/std": 0.2685026526451111,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.058047711849212646,
"fcm_dpo/q_t": 0.4985480308532715,
"grad_norm": 73.1733627319336,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.45881736278533936,
"logits/rejected": -0.40635204315185547,
"logps/chosen": -70.18849182128906,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.79878234863281,
"loss": 1.3808,
"margin_dpo/margin_mean": 0.05804792046546936,
"margin_dpo/margin_std": 0.3469369411468506,
"step": 19
},
{
"KL/chosen_KL_mean": 0.010059356689453125,
"KL/mean": -0.0335158109664917,
"KL/rejected_KL_mean": -0.07708358764648438,
"KL/std": 0.2843707203865051,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08714669942855835,
"fcm_dpo/q_t": 0.4978218078613281,
"grad_norm": 74.1033935546875,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5568749904632568,
"logits/rejected": -0.5010716915130615,
"logps/chosen": -50.793182373046875,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.90043640136719,
"loss": 1.378,
"margin_dpo/margin_mean": 0.08714690804481506,
"margin_dpo/margin_std": 0.3879868984222412,
"step": 20
},
{
"KL/chosen_KL_mean": 0.018102645874023438,
"KL/mean": -0.054987095296382904,
"KL/rejected_KL_mean": -0.12807846069335938,
"KL/std": 0.2719336450099945,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1461833268404007,
"fcm_dpo/q_t": 0.49634718894958496,
"grad_norm": 77.20344543457031,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5189667344093323,
"logits/rejected": -0.49694347381591797,
"logps/chosen": -50.04491424560547,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.99687194824219,
"loss": 1.372,
"margin_dpo/margin_mean": 0.14618362486362457,
"margin_dpo/margin_std": 0.33233124017715454,
"step": 21
},
{
"KL/chosen_KL_mean": 0.026437759399414062,
"KL/mean": -0.06921970844268799,
"KL/rejected_KL_mean": -0.16487503051757812,
"KL/std": 0.2893703877925873,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1913151741027832,
"fcm_dpo/q_t": 0.49521979689598083,
"grad_norm": 84.33903503417969,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.4624967873096466,
"logits/rejected": -0.41778409481048584,
"logps/chosen": -59.03119659423828,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.66954803466797,
"loss": 1.3677,
"margin_dpo/margin_mean": 0.1913146674633026,
"margin_dpo/margin_std": 0.4002724289894104,
"step": 22
},
{
"KL/chosen_KL_mean": 0.03466796875,
"KL/mean": -0.07599049806594849,
"KL/rejected_KL_mean": -0.1866474151611328,
"KL/std": 0.3174063563346863,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22131741046905518,
"fcm_dpo/q_t": 0.49447089433670044,
"grad_norm": 80.63067626953125,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.5079987645149231,
"logits/rejected": -0.48651230335235596,
"logps/chosen": -60.04302978515625,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.32620239257812,
"loss": 1.3647,
"margin_dpo/margin_mean": 0.221317857503891,
"margin_dpo/margin_std": 0.40781861543655396,
"step": 23
},
{
"KL/chosen_KL_mean": 0.024261474609375,
"KL/mean": -0.0957612544298172,
"KL/rejected_KL_mean": -0.21578598022460938,
"KL/std": 0.27903175354003906,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24004700779914856,
"fcm_dpo/q_t": 0.4940027594566345,
"grad_norm": 82.3310546875,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5110853910446167,
"logits/rejected": -0.49455416202545166,
"logps/chosen": -44.26677322387695,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.34100341796875,
"loss": 1.3628,
"margin_dpo/margin_mean": 0.240046888589859,
"margin_dpo/margin_std": 0.3710036873817444,
"step": 24
},
{
"KL/chosen_KL_mean": 0.06036949157714844,
"KL/mean": -0.07619968056678772,
"KL/rejected_KL_mean": -0.2127685546875,
"KL/std": 0.35055142641067505,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2731361985206604,
"fcm_dpo/q_t": 0.49317657947540283,
"grad_norm": 73.42306518554688,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5052649974822998,
"logits/rejected": -0.4755502939224243,
"logps/chosen": -52.4766845703125,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.55496215820312,
"loss": 1.3597,
"margin_dpo/margin_mean": 0.2731362283229828,
"margin_dpo/margin_std": 0.45307743549346924,
"step": 25
},
{
"KL/chosen_KL_mean": 0.07877922058105469,
"KL/mean": -0.11963352560997009,
"KL/rejected_KL_mean": -0.31804656982421875,
"KL/std": 0.45149433612823486,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3968273997306824,
"fcm_dpo/q_t": 0.4900946319103241,
"grad_norm": 87.18598937988281,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5580284595489502,
"logits/rejected": -0.5269525051116943,
"logps/chosen": -53.84402847290039,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.67776489257812,
"loss": 1.3479,
"margin_dpo/margin_mean": 0.396827757358551,
"margin_dpo/margin_std": 0.5908505916595459,
"step": 26
},
{
"KL/chosen_KL_mean": 0.11952590942382812,
"KL/mean": -0.12472324073314667,
"KL/rejected_KL_mean": -0.3689727783203125,
"KL/std": 0.4575558304786682,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4884994328022003,
"fcm_dpo/q_t": 0.4878042936325073,
"grad_norm": 93.66383361816406,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.522944450378418,
"logits/rejected": -0.48664578795433044,
"logps/chosen": -42.77900314331055,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.09317016601562,
"loss": 1.3389,
"margin_dpo/margin_mean": 0.4884990155696869,
"margin_dpo/margin_std": 0.5788041353225708,
"step": 27
},
{
"KL/chosen_KL_mean": 0.0053863525390625,
"KL/mean": -0.1380460411310196,
"KL/rejected_KL_mean": -0.2814826965332031,
"KL/std": 0.418185830116272,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.28686580061912537,
"fcm_dpo/q_t": 0.49283337593078613,
"grad_norm": 75.52678680419922,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5053206086158752,
"logits/rejected": -0.44998180866241455,
"logps/chosen": -60.551116943359375,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.68260192871094,
"loss": 1.3587,
"margin_dpo/margin_mean": 0.2868659198284149,
"margin_dpo/margin_std": 0.5646681785583496,
"step": 28
},
{
"KL/chosen_KL_mean": 0.10842704772949219,
"KL/mean": -0.14363093674182892,
"KL/rejected_KL_mean": -0.3956871032714844,
"KL/std": 0.4754260182380676,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5041146278381348,
"fcm_dpo/q_t": 0.4874116778373718,
"grad_norm": 91.20955657958984,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.549689531326294,
"logits/rejected": -0.503177285194397,
"logps/chosen": -57.69935607910156,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.7900390625,
"loss": 1.3374,
"margin_dpo/margin_mean": 0.5041155815124512,
"margin_dpo/margin_std": 0.5955373644828796,
"step": 29
},
{
"KL/chosen_KL_mean": 0.14672279357910156,
"KL/mean": -0.1726699322462082,
"KL/rejected_KL_mean": -0.4920654296875,
"KL/std": 0.5893620848655701,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6387929916381836,
"fcm_dpo/q_t": 0.48405760526657104,
"grad_norm": 87.42900848388672,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.47268378734588623,
"logits/rejected": -0.4410501718521118,
"logps/chosen": -52.43064880371094,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -98.98126983642578,
"loss": 1.3245,
"margin_dpo/margin_mean": 0.6387931108474731,
"margin_dpo/margin_std": 0.6522167921066284,
"step": 30
},
{
"KL/chosen_KL_mean": 0.12159347534179688,
"KL/mean": -0.13642428815364838,
"KL/rejected_KL_mean": -0.3944435119628906,
"KL/std": 0.5532899498939514,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5160378217697144,
"fcm_dpo/q_t": 0.4871293008327484,
"grad_norm": 67.98377990722656,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.503361165523529,
"logits/rejected": -0.45826011896133423,
"logps/chosen": -63.685325622558594,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.2884521484375,
"loss": 1.3366,
"margin_dpo/margin_mean": 0.5160377025604248,
"margin_dpo/margin_std": 0.6891189813613892,
"step": 31
},
{
"KL/chosen_KL_mean": 0.1765727996826172,
"KL/mean": -0.17987801134586334,
"KL/rejected_KL_mean": -0.5363311767578125,
"KL/std": 0.6821790933609009,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7129033803939819,
"fcm_dpo/q_t": 0.48223358392715454,
"grad_norm": 82.82377624511719,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.49510425329208374,
"logits/rejected": -0.45262959599494934,
"logps/chosen": -62.562950134277344,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.85383605957031,
"loss": 1.3182,
"margin_dpo/margin_mean": 0.7129031419754028,
"margin_dpo/margin_std": 0.8785291910171509,
"step": 32
},
{
"KL/chosen_KL_mean": 0.12042808532714844,
"KL/mean": -0.18817251920700073,
"KL/rejected_KL_mean": -0.49677276611328125,
"KL/std": 0.5911757946014404,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6171959638595581,
"fcm_dpo/q_t": 0.4845910668373108,
"grad_norm": 72.13778686523438,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.48218345642089844,
"logits/rejected": -0.4558228552341461,
"logps/chosen": -53.14054489135742,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.38191223144531,
"loss": 1.3265,
"margin_dpo/margin_mean": 0.6171959638595581,
"margin_dpo/margin_std": 0.6186330914497375,
"step": 33
},
{
"KL/chosen_KL_mean": 0.06296157836914062,
"KL/mean": -0.34480684995651245,
"KL/rejected_KL_mean": -0.7525749206542969,
"KL/std": 0.7686243057250977,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.815531849861145,
"fcm_dpo/q_t": 0.4796821177005768,
"grad_norm": 77.3115005493164,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.49842262268066406,
"logits/rejected": -0.4809451103210449,
"logps/chosen": -50.754364013671875,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.67442321777344,
"loss": 1.3086,
"margin_dpo/margin_mean": 0.8155316114425659,
"margin_dpo/margin_std": 0.9295395016670227,
"step": 34
},
{
"KL/chosen_KL_mean": 0.12413406372070312,
"KL/mean": -0.4548630118370056,
"KL/rejected_KL_mean": -1.0338554382324219,
"KL/std": 0.9395226240158081,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.157991647720337,
"fcm_dpo/q_t": 0.4711979627609253,
"grad_norm": 83.07164001464844,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.4888343811035156,
"logits/rejected": -0.4522944688796997,
"logps/chosen": -50.90035629272461,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.85828399658203,
"loss": 1.2764,
"margin_dpo/margin_mean": 1.1579914093017578,
"margin_dpo/margin_std": 0.9997678995132446,
"step": 35
},
{
"KL/chosen_KL_mean": 0.04621315002441406,
"KL/mean": -0.5572574138641357,
"KL/rejected_KL_mean": -1.1607246398925781,
"KL/std": 1.1042115688323975,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2069365978240967,
"fcm_dpo/q_t": 0.4699981212615967,
"grad_norm": 72.58307647705078,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5532280206680298,
"logits/rejected": -0.5167073011398315,
"logps/chosen": -51.94527816772461,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.20135498046875,
"loss": 1.2733,
"margin_dpo/margin_mean": 1.206936240196228,
"margin_dpo/margin_std": 1.2257723808288574,
"step": 36
},
{
"KL/chosen_KL_mean": 0.00463104248046875,
"KL/mean": -0.4934205710887909,
"KL/rejected_KL_mean": -0.9914779663085938,
"KL/std": 1.0511497259140015,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.996109127998352,
"fcm_dpo/q_t": 0.4753045439720154,
"grad_norm": 61.719268798828125,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5050272941589355,
"logits/rejected": -0.4607655107975006,
"logps/chosen": -62.80247497558594,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.88655090332031,
"loss": 1.2941,
"margin_dpo/margin_mean": 0.9961087703704834,
"margin_dpo/margin_std": 1.374046802520752,
"step": 37
},
{
"KL/chosen_KL_mean": 0.1590423583984375,
"KL/mean": -0.5295313596725464,
"KL/rejected_KL_mean": -1.2181015014648438,
"KL/std": 1.374910593032837,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3771462440490723,
"fcm_dpo/q_t": 0.4660264253616333,
"grad_norm": 69.59135437011719,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5226507186889648,
"logits/rejected": -0.4899941682815552,
"logps/chosen": -48.231475830078125,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.13053894042969,
"loss": 1.2606,
"margin_dpo/margin_mean": 1.3771460056304932,
"margin_dpo/margin_std": 1.6853830814361572,
"step": 38
},
{
"KL/chosen_KL_mean": 0.0872802734375,
"KL/mean": -0.7359923124313354,
"KL/rejected_KL_mean": -1.55926513671875,
"KL/std": 1.2973854541778564,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6465413570404053,
"fcm_dpo/q_t": 0.4591583013534546,
"grad_norm": 74.43284606933594,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5463048219680786,
"logits/rejected": -0.5056887269020081,
"logps/chosen": -50.66319274902344,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.12877655029297,
"loss": 1.2332,
"margin_dpo/margin_mean": 1.6465413570404053,
"margin_dpo/margin_std": 1.3756870031356812,
"step": 39
},
{
"KL/chosen_KL_mean": 0.2025127410888672,
"KL/mean": -0.5609214305877686,
"KL/rejected_KL_mean": -1.3243637084960938,
"KL/std": 1.3878769874572754,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5268778800964355,
"fcm_dpo/q_t": 0.4623277187347412,
"grad_norm": 60.651695251464844,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5206544399261475,
"logits/rejected": -0.4907876253128052,
"logps/chosen": -57.782554626464844,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.62443542480469,
"loss": 1.2468,
"margin_dpo/margin_mean": 1.5268781185150146,
"margin_dpo/margin_std": 1.691199779510498,
"step": 40
},
{
"KL/chosen_KL_mean": 0.06011772155761719,
"KL/mean": -0.9236688613891602,
"KL/rejected_KL_mean": -1.9074478149414062,
"KL/std": 1.8921372890472412,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10048507153987885,
"fcm_dpo/delta": 0.09654659032821655,
"fcm_dpo/margin": 1.967568039894104,
"fcm_dpo/q_t": 0.45162275433540344,
"grad_norm": 67.96371459960938,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5215292572975159,
"logits/rejected": -0.48398417234420776,
"logps/chosen": -62.63569641113281,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.93097686767578,
"loss": 1.2084,
"margin_dpo/margin_mean": 1.9675687551498413,
"margin_dpo/margin_std": 1.9688289165496826,
"step": 41
},
{
"KL/chosen_KL_mean": 0.2504749298095703,
"KL/mean": -1.0469075441360474,
"KL/rejected_KL_mean": -2.3442840576171875,
"KL/std": 2.180742025375366,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.10202906280755997,
"fcm_dpo/delta": 0.13704819977283478,
"fcm_dpo/margin": 2.5947656631469727,
"fcm_dpo/q_t": 0.4359305202960968,
"grad_norm": 80.4867935180664,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5434771776199341,
"logits/rejected": -0.49688485264778137,
"logps/chosen": -58.71595001220703,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.25265502929688,
"loss": 1.1538,
"margin_dpo/margin_mean": 2.5947659015655518,
"margin_dpo/margin_std": 2.4078125953674316,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5522232055664062,
"KL/mean": -0.6794946789741516,
"KL/rejected_KL_mean": -1.9112205505371094,
"KL/std": 1.821549654006958,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.10344166308641434,
"fcm_dpo/delta": 0.14711913466453552,
"fcm_dpo/margin": 2.463439464569092,
"fcm_dpo/q_t": 0.4379476308822632,
"grad_norm": 73.6222915649414,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5423204898834229,
"logits/rejected": -0.517362117767334,
"logps/chosen": -53.60377502441406,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.39141082763672,
"loss": 1.1583,
"margin_dpo/margin_mean": 2.463439702987671,
"margin_dpo/margin_std": 2.007230758666992,
"step": 43
},
{
"KL/chosen_KL_mean": 0.2297210693359375,
"KL/mean": -1.195556402206421,
"KL/rejected_KL_mean": -2.6208343505859375,
"KL/std": 2.280576705932617,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.10464578866958618,
"fcm_dpo/delta": 0.10327555239200592,
"fcm_dpo/margin": 2.8505544662475586,
"fcm_dpo/q_t": 0.4275043308734894,
"grad_norm": 81.44078063964844,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4537878632545471,
"logits/rejected": -0.43426036834716797,
"logps/chosen": -49.848777770996094,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.40459442138672,
"loss": 1.1233,
"margin_dpo/margin_mean": 2.8505539894104004,
"margin_dpo/margin_std": 2.22520112991333,
"step": 44
},
{
"KL/chosen_KL_mean": 0.09650421142578125,
"KL/mean": -1.0149602890014648,
"KL/rejected_KL_mean": -2.1264266967773438,
"KL/std": 2.0060698986053467,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.1063193678855896,
"fcm_dpo/delta": 0.1655273139476776,
"fcm_dpo/margin": 2.2229318618774414,
"fcm_dpo/q_t": 0.4429905414581299,
"grad_norm": 65.44628143310547,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.46523338556289673,
"logits/rejected": -0.4522145688533783,
"logps/chosen": -48.318424224853516,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.06285095214844,
"loss": 1.181,
"margin_dpo/margin_mean": 2.2229318618774414,
"margin_dpo/margin_std": 2.4767987728118896,
"step": 45
},
{
"KL/chosen_KL_mean": 0.18115234375,
"KL/mean": -1.2868940830230713,
"KL/rejected_KL_mean": -2.75494384765625,
"KL/std": 2.65130352973938,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.10749006271362305,
"fcm_dpo/delta": 0.08564689010381699,
"fcm_dpo/margin": 2.936098098754883,
"fcm_dpo/q_t": 0.4249163269996643,
"grad_norm": 71.94483184814453,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.5257543325424194,
"logits/rejected": -0.47459375858306885,
"logps/chosen": -55.818275451660156,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.40753173828125,
"loss": 1.1239,
"margin_dpo/margin_mean": 2.9360976219177246,
"margin_dpo/margin_std": 3.2204670906066895,
"step": 46
},
{
"KL/chosen_KL_mean": 0.41971397399902344,
"KL/mean": -1.100320816040039,
"KL/rejected_KL_mean": -2.6203575134277344,
"KL/std": 2.6521072387695312,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.10835415124893188,
"fcm_dpo/delta": 0.07160530984401703,
"fcm_dpo/margin": 3.040071487426758,
"fcm_dpo/q_t": 0.42072784900665283,
"grad_norm": 68.87890625,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5598181486129761,
"logits/rejected": -0.5062609910964966,
"logps/chosen": -57.50636291503906,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.299560546875,
"loss": 1.1053,
"margin_dpo/margin_mean": 3.040071487426758,
"margin_dpo/margin_std": 2.725712776184082,
"step": 47
},
{
"KL/chosen_KL_mean": 0.0533294677734375,
"KL/mean": -1.5111095905303955,
"KL/rejected_KL_mean": -3.075550079345703,
"KL/std": 2.7189888954162598,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.10873937606811523,
"fcm_dpo/delta": 0.06080545857548714,
"fcm_dpo/margin": 3.128880500793457,
"fcm_dpo/q_t": 0.41845259070396423,
"grad_norm": 75.4119873046875,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5846500396728516,
"logits/rejected": -0.5260331630706787,
"logps/chosen": -57.134742736816406,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.09214782714844,
"loss": 1.1042,
"margin_dpo/margin_mean": 3.128880262374878,
"margin_dpo/margin_std": 2.981379508972168,
"step": 48
},
{
"KL/chosen_KL_mean": 0.31061363220214844,
"KL/mean": -1.6151647567749023,
"KL/rejected_KL_mean": -3.540943145751953,
"KL/std": 3.465115785598755,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.10909098386764526,
"fcm_dpo/delta": -0.020696185529232025,
"fcm_dpo/margin": 3.8515536785125732,
"fcm_dpo/q_t": 0.4022524952888489,
"grad_norm": 79.5746841430664,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5361356139183044,
"logits/rejected": -0.4760180115699768,
"logps/chosen": -61.374656677246094,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.30841827392578,
"loss": 1.0546,
"margin_dpo/margin_mean": 3.8515536785125732,
"margin_dpo/margin_std": 3.974454164505005,
"step": 49
},
{
"KL/chosen_KL_mean": -0.15990066528320312,
"KL/mean": -2.221952438354492,
"KL/rejected_KL_mean": -4.284000396728516,
"KL/std": 3.5255508422851562,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.10840219259262085,
"fcm_dpo/delta": -0.048164162784814835,
"fcm_dpo/margin": 4.124100208282471,
"fcm_dpo/q_t": 0.395396888256073,
"grad_norm": 66.33158874511719,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5621310472488403,
"logits/rejected": -0.526812732219696,
"logps/chosen": -58.88404083251953,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.64215087890625,
"loss": 1.0337,
"margin_dpo/margin_mean": 4.1240997314453125,
"margin_dpo/margin_std": 4.0859832763671875,
"step": 50
},
{
"KL/chosen_KL_mean": -0.2936878204345703,
"KL/mean": -2.299487590789795,
"KL/rejected_KL_mean": -4.305290222167969,
"KL/std": 4.18043327331543,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.10777164250612259,
"fcm_dpo/delta": -0.03339504823088646,
"fcm_dpo/margin": 4.011598587036133,
"fcm_dpo/q_t": 0.4028114676475525,
"grad_norm": 55.99860382080078,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5201721787452698,
"logits/rejected": -0.48702794313430786,
"logps/chosen": -61.6673583984375,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.30729675292969,
"loss": 1.0701,
"margin_dpo/margin_mean": 4.011598587036133,
"margin_dpo/margin_std": 5.134487152099609,
"step": 51
},
{
"KL/chosen_KL_mean": 0.3607959747314453,
"KL/mean": -2.7140278816223145,
"KL/rejected_KL_mean": -5.788852691650391,
"KL/std": 4.7607879638671875,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.10591427981853485,
"fcm_dpo/delta": -0.2597273588180542,
"fcm_dpo/margin": 6.149646759033203,
"fcm_dpo/q_t": 0.3517502546310425,
"grad_norm": 61.00212860107422,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5540838241577148,
"logits/rejected": -0.4985507130622864,
"logps/chosen": -51.9765625,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.76277160644531,
"loss": 0.9009,
"margin_dpo/margin_mean": 6.149646759033203,
"margin_dpo/margin_std": 5.092857837677002,
"step": 52
},
{
"KL/chosen_KL_mean": -0.19704818725585938,
"KL/mean": -3.3417558670043945,
"KL/rejected_KL_mean": -6.486461639404297,
"KL/std": 5.458329677581787,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.10362114012241364,
"fcm_dpo/delta": -0.2598329186439514,
"fcm_dpo/margin": 6.289419174194336,
"fcm_dpo/q_t": 0.3608250916004181,
"grad_norm": 59.65618133544922,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6251329183578491,
"logits/rejected": -0.6039801836013794,
"logps/chosen": -53.51169967651367,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.27005767822266,
"loss": 0.9373,
"margin_dpo/margin_mean": 6.289420127868652,
"margin_dpo/margin_std": 6.279295921325684,
"step": 53
},
{
"KL/chosen_KL_mean": -0.4445972442626953,
"KL/mean": -3.137702703475952,
"KL/rejected_KL_mean": -5.830802917480469,
"KL/std": 5.121060848236084,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.10144082456827164,
"fcm_dpo/delta": -0.15038657188415527,
"fcm_dpo/margin": 5.386206150054932,
"fcm_dpo/q_t": 0.3753092288970947,
"grad_norm": 55.26798629760742,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5941368341445923,
"logits/rejected": -0.5407606363296509,
"logps/chosen": -51.13325881958008,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.54619598388672,
"loss": 0.9709,
"margin_dpo/margin_mean": 5.386206150054932,
"margin_dpo/margin_std": 5.081514835357666,
"step": 54
},
{
"KL/chosen_KL_mean": -0.927947998046875,
"KL/mean": -4.146857261657715,
"KL/rejected_KL_mean": -7.365764617919922,
"KL/std": 6.101950168609619,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.09932823479175568,
"fcm_dpo/delta": -0.24702517688274384,
"fcm_dpo/margin": 6.4378156661987305,
"fcm_dpo/q_t": 0.36172401905059814,
"grad_norm": 53.780818939208984,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6389350891113281,
"logits/rejected": -0.5758472084999084,
"logps/chosen": -63.543182373046875,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.35926818847656,
"loss": 0.9584,
"margin_dpo/margin_mean": 6.437814712524414,
"margin_dpo/margin_std": 7.308642387390137,
"step": 55
},
{
"KL/chosen_KL_mean": -0.7108230590820312,
"KL/mean": -3.80918550491333,
"KL/rejected_KL_mean": -6.907550811767578,
"KL/std": 6.075893402099609,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.09705003350973129,
"fcm_dpo/delta": -0.20751234889030457,
"fcm_dpo/margin": 6.196725845336914,
"fcm_dpo/q_t": 0.36958831548690796,
"grad_norm": 47.083831787109375,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.630185604095459,
"logits/rejected": -0.5887142419815063,
"logps/chosen": -58.643550872802734,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.08198547363281,
"loss": 0.9834,
"margin_dpo/margin_mean": 6.196725845336914,
"margin_dpo/margin_std": 7.354310035705566,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7268028259277344,
"KL/mean": -4.224173545837402,
"KL/rejected_KL_mean": -7.7215423583984375,
"KL/std": 6.010547161102295,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.09477832913398743,
"fcm_dpo/delta": -0.27153968811035156,
"fcm_dpo/margin": 6.994732856750488,
"fcm_dpo/q_t": 0.351573646068573,
"grad_norm": 52.91410827636719,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5690896511077881,
"logits/rejected": -0.5405641794204712,
"logps/chosen": -71.22209167480469,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.28700256347656,
"loss": 0.918,
"margin_dpo/margin_mean": 6.994732856750488,
"margin_dpo/margin_std": 6.630904674530029,
"step": 57
},
{
"KL/chosen_KL_mean": -1.010385513305664,
"KL/mean": -4.842105865478516,
"KL/rejected_KL_mean": -8.673824310302734,
"KL/std": 6.785933494567871,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.09193111956119537,
"fcm_dpo/delta": -0.31510716676712036,
"fcm_dpo/margin": 7.663437366485596,
"fcm_dpo/q_t": 0.349496066570282,
"grad_norm": 55.937931060791016,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5902031660079956,
"logits/rejected": -0.513656735420227,
"logps/chosen": -63.14332580566406,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.29112243652344,
"loss": 0.9191,
"margin_dpo/margin_mean": 7.663437843322754,
"margin_dpo/margin_std": 8.12877368927002,
"step": 58
},
{
"KL/chosen_KL_mean": -1.4919471740722656,
"KL/mean": -5.709989070892334,
"KL/rejected_KL_mean": -9.928028106689453,
"KL/std": 7.444372177124023,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.08817608654499054,
"fcm_dpo/delta": -0.35956332087516785,
"fcm_dpo/margin": 8.436084747314453,
"fcm_dpo/q_t": 0.34159553050994873,
"grad_norm": 52.03582763671875,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6292190551757812,
"logits/rejected": -0.5891801118850708,
"logps/chosen": -53.42447280883789,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.813232421875,
"loss": 0.8944,
"margin_dpo/margin_mean": 8.43608570098877,
"margin_dpo/margin_std": 8.554616928100586,
"step": 59
},
{
"KL/chosen_KL_mean": -2.6756553649902344,
"KL/mean": -6.041238784790039,
"KL/rejected_KL_mean": -9.406822204589844,
"KL/std": 6.815237998962402,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.08637680113315582,
"fcm_dpo/delta": -0.18736253678798676,
"fcm_dpo/margin": 6.731164932250977,
"fcm_dpo/q_t": 0.36844515800476074,
"grad_norm": 57.49814987182617,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.606688380241394,
"logits/rejected": -0.5464133024215698,
"logps/chosen": -63.61784362792969,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.80021667480469,
"loss": 0.9875,
"margin_dpo/margin_mean": 6.731164932250977,
"margin_dpo/margin_std": 7.683201313018799,
"step": 60
},
{
"KL/chosen_KL_mean": -1.5235176086425781,
"KL/mean": -5.651108264923096,
"KL/rejected_KL_mean": -9.778701782226562,
"KL/std": 9.094491004943848,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.08403058350086212,
"fcm_dpo/delta": -0.3050088882446289,
"fcm_dpo/margin": 8.255186080932617,
"fcm_dpo/q_t": 0.3618362545967102,
"grad_norm": 48.30548858642578,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6002434492111206,
"logits/rejected": -0.5660552382469177,
"logps/chosen": -62.15704345703125,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.63119506835938,
"loss": 0.9711,
"margin_dpo/margin_mean": 8.2551851272583,
"margin_dpo/margin_std": 11.305606842041016,
"step": 61
},
{
"KL/chosen_KL_mean": -1.6561450958251953,
"KL/mean": -4.769914150238037,
"KL/rejected_KL_mean": -7.8836822509765625,
"KL/std": 6.6822967529296875,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.08298887312412262,
"fcm_dpo/delta": -0.11985031515359879,
"fcm_dpo/margin": 6.227537155151367,
"fcm_dpo/q_t": 0.38729119300842285,
"grad_norm": 48.09984588623047,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.5964776873588562,
"logits/rejected": -0.5614856481552124,
"logps/chosen": -57.806915283203125,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.44987487792969,
"loss": 1.038,
"margin_dpo/margin_mean": 6.227537631988525,
"margin_dpo/margin_std": 8.184688568115234,
"step": 62
},
{
"KL/chosen_KL_mean": -2.681711196899414,
"KL/mean": -6.847625732421875,
"KL/rejected_KL_mean": -11.013534545898438,
"KL/std": 8.392871856689453,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.08056771755218506,
"fcm_dpo/delta": -0.2820628583431244,
"fcm_dpo/margin": 8.331822395324707,
"fcm_dpo/q_t": 0.35215944051742554,
"grad_norm": 49.44816970825195,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.6123620271682739,
"logits/rejected": -0.5682877898216248,
"logps/chosen": -75.8291015625,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -108.62359619140625,
"loss": 0.9343,
"margin_dpo/margin_mean": 8.33182144165039,
"margin_dpo/margin_std": 8.752732276916504,
"step": 63
},
{
"KL/chosen_KL_mean": -1.037282943725586,
"KL/mean": -5.9826202392578125,
"KL/rejected_KL_mean": -10.927955627441406,
"KL/std": 9.150436401367188,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.07755880057811737,
"fcm_dpo/delta": -0.3850880563259125,
"fcm_dpo/margin": 9.890669822692871,
"fcm_dpo/q_t": 0.3408496081829071,
"grad_norm": 44.27583694458008,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.605506181716919,
"logits/rejected": -0.5743293762207031,
"logps/chosen": -55.03588104248047,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.45816040039062,
"loss": 0.9102,
"margin_dpo/margin_mean": 9.890670776367188,
"margin_dpo/margin_std": 11.207742691040039,
"step": 64
},
{
"KL/chosen_KL_mean": -3.239490509033203,
"KL/mean": -8.281026840209961,
"KL/rejected_KL_mean": -13.322566986083984,
"KL/std": 10.2528076171875,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.07488435506820679,
"fcm_dpo/delta": -0.37056964635849,
"fcm_dpo/margin": 10.08306884765625,
"fcm_dpo/q_t": 0.3390854001045227,
"grad_norm": 44.772369384765625,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6510547995567322,
"logits/rejected": -0.6380674839019775,
"logps/chosen": -68.07549285888672,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.26902770996094,
"loss": 0.9165,
"margin_dpo/margin_mean": 10.083067893981934,
"margin_dpo/margin_std": 11.16361141204834,
"step": 65
},
{
"KL/chosen_KL_mean": -3.0490779876708984,
"KL/mean": -7.599049091339111,
"KL/rejected_KL_mean": -12.149017333984375,
"KL/std": 9.199509620666504,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.07262101769447327,
"fcm_dpo/delta": -0.271577388048172,
"fcm_dpo/margin": 9.099935531616211,
"fcm_dpo/q_t": 0.36167746782302856,
"grad_norm": 41.545040130615234,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.655698835849762,
"logits/rejected": -0.6236636638641357,
"logps/chosen": -54.49260330200195,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -87.78531646728516,
"loss": 0.9619,
"margin_dpo/margin_mean": 9.099935531616211,
"margin_dpo/margin_std": 11.122103691101074,
"step": 66
},
{
"KL/chosen_KL_mean": -2.6720447540283203,
"KL/mean": -7.3747406005859375,
"KL/rejected_KL_mean": -12.077438354492188,
"KL/std": 9.28006362915039,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.07105744630098343,
"fcm_dpo/delta": -0.27725303173065186,
"fcm_dpo/margin": 9.405391693115234,
"fcm_dpo/q_t": 0.3593307137489319,
"grad_norm": 41.94929885864258,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.6038053035736084,
"logits/rejected": -0.5614980459213257,
"logps/chosen": -62.012847900390625,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -84.8647232055664,
"loss": 0.9529,
"margin_dpo/margin_mean": 9.405391693115234,
"margin_dpo/margin_std": 11.189936637878418,
"step": 67
},
{
"KL/chosen_KL_mean": -3.055255889892578,
"KL/mean": -7.527899742126465,
"KL/rejected_KL_mean": -12.000545501708984,
"KL/std": 8.336349487304688,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.06936784088611603,
"fcm_dpo/delta": -0.22731636464595795,
"fcm_dpo/margin": 8.945289611816406,
"fcm_dpo/q_t": 0.36327165365219116,
"grad_norm": 40.98445129394531,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6367079019546509,
"logits/rejected": -0.5765199661254883,
"logps/chosen": -68.26109313964844,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -89.20779418945312,
"loss": 0.9496,
"margin_dpo/margin_mean": 8.945289611816406,
"margin_dpo/margin_std": 9.330879211425781,
"step": 68
},
{
"KL/chosen_KL_mean": -3.5810699462890625,
"KL/mean": -9.05520248413086,
"KL/rejected_KL_mean": -14.529338836669922,
"KL/std": 9.87652587890625,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.06693008542060852,
"fcm_dpo/delta": -0.346579372882843,
"fcm_dpo/margin": 10.948261260986328,
"fcm_dpo/q_t": 0.3431288003921509,
"grad_norm": 42.85259246826172,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6404001712799072,
"logits/rejected": -0.6165236830711365,
"logps/chosen": -63.40031051635742,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -117.91819763183594,
"loss": 0.8894,
"margin_dpo/margin_mean": 10.948262214660645,
"margin_dpo/margin_std": 10.901615142822266,
"step": 69
},
{
"KL/chosen_KL_mean": -5.2194671630859375,
"KL/mean": -11.088415145874023,
"KL/rejected_KL_mean": -16.957359313964844,
"KL/std": 12.024399757385254,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.0648043155670166,
"fcm_dpo/delta": -0.3743746280670166,
"fcm_dpo/margin": 11.73788833618164,
"fcm_dpo/q_t": 0.34193557500839233,
"grad_norm": 44.00492858886719,
"learning_rate": 5e-07,
"logits/chosen": -0.6287994384765625,
"logits/rejected": -0.5946310758590698,
"logps/chosen": -67.15010833740234,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -108.01814270019531,
"loss": 0.9036,
"margin_dpo/margin_mean": 11.737889289855957,
"margin_dpo/margin_std": 12.800237655639648,
"step": 70
},
{
"KL/chosen_KL_mean": -4.846014022827148,
"KL/mean": -11.838199615478516,
"KL/rejected_KL_mean": -18.830387115478516,
"KL/std": 12.62130355834961,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.061709873378276825,
"fcm_dpo/delta": -0.4846467971801758,
"fcm_dpo/margin": 13.984375,
"fcm_dpo/q_t": 0.32341229915618896,
"grad_norm": 39.75981903076172,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6493800282478333,
"logits/rejected": -0.6095075607299805,
"logps/chosen": -66.59635162353516,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -116.16700744628906,
"loss": 0.837,
"margin_dpo/margin_mean": 13.984375953674316,
"margin_dpo/margin_std": 13.748950004577637,
"step": 71
},
{
"KL/chosen_KL_mean": -5.580968856811523,
"KL/mean": -12.921693801879883,
"KL/rejected_KL_mean": -20.262420654296875,
"KL/std": 13.505867004394531,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.05879486724734306,
"fcm_dpo/delta": -0.4848293662071228,
"fcm_dpo/margin": 14.681446075439453,
"fcm_dpo/q_t": 0.3229914903640747,
"grad_norm": 41.22248840332031,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6457772254943848,
"logits/rejected": -0.6087461709976196,
"logps/chosen": -71.63438415527344,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -115.54940795898438,
"loss": 0.8629,
"margin_dpo/margin_mean": 14.681446075439453,
"margin_dpo/margin_std": 15.725846290588379,
"step": 72
},
{
"KL/chosen_KL_mean": -7.517843246459961,
"KL/mean": -14.800748825073242,
"KL/rejected_KL_mean": -22.083660125732422,
"KL/std": 17.561433792114258,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.05613835155963898,
"fcm_dpo/delta": -0.43695318698883057,
"fcm_dpo/margin": 14.565816879272461,
"fcm_dpo/q_t": 0.3509829044342041,
"grad_norm": 45.127349853515625,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.6935266256332397,
"logits/rejected": -0.653118908405304,
"logps/chosen": -73.77411651611328,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -112.539794921875,
"loss": 1.0029,
"margin_dpo/margin_mean": 14.565816879272461,
"margin_dpo/margin_std": 21.94605255126953,
"step": 73
},
{
"KL/chosen_KL_mean": -7.943721771240234,
"KL/mean": -15.891912460327148,
"KL/rejected_KL_mean": -23.840110778808594,
"KL/std": 19.308032989501953,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.05364230275154114,
"fcm_dpo/delta": -0.4738168716430664,
"fcm_dpo/margin": 15.896381378173828,
"fcm_dpo/q_t": 0.3405853509902954,
"grad_norm": 44.694664001464844,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6524510383605957,
"logits/rejected": -0.6433727741241455,
"logps/chosen": -61.36860656738281,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -119.78704833984375,
"loss": 0.9488,
"margin_dpo/margin_mean": 15.896381378173828,
"margin_dpo/margin_std": 23.962190628051758,
"step": 74
},
{
"KL/chosen_KL_mean": -7.722936630249023,
"KL/mean": -18.847137451171875,
"KL/rejected_KL_mean": -29.971328735351562,
"KL/std": 19.825923919677734,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.050399381667375565,
"fcm_dpo/delta": -0.761603832244873,
"fcm_dpo/margin": 22.24839973449707,
"fcm_dpo/q_t": 0.2916664481163025,
"grad_norm": 36.36934280395508,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6783813238143921,
"logits/rejected": -0.6700829267501831,
"logps/chosen": -59.584598541259766,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -141.22531127929688,
"loss": 0.7699,
"margin_dpo/margin_mean": 22.248397827148438,
"margin_dpo/margin_std": 22.400943756103516,
"step": 75
},
{
"KL/chosen_KL_mean": -9.420347213745117,
"KL/mean": -16.42698097229004,
"KL/rejected_KL_mean": -23.433605194091797,
"KL/std": 15.946723937988281,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.04817916452884674,
"fcm_dpo/delta": -0.2846575379371643,
"fcm_dpo/margin": 14.013269424438477,
"fcm_dpo/q_t": 0.35380053520202637,
"grad_norm": 39.79032516479492,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.7044748067855835,
"logits/rejected": -0.6629197597503662,
"logps/chosen": -62.68638610839844,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -101.65023803710938,
"loss": 0.9645,
"margin_dpo/margin_mean": 14.013269424438477,
"margin_dpo/margin_std": 17.342012405395508,
"step": 76
},
{
"KL/chosen_KL_mean": -8.207674026489258,
"KL/mean": -19.680038452148438,
"KL/rejected_KL_mean": -31.15240478515625,
"KL/std": 21.73928451538086,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.04527886211872101,
"fcm_dpo/delta": -0.677450954914093,
"fcm_dpo/margin": 22.944732666015625,
"fcm_dpo/q_t": 0.29555851221084595,
"grad_norm": 41.42151641845703,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6860731244087219,
"logits/rejected": -0.647629976272583,
"logps/chosen": -66.30435180664062,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -124.92601776123047,
"loss": 0.8147,
"margin_dpo/margin_mean": 22.944730758666992,
"margin_dpo/margin_std": 23.827333450317383,
"step": 77
},
{
"KL/chosen_KL_mean": -8.151559829711914,
"KL/mean": -16.818445205688477,
"KL/rejected_KL_mean": -25.485332489013672,
"KL/std": 17.356788635253906,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.043553952127695084,
"fcm_dpo/delta": -0.368335485458374,
"fcm_dpo/margin": 17.33376693725586,
"fcm_dpo/q_t": 0.34413814544677734,
"grad_norm": 36.57780838012695,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6453582048416138,
"logits/rejected": -0.6197341680526733,
"logps/chosen": -63.765342712402344,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -110.41969299316406,
"loss": 0.9188,
"margin_dpo/margin_mean": 17.33376693725586,
"margin_dpo/margin_std": 19.793365478515625,
"step": 78
},
{
"KL/chosen_KL_mean": -8.366096496582031,
"KL/mean": -17.763093948364258,
"KL/rejected_KL_mean": -27.16008758544922,
"KL/std": 19.61853790283203,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.04178548604249954,
"fcm_dpo/delta": -0.4011922776699066,
"fcm_dpo/margin": 18.793994903564453,
"fcm_dpo/q_t": 0.34827283024787903,
"grad_norm": 32.12797164916992,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6654260158538818,
"logits/rejected": -0.6358423829078674,
"logps/chosen": -63.81658172607422,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -114.80764770507812,
"loss": 0.9432,
"margin_dpo/margin_mean": 18.793994903564453,
"margin_dpo/margin_std": 24.024066925048828,
"step": 79
},
{
"KL/chosen_KL_mean": -11.082340240478516,
"KL/mean": -19.782291412353516,
"KL/rejected_KL_mean": -28.48224639892578,
"KL/std": 20.413394927978516,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.0402105450630188,
"fcm_dpo/delta": -0.31318235397338867,
"fcm_dpo/margin": 17.399906158447266,
"fcm_dpo/q_t": 0.3642631471157074,
"grad_norm": 35.137020111083984,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7078701853752136,
"logits/rejected": -0.6634222269058228,
"logps/chosen": -69.60163116455078,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -116.02974700927734,
"loss": 0.9904,
"margin_dpo/margin_mean": 17.399904251098633,
"margin_dpo/margin_std": 23.675479888916016,
"step": 80
},
{
"KL/chosen_KL_mean": -9.967578887939453,
"KL/mean": -21.815593719482422,
"KL/rejected_KL_mean": -33.663612365722656,
"KL/std": 24.03949737548828,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.03838124871253967,
"fcm_dpo/delta": -0.5364028811454773,
"fcm_dpo/margin": 23.69603729248047,
"fcm_dpo/q_t": 0.3254055976867676,
"grad_norm": 35.88831329345703,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7091889381408691,
"logits/rejected": -0.6791235208511353,
"logps/chosen": -76.41644287109375,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -163.32632446289062,
"loss": 0.8792,
"margin_dpo/margin_mean": 23.69603729248047,
"margin_dpo/margin_std": 28.50186538696289,
"step": 81
},
{
"KL/chosen_KL_mean": -12.100605010986328,
"KL/mean": -21.5577392578125,
"KL/rejected_KL_mean": -31.014873504638672,
"KL/std": 21.757911682128906,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.03735063970088959,
"fcm_dpo/delta": -0.3172946870326996,
"fcm_dpo/margin": 18.914270401000977,
"fcm_dpo/q_t": 0.3623591363430023,
"grad_norm": 41.56498336791992,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6205165386199951,
"logits/rejected": -0.6074250340461731,
"logps/chosen": -64.33299255371094,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -121.75812530517578,
"loss": 1.044,
"margin_dpo/margin_mean": 18.914270401000977,
"margin_dpo/margin_std": 29.809494018554688,
"step": 82
},
{
"KL/chosen_KL_mean": -12.361867904663086,
"KL/mean": -23.320655822753906,
"KL/rejected_KL_mean": -34.27945327758789,
"KL/std": 22.589454650878906,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.035669997334480286,
"fcm_dpo/delta": -0.39825981855392456,
"fcm_dpo/margin": 21.917579650878906,
"fcm_dpo/q_t": 0.3431558609008789,
"grad_norm": 36.22183609008789,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6784165501594543,
"logits/rejected": -0.6415207982063293,
"logps/chosen": -68.18925476074219,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -137.995361328125,
"loss": 0.9297,
"margin_dpo/margin_mean": 21.917579650878906,
"margin_dpo/margin_std": 26.579288482666016,
"step": 83
},
{
"KL/chosen_KL_mean": -10.984846115112305,
"KL/mean": -20.518203735351562,
"KL/rejected_KL_mean": -30.051559448242188,
"KL/std": 19.22481346130371,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.034514445811510086,
"fcm_dpo/delta": -0.268974244594574,
"fcm_dpo/margin": 19.066707611083984,
"fcm_dpo/q_t": 0.35393911600112915,
"grad_norm": 31.12311553955078,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6781740188598633,
"logits/rejected": -0.6409305334091187,
"logps/chosen": -78.1610107421875,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -117.35015869140625,
"loss": 0.9372,
"margin_dpo/margin_mean": 19.066707611083984,
"margin_dpo/margin_std": 19.633758544921875,
"step": 84
},
{
"KL/chosen_KL_mean": -11.446046829223633,
"KL/mean": -21.150569915771484,
"KL/rejected_KL_mean": -30.855098724365234,
"KL/std": 19.64529037475586,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.03391923010349274,
"fcm_dpo/delta": -0.2667713761329651,
"fcm_dpo/margin": 19.409046173095703,
"fcm_dpo/q_t": 0.36208292841911316,
"grad_norm": 31.290390014648438,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6304664611816406,
"logits/rejected": -0.5972647070884705,
"logps/chosen": -69.8526611328125,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -109.493896484375,
"loss": 0.9736,
"margin_dpo/margin_mean": 19.409046173095703,
"margin_dpo/margin_std": 23.33486557006836,
"step": 85
},
{
"KL/chosen_KL_mean": -14.669717788696289,
"KL/mean": -24.960693359375,
"KL/rejected_KL_mean": -35.25166702270508,
"KL/std": 25.916988372802734,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.032901618629693985,
"fcm_dpo/delta": -0.2864453196525574,
"fcm_dpo/margin": 20.581951141357422,
"fcm_dpo/q_t": 0.37629234790802,
"grad_norm": 39.49543380737305,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.7171605825424194,
"logits/rejected": -0.7046980857849121,
"logps/chosen": -70.80718231201172,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -123.37332153320312,
"loss": 1.0691,
"margin_dpo/margin_mean": 20.581951141357422,
"margin_dpo/margin_std": 35.439178466796875,
"step": 86
},
{
"KL/chosen_KL_mean": -14.323598861694336,
"KL/mean": -26.19344711303711,
"KL/rejected_KL_mean": -38.063297271728516,
"KL/std": 25.82670021057129,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.03160586953163147,
"fcm_dpo/delta": -0.36560332775115967,
"fcm_dpo/margin": 23.739696502685547,
"fcm_dpo/q_t": 0.34734636545181274,
"grad_norm": 31.173131942749023,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7083604335784912,
"logits/rejected": -0.6805447340011597,
"logps/chosen": -69.9596939086914,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -133.5308837890625,
"loss": 0.9671,
"margin_dpo/margin_mean": 23.739696502685547,
"margin_dpo/margin_std": 31.024333953857422,
"step": 87
},
{
"KL/chosen_KL_mean": -18.130157470703125,
"KL/mean": -27.014564514160156,
"KL/rejected_KL_mean": -35.89897155761719,
"KL/std": 26.533187866210938,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.030946895480155945,
"fcm_dpo/delta": -0.15513765811920166,
"fcm_dpo/margin": 17.768814086914062,
"fcm_dpo/q_t": 0.3872056007385254,
"grad_norm": 33.197364807128906,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.658212423324585,
"logits/rejected": -0.6492829322814941,
"logps/chosen": -91.80130767822266,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -142.6074676513672,
"loss": 1.0846,
"margin_dpo/margin_mean": 17.768814086914062,
"margin_dpo/margin_std": 27.587907791137695,
"step": 88
},
{
"KL/chosen_KL_mean": -10.722139358520508,
"KL/mean": -20.540828704833984,
"KL/rejected_KL_mean": -30.359527587890625,
"KL/std": 22.09980010986328,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.03054092824459076,
"fcm_dpo/delta": -0.2056586593389511,
"fcm_dpo/margin": 19.63738250732422,
"fcm_dpo/q_t": 0.3710666000843048,
"grad_norm": 28.453542709350586,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7179343104362488,
"logits/rejected": -0.6833662986755371,
"logps/chosen": -71.34705352783203,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -112.44307708740234,
"loss": 0.9914,
"margin_dpo/margin_mean": 19.63738250732422,
"margin_dpo/margin_std": 24.839431762695312,
"step": 89
},
{
"KL/chosen_KL_mean": -14.105724334716797,
"KL/mean": -27.13547706604004,
"KL/rejected_KL_mean": -40.16523361206055,
"KL/std": 29.942901611328125,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.029609836637973785,
"fcm_dpo/delta": -0.38564369082450867,
"fcm_dpo/margin": 26.059507369995117,
"fcm_dpo/q_t": 0.3560681939125061,
"grad_norm": 30.310239791870117,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6555777192115784,
"logits/rejected": -0.6643766164779663,
"logps/chosen": -67.39103698730469,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -151.70994567871094,
"loss": 0.9857,
"margin_dpo/margin_mean": 26.05950927734375,
"margin_dpo/margin_std": 36.949066162109375,
"step": 90
},
{
"KL/chosen_KL_mean": -14.317174911499023,
"KL/mean": -25.541458129882812,
"KL/rejected_KL_mean": -36.76573944091797,
"KL/std": 24.45753288269043,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.028681177645921707,
"fcm_dpo/delta": -0.2515973150730133,
"fcm_dpo/margin": 22.448570251464844,
"fcm_dpo/q_t": 0.3691549599170685,
"grad_norm": 29.94892692565918,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6794449090957642,
"logits/rejected": -0.643684983253479,
"logps/chosen": -76.12013244628906,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -124.63969421386719,
"loss": 1.0094,
"margin_dpo/margin_mean": 22.448570251464844,
"margin_dpo/margin_std": 31.180084228515625,
"step": 91
},
{
"KL/chosen_KL_mean": -12.907651901245117,
"KL/mean": -24.19264030456543,
"KL/rejected_KL_mean": -35.477630615234375,
"KL/std": 25.02846908569336,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.028016380965709686,
"fcm_dpo/delta": -0.23956279456615448,
"fcm_dpo/margin": 22.56998062133789,
"fcm_dpo/q_t": 0.3695027232170105,
"grad_norm": 27.0372371673584,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7169160842895508,
"logits/rejected": -0.693396270275116,
"logps/chosen": -64.54841613769531,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -113.35881042480469,
"loss": 1.0017,
"margin_dpo/margin_mean": 22.56998062133789,
"margin_dpo/margin_std": 30.188648223876953,
"step": 92
},
{
"KL/chosen_KL_mean": -14.480842590332031,
"KL/mean": -25.734779357910156,
"KL/rejected_KL_mean": -36.98870849609375,
"KL/std": 23.70608901977539,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.027309108525514603,
"fcm_dpo/delta": -0.22143405675888062,
"fcm_dpo/margin": 22.507863998413086,
"fcm_dpo/q_t": 0.3679991662502289,
"grad_norm": 27.40256118774414,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.738601803779602,
"logits/rejected": -0.6867475509643555,
"logps/chosen": -67.01008605957031,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -114.14945983886719,
"loss": 0.9724,
"margin_dpo/margin_mean": 22.507863998413086,
"margin_dpo/margin_std": 26.947500228881836,
"step": 93
},
{
"KL/chosen_KL_mean": -15.58909797668457,
"KL/mean": -28.327407836914062,
"KL/rejected_KL_mean": -41.06570816040039,
"KL/std": 26.550617218017578,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.026492692530155182,
"fcm_dpo/delta": -0.28588011860847473,
"fcm_dpo/margin": 25.476613998413086,
"fcm_dpo/q_t": 0.35494130849838257,
"grad_norm": 27.439083099365234,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6954045295715332,
"logits/rejected": -0.6782183647155762,
"logps/chosen": -76.81170654296875,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -140.66473388671875,
"loss": 0.9351,
"margin_dpo/margin_mean": 25.476612091064453,
"margin_dpo/margin_std": 27.425670623779297,
"step": 94
},
{
"KL/chosen_KL_mean": -14.52149772644043,
"KL/mean": -24.788345336914062,
"KL/rejected_KL_mean": -35.055198669433594,
"KL/std": 24.6375732421875,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.026066306978464127,
"fcm_dpo/delta": -0.13932469487190247,
"fcm_dpo/margin": 20.533700942993164,
"fcm_dpo/q_t": 0.3804051876068115,
"grad_norm": 26.025171279907227,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.7163432836532593,
"logits/rejected": -0.68684983253479,
"logps/chosen": -67.0451431274414,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -110.935546875,
"loss": 1.0224,
"margin_dpo/margin_mean": 20.533700942993164,
"margin_dpo/margin_std": 25.788597106933594,
"step": 95
},
{
"KL/chosen_KL_mean": -13.763811111450195,
"KL/mean": -27.650653839111328,
"KL/rejected_KL_mean": -41.537498474121094,
"KL/std": 28.800777435302734,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.025319525972008705,
"fcm_dpo/delta": -0.31601524353027344,
"fcm_dpo/margin": 27.773691177368164,
"fcm_dpo/q_t": 0.35219550132751465,
"grad_norm": 26.12571144104004,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.7276456356048584,
"logits/rejected": -0.6974596977233887,
"logps/chosen": -75.92078399658203,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -138.13351440429688,
"loss": 0.9254,
"margin_dpo/margin_mean": 27.773693084716797,
"margin_dpo/margin_std": 31.26250457763672,
"step": 96
},
{
"KL/chosen_KL_mean": -15.030601501464844,
"KL/mean": -25.972335815429688,
"KL/rejected_KL_mean": -36.91407012939453,
"KL/std": 23.70777702331543,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.025046605616807938,
"fcm_dpo/delta": -0.15248806774616241,
"fcm_dpo/margin": 21.88347625732422,
"fcm_dpo/q_t": 0.37621963024139404,
"grad_norm": 27.10018539428711,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6382083892822266,
"logits/rejected": -0.5908419489860535,
"logps/chosen": -69.67697143554688,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -113.87882232666016,
"loss": 0.9984,
"margin_dpo/margin_mean": 21.88347625732422,
"margin_dpo/margin_std": 23.601760864257812,
"step": 97
},
{
"KL/chosen_KL_mean": -18.270347595214844,
"KL/mean": -30.684240341186523,
"KL/rejected_KL_mean": -43.09813690185547,
"KL/std": 26.8064022064209,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.02441706880927086,
"fcm_dpo/delta": -0.21288006007671356,
"fcm_dpo/margin": 24.82779312133789,
"fcm_dpo/q_t": 0.36630043387413025,
"grad_norm": 27.9980525970459,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.6928224563598633,
"logits/rejected": -0.6749119758605957,
"logps/chosen": -83.52897644042969,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -139.62562561035156,
"loss": 1.0038,
"margin_dpo/margin_mean": 24.82779312133789,
"margin_dpo/margin_std": 31.736007690429688,
"step": 98
},
{
"KL/chosen_KL_mean": -14.74056625366211,
"KL/mean": -29.577295303344727,
"KL/rejected_KL_mean": -44.41402053833008,
"KL/std": 29.210227966308594,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.0237110685557127,
"fcm_dpo/delta": -0.31514137983322144,
"fcm_dpo/margin": 29.67345428466797,
"fcm_dpo/q_t": 0.3556155562400818,
"grad_norm": 24.59937858581543,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.657272458076477,
"logits/rejected": -0.6558362245559692,
"logps/chosen": -60.379051208496094,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -130.85195922851562,
"loss": 0.9386,
"margin_dpo/margin_mean": 29.6734561920166,
"margin_dpo/margin_std": 35.876800537109375,
"step": 99
},
{
"KL/chosen_KL_mean": -16.435449600219727,
"KL/mean": -25.48604393005371,
"KL/rejected_KL_mean": -34.53663635253906,
"KL/std": 23.471981048583984,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.02355112135410309,
"fcm_dpo/delta": -0.026980634778738022,
"fcm_dpo/margin": 18.101184844970703,
"fcm_dpo/q_t": 0.40622931718826294,
"grad_norm": 27.36819839477539,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6757396459579468,
"logits/rejected": -0.6324243545532227,
"logps/chosen": -74.02943420410156,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -108.59684753417969,
"loss": 1.1247,
"margin_dpo/margin_mean": 18.101184844970703,
"margin_dpo/margin_std": 30.202056884765625,
"step": 100
},
{
"KL/chosen_KL_mean": -20.00939178466797,
"KL/mean": -30.913528442382812,
"KL/rejected_KL_mean": -41.817665100097656,
"KL/std": 28.39974021911621,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.02322225831449032,
"fcm_dpo/delta": -0.109965018928051,
"fcm_dpo/margin": 21.80826759338379,
"fcm_dpo/q_t": 0.39490771293640137,
"grad_norm": 27.217756271362305,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6777423620223999,
"logits/rejected": -0.6437240242958069,
"logps/chosen": -81.65824890136719,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -125.00735473632812,
"loss": 1.0806,
"margin_dpo/margin_mean": 21.808269500732422,
"margin_dpo/margin_std": 34.3305778503418,
"step": 101
},
{
"KL/chosen_KL_mean": -22.16390037536621,
"KL/mean": -30.27094268798828,
"KL/rejected_KL_mean": -38.377986907958984,
"KL/std": 27.00689697265625,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.023358814418315887,
"fcm_dpo/delta": 0.020977407693862915,
"fcm_dpo/margin": 16.214086532592773,
"fcm_dpo/q_t": 0.4182248115539551,
"grad_norm": 30.8178653717041,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.7105182409286499,
"logits/rejected": -0.6647392511367798,
"logps/chosen": -86.24278259277344,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -106.5650634765625,
"loss": 1.1713,
"margin_dpo/margin_mean": 16.214086532592773,
"margin_dpo/margin_std": 31.576683044433594,
"step": 102
},
{
"KL/chosen_KL_mean": -16.63315200805664,
"KL/mean": -32.39291763305664,
"KL/rejected_KL_mean": -48.15269088745117,
"KL/std": 29.840614318847656,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.02269843965768814,
"fcm_dpo/delta": -0.32748711109161377,
"fcm_dpo/margin": 31.519527435302734,
"fcm_dpo/q_t": 0.34971487522125244,
"grad_norm": 25.520038604736328,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6867813467979431,
"logits/rejected": -0.6520103216171265,
"logps/chosen": -77.93243408203125,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -141.72540283203125,
"loss": 0.9196,
"margin_dpo/margin_mean": 31.5195255279541,
"margin_dpo/margin_std": 35.207923889160156,
"step": 103
},
{
"KL/chosen_KL_mean": -18.6593074798584,
"KL/mean": -33.599281311035156,
"KL/rejected_KL_mean": -48.53925323486328,
"KL/std": 33.02933120727539,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.022114867344498634,
"fcm_dpo/delta": -0.2693977355957031,
"fcm_dpo/margin": 29.879947662353516,
"fcm_dpo/q_t": 0.36667945981025696,
"grad_norm": 25.26581382751465,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.7141636610031128,
"logits/rejected": -0.6804061532020569,
"logps/chosen": -73.0320816040039,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -138.10397338867188,
"loss": 0.988,
"margin_dpo/margin_mean": 29.879947662353516,
"margin_dpo/margin_std": 39.69654846191406,
"step": 104
},
{
"KL/chosen_KL_mean": -17.245506286621094,
"KL/mean": -38.19187545776367,
"KL/rejected_KL_mean": -59.13824462890625,
"KL/std": 32.81090545654297,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.021134644746780396,
"fcm_dpo/delta": -0.5075461268424988,
"fcm_dpo/margin": 41.89273452758789,
"fcm_dpo/q_t": 0.31017786264419556,
"grad_norm": 25.08208656311035,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.7256340980529785,
"logits/rejected": -0.7023687958717346,
"logps/chosen": -71.88445281982422,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -157.11175537109375,
"loss": 0.7999,
"margin_dpo/margin_mean": 41.89273452758789,
"margin_dpo/margin_std": 35.73217010498047,
"step": 105
},
{
"KL/chosen_KL_mean": -19.42215919494629,
"KL/mean": -32.675418853759766,
"KL/rejected_KL_mean": -45.928688049316406,
"KL/std": 28.108577728271484,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.020594779402017593,
"fcm_dpo/delta": -0.15016362071037292,
"fcm_dpo/margin": 26.506515502929688,
"fcm_dpo/q_t": 0.37773293256759644,
"grad_norm": 24.609127044677734,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.662516713142395,
"logits/rejected": -0.6267033815383911,
"logps/chosen": -74.25505065917969,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -131.15330505371094,
"loss": 1.0027,
"margin_dpo/margin_mean": 26.506515502929688,
"margin_dpo/margin_std": 30.649978637695312,
"step": 106
},
{
"KL/chosen_KL_mean": -23.350513458251953,
"KL/mean": -38.28211975097656,
"KL/rejected_KL_mean": -53.213722229003906,
"KL/std": 33.904205322265625,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.02017119899392128,
"fcm_dpo/delta": -0.20935019850730896,
"fcm_dpo/margin": 29.863208770751953,
"fcm_dpo/q_t": 0.367611289024353,
"grad_norm": 24.215755462646484,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.685881495475769,
"logits/rejected": -0.6361589431762695,
"logps/chosen": -93.0583267211914,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -147.95323181152344,
"loss": 0.9882,
"margin_dpo/margin_mean": 29.86320686340332,
"margin_dpo/margin_std": 35.658416748046875,
"step": 107
},
{
"KL/chosen_KL_mean": -21.30466651916504,
"KL/mean": -38.65107727050781,
"KL/rejected_KL_mean": -55.99748992919922,
"KL/std": 37.09898376464844,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.01967495307326317,
"fcm_dpo/delta": -0.2925887405872345,
"fcm_dpo/margin": 34.69282531738281,
"fcm_dpo/q_t": 0.36451563239097595,
"grad_norm": 24.30539894104004,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6991877555847168,
"logits/rejected": -0.6731090545654297,
"logps/chosen": -77.3145523071289,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -151.7935028076172,
"loss": 0.9954,
"margin_dpo/margin_mean": 34.69282531738281,
"margin_dpo/margin_std": 47.79713439941406,
"step": 108
},
{
"KL/chosen_KL_mean": -19.659822463989258,
"KL/mean": -38.451622009277344,
"KL/rejected_KL_mean": -57.24341583251953,
"KL/std": 36.00904846191406,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.019026821479201317,
"fcm_dpo/delta": -0.32746434211730957,
"fcm_dpo/margin": 37.583595275878906,
"fcm_dpo/q_t": 0.34695935249328613,
"grad_norm": 24.934322357177734,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.6155581474304199,
"logits/rejected": -0.5928876996040344,
"logps/chosen": -82.5453109741211,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -155.9291534423828,
"loss": 0.923,
"margin_dpo/margin_mean": 37.58359909057617,
"margin_dpo/margin_std": 40.89533996582031,
"step": 109
},
{
"KL/chosen_KL_mean": -19.98564910888672,
"KL/mean": -37.01270294189453,
"KL/rejected_KL_mean": -54.039756774902344,
"KL/std": 35.79534912109375,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.018443116918206215,
"fcm_dpo/delta": -0.2388836145401001,
"fcm_dpo/margin": 34.054107666015625,
"fcm_dpo/q_t": 0.3669545650482178,
"grad_norm": 22.622568130493164,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6140676736831665,
"logits/rejected": -0.5725805163383484,
"logps/chosen": -78.73933410644531,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -133.78976440429688,
"loss": 0.9958,
"margin_dpo/margin_mean": 34.054107666015625,
"margin_dpo/margin_std": 42.70184326171875,
"step": 110
},
{
"KL/chosen_KL_mean": -23.330896377563477,
"KL/mean": -42.295196533203125,
"KL/rejected_KL_mean": -61.259498596191406,
"KL/std": 40.184547424316406,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.01799844577908516,
"fcm_dpo/delta": -0.29396289587020874,
"fcm_dpo/margin": 37.92860412597656,
"fcm_dpo/q_t": 0.35755258798599243,
"grad_norm": 26.07427978515625,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.6798889636993408,
"logits/rejected": -0.6572076082229614,
"logps/chosen": -91.95500183105469,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -159.6883544921875,
"loss": 0.979,
"margin_dpo/margin_mean": 37.92860794067383,
"margin_dpo/margin_std": 48.289306640625,
"step": 111
},
{
"KL/chosen_KL_mean": -21.652713775634766,
"KL/mean": -34.00275421142578,
"KL/rejected_KL_mean": -46.35279083251953,
"KL/std": 28.837787628173828,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.017818715423345566,
"fcm_dpo/delta": -0.04140431433916092,
"fcm_dpo/margin": 24.700084686279297,
"fcm_dpo/q_t": 0.40135613083839417,
"grad_norm": 22.577350616455078,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6432993412017822,
"logits/rejected": -0.5998743772506714,
"logps/chosen": -71.90235900878906,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -111.1272201538086,
"loss": 1.0813,
"margin_dpo/margin_mean": 24.700084686279297,
"margin_dpo/margin_std": 34.30575180053711,
"step": 112
},
{
"KL/chosen_KL_mean": -26.370677947998047,
"KL/mean": -39.9312744140625,
"KL/rejected_KL_mean": -53.49187469482422,
"KL/std": 28.95420265197754,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.017736122012138367,
"fcm_dpo/delta": -0.08302031457424164,
"fcm_dpo/margin": 27.121187210083008,
"fcm_dpo/q_t": 0.39081066846847534,
"grad_norm": 22.69746208190918,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.6275451183319092,
"logits/rejected": -0.5815028548240662,
"logps/chosen": -93.08363342285156,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -131.4605712890625,
"loss": 1.0368,
"margin_dpo/margin_mean": 27.121185302734375,
"margin_dpo/margin_std": 32.2321891784668,
"step": 113
},
{
"KL/chosen_KL_mean": -24.161130905151367,
"KL/mean": -45.25044250488281,
"KL/rejected_KL_mean": -66.33973693847656,
"KL/std": 44.154300689697266,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.017194289714097977,
"fcm_dpo/delta": -0.3392394781112671,
"fcm_dpo/margin": 42.178619384765625,
"fcm_dpo/q_t": 0.3535785377025604,
"grad_norm": 23.69581413269043,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.6525863409042358,
"logits/rejected": -0.6185038089752197,
"logps/chosen": -81.94621276855469,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -153.44940185546875,
"loss": 0.9378,
"margin_dpo/margin_mean": 42.178619384765625,
"margin_dpo/margin_std": 53.02991485595703,
"step": 114
},
{
"KL/chosen_KL_mean": -32.144317626953125,
"KL/mean": -46.77862548828125,
"KL/rejected_KL_mean": -61.412940979003906,
"KL/std": 41.4689826965332,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.01700768992304802,
"fcm_dpo/delta": -0.1002306193113327,
"fcm_dpo/margin": 29.26862335205078,
"fcm_dpo/q_t": 0.3986138701438904,
"grad_norm": 32.29568862915039,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.6790868043899536,
"logits/rejected": -0.6547123193740845,
"logps/chosen": -97.72695922851562,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -159.97845458984375,
"loss": 1.1207,
"margin_dpo/margin_mean": 29.26862335205078,
"margin_dpo/margin_std": 53.58454895019531,
"step": 115
},
{
"KL/chosen_KL_mean": -24.143800735473633,
"KL/mean": -39.930686950683594,
"KL/rejected_KL_mean": -55.71758270263672,
"KL/std": 35.130592346191406,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.016804486513137817,
"fcm_dpo/delta": -0.13403692841529846,
"fcm_dpo/margin": 31.573772430419922,
"fcm_dpo/q_t": 0.38285309076309204,
"grad_norm": 24.514631271362305,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.64164799451828,
"logits/rejected": -0.6261519193649292,
"logps/chosen": -75.54411315917969,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -136.23944091796875,
"loss": 1.0119,
"margin_dpo/margin_mean": 31.573772430419922,
"margin_dpo/margin_std": 37.68627166748047,
"step": 116
},
{
"KL/chosen_KL_mean": -30.229568481445312,
"KL/mean": -43.88397216796875,
"KL/rejected_KL_mean": -57.53838348388672,
"KL/std": 37.56050491333008,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.01664135418832302,
"fcm_dpo/delta": -0.055726565420627594,
"fcm_dpo/margin": 27.30881118774414,
"fcm_dpo/q_t": 0.39999547600746155,
"grad_norm": 29.87828254699707,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.6171888113021851,
"logits/rejected": -0.5780812501907349,
"logps/chosen": -99.5279769897461,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -124.12237548828125,
"loss": 1.0906,
"margin_dpo/margin_mean": 27.30881118774414,
"margin_dpo/margin_std": 42.070045471191406,
"step": 117
},
{
"KL/chosen_KL_mean": -24.80445098876953,
"KL/mean": -40.591796875,
"KL/rejected_KL_mean": -56.37914276123047,
"KL/std": 33.962181091308594,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.016470149159431458,
"fcm_dpo/delta": -0.12322327494621277,
"fcm_dpo/margin": 31.574687957763672,
"fcm_dpo/q_t": 0.38508450984954834,
"grad_norm": 23.370370864868164,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6490514278411865,
"logits/rejected": -0.6261621117591858,
"logps/chosen": -80.4454345703125,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -132.04820251464844,
"loss": 1.0199,
"margin_dpo/margin_mean": 31.574687957763672,
"margin_dpo/margin_std": 38.370567321777344,
"step": 118
},
{
"KL/chosen_KL_mean": -33.29602813720703,
"KL/mean": -48.99560546875,
"KL/rejected_KL_mean": -64.69517517089844,
"KL/std": 37.841270446777344,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.016222678124904633,
"fcm_dpo/delta": -0.11308414489030838,
"fcm_dpo/margin": 31.399154663085938,
"fcm_dpo/q_t": 0.39160895347595215,
"grad_norm": 25.578641891479492,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6672037839889526,
"logits/rejected": -0.6525957584381104,
"logps/chosen": -106.80622863769531,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -167.67247009277344,
"loss": 1.0591,
"margin_dpo/margin_mean": 31.399154663085938,
"margin_dpo/margin_std": 45.50007247924805,
"step": 119
},
{
"KL/chosen_KL_mean": -34.85655975341797,
"KL/mean": -56.261322021484375,
"KL/rejected_KL_mean": -77.66609191894531,
"KL/std": 47.26749801635742,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.015881016850471497,
"fcm_dpo/delta": -0.2897739112377167,
"fcm_dpo/margin": 42.80952453613281,
"fcm_dpo/q_t": 0.3552402853965759,
"grad_norm": 24.126750946044922,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6502152681350708,
"logits/rejected": -0.6147547960281372,
"logps/chosen": -111.63739013671875,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -185.6898193359375,
"loss": 0.9457,
"margin_dpo/margin_mean": 42.80952453613281,
"margin_dpo/margin_std": 49.247589111328125,
"step": 120
},
{
"KL/chosen_KL_mean": -33.858795166015625,
"KL/mean": -57.48423767089844,
"KL/rejected_KL_mean": -81.10968017578125,
"KL/std": 46.71514892578125,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.015335135161876678,
"fcm_dpo/delta": -0.33797329664230347,
"fcm_dpo/margin": 47.250885009765625,
"fcm_dpo/q_t": 0.35153231024742126,
"grad_norm": 25.32863998413086,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.612008810043335,
"logits/rejected": -0.6054136753082275,
"logps/chosen": -95.64868927001953,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -191.104248046875,
"loss": 0.9492,
"margin_dpo/margin_mean": 47.250885009765625,
"margin_dpo/margin_std": 58.144596099853516,
"step": 121
},
{
"KL/chosen_KL_mean": -31.168033599853516,
"KL/mean": -60.511810302734375,
"KL/rejected_KL_mean": -89.85560607910156,
"KL/std": 53.64841842651367,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.014709394425153732,
"fcm_dpo/delta": -0.4840208888053894,
"fcm_dpo/margin": 58.68756103515625,
"fcm_dpo/q_t": 0.3186884820461273,
"grad_norm": 25.857219696044922,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.5904278755187988,
"logits/rejected": -0.6040619015693665,
"logps/chosen": -78.07024383544922,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -196.56979370117188,
"loss": 0.8285,
"margin_dpo/margin_mean": 58.68756103515625,
"margin_dpo/margin_std": 54.87202072143555,
"step": 122
},
{
"KL/chosen_KL_mean": -33.60801696777344,
"KL/mean": -53.090492248535156,
"KL/rejected_KL_mean": -72.57296752929688,
"KL/std": 46.223365783691406,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.014308318495750427,
"fcm_dpo/delta": -0.16279909014701843,
"fcm_dpo/margin": 38.964935302734375,
"fcm_dpo/q_t": 0.38503628969192505,
"grad_norm": 23.776098251342773,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.5932154655456543,
"logits/rejected": -0.5714948773384094,
"logps/chosen": -94.9466552734375,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -160.34835815429688,
"loss": 1.0682,
"margin_dpo/margin_mean": 38.964935302734375,
"margin_dpo/margin_std": 61.33662414550781,
"step": 123
},
{
"KL/chosen_KL_mean": -37.506778717041016,
"KL/mean": -65.6749267578125,
"KL/rejected_KL_mean": -93.84307861328125,
"KL/std": 61.69428253173828,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.013852743431925774,
"fcm_dpo/delta": -0.396537721157074,
"fcm_dpo/margin": 56.33629608154297,
"fcm_dpo/q_t": 0.35203444957733154,
"grad_norm": 27.827972412109375,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.5845953226089478,
"logits/rejected": -0.5840995907783508,
"logps/chosen": -108.95510864257812,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -211.42364501953125,
"loss": 0.9658,
"margin_dpo/margin_mean": 56.33629608154297,
"margin_dpo/margin_std": 77.30990600585938,
"step": 124
},
{
"KL/chosen_KL_mean": -31.203296661376953,
"KL/mean": -56.6873779296875,
"KL/rejected_KL_mean": -82.17146301269531,
"KL/std": 51.79102325439453,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.013450113125145435,
"fcm_dpo/delta": -0.29537516832351685,
"fcm_dpo/margin": 50.968162536621094,
"fcm_dpo/q_t": 0.35877731442451477,
"grad_norm": 22.051082611083984,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.5767650604248047,
"logits/rejected": -0.5755800008773804,
"logps/chosen": -81.34024047851562,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -166.1600799560547,
"loss": 0.9621,
"margin_dpo/margin_mean": 50.968162536621094,
"margin_dpo/margin_std": 62.841583251953125,
"step": 125
},
{
"KL/chosen_KL_mean": -34.27449035644531,
"KL/mean": -57.64450454711914,
"KL/rejected_KL_mean": -81.01451110839844,
"KL/std": 46.706634521484375,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.013104382902383804,
"fcm_dpo/delta": -0.21946457028388977,
"fcm_dpo/margin": 46.74002456665039,
"fcm_dpo/q_t": 0.369617223739624,
"grad_norm": 23.553150177001953,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.5703192949295044,
"logits/rejected": -0.5591797232627869,
"logps/chosen": -89.94155883789062,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -179.14427185058594,
"loss": 0.9888,
"margin_dpo/margin_mean": 46.74002456665039,
"margin_dpo/margin_std": 58.35154724121094,
"step": 126
},
{
"KL/chosen_KL_mean": -32.794464111328125,
"KL/mean": -50.73387145996094,
"KL/rejected_KL_mean": -68.67326354980469,
"KL/std": 40.8072509765625,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.012982767075300217,
"fcm_dpo/delta": -0.0673581212759018,
"fcm_dpo/margin": 35.87880325317383,
"fcm_dpo/q_t": 0.3947966694831848,
"grad_norm": 23.849796295166016,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.579119086265564,
"logits/rejected": -0.5568236112594604,
"logps/chosen": -89.3491439819336,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -145.46905517578125,
"loss": 1.073,
"margin_dpo/margin_mean": 35.87880325317383,
"margin_dpo/margin_std": 50.529823303222656,
"step": 127
},
{
"KL/chosen_KL_mean": -35.758731842041016,
"KL/mean": -54.428401947021484,
"KL/rejected_KL_mean": -73.09808349609375,
"KL/std": 48.978797912597656,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.012904481962323189,
"fcm_dpo/delta": -0.08387961983680725,
"fcm_dpo/margin": 37.3393440246582,
"fcm_dpo/q_t": 0.3962070941925049,
"grad_norm": 35.115657806396484,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.5600550174713135,
"logits/rejected": -0.5325770378112793,
"logps/chosen": -93.87968444824219,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -149.53704833984375,
"loss": 1.0968,
"margin_dpo/margin_mean": 37.3393440246582,
"margin_dpo/margin_std": 60.14480972290039,
"step": 128
},
{
"KL/chosen_KL_mean": -44.086055755615234,
"KL/mean": -63.96501922607422,
"KL/rejected_KL_mean": -83.84398651123047,
"KL/std": 53.29756164550781,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.012750450521707535,
"fcm_dpo/delta": -0.10966508090496063,
"fcm_dpo/margin": 39.75792694091797,
"fcm_dpo/q_t": 0.396596223115921,
"grad_norm": 24.342992782592773,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.5686518549919128,
"logits/rejected": -0.5536836385726929,
"logps/chosen": -111.00242614746094,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -180.4862060546875,
"loss": 1.1002,
"margin_dpo/margin_mean": 39.75792694091797,
"margin_dpo/margin_std": 68.46754455566406,
"step": 129
},
{
"KL/chosen_KL_mean": -33.01520919799805,
"KL/mean": -62.57740020751953,
"KL/rejected_KL_mean": -92.13957977294922,
"KL/std": 57.29778289794922,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.012459097430109978,
"fcm_dpo/delta": -0.34889647364616394,
"fcm_dpo/margin": 59.1243782043457,
"fcm_dpo/q_t": 0.34880900382995605,
"grad_norm": 21.42711639404297,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.5372148752212524,
"logits/rejected": -0.5315117835998535,
"logps/chosen": -77.68206024169922,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -174.92123413085938,
"loss": 0.9393,
"margin_dpo/margin_mean": 59.1243782043457,
"margin_dpo/margin_std": 68.27285766601562,
"step": 130
},
{
"KL/chosen_KL_mean": -28.63924217224121,
"KL/mean": -57.72308349609375,
"KL/rejected_KL_mean": -86.80691528320312,
"KL/std": 54.85826110839844,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.011984329670667648,
"fcm_dpo/delta": -0.30810654163360596,
"fcm_dpo/margin": 58.167686462402344,
"fcm_dpo/q_t": 0.3473639190196991,
"grad_norm": 27.132959365844727,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.5333915948867798,
"logits/rejected": -0.5392862558364868,
"logps/chosen": -73.56382751464844,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -175.2509307861328,
"loss": 0.9189,
"margin_dpo/margin_mean": 58.167686462402344,
"margin_dpo/margin_std": 60.25544738769531,
"step": 131
},
{
"KL/chosen_KL_mean": -38.82572937011719,
"KL/mean": -63.184059143066406,
"KL/rejected_KL_mean": -87.54237365722656,
"KL/std": 55.58538818359375,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.011730792000889778,
"fcm_dpo/delta": -0.1767834722995758,
"fcm_dpo/margin": 48.716651916503906,
"fcm_dpo/q_t": 0.3818228244781494,
"grad_norm": 23.049833297729492,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.5593503713607788,
"logits/rejected": -0.5511766672134399,
"logps/chosen": -97.82681274414062,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -175.43453979492188,
"loss": 1.034,
"margin_dpo/margin_mean": 48.716651916503906,
"margin_dpo/margin_std": 71.09864807128906,
"step": 132
},
{
"KL/chosen_KL_mean": -45.97046661376953,
"KL/mean": -67.64411163330078,
"KL/rejected_KL_mean": -89.3177490234375,
"KL/std": 51.025794982910156,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.011588573455810547,
"fcm_dpo/delta": -0.1051153689622879,
"fcm_dpo/margin": 43.3472900390625,
"fcm_dpo/q_t": 0.3930418789386749,
"grad_norm": 26.683420181274414,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.5510296821594238,
"logits/rejected": -0.5348105430603027,
"logps/chosen": -112.57496643066406,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -185.65130615234375,
"loss": 1.0613,
"margin_dpo/margin_mean": 43.3472900390625,
"margin_dpo/margin_std": 63.578800201416016,
"step": 133
},
{
"KL/chosen_KL_mean": -36.70982360839844,
"KL/mean": -60.16697311401367,
"KL/rejected_KL_mean": -83.62411499023438,
"KL/std": 50.47258758544922,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.011467832140624523,
"fcm_dpo/delta": -0.14169654250144958,
"fcm_dpo/margin": 46.91429901123047,
"fcm_dpo/q_t": 0.38246801495552063,
"grad_norm": 22.107864379882812,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.5395177006721497,
"logits/rejected": -0.5207037329673767,
"logps/chosen": -88.77908325195312,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -171.27862548828125,
"loss": 1.0101,
"margin_dpo/margin_mean": 46.91429901123047,
"margin_dpo/margin_std": 57.2902946472168,
"step": 134
},
{
"KL/chosen_KL_mean": -41.4932861328125,
"KL/mean": -73.72633361816406,
"KL/rejected_KL_mean": -105.95936584472656,
"KL/std": 65.24856567382812,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.011130950413644314,
"fcm_dpo/delta": -0.3293672502040863,
"fcm_dpo/margin": 64.46609497070312,
"fcm_dpo/q_t": 0.35333138704299927,
"grad_norm": 23.30121612548828,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.5058168768882751,
"logits/rejected": -0.5387102961540222,
"logps/chosen": -91.84715270996094,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -221.93911743164062,
"loss": 0.9469,
"margin_dpo/margin_mean": 64.46609497070312,
"margin_dpo/margin_std": 78.12184143066406,
"step": 135
},
{
"KL/chosen_KL_mean": -48.40357208251953,
"KL/mean": -70.24093627929688,
"KL/rejected_KL_mean": -92.07830810546875,
"KL/std": 58.511390686035156,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.010931683704257011,
"fcm_dpo/delta": -0.08014538139104843,
"fcm_dpo/margin": 43.674739837646484,
"fcm_dpo/q_t": 0.4048768877983093,
"grad_norm": 23.915916442871094,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.5324341654777527,
"logits/rejected": -0.5198547840118408,
"logps/chosen": -113.47608184814453,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -188.3995361328125,
"loss": 1.1016,
"margin_dpo/margin_mean": 43.67473602294922,
"margin_dpo/margin_std": 73.63079833984375,
"step": 136
},
{
"KL/chosen_KL_mean": -48.21935272216797,
"KL/mean": -82.04722595214844,
"KL/rejected_KL_mean": -115.87509155273438,
"KL/std": 78.75631713867188,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.010671773925423622,
"fcm_dpo/delta": -0.3343368172645569,
"fcm_dpo/margin": 67.65573120117188,
"fcm_dpo/q_t": 0.35995662212371826,
"grad_norm": 21.44792938232422,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.49152958393096924,
"logits/rejected": -0.5171319246292114,
"logps/chosen": -96.97846984863281,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -229.73886108398438,
"loss": 0.9823,
"margin_dpo/margin_mean": 67.65573120117188,
"margin_dpo/margin_std": 93.46538543701172,
"step": 137
},
{
"KL/chosen_KL_mean": -49.967342376708984,
"KL/mean": -75.7919692993164,
"KL/rejected_KL_mean": -101.61659240722656,
"KL/std": 60.058433532714844,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.010484375059604645,
"fcm_dpo/delta": -0.14534850418567657,
"fcm_dpo/margin": 51.649253845214844,
"fcm_dpo/q_t": 0.37925344705581665,
"grad_norm": 23.9995174407959,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.4553467631340027,
"logits/rejected": -0.4406394362449646,
"logps/chosen": -110.48699188232422,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -194.81353759765625,
"loss": 1.0095,
"margin_dpo/margin_mean": 51.649253845214844,
"margin_dpo/margin_std": 61.569252014160156,
"step": 138
},
{
"KL/chosen_KL_mean": -40.474876403808594,
"KL/mean": -71.97026062011719,
"KL/rejected_KL_mean": -103.46565246582031,
"KL/std": 57.427764892578125,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.010253118351101875,
"fcm_dpo/delta": -0.25380727648735046,
"fcm_dpo/margin": 62.99077224731445,
"fcm_dpo/q_t": 0.3614668846130371,
"grad_norm": 20.68395233154297,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.5000328421592712,
"logits/rejected": -0.48780351877212524,
"logps/chosen": -87.36625671386719,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -183.19363403320312,
"loss": 0.9557,
"margin_dpo/margin_mean": 62.99076843261719,
"margin_dpo/margin_std": 72.46928405761719,
"step": 139
},
{
"KL/chosen_KL_mean": -49.33660125732422,
"KL/mean": -78.02191925048828,
"KL/rejected_KL_mean": -106.70724487304688,
"KL/std": 64.60366821289062,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.010033691301941872,
"fcm_dpo/delta": -0.1809324026107788,
"fcm_dpo/margin": 57.37063980102539,
"fcm_dpo/q_t": 0.37695828080177307,
"grad_norm": 23.712533950805664,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.4792005717754364,
"logits/rejected": -0.46222788095474243,
"logps/chosen": -108.31130981445312,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -189.99134826660156,
"loss": 1.0174,
"margin_dpo/margin_mean": 57.370643615722656,
"margin_dpo/margin_std": 76.40518188476562,
"step": 140
},
{
"KL/chosen_KL_mean": -53.7377815246582,
"KL/mean": -81.11244201660156,
"KL/rejected_KL_mean": -108.48710632324219,
"KL/std": 67.55751037597656,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.009888198226690292,
"fcm_dpo/delta": -0.14525547623634338,
"fcm_dpo/margin": 54.74932098388672,
"fcm_dpo/q_t": 0.3810844421386719,
"grad_norm": 26.525779724121094,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.4832468628883362,
"logits/rejected": -0.46669474244117737,
"logps/chosen": -128.81344604492188,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -206.6793670654297,
"loss": 1.0432,
"margin_dpo/margin_mean": 54.74932098388672,
"margin_dpo/margin_std": 76.88491821289062,
"step": 141
},
{
"KL/chosen_KL_mean": -56.92424774169922,
"KL/mean": -88.21382141113281,
"KL/rejected_KL_mean": -119.50340270996094,
"KL/std": 76.11981201171875,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.009715566411614418,
"fcm_dpo/delta": -0.2142324447631836,
"fcm_dpo/margin": 62.57915496826172,
"fcm_dpo/q_t": 0.3745476007461548,
"grad_norm": 27.3022403717041,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.4715278446674347,
"logits/rejected": -0.4681225121021271,
"logps/chosen": -114.95217895507812,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -214.08563232421875,
"loss": 1.0324,
"margin_dpo/margin_mean": 62.57914733886719,
"margin_dpo/margin_std": 89.07696533203125,
"step": 142
},
{
"KL/chosen_KL_mean": -61.330406188964844,
"KL/mean": -81.28429412841797,
"KL/rejected_KL_mean": -101.2381820678711,
"KL/std": 67.98274230957031,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.009628836065530777,
"fcm_dpo/delta": 0.015870332717895508,
"fcm_dpo/margin": 39.907772064208984,
"fcm_dpo/q_t": 0.4193767309188843,
"grad_norm": 26.948379516601562,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.42380428314208984,
"logits/rejected": -0.4101778268814087,
"logps/chosen": -118.9268569946289,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -180.23776245117188,
"loss": 1.1652,
"margin_dpo/margin_mean": 39.907772064208984,
"margin_dpo/margin_std": 78.14988708496094,
"step": 143
},
{
"KL/chosen_KL_mean": -52.056514739990234,
"KL/mean": -77.15028381347656,
"KL/rejected_KL_mean": -102.24404907226562,
"KL/std": 56.15443420410156,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.009588293731212616,
"fcm_dpo/delta": -0.08328879624605179,
"fcm_dpo/margin": 50.187538146972656,
"fcm_dpo/q_t": 0.39288330078125,
"grad_norm": 22.615461349487305,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.45128071308135986,
"logits/rejected": -0.42970043420791626,
"logps/chosen": -111.96287536621094,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -184.24432373046875,
"loss": 1.0523,
"margin_dpo/margin_mean": 50.187538146972656,
"margin_dpo/margin_std": 65.81391143798828,
"step": 144
},
{
"KL/chosen_KL_mean": -52.327789306640625,
"KL/mean": -79.01912689208984,
"KL/rejected_KL_mean": -105.71046447753906,
"KL/std": 58.76500701904297,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.009492561221122742,
"fcm_dpo/delta": -0.10956543684005737,
"fcm_dpo/margin": 53.38267517089844,
"fcm_dpo/q_t": 0.3878716230392456,
"grad_norm": 26.634552001953125,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.4835849404335022,
"logits/rejected": -0.46149128675460815,
"logps/chosen": -108.92845153808594,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -183.5767822265625,
"loss": 1.0402,
"margin_dpo/margin_mean": 53.38267517089844,
"margin_dpo/margin_std": 69.78041076660156,
"step": 145
},
{
"KL/chosen_KL_mean": -71.99324035644531,
"KL/mean": -92.98985290527344,
"KL/rejected_KL_mean": -113.98645782470703,
"KL/std": 62.72893524169922,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.009455716237425804,
"fcm_dpo/delta": 0.002865489572286606,
"fcm_dpo/margin": 41.99322509765625,
"fcm_dpo/q_t": 0.41135260462760925,
"grad_norm": 31.551198959350586,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.4793580174446106,
"logits/rejected": -0.44883793592453003,
"logps/chosen": -137.99368286132812,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -195.68923950195312,
"loss": 1.1451,
"margin_dpo/margin_mean": 41.99322509765625,
"margin_dpo/margin_std": 74.72547912597656,
"step": 146
},
{
"KL/chosen_KL_mean": -52.66679000854492,
"KL/mean": -81.16831970214844,
"KL/rejected_KL_mean": -109.66986083984375,
"KL/std": 64.48698425292969,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.009364601224660873,
"fcm_dpo/delta": -0.13770800828933716,
"fcm_dpo/margin": 57.00305938720703,
"fcm_dpo/q_t": 0.38518407940864563,
"grad_norm": 22.39113426208496,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.4767155647277832,
"logits/rejected": -0.4504894018173218,
"logps/chosen": -106.07228088378906,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -181.0604705810547,
"loss": 1.042,
"margin_dpo/margin_mean": 57.0030632019043,
"margin_dpo/margin_std": 79.139892578125,
"step": 147
},
{
"KL/chosen_KL_mean": -51.239707946777344,
"KL/mean": -74.8543701171875,
"KL/rejected_KL_mean": -98.46903991699219,
"KL/std": 62.515411376953125,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.009360980242490768,
"fcm_dpo/delta": -0.043911248445510864,
"fcm_dpo/margin": 47.22932434082031,
"fcm_dpo/q_t": 0.40432801842689514,
"grad_norm": 21.143596649169922,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.5241053104400635,
"logits/rejected": -0.5143436789512634,
"logps/chosen": -116.17678833007812,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -201.56288146972656,
"loss": 1.0885,
"margin_dpo/margin_mean": 47.22932434082031,
"margin_dpo/margin_std": 65.89462280273438,
"step": 148
},
{
"KL/chosen_KL_mean": -49.8416633605957,
"KL/mean": -79.41673278808594,
"KL/rejected_KL_mean": -108.9917984008789,
"KL/std": 57.800533294677734,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.009160520508885384,
"fcm_dpo/delta": -0.14721731841564178,
"fcm_dpo/margin": 59.15013122558594,
"fcm_dpo/q_t": 0.3789201080799103,
"grad_norm": 20.35516357421875,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.5416568517684937,
"logits/rejected": -0.5244969129562378,
"logps/chosen": -108.3154296875,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -208.30654907226562,
"loss": 1.0075,
"margin_dpo/margin_mean": 59.150123596191406,
"margin_dpo/margin_std": 67.78451538085938,
"step": 149
},
{
"KL/chosen_KL_mean": -44.17631912231445,
"KL/mean": -72.84591674804688,
"KL/rejected_KL_mean": -101.5155029296875,
"KL/std": 67.9078598022461,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.009045644663274288,
"fcm_dpo/delta": -0.12300632148981094,
"fcm_dpo/margin": 57.33917999267578,
"fcm_dpo/q_t": 0.3912937641143799,
"grad_norm": 20.950302124023438,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.472980797290802,
"logits/rejected": -0.4736313223838806,
"logps/chosen": -89.88212585449219,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -184.86309814453125,
"loss": 1.0437,
"margin_dpo/margin_mean": 57.33917999267578,
"margin_dpo/margin_std": 80.5770034790039,
"step": 150
},
{
"KL/chosen_KL_mean": -54.050907135009766,
"KL/mean": -83.80128479003906,
"KL/rejected_KL_mean": -113.55165100097656,
"KL/std": 63.99048614501953,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.008962772786617279,
"fcm_dpo/delta": -0.13693660497665405,
"fcm_dpo/margin": 59.50074768066406,
"fcm_dpo/q_t": 0.38013043999671936,
"grad_norm": 24.177379608154297,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.5812417268753052,
"logits/rejected": -0.5512596368789673,
"logps/chosen": -124.62174224853516,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -214.01547241210938,
"loss": 1.0088,
"margin_dpo/margin_mean": 59.50074768066406,
"margin_dpo/margin_std": 69.27177429199219,
"step": 151
},
{
"KL/chosen_KL_mean": -47.178958892822266,
"KL/mean": -82.26052856445312,
"KL/rejected_KL_mean": -117.34210968017578,
"KL/std": 66.30995178222656,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.008797157555818558,
"fcm_dpo/delta": -0.22390428185462952,
"fcm_dpo/margin": 70.16314697265625,
"fcm_dpo/q_t": 0.3667450547218323,
"grad_norm": 22.298110961914062,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.5535380840301514,
"logits/rejected": -0.5499355792999268,
"logps/chosen": -107.34334564208984,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -223.48257446289062,
"loss": 0.9731,
"margin_dpo/margin_mean": 70.16314697265625,
"margin_dpo/margin_std": 80.07929992675781,
"step": 152
},
{
"KL/chosen_KL_mean": -48.634864807128906,
"KL/mean": -78.62088012695312,
"KL/rejected_KL_mean": -108.60690307617188,
"KL/std": 71.24372863769531,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.008663169108331203,
"fcm_dpo/delta": -0.12265150249004364,
"fcm_dpo/margin": 59.97203063964844,
"fcm_dpo/q_t": 0.38973453640937805,
"grad_norm": 17.89531898498535,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.5327095985412598,
"logits/rejected": -0.5256816744804382,
"logps/chosen": -104.95014190673828,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -194.26272583007812,
"loss": 1.0512,
"margin_dpo/margin_mean": 59.97203063964844,
"margin_dpo/margin_std": 86.57862854003906,
"step": 153
},
{
"KL/chosen_KL_mean": -57.993751525878906,
"KL/mean": -87.43338775634766,
"KL/rejected_KL_mean": -116.87301635742188,
"KL/std": 72.61052703857422,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.008578930050134659,
"fcm_dpo/delta": -0.10782384127378464,
"fcm_dpo/margin": 58.87928009033203,
"fcm_dpo/q_t": 0.3921113610267639,
"grad_norm": 21.391340255737305,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.4942118525505066,
"logits/rejected": -0.492919921875,
"logps/chosen": -120.73632049560547,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -221.11721801757812,
"loss": 1.0787,
"margin_dpo/margin_mean": 58.87928009033203,
"margin_dpo/margin_std": 90.07198333740234,
"step": 154
},
{
"KL/chosen_KL_mean": -52.702293395996094,
"KL/mean": -82.81259155273438,
"KL/rejected_KL_mean": -112.92288208007812,
"KL/std": 68.29090118408203,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.0084763802587986,
"fcm_dpo/delta": -0.11326827853918076,
"fcm_dpo/margin": 60.2205924987793,
"fcm_dpo/q_t": 0.38809964060783386,
"grad_norm": 21.07434844970703,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.5273807048797607,
"logits/rejected": -0.5057998895645142,
"logps/chosen": -113.35547637939453,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -190.4150848388672,
"loss": 1.0476,
"margin_dpo/margin_mean": 60.22058868408203,
"margin_dpo/margin_std": 82.90054321289062,
"step": 155
},
{
"KL/chosen_KL_mean": -74.92457580566406,
"KL/mean": -92.70379638671875,
"KL/rejected_KL_mean": -110.48301696777344,
"KL/std": 70.93622589111328,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.008492467924952507,
"fcm_dpo/delta": 0.09958405047655106,
"fcm_dpo/margin": 35.558448791503906,
"fcm_dpo/q_t": 0.43309691548347473,
"grad_norm": 32.95204162597656,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.4810073971748352,
"logits/rejected": -0.468288779258728,
"logps/chosen": -144.41644287109375,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -187.65231323242188,
"loss": 1.252,
"margin_dpo/margin_mean": 35.55845260620117,
"margin_dpo/margin_std": 93.71145629882812,
"step": 156
},
{
"KL/chosen_KL_mean": -68.62439727783203,
"KL/mean": -105.9276123046875,
"KL/rejected_KL_mean": -143.2308349609375,
"KL/std": 78.11910247802734,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.008352255448698997,
"fcm_dpo/delta": -0.23134897649288177,
"fcm_dpo/margin": 74.60643768310547,
"fcm_dpo/q_t": 0.3620302677154541,
"grad_norm": 25.482498168945312,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.43237775564193726,
"logits/rejected": -0.4356487989425659,
"logps/chosen": -129.99282836914062,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -250.877197265625,
"loss": 0.9816,
"margin_dpo/margin_mean": 74.6064453125,
"margin_dpo/margin_std": 87.3945541381836,
"step": 157
},
{
"KL/chosen_KL_mean": -68.26705932617188,
"KL/mean": -107.6445083618164,
"KL/rejected_KL_mean": -147.02197265625,
"KL/std": 97.27963256835938,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.008176269009709358,
"fcm_dpo/delta": -0.2519432008266449,
"fcm_dpo/margin": 78.75489807128906,
"fcm_dpo/q_t": 0.37355363368988037,
"grad_norm": 22.437089920043945,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.41110551357269287,
"logits/rejected": -0.4253670275211334,
"logps/chosen": -125.87997436523438,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -260.7166442871094,
"loss": 1.0189,
"margin_dpo/margin_mean": 78.75489807128906,
"margin_dpo/margin_std": 116.05311584472656,
"step": 158
},
{
"KL/chosen_KL_mean": -77.33633422851562,
"KL/mean": -105.33822631835938,
"KL/rejected_KL_mean": -133.34011840820312,
"KL/std": 85.79032897949219,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.008105665445327759,
"fcm_dpo/delta": -0.05528353154659271,
"fcm_dpo/margin": 56.00377655029297,
"fcm_dpo/q_t": 0.40175575017929077,
"grad_norm": 24.443878173828125,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.46176522970199585,
"logits/rejected": -0.42901933193206787,
"logps/chosen": -158.89666748046875,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -222.23883056640625,
"loss": 1.1099,
"margin_dpo/margin_mean": 56.00377655029297,
"margin_dpo/margin_std": 92.10934448242188,
"step": 159
},
{
"KL/chosen_KL_mean": -79.1042251586914,
"KL/mean": -115.31195068359375,
"KL/rejected_KL_mean": -151.51966857910156,
"KL/std": 91.41381072998047,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.007949615828692913,
"fcm_dpo/delta": -0.18148598074913025,
"fcm_dpo/margin": 72.41543579101562,
"fcm_dpo/q_t": 0.38441920280456543,
"grad_norm": 28.788623809814453,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.4013976454734802,
"logits/rejected": -0.38853079080581665,
"logps/chosen": -144.83511352539062,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -248.73748779296875,
"loss": 1.0617,
"margin_dpo/margin_mean": 72.41543579101562,
"margin_dpo/margin_std": 113.82682037353516,
"step": 160
},
{
"KL/chosen_KL_mean": -68.4842529296875,
"KL/mean": -100.51941680908203,
"KL/rejected_KL_mean": -132.55458068847656,
"KL/std": 72.564453125,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.007867861539125443,
"fcm_dpo/delta": -0.10678651928901672,
"fcm_dpo/margin": 64.07034301757812,
"fcm_dpo/q_t": 0.3917103111743927,
"grad_norm": 26.057464599609375,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.41672807931900024,
"logits/rejected": -0.41233956813812256,
"logps/chosen": -120.92071533203125,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -215.98553466796875,
"loss": 1.0604,
"margin_dpo/margin_mean": 64.07034301757812,
"margin_dpo/margin_std": 92.08967590332031,
"step": 161
},
{
"KL/chosen_KL_mean": -64.87035369873047,
"KL/mean": -97.34904479980469,
"KL/rejected_KL_mean": -129.82774353027344,
"KL/std": 81.12202453613281,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.0077486904338002205,
"fcm_dpo/delta": -0.1073184385895729,
"fcm_dpo/margin": 64.9573974609375,
"fcm_dpo/q_t": 0.39209944009780884,
"grad_norm": 23.717235565185547,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.46293187141418457,
"logits/rejected": -0.44232678413391113,
"logps/chosen": -127.48094177246094,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -219.21832275390625,
"loss": 1.0674,
"margin_dpo/margin_mean": 64.9573974609375,
"margin_dpo/margin_std": 94.93345642089844,
"step": 162
},
{
"KL/chosen_KL_mean": -73.33845520019531,
"KL/mean": -104.88301086425781,
"KL/rejected_KL_mean": -136.42755126953125,
"KL/std": 80.35075378417969,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.007712847553193569,
"fcm_dpo/delta": -0.0887412428855896,
"fcm_dpo/margin": 63.08909606933594,
"fcm_dpo/q_t": 0.3963692784309387,
"grad_norm": 25.23561668395996,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.3967311382293701,
"logits/rejected": -0.37458351254463196,
"logps/chosen": -128.36007690429688,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -211.84576416015625,
"loss": 1.0769,
"margin_dpo/margin_mean": 63.08909606933594,
"margin_dpo/margin_std": 95.95596313476562,
"step": 163
},
{
"KL/chosen_KL_mean": -66.4931411743164,
"KL/mean": -104.46180725097656,
"KL/rejected_KL_mean": -142.43045043945312,
"KL/std": 79.09434509277344,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.007562342099845409,
"fcm_dpo/delta": -0.18068404495716095,
"fcm_dpo/margin": 75.93731689453125,
"fcm_dpo/q_t": 0.3738352656364441,
"grad_norm": 23.940841674804688,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.3866020441055298,
"logits/rejected": -0.3667601943016052,
"logps/chosen": -122.1338119506836,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -222.0950927734375,
"loss": 0.9952,
"margin_dpo/margin_mean": 75.93731689453125,
"margin_dpo/margin_std": 90.33500671386719,
"step": 164
},
{
"KL/chosen_KL_mean": -70.58739471435547,
"KL/mean": -95.22163391113281,
"KL/rejected_KL_mean": -119.85586547851562,
"KL/std": 65.43978881835938,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.007562550250440836,
"fcm_dpo/delta": 0.027895405888557434,
"fcm_dpo/margin": 49.26846694946289,
"fcm_dpo/q_t": 0.41699641942977905,
"grad_norm": 28.192373275756836,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.4508388042449951,
"logits/rejected": -0.41956329345703125,
"logps/chosen": -131.89808654785156,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -193.52647399902344,
"loss": 1.1451,
"margin_dpo/margin_mean": 49.26846694946289,
"margin_dpo/margin_std": 85.33627319335938,
"step": 165
},
{
"KL/chosen_KL_mean": -62.29485321044922,
"KL/mean": -107.13589477539062,
"KL/rejected_KL_mean": -151.9769287109375,
"KL/std": 95.33642578125,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.007406267803162336,
"fcm_dpo/delta": -0.27418437600135803,
"fcm_dpo/margin": 89.68207550048828,
"fcm_dpo/q_t": 0.3678908348083496,
"grad_norm": 19.755449295043945,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.38571715354919434,
"logits/rejected": -0.395542174577713,
"logps/chosen": -113.27845764160156,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -250.07205200195312,
"loss": 0.9781,
"margin_dpo/margin_mean": 89.68207550048828,
"margin_dpo/margin_std": 119.04489135742188,
"step": 166
},
{
"KL/chosen_KL_mean": -62.64759826660156,
"KL/mean": -109.76869201660156,
"KL/rejected_KL_mean": -156.8897705078125,
"KL/std": 87.04228210449219,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.007213749922811985,
"fcm_dpo/delta": -0.2897433936595917,
"fcm_dpo/margin": 94.24217987060547,
"fcm_dpo/q_t": 0.3522365093231201,
"grad_norm": 21.15924072265625,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.3807927370071411,
"logits/rejected": -0.3821834325790405,
"logps/chosen": -113.07169342041016,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -252.92019653320312,
"loss": 0.9275,
"margin_dpo/margin_mean": 94.24217224121094,
"margin_dpo/margin_std": 99.01016235351562,
"step": 167
},
{
"KL/chosen_KL_mean": -68.97935485839844,
"KL/mean": -104.1439208984375,
"KL/rejected_KL_mean": -139.30848693847656,
"KL/std": 81.96823120117188,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.007100098766386509,
"fcm_dpo/delta": -0.10204099118709564,
"fcm_dpo/margin": 70.32914733886719,
"fcm_dpo/q_t": 0.3902486562728882,
"grad_norm": 21.94905662536621,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.405794233083725,
"logits/rejected": -0.39295631647109985,
"logps/chosen": -118.44217681884766,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -214.61705017089844,
"loss": 1.0391,
"margin_dpo/margin_mean": 70.32914733886719,
"margin_dpo/margin_std": 90.95465850830078,
"step": 168
},
{
"KL/chosen_KL_mean": -69.83270263671875,
"KL/mean": -100.67288208007812,
"KL/rejected_KL_mean": -131.5130615234375,
"KL/std": 81.13442993164062,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.007038552314043045,
"fcm_dpo/delta": -0.03588036447763443,
"fcm_dpo/margin": 61.68037414550781,
"fcm_dpo/q_t": 0.4094877243041992,
"grad_norm": 21.894031524658203,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.42251360416412354,
"logits/rejected": -0.403925359249115,
"logps/chosen": -129.63613891601562,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -214.85882568359375,
"loss": 1.1309,
"margin_dpo/margin_mean": 61.68037414550781,
"margin_dpo/margin_std": 110.4356460571289,
"step": 169
},
{
"KL/chosen_KL_mean": -62.970611572265625,
"KL/mean": -99.88447570800781,
"KL/rejected_KL_mean": -136.79833984375,
"KL/std": 76.65866088867188,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.0069792973808944225,
"fcm_dpo/delta": -0.11909201741218567,
"fcm_dpo/margin": 73.82772827148438,
"fcm_dpo/q_t": 0.3844184875488281,
"grad_norm": 18.95234489440918,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.3876168131828308,
"logits/rejected": -0.37476158142089844,
"logps/chosen": -112.4423828125,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -212.71568298339844,
"loss": 1.0306,
"margin_dpo/margin_mean": 73.82772827148438,
"margin_dpo/margin_std": 91.29777526855469,
"step": 170
},
{
"KL/chosen_KL_mean": -92.04759979248047,
"KL/mean": -121.734619140625,
"KL/rejected_KL_mean": -151.42166137695312,
"KL/std": 87.79399108886719,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.006971217226237059,
"fcm_dpo/delta": -0.01420888677239418,
"fcm_dpo/margin": 59.374053955078125,
"fcm_dpo/q_t": 0.41368788480758667,
"grad_norm": 30.997112274169922,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.3963302969932556,
"logits/rejected": -0.3728923797607422,
"logps/chosen": -176.54690551757812,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -260.80377197265625,
"loss": 1.1435,
"margin_dpo/margin_mean": 59.374053955078125,
"margin_dpo/margin_std": 111.98719787597656,
"step": 171
},
{
"KL/chosen_KL_mean": -83.34523010253906,
"KL/mean": -114.48237609863281,
"KL/rejected_KL_mean": -145.6195068359375,
"KL/std": 86.62696838378906,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.00694639515131712,
"fcm_dpo/delta": -0.03337009996175766,
"fcm_dpo/margin": 62.2743034362793,
"fcm_dpo/q_t": 0.4046783447265625,
"grad_norm": 21.863208770751953,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.44137507677078247,
"logits/rejected": -0.4202975630760193,
"logps/chosen": -151.9991455078125,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -231.05618286132812,
"loss": 1.1028,
"margin_dpo/margin_mean": 62.2743034362793,
"margin_dpo/margin_std": 97.67938232421875,
"step": 172
},
{
"KL/chosen_KL_mean": -75.67850494384766,
"KL/mean": -107.38485717773438,
"KL/rejected_KL_mean": -139.09121704101562,
"KL/std": 80.97750854492188,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.006938562728464603,
"fcm_dpo/delta": -0.04095254838466644,
"fcm_dpo/margin": 63.4127082824707,
"fcm_dpo/q_t": 0.40140169858932495,
"grad_norm": 22.291126251220703,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.4557979702949524,
"logits/rejected": -0.4270375072956085,
"logps/chosen": -138.72938537597656,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -217.775146484375,
"loss": 1.0729,
"margin_dpo/margin_mean": 63.41270446777344,
"margin_dpo/margin_std": 85.16667175292969,
"step": 173
},
{
"KL/chosen_KL_mean": -73.96533203125,
"KL/mean": -110.90480041503906,
"KL/rejected_KL_mean": -147.84425354003906,
"KL/std": 88.3280258178711,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.006883748807013035,
"fcm_dpo/delta": -0.1114603653550148,
"fcm_dpo/margin": 73.87892150878906,
"fcm_dpo/q_t": 0.3911077380180359,
"grad_norm": 23.334630966186523,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.3489905595779419,
"logits/rejected": -0.3669482469558716,
"logps/chosen": -127.32830810546875,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -249.75546264648438,
"loss": 1.0572,
"margin_dpo/margin_mean": 73.87892150878906,
"margin_dpo/margin_std": 104.1011962890625,
"step": 174
},
{
"KL/chosen_KL_mean": -69.70106506347656,
"KL/mean": -120.88041687011719,
"KL/rejected_KL_mean": -172.05975341796875,
"KL/std": 95.63670349121094,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.00668959878385067,
"fcm_dpo/delta": -0.29501020908355713,
"fcm_dpo/margin": 102.35872650146484,
"fcm_dpo/q_t": 0.34925341606140137,
"grad_norm": 25.57117462158203,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.3187200427055359,
"logits/rejected": -0.30461329221725464,
"logps/chosen": -115.11882019042969,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -261.5655517578125,
"loss": 0.9085,
"margin_dpo/margin_mean": 102.35871887207031,
"margin_dpo/margin_std": 99.24336242675781,
"step": 175
},
{
"KL/chosen_KL_mean": -75.16090393066406,
"KL/mean": -117.7568359375,
"KL/rejected_KL_mean": -160.35279846191406,
"KL/std": 92.66831970214844,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.0065691666677594185,
"fcm_dpo/delta": -0.16410863399505615,
"fcm_dpo/margin": 85.19189453125,
"fcm_dpo/q_t": 0.380332350730896,
"grad_norm": 22.5208683013916,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.3513135313987732,
"logits/rejected": -0.369649738073349,
"logps/chosen": -125.61373901367188,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -255.91177368164062,
"loss": 1.0084,
"margin_dpo/margin_mean": 85.19189453125,
"margin_dpo/margin_std": 107.12543487548828,
"step": 176
},
{
"KL/chosen_KL_mean": -87.45088195800781,
"KL/mean": -129.8768310546875,
"KL/rejected_KL_mean": -172.30276489257812,
"KL/std": 101.22750854492188,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.006470865570008755,
"fcm_dpo/delta": -0.15313297510147095,
"fcm_dpo/margin": 84.85188293457031,
"fcm_dpo/q_t": 0.3814983367919922,
"grad_norm": 33.10342025756836,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.38505709171295166,
"logits/rejected": -0.37769830226898193,
"logps/chosen": -148.66734313964844,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -268.196533203125,
"loss": 1.0131,
"margin_dpo/margin_mean": 84.85188293457031,
"margin_dpo/margin_std": 107.24163818359375,
"step": 177
},
{
"KL/chosen_KL_mean": -96.93707275390625,
"KL/mean": -150.7945098876953,
"KL/rejected_KL_mean": -204.65194702148438,
"KL/std": 121.38895416259766,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.006285130977630615,
"fcm_dpo/delta": -0.28755688667297363,
"fcm_dpo/margin": 107.7148666381836,
"fcm_dpo/q_t": 0.36004963517189026,
"grad_norm": 27.839982986450195,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.27259236574172974,
"logits/rejected": -0.26497703790664673,
"logps/chosen": -155.20187377929688,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -310.01727294921875,
"loss": 0.9589,
"margin_dpo/margin_mean": 107.7148666381836,
"margin_dpo/margin_std": 132.7176971435547,
"step": 178
},
{
"KL/chosen_KL_mean": -97.64852905273438,
"KL/mean": -136.50204467773438,
"KL/rejected_KL_mean": -175.3555908203125,
"KL/std": 101.76957702636719,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.006197200156748295,
"fcm_dpo/delta": -0.08410830795764923,
"fcm_dpo/margin": 77.70704650878906,
"fcm_dpo/q_t": 0.39651763439178467,
"grad_norm": 37.91785430908203,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.29223281145095825,
"logits/rejected": -0.29845482110977173,
"logps/chosen": -158.70684814453125,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -265.8834228515625,
"loss": 1.0988,
"margin_dpo/margin_mean": 77.70704650878906,
"margin_dpo/margin_std": 125.48446655273438,
"step": 179
},
{
"KL/chosen_KL_mean": -83.23408508300781,
"KL/mean": -131.74658203125,
"KL/rejected_KL_mean": -180.2590789794922,
"KL/std": 91.78398132324219,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.006104484666138887,
"fcm_dpo/delta": -0.19836077094078064,
"fcm_dpo/margin": 97.02499389648438,
"fcm_dpo/q_t": 0.3690943717956543,
"grad_norm": 21.05206871032715,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.2990095019340515,
"logits/rejected": -0.29073792695999146,
"logps/chosen": -137.57681274414062,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -278.4709167480469,
"loss": 0.9838,
"margin_dpo/margin_mean": 97.02499389648438,
"margin_dpo/margin_std": 113.17401123046875,
"step": 180
},
{
"KL/chosen_KL_mean": -71.93513488769531,
"KL/mean": -102.62535095214844,
"KL/rejected_KL_mean": -133.31556701660156,
"KL/std": 82.91169738769531,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.006095272954553366,
"fcm_dpo/delta": 0.02632547914981842,
"fcm_dpo/margin": 61.38043212890625,
"fcm_dpo/q_t": 0.41590094566345215,
"grad_norm": 20.209400177001953,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.37073230743408203,
"logits/rejected": -0.34592998027801514,
"logps/chosen": -126.93559265136719,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -194.97174072265625,
"loss": 1.1374,
"margin_dpo/margin_mean": 61.38043212890625,
"margin_dpo/margin_std": 102.93232727050781,
"step": 181
},
{
"KL/chosen_KL_mean": -68.23200225830078,
"KL/mean": -119.69134521484375,
"KL/rejected_KL_mean": -171.15066528320312,
"KL/std": 100.23439025878906,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.005980286747217178,
"fcm_dpo/delta": -0.2230871319770813,
"fcm_dpo/margin": 102.91866302490234,
"fcm_dpo/q_t": 0.3642627000808716,
"grad_norm": 20.144750595092773,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.31923383474349976,
"logits/rejected": -0.32709991931915283,
"logps/chosen": -109.33985900878906,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -260.6722412109375,
"loss": 0.9789,
"margin_dpo/margin_mean": 102.91866302490234,
"margin_dpo/margin_std": 120.07926940917969,
"step": 182
},
{
"KL/chosen_KL_mean": -99.28335571289062,
"KL/mean": -128.1656036376953,
"KL/rejected_KL_mean": -157.0478515625,
"KL/std": 81.12559509277344,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.00591336889192462,
"fcm_dpo/delta": -0.07965878397226334,
"fcm_dpo/margin": 57.76450729370117,
"fcm_dpo/q_t": 0.42185482382774353,
"grad_norm": 21.069660186767578,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.30978119373321533,
"logits/rejected": -0.29306042194366455,
"logps/chosen": -156.80792236328125,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -233.02359008789062,
"loss": 1.1494,
"margin_dpo/margin_mean": 57.764503479003906,
"margin_dpo/margin_std": 89.65037536621094,
"step": 183
},
{
"KL/chosen_KL_mean": -80.81704711914062,
"KL/mean": -110.47319030761719,
"KL/rejected_KL_mean": -140.12933349609375,
"KL/std": 76.08272552490234,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.005951171740889549,
"fcm_dpo/delta": 0.04741087928414345,
"fcm_dpo/margin": 59.312294006347656,
"fcm_dpo/q_t": 0.4209594130516052,
"grad_norm": 21.374256134033203,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.41061311960220337,
"logits/rejected": -0.4004828929901123,
"logps/chosen": -139.36199951171875,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -216.76339721679688,
"loss": 1.1399,
"margin_dpo/margin_mean": 59.312294006347656,
"margin_dpo/margin_std": 94.55437469482422,
"step": 184
},
{
"KL/chosen_KL_mean": -88.34465026855469,
"KL/mean": -112.53829193115234,
"KL/rejected_KL_mean": -136.73193359375,
"KL/std": 88.54534912109375,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.005989473778754473,
"fcm_dpo/delta": 0.1118379533290863,
"fcm_dpo/margin": 48.38727951049805,
"fcm_dpo/q_t": 0.4377959966659546,
"grad_norm": 20.70436668395996,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.3213365077972412,
"logits/rejected": -0.2884293794631958,
"logps/chosen": -150.37049865722656,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -210.49453735351562,
"loss": 1.2074,
"margin_dpo/margin_mean": 48.38727569580078,
"margin_dpo/margin_std": 105.9352798461914,
"step": 185
},
{
"KL/chosen_KL_mean": -81.42449188232422,
"KL/mean": -126.01622009277344,
"KL/rejected_KL_mean": -170.60794067382812,
"KL/std": 91.03404235839844,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.005945336539298296,
"fcm_dpo/delta": -0.13385680317878723,
"fcm_dpo/margin": 89.18345642089844,
"fcm_dpo/q_t": 0.3814866244792938,
"grad_norm": 34.85935974121094,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.41816607117652893,
"logits/rejected": -0.3946566581726074,
"logps/chosen": -150.7779541015625,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -258.6803894042969,
"loss": 1.0069,
"margin_dpo/margin_mean": 89.18345642089844,
"margin_dpo/margin_std": 102.38252258300781,
"step": 186
},
{
"KL/chosen_KL_mean": -77.34542846679688,
"KL/mean": -114.29693603515625,
"KL/rejected_KL_mean": -151.24844360351562,
"KL/std": 86.13810729980469,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.005925598554313183,
"fcm_dpo/delta": -0.038912609219551086,
"fcm_dpo/margin": 73.90303039550781,
"fcm_dpo/q_t": 0.4008300006389618,
"grad_norm": 29.05039405822754,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.3810155987739563,
"logits/rejected": -0.37561988830566406,
"logps/chosen": -130.1018829345703,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -233.21754455566406,
"loss": 1.0696,
"margin_dpo/margin_mean": 73.90303039550781,
"margin_dpo/margin_std": 94.80323791503906,
"step": 187
},
{
"KL/chosen_KL_mean": -73.07807922363281,
"KL/mean": -118.4848403930664,
"KL/rejected_KL_mean": -163.8916015625,
"KL/std": 96.97216796875,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.005849046632647514,
"fcm_dpo/delta": -0.13469749689102173,
"fcm_dpo/margin": 90.81352233886719,
"fcm_dpo/q_t": 0.3821578025817871,
"grad_norm": 35.57075881958008,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.35656416416168213,
"logits/rejected": -0.36152058839797974,
"logps/chosen": -122.49356842041016,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -253.43203735351562,
"loss": 1.0084,
"margin_dpo/margin_mean": 90.81352233886719,
"margin_dpo/margin_std": 105.5975341796875,
"step": 188
},
{
"KL/chosen_KL_mean": -88.35533142089844,
"KL/mean": -125.64856719970703,
"KL/rejected_KL_mean": -162.94180297851562,
"KL/std": 98.73692321777344,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.005813870579004288,
"fcm_dpo/delta": -0.0343741774559021,
"fcm_dpo/margin": 74.58646392822266,
"fcm_dpo/q_t": 0.4062124490737915,
"grad_norm": 29.83676528930664,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.3425024151802063,
"logits/rejected": -0.32220566272735596,
"logps/chosen": -140.75428771972656,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -235.10916137695312,
"loss": 1.1033,
"margin_dpo/margin_mean": 74.58646392822266,
"margin_dpo/margin_std": 120.13783264160156,
"step": 189
},
{
"KL/chosen_KL_mean": -93.82635498046875,
"KL/mean": -138.6332550048828,
"KL/rejected_KL_mean": -183.44015502929688,
"KL/std": 105.86849975585938,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.0057814717292785645,
"fcm_dpo/delta": -0.12157705426216125,
"fcm_dpo/margin": 89.61378479003906,
"fcm_dpo/q_t": 0.3877827227115631,
"grad_norm": 21.399682998657227,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.3890034854412079,
"logits/rejected": -0.37827688455581665,
"logps/chosen": -158.50942993164062,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -285.99066162109375,
"loss": 1.0533,
"margin_dpo/margin_mean": 89.61378479003906,
"margin_dpo/margin_std": 122.82894897460938,
"step": 190
},
{
"KL/chosen_KL_mean": -86.75233459472656,
"KL/mean": -150.7803955078125,
"KL/rejected_KL_mean": -214.80844116210938,
"KL/std": 124.46101379394531,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.005586773157119751,
"fcm_dpo/delta": -0.32792675495147705,
"fcm_dpo/margin": 128.05612182617188,
"fcm_dpo/q_t": 0.34851568937301636,
"grad_norm": 24.331701278686523,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.3571997880935669,
"logits/rejected": -0.3361099660396576,
"logps/chosen": -155.4112091064453,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -324.9481201171875,
"loss": 0.912,
"margin_dpo/margin_mean": 128.05612182617188,
"margin_dpo/margin_std": 137.22451782226562,
"step": 191
},
{
"KL/chosen_KL_mean": -114.01398468017578,
"KL/mean": -156.28001403808594,
"KL/rejected_KL_mean": -198.54605102539062,
"KL/std": 106.98095703125,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.005530376918613911,
"fcm_dpo/delta": -0.06909237802028656,
"fcm_dpo/margin": 84.53204345703125,
"fcm_dpo/q_t": 0.39953508973121643,
"grad_norm": 28.160974502563477,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.31193774938583374,
"logits/rejected": -0.31198960542678833,
"logps/chosen": -183.74090576171875,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -301.86737060546875,
"loss": 1.089,
"margin_dpo/margin_mean": 84.53204345703125,
"margin_dpo/margin_std": 131.72988891601562,
"step": 192
},
{
"KL/chosen_KL_mean": -114.42250061035156,
"KL/mean": -138.80294799804688,
"KL/rejected_KL_mean": -163.18338012695312,
"KL/std": 97.2500228881836,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.005518913269042969,
"fcm_dpo/delta": 0.021666744723916054,
"fcm_dpo/margin": 48.76087951660156,
"fcm_dpo/q_t": 0.43647685647010803,
"grad_norm": 31.76787567138672,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.3469330072402954,
"logits/rejected": -0.33776986598968506,
"logps/chosen": -174.61300659179688,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -239.5909423828125,
"loss": 1.2443,
"margin_dpo/margin_mean": 48.76087951660156,
"margin_dpo/margin_std": 123.41360473632812,
"step": 193
},
{
"KL/chosen_KL_mean": -71.24337768554688,
"KL/mean": -112.21340942382812,
"KL/rejected_KL_mean": -153.18345642089844,
"KL/std": 83.32481384277344,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.0054954723455011845,
"fcm_dpo/delta": -0.051623422652482986,
"fcm_dpo/margin": 81.94007873535156,
"fcm_dpo/q_t": 0.395707905292511,
"grad_norm": 20.26607894897461,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.2368777096271515,
"logits/rejected": -0.21953874826431274,
"logps/chosen": -109.08374786376953,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -213.8682403564453,
"loss": 1.0437,
"margin_dpo/margin_mean": 81.94007873535156,
"margin_dpo/margin_std": 90.14231872558594,
"step": 194
},
{
"KL/chosen_KL_mean": -112.94318389892578,
"KL/mean": -156.35760498046875,
"KL/rejected_KL_mean": -199.77203369140625,
"KL/std": 102.81307983398438,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.0054633598774671555,
"fcm_dpo/delta": -0.07621531188488007,
"fcm_dpo/margin": 86.82884216308594,
"fcm_dpo/q_t": 0.3926421105861664,
"grad_norm": 26.608348846435547,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.27807819843292236,
"logits/rejected": -0.29660797119140625,
"logps/chosen": -167.83474731445312,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -296.54296875,
"loss": 1.0427,
"margin_dpo/margin_mean": 86.82884216308594,
"margin_dpo/margin_std": 105.82039642333984,
"step": 195
},
{
"KL/chosen_KL_mean": -87.1651382446289,
"KL/mean": -137.2007598876953,
"KL/rejected_KL_mean": -187.23635864257812,
"KL/std": 106.05245208740234,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.005381887778639793,
"fcm_dpo/delta": -0.14310941100120544,
"fcm_dpo/margin": 100.07121276855469,
"fcm_dpo/q_t": 0.3824601173400879,
"grad_norm": 20.205522537231445,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.3048306703567505,
"logits/rejected": -0.2887340188026428,
"logps/chosen": -140.41038513183594,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -263.289306640625,
"loss": 1.0251,
"margin_dpo/margin_mean": 100.07121276855469,
"margin_dpo/margin_std": 125.76506042480469,
"step": 196
},
{
"KL/chosen_KL_mean": -87.905029296875,
"KL/mean": -126.29278564453125,
"KL/rejected_KL_mean": -164.6805419921875,
"KL/std": 89.6479263305664,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.005379960872232914,
"fcm_dpo/delta": -0.01339656300842762,
"fcm_dpo/margin": 76.7755126953125,
"fcm_dpo/q_t": 0.40633296966552734,
"grad_norm": 25.854190826416016,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.2854430675506592,
"logits/rejected": -0.2793597877025604,
"logps/chosen": -148.3253631591797,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -241.8894500732422,
"loss": 1.0832,
"margin_dpo/margin_mean": 76.7755126953125,
"margin_dpo/margin_std": 100.7091064453125,
"step": 197
},
{
"KL/chosen_KL_mean": -97.10411071777344,
"KL/mean": -145.3472900390625,
"KL/rejected_KL_mean": -193.59048461914062,
"KL/std": 110.99395751953125,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.005326003301888704,
"fcm_dpo/delta": -0.11684248596429825,
"fcm_dpo/margin": 96.48635864257812,
"fcm_dpo/q_t": 0.38827261328697205,
"grad_norm": 22.704126358032227,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.32224997878074646,
"logits/rejected": -0.32293713092803955,
"logps/chosen": -152.14028930664062,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -290.833740234375,
"loss": 1.0307,
"margin_dpo/margin_mean": 96.48635864257812,
"margin_dpo/margin_std": 124.41645050048828,
"step": 198
},
{
"KL/chosen_KL_mean": -93.67039489746094,
"KL/mean": -140.52859497070312,
"KL/rejected_KL_mean": -187.3867950439453,
"KL/std": 99.46040344238281,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.0052545820362865925,
"fcm_dpo/delta": -0.09539773315191269,
"fcm_dpo/margin": 93.71640014648438,
"fcm_dpo/q_t": 0.38956761360168457,
"grad_norm": 34.478763580322266,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.3307437300682068,
"logits/rejected": -0.31390073895454407,
"logps/chosen": -150.4992218017578,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -272.0350036621094,
"loss": 1.0441,
"margin_dpo/margin_mean": 93.71640014648438,
"margin_dpo/margin_std": 118.5833969116211,
"step": 199
},
{
"KL/chosen_KL_mean": -91.72415161132812,
"KL/mean": -139.52906799316406,
"KL/rejected_KL_mean": -187.333984375,
"KL/std": 108.85971069335938,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.0052094850689172745,
"fcm_dpo/delta": -0.10084635764360428,
"fcm_dpo/margin": 95.60984802246094,
"fcm_dpo/q_t": 0.3912206292152405,
"grad_norm": 20.905105590820312,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.29428964853286743,
"logits/rejected": -0.2763988971710205,
"logps/chosen": -144.79119873046875,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -267.9424133300781,
"loss": 1.0584,
"margin_dpo/margin_mean": 95.60984802246094,
"margin_dpo/margin_std": 135.0447998046875,
"step": 200
},
{
"KL/chosen_KL_mean": -94.4271469116211,
"KL/mean": -142.72763061523438,
"KL/rejected_KL_mean": -191.028076171875,
"KL/std": 113.81238555908203,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.005169622600078583,
"fcm_dpo/delta": -0.1018773764371872,
"fcm_dpo/margin": 96.60094451904297,
"fcm_dpo/q_t": 0.389159232378006,
"grad_norm": 26.09699249267578,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.34645718336105347,
"logits/rejected": -0.34164953231811523,
"logps/chosen": -169.82937622070312,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -305.8363037109375,
"loss": 1.0446,
"margin_dpo/margin_mean": 96.60094451904297,
"margin_dpo/margin_std": 127.17774963378906,
"step": 201
},
{
"KL/chosen_KL_mean": -98.94967651367188,
"KL/mean": -132.004150390625,
"KL/rejected_KL_mean": -165.05862426757812,
"KL/std": 95.95150756835938,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.0051773954182863235,
"fcm_dpo/delta": 0.05868521332740784,
"fcm_dpo/margin": 66.10896301269531,
"fcm_dpo/q_t": 0.42411088943481445,
"grad_norm": 20.637813568115234,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.30686646699905396,
"logits/rejected": -0.31850093603134155,
"logps/chosen": -149.05099487304688,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -252.04367065429688,
"loss": 1.1671,
"margin_dpo/margin_mean": 66.10896301269531,
"margin_dpo/margin_std": 123.44126892089844,
"step": 202
},
{
"KL/chosen_KL_mean": -98.62371826171875,
"KL/mean": -140.30938720703125,
"KL/rejected_KL_mean": -181.99505615234375,
"KL/std": 101.09996795654297,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.005167345050722361,
"fcm_dpo/delta": -0.03151167184114456,
"fcm_dpo/margin": 83.371337890625,
"fcm_dpo/q_t": 0.4019864797592163,
"grad_norm": 25.103071212768555,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.31344640254974365,
"logits/rejected": -0.30354416370391846,
"logps/chosen": -159.23341369628906,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -267.8910217285156,
"loss": 1.0711,
"margin_dpo/margin_mean": 83.371337890625,
"margin_dpo/margin_std": 107.46673583984375,
"step": 203
},
{
"KL/chosen_KL_mean": -110.24484252929688,
"KL/mean": -148.1464385986328,
"KL/rejected_KL_mean": -186.04803466796875,
"KL/std": 108.95531463623047,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.005168107338249683,
"fcm_dpo/delta": 0.008405912667512894,
"fcm_dpo/margin": 75.80317687988281,
"fcm_dpo/q_t": 0.41412895917892456,
"grad_norm": 25.396324157714844,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.40366190671920776,
"logits/rejected": -0.3658892512321472,
"logps/chosen": -190.40980529785156,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -273.74395751953125,
"loss": 1.1248,
"margin_dpo/margin_mean": 75.80317687988281,
"margin_dpo/margin_std": 123.09947204589844,
"step": 204
},
{
"KL/chosen_KL_mean": -108.8935775756836,
"KL/mean": -159.8162841796875,
"KL/rejected_KL_mean": -210.739013671875,
"KL/std": 114.94783020019531,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.005101551301777363,
"fcm_dpo/delta": -0.12394419312477112,
"fcm_dpo/margin": 101.84541320800781,
"fcm_dpo/q_t": 0.38280677795410156,
"grad_norm": 25.676708221435547,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.2786010801792145,
"logits/rejected": -0.2449745088815689,
"logps/chosen": -168.2783203125,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -295.86407470703125,
"loss": 1.0303,
"margin_dpo/margin_mean": 101.84541320800781,
"margin_dpo/margin_std": 124.0643310546875,
"step": 205
},
{
"KL/chosen_KL_mean": -103.83800506591797,
"KL/mean": -158.62493896484375,
"KL/rejected_KL_mean": -213.41188049316406,
"KL/std": 111.25010681152344,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.005038215778768063,
"fcm_dpo/delta": -0.15676981210708618,
"fcm_dpo/margin": 109.57386779785156,
"fcm_dpo/q_t": 0.374674528837204,
"grad_norm": 31.18449592590332,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.2506366968154907,
"logits/rejected": -0.25460052490234375,
"logps/chosen": -150.80250549316406,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -312.3653564453125,
"loss": 0.9924,
"margin_dpo/margin_mean": 109.57386016845703,
"margin_dpo/margin_std": 120.0478515625,
"step": 206
},
{
"KL/chosen_KL_mean": -93.25586700439453,
"KL/mean": -158.32984924316406,
"KL/rejected_KL_mean": -223.40382385253906,
"KL/std": 127.83676147460938,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.004938926547765732,
"fcm_dpo/delta": -0.25064384937286377,
"fcm_dpo/margin": 130.14794921875,
"fcm_dpo/q_t": 0.36273083090782166,
"grad_norm": 25.51757049560547,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.3203808069229126,
"logits/rejected": -0.29214242100715637,
"logps/chosen": -149.3121337890625,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -307.85162353515625,
"loss": 0.955,
"margin_dpo/margin_mean": 130.14796447753906,
"margin_dpo/margin_std": 148.46749877929688,
"step": 207
},
{
"KL/chosen_KL_mean": -143.90771484375,
"KL/mean": -192.93313598632812,
"KL/rejected_KL_mean": -241.9585418701172,
"KL/std": 121.01240539550781,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.004868675023317337,
"fcm_dpo/delta": -0.07939005643129349,
"fcm_dpo/margin": 98.05082702636719,
"fcm_dpo/q_t": 0.39406818151474,
"grad_norm": 27.979568481445312,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.35994622111320496,
"logits/rejected": -0.3440352976322174,
"logps/chosen": -210.975341796875,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -336.24542236328125,
"loss": 1.08,
"margin_dpo/margin_mean": 98.05082702636719,
"margin_dpo/margin_std": 146.68267822265625,
"step": 208
},
{
"KL/chosen_KL_mean": -118.966064453125,
"KL/mean": -162.17567443847656,
"KL/rejected_KL_mean": -205.38528442382812,
"KL/std": 105.17567443847656,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.004856686107814312,
"fcm_dpo/delta": -0.02013271115720272,
"fcm_dpo/margin": 86.41921997070312,
"fcm_dpo/q_t": 0.40564462542533875,
"grad_norm": 29.448095321655273,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.24659614264965057,
"logits/rejected": -0.23566904664039612,
"logps/chosen": -175.14776611328125,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -286.3267822265625,
"loss": 1.1089,
"margin_dpo/margin_mean": 86.41921997070312,
"margin_dpo/margin_std": 132.27151489257812,
"step": 209
},
{
"KL/chosen_KL_mean": -103.5418472290039,
"KL/mean": -154.00094604492188,
"KL/rejected_KL_mean": -204.46005249023438,
"KL/std": 109.31126403808594,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.004831655882298946,
"fcm_dpo/delta": -0.0897798091173172,
"fcm_dpo/margin": 100.91820526123047,
"fcm_dpo/q_t": 0.3923318386077881,
"grad_norm": 25.811580657958984,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.2856072783470154,
"logits/rejected": -0.2767780125141144,
"logps/chosen": -149.91366577148438,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -281.14166259765625,
"loss": 1.0508,
"margin_dpo/margin_mean": 100.91820526123047,
"margin_dpo/margin_std": 132.74383544921875,
"step": 210
},
{
"KL/chosen_KL_mean": -145.32662963867188,
"KL/mean": -188.92605590820312,
"KL/rejected_KL_mean": -232.5254669189453,
"KL/std": 120.26460266113281,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.004806230776011944,
"fcm_dpo/delta": -0.019507823511958122,
"fcm_dpo/margin": 87.1988296508789,
"fcm_dpo/q_t": 0.40984851121902466,
"grad_norm": 27.256078720092773,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.35860946774482727,
"logits/rejected": -0.31967782974243164,
"logps/chosen": -224.25897216796875,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -319.346435546875,
"loss": 1.1129,
"margin_dpo/margin_mean": 87.19883728027344,
"margin_dpo/margin_std": 139.08685302734375,
"step": 211
},
{
"KL/chosen_KL_mean": -117.86914825439453,
"KL/mean": -179.65806579589844,
"KL/rejected_KL_mean": -241.44700622558594,
"KL/std": 132.9044952392578,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.004715276416391134,
"fcm_dpo/delta": -0.18955346941947937,
"fcm_dpo/margin": 123.57785034179688,
"fcm_dpo/q_t": 0.372633695602417,
"grad_norm": 25.3454647064209,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.34758710861206055,
"logits/rejected": -0.33722245693206787,
"logps/chosen": -176.066162109375,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -344.5048522949219,
"loss": 1.0062,
"margin_dpo/margin_mean": 123.57785034179688,
"margin_dpo/margin_std": 148.7064208984375,
"step": 212
},
{
"KL/chosen_KL_mean": -108.09127807617188,
"KL/mean": -165.18792724609375,
"KL/rejected_KL_mean": -222.28460693359375,
"KL/std": 112.09000396728516,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.004654336720705032,
"fcm_dpo/delta": -0.13551779091358185,
"fcm_dpo/margin": 114.19332885742188,
"fcm_dpo/q_t": 0.38082438707351685,
"grad_norm": 22.01609992980957,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.3316152095794678,
"logits/rejected": -0.3008173108100891,
"logps/chosen": -175.60398864746094,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -316.1993408203125,
"loss": 1.0141,
"margin_dpo/margin_mean": 114.19332885742188,
"margin_dpo/margin_std": 134.8514404296875,
"step": 213
},
{
"KL/chosen_KL_mean": -97.71200561523438,
"KL/mean": -148.03985595703125,
"KL/rejected_KL_mean": -198.36767578125,
"KL/std": 111.36173248291016,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.004625334870070219,
"fcm_dpo/delta": -0.0671173483133316,
"fcm_dpo/margin": 100.65567016601562,
"fcm_dpo/q_t": 0.3962777256965637,
"grad_norm": 21.930564880371094,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.2828127145767212,
"logits/rejected": -0.2885119915008545,
"logps/chosen": -139.31689453125,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -275.88507080078125,
"loss": 1.0601,
"margin_dpo/margin_mean": 100.65567016601562,
"margin_dpo/margin_std": 133.34600830078125,
"step": 214
},
{
"KL/chosen_KL_mean": -108.55481719970703,
"KL/mean": -162.83993530273438,
"KL/rejected_KL_mean": -217.1250762939453,
"KL/std": 115.2216796875,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.0045753284357488155,
"fcm_dpo/delta": -0.09945982694625854,
"fcm_dpo/margin": 108.57025909423828,
"fcm_dpo/q_t": 0.38728252053260803,
"grad_norm": 26.600690841674805,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.30017614364624023,
"logits/rejected": -0.2858251929283142,
"logps/chosen": -161.83407592773438,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -307.0897216796875,
"loss": 1.0211,
"margin_dpo/margin_mean": 108.57025909423828,
"margin_dpo/margin_std": 121.32503509521484,
"step": 215
},
{
"KL/chosen_KL_mean": -110.91032409667969,
"KL/mean": -164.44400024414062,
"KL/rejected_KL_mean": -217.9776611328125,
"KL/std": 116.61207580566406,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.004546986427158117,
"fcm_dpo/delta": -0.08895479142665863,
"fcm_dpo/margin": 107.06733703613281,
"fcm_dpo/q_t": 0.391222208738327,
"grad_norm": 28.687511444091797,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.31943368911743164,
"logits/rejected": -0.30127909779548645,
"logps/chosen": -159.79812622070312,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -295.17657470703125,
"loss": 1.0535,
"margin_dpo/margin_mean": 107.06732940673828,
"margin_dpo/margin_std": 142.55877685546875,
"step": 216
},
{
"KL/chosen_KL_mean": -110.0457763671875,
"KL/mean": -171.7996826171875,
"KL/rejected_KL_mean": -233.55361938476562,
"KL/std": 119.768310546875,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.004474131390452385,
"fcm_dpo/delta": -0.15707343816757202,
"fcm_dpo/margin": 123.50784301757812,
"fcm_dpo/q_t": 0.376447856426239,
"grad_norm": 21.844968795776367,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.32586967945098877,
"logits/rejected": -0.3191707134246826,
"logps/chosen": -159.89108276367188,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -333.6319580078125,
"loss": 0.9878,
"margin_dpo/margin_mean": 123.50784301757812,
"margin_dpo/margin_std": 134.11033630371094,
"step": 217
},
{
"KL/chosen_KL_mean": -115.66281127929688,
"KL/mean": -165.30850219726562,
"KL/rejected_KL_mean": -214.9542236328125,
"KL/std": 118.33514404296875,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.00444306805729866,
"fcm_dpo/delta": -0.04212786629796028,
"fcm_dpo/margin": 99.2914047241211,
"fcm_dpo/q_t": 0.40180084109306335,
"grad_norm": 22.195404052734375,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.3458363711833954,
"logits/rejected": -0.33776670694351196,
"logps/chosen": -174.239501953125,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -302.80059814453125,
"loss": 1.0785,
"margin_dpo/margin_mean": 99.2914047241211,
"margin_dpo/margin_std": 137.7327423095703,
"step": 218
},
{
"KL/chosen_KL_mean": -124.46105194091797,
"KL/mean": -168.70489501953125,
"KL/rejected_KL_mean": -212.94874572753906,
"KL/std": 120.42536163330078,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.00445276964455843,
"fcm_dpo/delta": 0.005898263305425644,
"fcm_dpo/margin": 88.48770141601562,
"fcm_dpo/q_t": 0.4121992588043213,
"grad_norm": 22.444704055786133,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.3449735641479492,
"logits/rejected": -0.33431658148765564,
"logps/chosen": -185.54490661621094,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -298.7791748046875,
"loss": 1.1278,
"margin_dpo/margin_mean": 88.4876937866211,
"margin_dpo/margin_std": 145.41778564453125,
"step": 219
},
{
"KL/chosen_KL_mean": -146.02963256835938,
"KL/mean": -183.63616943359375,
"KL/rejected_KL_mean": -221.24269104003906,
"KL/std": 107.1483154296875,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.004458419047296047,
"fcm_dpo/delta": 0.06577261537313461,
"fcm_dpo/margin": 75.21304321289062,
"fcm_dpo/q_t": 0.42131632566452026,
"grad_norm": 22.67669677734375,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.31147876381874084,
"logits/rejected": -0.2866259217262268,
"logps/chosen": -216.06092834472656,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -308.92822265625,
"loss": 1.1337,
"margin_dpo/margin_mean": 75.21304321289062,
"margin_dpo/margin_std": 106.4920654296875,
"step": 220
},
{
"KL/chosen_KL_mean": -136.16143798828125,
"KL/mean": -213.4010009765625,
"KL/rejected_KL_mean": -290.6405944824219,
"KL/std": 141.66920471191406,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.004385126754641533,
"fcm_dpo/delta": -0.2867964208126068,
"fcm_dpo/margin": 154.47915649414062,
"fcm_dpo/q_t": 0.3487062454223633,
"grad_norm": 29.397729873657227,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.2791799306869507,
"logits/rejected": -0.28570184111595154,
"logps/chosen": -188.31613159179688,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -397.1082763671875,
"loss": 0.9085,
"margin_dpo/margin_mean": 154.47915649414062,
"margin_dpo/margin_std": 143.16314697265625,
"step": 221
},
{
"KL/chosen_KL_mean": -145.255126953125,
"KL/mean": -204.62652587890625,
"KL/rejected_KL_mean": -263.99798583984375,
"KL/std": 128.56329345703125,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.004318445920944214,
"fcm_dpo/delta": -0.11569039523601532,
"fcm_dpo/margin": 118.74282836914062,
"fcm_dpo/q_t": 0.3848910629749298,
"grad_norm": 26.170949935913086,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.33129405975341797,
"logits/rejected": -0.32808414101600647,
"logps/chosen": -206.22622680664062,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -363.9991149902344,
"loss": 1.0234,
"margin_dpo/margin_mean": 118.74282836914062,
"margin_dpo/margin_std": 141.51174926757812,
"step": 222
},
{
"KL/chosen_KL_mean": -161.82325744628906,
"KL/mean": -213.307861328125,
"KL/rejected_KL_mean": -264.79248046875,
"KL/std": 131.9065399169922,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.004278381355106831,
"fcm_dpo/delta": -0.04155343025922775,
"fcm_dpo/margin": 102.96919250488281,
"fcm_dpo/q_t": 0.40245696902275085,
"grad_norm": 34.5849723815918,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.2450229823589325,
"logits/rejected": -0.23633867502212524,
"logps/chosen": -214.46383666992188,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -347.61749267578125,
"loss": 1.1098,
"margin_dpo/margin_mean": 102.96919250488281,
"margin_dpo/margin_std": 167.57089233398438,
"step": 223
},
{
"KL/chosen_KL_mean": -148.254150390625,
"KL/mean": -204.7643280029297,
"KL/rejected_KL_mean": -261.2745361328125,
"KL/std": 144.42315673828125,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.004234839230775833,
"fcm_dpo/delta": -0.08161487430334091,
"fcm_dpo/margin": 113.02037048339844,
"fcm_dpo/q_t": 0.39719724655151367,
"grad_norm": 37.836830139160156,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.26897698640823364,
"logits/rejected": -0.24652792513370514,
"logps/chosen": -196.84954833984375,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -338.3910217285156,
"loss": 1.0801,
"margin_dpo/margin_mean": 113.02037048339844,
"margin_dpo/margin_std": 168.1210479736328,
"step": 224
},
{
"KL/chosen_KL_mean": -170.34971618652344,
"KL/mean": -240.19332885742188,
"KL/rejected_KL_mean": -310.03692626953125,
"KL/std": 145.15887451171875,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.0041815126314759254,
"fcm_dpo/delta": -0.1897869110107422,
"fcm_dpo/margin": 139.68719482421875,
"fcm_dpo/q_t": 0.37290918827056885,
"grad_norm": 30.018829345703125,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.2817176878452301,
"logits/rejected": -0.2691669464111328,
"logps/chosen": -228.35018920898438,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -409.9398193359375,
"loss": 1.0032,
"margin_dpo/margin_mean": 139.6872100830078,
"margin_dpo/margin_std": 175.74925231933594,
"step": 225
},
{
"KL/chosen_KL_mean": -143.76260375976562,
"KL/mean": -200.83154296875,
"KL/rejected_KL_mean": -257.90045166015625,
"KL/std": 143.9960479736328,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.004126027226448059,
"fcm_dpo/delta": -0.07347771525382996,
"fcm_dpo/margin": 114.13786315917969,
"fcm_dpo/q_t": 0.3982742428779602,
"grad_norm": 30.518173217773438,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.35959500074386597,
"logits/rejected": -0.3191676139831543,
"logps/chosen": -202.66140747070312,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -336.5882263183594,
"loss": 1.0742,
"margin_dpo/margin_mean": 114.13786315917969,
"margin_dpo/margin_std": 164.3631591796875,
"step": 226
},
{
"KL/chosen_KL_mean": -155.82469177246094,
"KL/mean": -225.405029296875,
"KL/rejected_KL_mean": -294.9853515625,
"KL/std": 156.4764404296875,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.0040718745440244675,
"fcm_dpo/delta": -0.1720867156982422,
"fcm_dpo/margin": 139.16065979003906,
"fcm_dpo/q_t": 0.37408214807510376,
"grad_norm": 28.692975997924805,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.3732532262802124,
"logits/rejected": -0.35874879360198975,
"logps/chosen": -214.89688110351562,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -394.3977355957031,
"loss": 0.9953,
"margin_dpo/margin_mean": 139.16065979003906,
"margin_dpo/margin_std": 163.25885009765625,
"step": 227
},
{
"KL/chosen_KL_mean": -154.69204711914062,
"KL/mean": -203.87847900390625,
"KL/rejected_KL_mean": -253.0648956298828,
"KL/std": 127.062255859375,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.004066440276801586,
"fcm_dpo/delta": -5.455687642097473e-05,
"fcm_dpo/margin": 98.37286376953125,
"fcm_dpo/q_t": 0.40937620401382446,
"grad_norm": 25.941659927368164,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.3328084647655487,
"logits/rejected": -0.31316232681274414,
"logps/chosen": -220.5833282470703,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -344.1136474609375,
"loss": 1.1158,
"margin_dpo/margin_mean": 98.37286376953125,
"margin_dpo/margin_std": 152.87173461914062,
"step": 228
},
{
"KL/chosen_KL_mean": -146.78775024414062,
"KL/mean": -200.69815063476562,
"KL/rejected_KL_mean": -254.60853576660156,
"KL/std": 140.32000732421875,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.004051582887768745,
"fcm_dpo/delta": -0.03766883909702301,
"fcm_dpo/margin": 107.82078552246094,
"fcm_dpo/q_t": 0.4005894660949707,
"grad_norm": 39.35184860229492,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.409065306186676,
"logits/rejected": -0.3768808841705322,
"logps/chosen": -217.494140625,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -339.1359558105469,
"loss": 1.082,
"margin_dpo/margin_mean": 107.82078552246094,
"margin_dpo/margin_std": 151.07818603515625,
"step": 229
},
{
"KL/chosen_KL_mean": -113.96424102783203,
"KL/mean": -189.5445556640625,
"KL/rejected_KL_mean": -265.1248779296875,
"KL/std": 134.1195068359375,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.003982962109148502,
"fcm_dpo/delta": -0.2084825038909912,
"fcm_dpo/margin": 151.16061401367188,
"fcm_dpo/q_t": 0.3635350465774536,
"grad_norm": 26.65117645263672,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.28100255131721497,
"logits/rejected": -0.2840505540370941,
"logps/chosen": -153.24624633789062,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -350.74676513671875,
"loss": 0.9531,
"margin_dpo/margin_mean": 151.16061401367188,
"margin_dpo/margin_std": 151.04591369628906,
"step": 230
},
{
"KL/chosen_KL_mean": -125.48028564453125,
"KL/mean": -177.44976806640625,
"KL/rejected_KL_mean": -229.41925048828125,
"KL/std": 115.8082275390625,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.003960724454373121,
"fcm_dpo/delta": -0.011919192038476467,
"fcm_dpo/margin": 103.93896484375,
"fcm_dpo/q_t": 0.4071752727031708,
"grad_norm": 28.911128997802734,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.3797636032104492,
"logits/rejected": -0.35482439398765564,
"logps/chosen": -188.75672912597656,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -303.54315185546875,
"loss": 1.0837,
"margin_dpo/margin_mean": 103.93896484375,
"margin_dpo/margin_std": 136.3351287841797,
"step": 231
},
{
"KL/chosen_KL_mean": -152.2584228515625,
"KL/mean": -195.20050048828125,
"KL/rejected_KL_mean": -238.14256286621094,
"KL/std": 134.2640838623047,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.0039717936888337135,
"fcm_dpo/delta": 0.059911780059337616,
"fcm_dpo/margin": 85.88412475585938,
"fcm_dpo/q_t": 0.4243953824043274,
"grad_norm": 27.307865142822266,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.3296153247356415,
"logits/rejected": -0.3048354983329773,
"logps/chosen": -223.00718688964844,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -322.11962890625,
"loss": 1.1477,
"margin_dpo/margin_mean": 85.88412475585938,
"margin_dpo/margin_std": 142.83050537109375,
"step": 232
},
{
"KL/chosen_KL_mean": -135.91783142089844,
"KL/mean": -204.65484619140625,
"KL/rejected_KL_mean": -273.3918762207031,
"KL/std": 146.017333984375,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.003943287767469883,
"fcm_dpo/delta": -0.14597077667713165,
"fcm_dpo/margin": 137.4740447998047,
"fcm_dpo/q_t": 0.38463860750198364,
"grad_norm": 24.649152755737305,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.34792935848236084,
"logits/rejected": -0.3480525016784668,
"logps/chosen": -190.80076599121094,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -380.8719482421875,
"loss": 1.0429,
"margin_dpo/margin_mean": 137.47402954101562,
"margin_dpo/margin_std": 192.8949737548828,
"step": 233
},
{
"KL/chosen_KL_mean": -127.90108489990234,
"KL/mean": -198.03463745117188,
"KL/rejected_KL_mean": -268.168212890625,
"KL/std": 129.72488403320312,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.0038681491278111935,
"fcm_dpo/delta": -0.14766976237297058,
"fcm_dpo/margin": 140.26710510253906,
"fcm_dpo/q_t": 0.3764752745628357,
"grad_norm": 24.525766372680664,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.3588540554046631,
"logits/rejected": -0.3863643705844879,
"logps/chosen": -171.99560546875,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -368.1748352050781,
"loss": 0.9883,
"margin_dpo/margin_mean": 140.26710510253906,
"margin_dpo/margin_std": 142.52896118164062,
"step": 234
},
{
"KL/chosen_KL_mean": -163.20513916015625,
"KL/mean": -208.35916137695312,
"KL/rejected_KL_mean": -253.51319885253906,
"KL/std": 124.54771423339844,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.0038875495083630085,
"fcm_dpo/delta": 0.049665797501802444,
"fcm_dpo/margin": 90.30805969238281,
"fcm_dpo/q_t": 0.42041072249412537,
"grad_norm": 26.740400314331055,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.3990609049797058,
"logits/rejected": -0.37867674231529236,
"logps/chosen": -225.44305419921875,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -343.90826416015625,
"loss": 1.1485,
"margin_dpo/margin_mean": 90.30805969238281,
"margin_dpo/margin_std": 152.52548217773438,
"step": 235
},
{
"KL/chosen_KL_mean": -124.98179626464844,
"KL/mean": -199.99957275390625,
"KL/rejected_KL_mean": -275.0173645019531,
"KL/std": 138.9434051513672,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.0038370825350284576,
"fcm_dpo/delta": -0.18083232641220093,
"fcm_dpo/margin": 150.0355682373047,
"fcm_dpo/q_t": 0.3679265081882477,
"grad_norm": 33.604164123535156,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.37230125069618225,
"logits/rejected": -0.3716471195220947,
"logps/chosen": -174.32315063476562,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -378.52899169921875,
"loss": 0.9526,
"margin_dpo/margin_mean": 150.0355682373047,
"margin_dpo/margin_std": 135.6669464111328,
"step": 236
},
{
"KL/chosen_KL_mean": -173.83096313476562,
"KL/mean": -229.819580078125,
"KL/rejected_KL_mean": -285.8081970214844,
"KL/std": 140.05715942382812,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.003812191542237997,
"fcm_dpo/delta": -0.027462899684906006,
"fcm_dpo/margin": 111.97723388671875,
"fcm_dpo/q_t": 0.40431371331214905,
"grad_norm": 30.839101791381836,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.3602313995361328,
"logits/rejected": -0.35376355051994324,
"logps/chosen": -227.99908447265625,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -380.58856201171875,
"loss": 1.0919,
"margin_dpo/margin_mean": 111.97724151611328,
"margin_dpo/margin_std": 162.9444580078125,
"step": 237
},
{
"KL/chosen_KL_mean": -161.74685668945312,
"KL/mean": -218.63809204101562,
"KL/rejected_KL_mean": -275.5293273925781,
"KL/std": 138.28067016601562,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.0037880742456763983,
"fcm_dpo/delta": -0.03207089379429817,
"fcm_dpo/margin": 113.78250122070312,
"fcm_dpo/q_t": 0.4048614501953125,
"grad_norm": 25.45376968383789,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.3611815571784973,
"logits/rejected": -0.3783150911331177,
"logps/chosen": -215.719970703125,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -364.947265625,
"loss": 1.0809,
"margin_dpo/margin_mean": 113.7824935913086,
"margin_dpo/margin_std": 156.38768005371094,
"step": 238
},
{
"KL/chosen_KL_mean": -173.99893188476562,
"KL/mean": -237.2444305419922,
"KL/rejected_KL_mean": -300.4898986816406,
"KL/std": 130.01622009277344,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.0037591117434203625,
"fcm_dpo/delta": -0.07816985249519348,
"fcm_dpo/margin": 126.49100494384766,
"fcm_dpo/q_t": 0.39054858684539795,
"grad_norm": 27.46272087097168,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3645399808883667,
"logits/rejected": -0.34787994623184204,
"logps/chosen": -232.09674072265625,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -394.0828552246094,
"loss": 1.0356,
"margin_dpo/margin_mean": 126.49099731445312,
"margin_dpo/margin_std": 139.7375030517578,
"step": 239
},
{
"KL/chosen_KL_mean": -181.10501098632812,
"KL/mean": -235.21560668945312,
"KL/rejected_KL_mean": -289.326171875,
"KL/std": 143.65615844726562,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.003757942235097289,
"fcm_dpo/delta": -0.006928959861397743,
"fcm_dpo/margin": 108.22117614746094,
"fcm_dpo/q_t": 0.40816569328308105,
"grad_norm": 32.250770568847656,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.35770127177238464,
"logits/rejected": -0.3304799795150757,
"logps/chosen": -241.719482421875,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -363.4447021484375,
"loss": 1.1151,
"margin_dpo/margin_mean": 108.2211685180664,
"margin_dpo/margin_std": 168.62789916992188,
"step": 240
},
{
"KL/chosen_KL_mean": -149.57997131347656,
"KL/mean": -231.7826690673828,
"KL/rejected_KL_mean": -313.9853820800781,
"KL/std": 156.34739685058594,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.0037074610590934753,
"fcm_dpo/delta": -0.21590769290924072,
"fcm_dpo/margin": 164.4053955078125,
"fcm_dpo/q_t": 0.3671102821826935,
"grad_norm": 24.17862892150879,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.4300222396850586,
"logits/rejected": -0.4012630581855774,
"logps/chosen": -215.67105102539062,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -402.0462646484375,
"loss": 0.9689,
"margin_dpo/margin_mean": 164.4053955078125,
"margin_dpo/margin_std": 181.3541717529297,
"step": 241
},
{
"KL/chosen_KL_mean": -168.50222778320312,
"KL/mean": -227.4334716796875,
"KL/rejected_KL_mean": -286.3647155761719,
"KL/std": 131.76930236816406,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.0036807293072342873,
"fcm_dpo/delta": -0.03467430919408798,
"fcm_dpo/margin": 117.86245727539062,
"fcm_dpo/q_t": 0.4024999737739563,
"grad_norm": 26.07633399963379,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.35617491602897644,
"logits/rejected": -0.33020225167274475,
"logps/chosen": -236.3661651611328,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -369.72503662109375,
"loss": 1.0726,
"margin_dpo/margin_mean": 117.86245727539062,
"margin_dpo/margin_std": 153.87033081054688,
"step": 242
},
{
"KL/chosen_KL_mean": -169.4001922607422,
"KL/mean": -241.8457794189453,
"KL/rejected_KL_mean": -314.2913818359375,
"KL/std": 151.13641357421875,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.003632163628935814,
"fcm_dpo/delta": -0.12977714836597443,
"fcm_dpo/margin": 144.89117431640625,
"fcm_dpo/q_t": 0.3812503218650818,
"grad_norm": 23.20541763305664,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.3599437475204468,
"logits/rejected": -0.32367074489593506,
"logps/chosen": -232.48443603515625,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -390.62701416015625,
"loss": 1.0077,
"margin_dpo/margin_mean": 144.89117431640625,
"margin_dpo/margin_std": 165.2962646484375,
"step": 243
},
{
"KL/chosen_KL_mean": -147.78445434570312,
"KL/mean": -222.11541748046875,
"KL/rejected_KL_mean": -296.4464111328125,
"KL/std": 144.9271240234375,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.0035813949070870876,
"fcm_dpo/delta": -0.13626864552497864,
"fcm_dpo/margin": 148.6619415283203,
"fcm_dpo/q_t": 0.3784053921699524,
"grad_norm": 27.900978088378906,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.42759430408477783,
"logits/rejected": -0.41793277859687805,
"logps/chosen": -208.92514038085938,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -391.33831787109375,
"loss": 0.9878,
"margin_dpo/margin_mean": 148.66195678710938,
"margin_dpo/margin_std": 146.8434295654297,
"step": 244
},
{
"KL/chosen_KL_mean": -168.794677734375,
"KL/mean": -229.46194458007812,
"KL/rejected_KL_mean": -290.12921142578125,
"KL/std": 146.95806884765625,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.003548422595486045,
"fcm_dpo/delta": -0.03223326802253723,
"fcm_dpo/margin": 121.33451843261719,
"fcm_dpo/q_t": 0.4033067524433136,
"grad_norm": 29.785207748413086,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.4032437801361084,
"logits/rejected": -0.3697701692581177,
"logps/chosen": -236.05697631835938,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -377.769287109375,
"loss": 1.0876,
"margin_dpo/margin_mean": 121.33451843261719,
"margin_dpo/margin_std": 165.6243438720703,
"step": 245
},
{
"KL/chosen_KL_mean": -169.9764404296875,
"KL/mean": -230.96017456054688,
"KL/rejected_KL_mean": -291.94390869140625,
"KL/std": 162.06460571289062,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.003547412808984518,
"fcm_dpo/delta": -0.0335458442568779,
"fcm_dpo/margin": 121.96745300292969,
"fcm_dpo/q_t": 0.4048032760620117,
"grad_norm": 29.99637222290039,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.4445304274559021,
"logits/rejected": -0.4077053964138031,
"logps/chosen": -236.6734161376953,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -376.2902526855469,
"loss": 1.0863,
"margin_dpo/margin_mean": 121.96745300292969,
"margin_dpo/margin_std": 176.64109802246094,
"step": 246
},
{
"KL/chosen_KL_mean": -202.48690795898438,
"KL/mean": -290.2227478027344,
"KL/rejected_KL_mean": -377.9586181640625,
"KL/std": 172.92800903320312,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.0034898994490504265,
"fcm_dpo/delta": -0.21926915645599365,
"fcm_dpo/margin": 175.4716796875,
"fcm_dpo/q_t": 0.3639224171638489,
"grad_norm": 31.32638931274414,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.34930846095085144,
"logits/rejected": -0.3457414507865906,
"logps/chosen": -259.0922546386719,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -484.2518615722656,
"loss": 0.9621,
"margin_dpo/margin_mean": 175.4716796875,
"margin_dpo/margin_std": 188.98797607421875,
"step": 247
},
{
"KL/chosen_KL_mean": -176.61392211914062,
"KL/mean": -252.390625,
"KL/rejected_KL_mean": -328.1673278808594,
"KL/std": 141.5336151123047,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.003434403333812952,
"fcm_dpo/delta": -0.12410237640142441,
"fcm_dpo/margin": 151.55343627929688,
"fcm_dpo/q_t": 0.3788222670555115,
"grad_norm": 26.931861877441406,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.35363906621932983,
"logits/rejected": -0.37153977155685425,
"logps/chosen": -220.65713500976562,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -420.02423095703125,
"loss": 0.9855,
"margin_dpo/margin_mean": 151.55343627929688,
"margin_dpo/margin_std": 138.50741577148438,
"step": 248
},
{
"KL/chosen_KL_mean": -223.4722137451172,
"KL/mean": -264.677734375,
"KL/rejected_KL_mean": -305.8832702636719,
"KL/std": 150.7882080078125,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.0034578610211610794,
"fcm_dpo/delta": 0.11675933003425598,
"fcm_dpo/margin": 82.41104888916016,
"fcm_dpo/q_t": 0.43573471903800964,
"grad_norm": 46.4980583190918,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.3905929923057556,
"logits/rejected": -0.3669302463531494,
"logps/chosen": -285.91455078125,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -386.351318359375,
"loss": 1.2111,
"margin_dpo/margin_mean": 82.41105651855469,
"margin_dpo/margin_std": 178.0140380859375,
"step": 249
},
{
"KL/chosen_KL_mean": -191.8778076171875,
"KL/mean": -268.19073486328125,
"KL/rejected_KL_mean": -344.503662109375,
"KL/std": 147.71640014648438,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.003444279544055462,
"fcm_dpo/delta": -0.12898439168930054,
"fcm_dpo/margin": 152.62586975097656,
"fcm_dpo/q_t": 0.3783637285232544,
"grad_norm": 38.140594482421875,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.37574708461761475,
"logits/rejected": -0.3200438618659973,
"logps/chosen": -257.5145263671875,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -418.3755187988281,
"loss": 0.9902,
"margin_dpo/margin_mean": 152.62586975097656,
"margin_dpo/margin_std": 148.07838439941406,
"step": 250
},
{
"KL/chosen_KL_mean": -210.373291015625,
"KL/mean": -262.0963134765625,
"KL/rejected_KL_mean": -313.8193664550781,
"KL/std": 158.57400512695312,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.003431498073041439,
"fcm_dpo/delta": 0.045787617564201355,
"fcm_dpo/margin": 103.44607543945312,
"fcm_dpo/q_t": 0.4200136065483093,
"grad_norm": 31.18090057373047,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3738080859184265,
"logits/rejected": -0.34553277492523193,
"logps/chosen": -267.5559997558594,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -391.4827880859375,
"loss": 1.1472,
"margin_dpo/margin_mean": 103.44607543945312,
"margin_dpo/margin_std": 171.2592010498047,
"step": 251
},
{
"KL/chosen_KL_mean": -202.80035400390625,
"KL/mean": -273.161865234375,
"KL/rejected_KL_mean": -343.52337646484375,
"KL/std": 140.35821533203125,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.003416198305785656,
"fcm_dpo/delta": -0.08289383351802826,
"fcm_dpo/margin": 140.72303771972656,
"fcm_dpo/q_t": 0.38559865951538086,
"grad_norm": 29.2058048248291,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.413099467754364,
"logits/rejected": -0.3722429573535919,
"logps/chosen": -274.4859924316406,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -428.2813720703125,
"loss": 1.0164,
"margin_dpo/margin_mean": 140.72303771972656,
"margin_dpo/margin_std": 137.57774353027344,
"step": 252
},
{
"KL/chosen_KL_mean": -172.536865234375,
"KL/mean": -240.89328002929688,
"KL/rejected_KL_mean": -309.2496643066406,
"KL/std": 148.74118041992188,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.0034063609782606363,
"fcm_dpo/delta": -0.06741885095834732,
"fcm_dpo/margin": 136.71279907226562,
"fcm_dpo/q_t": 0.392569899559021,
"grad_norm": 23.644466400146484,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.414895236492157,
"logits/rejected": -0.4082959294319153,
"logps/chosen": -241.67080688476562,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -407.95220947265625,
"loss": 1.0485,
"margin_dpo/margin_mean": 136.71279907226562,
"margin_dpo/margin_std": 162.5023193359375,
"step": 253
},
{
"KL/chosen_KL_mean": -175.3002166748047,
"KL/mean": -234.9326171875,
"KL/rejected_KL_mean": -294.5650329589844,
"KL/std": 157.5005645751953,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.003387467935681343,
"fcm_dpo/delta": -0.0041029490530490875,
"fcm_dpo/margin": 119.26481628417969,
"fcm_dpo/q_t": 0.4124619662761688,
"grad_norm": 26.61585235595703,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.377765953540802,
"logits/rejected": -0.3579869270324707,
"logps/chosen": -229.45521545410156,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -384.8726806640625,
"loss": 1.1203,
"margin_dpo/margin_mean": 119.26481628417969,
"margin_dpo/margin_std": 196.88693237304688,
"step": 254
},
{
"KL/chosen_KL_mean": -181.1912841796875,
"KL/mean": -242.2811279296875,
"KL/rejected_KL_mean": -303.3709716796875,
"KL/std": 136.550048828125,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.003378466470167041,
"fcm_dpo/delta": -0.013259213417768478,
"fcm_dpo/margin": 122.17970275878906,
"fcm_dpo/q_t": 0.4052705764770508,
"grad_norm": 23.39793586730957,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.36594948172569275,
"logits/rejected": -0.35779714584350586,
"logps/chosen": -238.33294677734375,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -393.57958984375,
"loss": 1.0967,
"margin_dpo/margin_mean": 122.17970275878906,
"margin_dpo/margin_std": 172.21868896484375,
"step": 255
},
{
"KL/chosen_KL_mean": -156.4071044921875,
"KL/mean": -223.8316650390625,
"KL/rejected_KL_mean": -291.2562561035156,
"KL/std": 148.6212158203125,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.003370137419551611,
"fcm_dpo/delta": -0.05573172867298126,
"fcm_dpo/margin": 134.84915161132812,
"fcm_dpo/q_t": 0.3962138891220093,
"grad_norm": 22.66250228881836,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.3113047480583191,
"logits/rejected": -0.31630462408065796,
"logps/chosen": -211.57058715820312,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -383.81915283203125,
"loss": 1.0567,
"margin_dpo/margin_mean": 134.84913635253906,
"margin_dpo/margin_std": 168.68287658691406,
"step": 256
},
{
"KL/chosen_KL_mean": -151.5054931640625,
"KL/mean": -221.162109375,
"KL/rejected_KL_mean": -290.8187255859375,
"KL/std": 151.1150665283203,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.0033370412420481443,
"fcm_dpo/delta": -0.06718017160892487,
"fcm_dpo/margin": 139.313232421875,
"fcm_dpo/q_t": 0.39490461349487305,
"grad_norm": 20.624574661254883,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.32282236218452454,
"logits/rejected": -0.3079400360584259,
"logps/chosen": -200.92919921875,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -370.35662841796875,
"loss": 1.0528,
"margin_dpo/margin_mean": 139.313232421875,
"margin_dpo/margin_std": 168.8369598388672,
"step": 257
},
{
"KL/chosen_KL_mean": -196.22232055664062,
"KL/mean": -263.02752685546875,
"KL/rejected_KL_mean": -329.832763671875,
"KL/std": 156.14871215820312,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.0033341706730425358,
"fcm_dpo/delta": -0.0465044304728508,
"fcm_dpo/margin": 133.61041259765625,
"fcm_dpo/q_t": 0.3972247242927551,
"grad_norm": 24.023584365844727,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.316849946975708,
"logits/rejected": -0.3123030960559845,
"logps/chosen": -255.6064453125,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -425.8228454589844,
"loss": 1.0785,
"margin_dpo/margin_mean": 133.61041259765625,
"margin_dpo/margin_std": 185.94827270507812,
"step": 258
},
{
"KL/chosen_KL_mean": -187.66934204101562,
"KL/mean": -247.58529663085938,
"KL/rejected_KL_mean": -307.50128173828125,
"KL/std": 151.50286865234375,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.0033223240170627832,
"fcm_dpo/delta": 0.001799125224351883,
"fcm_dpo/margin": 119.83191680908203,
"fcm_dpo/q_t": 0.409615159034729,
"grad_norm": 26.094667434692383,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.3730711042881012,
"logits/rejected": -0.36661672592163086,
"logps/chosen": -240.4976806640625,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -396.69293212890625,
"loss": 1.1018,
"margin_dpo/margin_mean": 119.83192443847656,
"margin_dpo/margin_std": 169.11459350585938,
"step": 259
},
{
"KL/chosen_KL_mean": -195.3187255859375,
"KL/mean": -271.7413330078125,
"KL/rejected_KL_mean": -348.1639404296875,
"KL/std": 158.65463256835938,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.003307567210868001,
"fcm_dpo/delta": -0.10822096467018127,
"fcm_dpo/margin": 152.84521484375,
"fcm_dpo/q_t": 0.3863574266433716,
"grad_norm": 33.96175003051758,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.35657697916030884,
"logits/rejected": -0.3744693994522095,
"logps/chosen": -242.73638916015625,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -443.25372314453125,
"loss": 1.013,
"margin_dpo/margin_mean": 152.84521484375,
"margin_dpo/margin_std": 169.8836212158203,
"step": 260
},
{
"KL/chosen_KL_mean": -197.50112915039062,
"KL/mean": -269.7708435058594,
"KL/rejected_KL_mean": -342.04052734375,
"KL/std": 168.48382568359375,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.0032701774034649134,
"fcm_dpo/delta": -0.07457753270864487,
"fcm_dpo/margin": 144.5394287109375,
"fcm_dpo/q_t": 0.39346715807914734,
"grad_norm": 23.455717086791992,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.3204799294471741,
"logits/rejected": -0.31587928533554077,
"logps/chosen": -250.5325164794922,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -430.55548095703125,
"loss": 1.0506,
"margin_dpo/margin_mean": 144.5394287109375,
"margin_dpo/margin_std": 183.651611328125,
"step": 261
},
{
"KL/chosen_KL_mean": -230.0545654296875,
"KL/mean": -290.9325256347656,
"KL/rejected_KL_mean": -351.8105163574219,
"KL/std": 155.808349609375,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.0032733541447669268,
"fcm_dpo/delta": 0.0013803504407405853,
"fcm_dpo/margin": 121.75595092773438,
"fcm_dpo/q_t": 0.4075956344604492,
"grad_norm": 29.473356246948242,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.3197232484817505,
"logits/rejected": -0.28875744342803955,
"logps/chosen": -289.6746826171875,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -438.22906494140625,
"loss": 1.086,
"margin_dpo/margin_mean": 121.75595092773438,
"margin_dpo/margin_std": 153.61898803710938,
"step": 262
},
{
"KL/chosen_KL_mean": -204.87518310546875,
"KL/mean": -291.5097351074219,
"KL/rejected_KL_mean": -378.1442565917969,
"KL/std": 180.58358764648438,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.003221530467271805,
"fcm_dpo/delta": -0.16330862045288086,
"fcm_dpo/margin": 173.26904296875,
"fcm_dpo/q_t": 0.3738434314727783,
"grad_norm": 23.88220977783203,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.3573540151119232,
"logits/rejected": -0.3329017460346222,
"logps/chosen": -264.296142578125,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -475.00146484375,
"loss": 0.9867,
"margin_dpo/margin_mean": 173.26905822753906,
"margin_dpo/margin_std": 186.5802764892578,
"step": 263
},
{
"KL/chosen_KL_mean": -210.99795532226562,
"KL/mean": -283.88690185546875,
"KL/rejected_KL_mean": -356.77581787109375,
"KL/std": 164.59521484375,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.0031880121678113937,
"fcm_dpo/delta": -0.06703611463308334,
"fcm_dpo/margin": 145.77786254882812,
"fcm_dpo/q_t": 0.39508941769599915,
"grad_norm": 27.110586166381836,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.374176949262619,
"logits/rejected": -0.3629017472267151,
"logps/chosen": -273.72003173828125,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -450.6320495605469,
"loss": 1.0561,
"margin_dpo/margin_mean": 145.77786254882812,
"margin_dpo/margin_std": 182.66200256347656,
"step": 264
},
{
"KL/chosen_KL_mean": -237.04635620117188,
"KL/mean": -308.437744140625,
"KL/rejected_KL_mean": -379.8291015625,
"KL/std": 194.23110961914062,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.0031799268908798695,
"fcm_dpo/delta": -0.0553123913705349,
"fcm_dpo/margin": 142.78271484375,
"fcm_dpo/q_t": 0.40076446533203125,
"grad_norm": 29.41632843017578,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.33580052852630615,
"logits/rejected": -0.31790751218795776,
"logps/chosen": -299.017822265625,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -467.8497009277344,
"loss": 1.0958,
"margin_dpo/margin_mean": 142.78274536132812,
"margin_dpo/margin_std": 224.72232055664062,
"step": 265
},
{
"KL/chosen_KL_mean": -247.69691467285156,
"KL/mean": -305.4302978515625,
"KL/rejected_KL_mean": -363.1636962890625,
"KL/std": 160.6911163330078,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0031850021332502365,
"fcm_dpo/delta": 0.03283064067363739,
"fcm_dpo/margin": 115.46676635742188,
"fcm_dpo/q_t": 0.4174157381057739,
"grad_norm": 63.25857925415039,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.36059173941612244,
"logits/rejected": -0.3145703673362732,
"logps/chosen": -314.79656982421875,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -431.1349182128906,
"loss": 1.1341,
"margin_dpo/margin_mean": 115.46676635742188,
"margin_dpo/margin_std": 187.57647705078125,
"step": 266
},
{
"KL/chosen_KL_mean": -224.26531982421875,
"KL/mean": -294.53692626953125,
"KL/rejected_KL_mean": -364.8084716796875,
"KL/std": 172.51197814941406,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.00316983787342906,
"fcm_dpo/delta": -0.046759672462940216,
"fcm_dpo/margin": 140.54315185546875,
"fcm_dpo/q_t": 0.39952901005744934,
"grad_norm": 43.019325256347656,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3794315457344055,
"logits/rejected": -0.342626690864563,
"logps/chosen": -293.236083984375,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -454.9769287109375,
"loss": 1.0756,
"margin_dpo/margin_mean": 140.5431365966797,
"margin_dpo/margin_std": 194.42465209960938,
"step": 267
},
{
"KL/chosen_KL_mean": -234.8605194091797,
"KL/mean": -304.49365234375,
"KL/rejected_KL_mean": -374.12677001953125,
"KL/std": 169.82725524902344,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.0031569194979965687,
"fcm_dpo/delta": -0.04073052108287811,
"fcm_dpo/margin": 139.26629638671875,
"fcm_dpo/q_t": 0.402080237865448,
"grad_norm": 35.164588928222656,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.36576220393180847,
"logits/rejected": -0.3701912760734558,
"logps/chosen": -290.7608337402344,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -475.7744140625,
"loss": 1.0845,
"margin_dpo/margin_mean": 139.26629638671875,
"margin_dpo/margin_std": 201.86578369140625,
"step": 268
},
{
"KL/chosen_KL_mean": -243.96658325195312,
"KL/mean": -328.11651611328125,
"KL/rejected_KL_mean": -412.26641845703125,
"KL/std": 171.32447814941406,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.0031313346698880196,
"fcm_dpo/delta": -0.13035503029823303,
"fcm_dpo/margin": 168.2998504638672,
"fcm_dpo/q_t": 0.38259977102279663,
"grad_norm": 30.891639709472656,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.4082416594028473,
"logits/rejected": -0.3874931335449219,
"logps/chosen": -314.00616455078125,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -519.6158447265625,
"loss": 1.0322,
"margin_dpo/margin_mean": 168.29983520507812,
"margin_dpo/margin_std": 217.715576171875,
"step": 269
},
{
"KL/chosen_KL_mean": -209.37474060058594,
"KL/mean": -271.95452880859375,
"KL/rejected_KL_mean": -334.53436279296875,
"KL/std": 150.77197265625,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.0031246920116245747,
"fcm_dpo/delta": 0.008997350931167603,
"fcm_dpo/margin": 125.15963745117188,
"fcm_dpo/q_t": 0.41077637672424316,
"grad_norm": 45.283546447753906,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.4233609139919281,
"logits/rejected": -0.41205936670303345,
"logps/chosen": -278.908203125,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -444.4630126953125,
"loss": 1.1118,
"margin_dpo/margin_mean": 125.15963745117188,
"margin_dpo/margin_std": 185.95555114746094,
"step": 270
},
{
"KL/chosen_KL_mean": -188.9541473388672,
"KL/mean": -265.62396240234375,
"KL/rejected_KL_mean": -342.2938232421875,
"KL/std": 149.54208374023438,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.0030992082320153713,
"fcm_dpo/delta": -0.07721711695194244,
"fcm_dpo/margin": 153.33963012695312,
"fcm_dpo/q_t": 0.39120548963546753,
"grad_norm": 27.018428802490234,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3589969277381897,
"logits/rejected": -0.33432579040527344,
"logps/chosen": -245.71871948242188,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -434.8076477050781,
"loss": 1.0279,
"margin_dpo/margin_mean": 153.3396453857422,
"margin_dpo/margin_std": 167.06747436523438,
"step": 271
},
{
"KL/chosen_KL_mean": -173.87306213378906,
"KL/mean": -266.062744140625,
"KL/rejected_KL_mean": -358.25244140625,
"KL/std": 168.98089599609375,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.003064734861254692,
"fcm_dpo/delta": -0.16969643533229828,
"fcm_dpo/margin": 184.37937927246094,
"fcm_dpo/q_t": 0.3702283501625061,
"grad_norm": 34.31309509277344,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.31992781162261963,
"logits/rejected": -0.3296176791191101,
"logps/chosen": -223.37022399902344,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -463.79522705078125,
"loss": 0.9675,
"margin_dpo/margin_mean": 184.37937927246094,
"margin_dpo/margin_std": 175.53350830078125,
"step": 272
},
{
"KL/chosen_KL_mean": -202.81802368164062,
"KL/mean": -292.4516296386719,
"KL/rejected_KL_mean": -382.08526611328125,
"KL/std": 168.8096923828125,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.0030071022920310497,
"fcm_dpo/delta": -0.14330768585205078,
"fcm_dpo/margin": 179.26719665527344,
"fcm_dpo/q_t": 0.37718284130096436,
"grad_norm": 23.817079544067383,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.3086550533771515,
"logits/rejected": -0.28471726179122925,
"logps/chosen": -265.7934265136719,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -474.5838317871094,
"loss": 0.9925,
"margin_dpo/margin_mean": 179.26719665527344,
"margin_dpo/margin_std": 188.98374938964844,
"step": 273
},
{
"KL/chosen_KL_mean": -239.08636474609375,
"KL/mean": -309.47210693359375,
"KL/rejected_KL_mean": -379.85784912109375,
"KL/std": 157.9844970703125,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.003000047989189625,
"fcm_dpo/delta": -0.02283497154712677,
"fcm_dpo/margin": 140.77145385742188,
"fcm_dpo/q_t": 0.4034059941768646,
"grad_norm": 29.533851623535156,
"learning_rate": 3.75e-07,
"logits/chosen": -0.2807186245918274,
"logits/rejected": -0.260738730430603,
"logps/chosen": -294.75408935546875,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -457.19091796875,
"loss": 1.077,
"margin_dpo/margin_mean": 140.77145385742188,
"margin_dpo/margin_std": 181.21035766601562,
"step": 274
},
{
"KL/chosen_KL_mean": -190.72259521484375,
"KL/mean": -263.8143005371094,
"KL/rejected_KL_mean": -336.906005859375,
"KL/std": 159.49850463867188,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.002990099135786295,
"fcm_dpo/delta": -0.03795165196061134,
"fcm_dpo/margin": 146.18341064453125,
"fcm_dpo/q_t": 0.39963993430137634,
"grad_norm": 28.2340145111084,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.3173510432243347,
"logits/rejected": -0.3210340142250061,
"logps/chosen": -239.31729125976562,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -430.209716796875,
"loss": 1.0664,
"margin_dpo/margin_mean": 146.18341064453125,
"margin_dpo/margin_std": 182.6461181640625,
"step": 275
},
{
"KL/chosen_KL_mean": -207.98233032226562,
"KL/mean": -282.283203125,
"KL/rejected_KL_mean": -356.5841064453125,
"KL/std": 165.94070434570312,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.002971104346215725,
"fcm_dpo/delta": -0.04252216964960098,
"fcm_dpo/margin": 148.60177612304688,
"fcm_dpo/q_t": 0.4004755914211273,
"grad_norm": 30.543088912963867,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.31873783469200134,
"logits/rejected": -0.29504579305648804,
"logps/chosen": -264.55975341796875,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -426.94976806640625,
"loss": 1.0728,
"margin_dpo/margin_mean": 148.60177612304688,
"margin_dpo/margin_std": 199.07870483398438,
"step": 276
},
{
"KL/chosen_KL_mean": -230.2869110107422,
"KL/mean": -309.2855529785156,
"KL/rejected_KL_mean": -388.2841796875,
"KL/std": 167.18478393554688,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.0029588094912469387,
"fcm_dpo/delta": -0.0690733790397644,
"fcm_dpo/margin": 157.99728393554688,
"fcm_dpo/q_t": 0.39369434118270874,
"grad_norm": 26.234636306762695,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.29924240708351135,
"logits/rejected": -0.2815479636192322,
"logps/chosen": -286.5584716796875,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -481.16546630859375,
"loss": 1.0493,
"margin_dpo/margin_mean": 157.99728393554688,
"margin_dpo/margin_std": 193.92010498046875,
"step": 277
},
{
"KL/chosen_KL_mean": -206.88705444335938,
"KL/mean": -296.92340087890625,
"KL/rejected_KL_mean": -386.95977783203125,
"KL/std": 179.54661560058594,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.0029181232675909996,
"fcm_dpo/delta": -0.12909561395645142,
"fcm_dpo/margin": 180.07273864746094,
"fcm_dpo/q_t": 0.38142508268356323,
"grad_norm": 31.345380783081055,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.2834761142730713,
"logits/rejected": -0.28856098651885986,
"logps/chosen": -259.8289794921875,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -478.21337890625,
"loss": 1.004,
"margin_dpo/margin_mean": 180.07275390625,
"margin_dpo/margin_std": 198.38058471679688,
"step": 278
},
{
"KL/chosen_KL_mean": -245.1502227783203,
"KL/mean": -334.9271240234375,
"KL/rejected_KL_mean": -424.7039794921875,
"KL/std": 189.08995056152344,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.0028769150376319885,
"fcm_dpo/delta": -0.12047116458415985,
"fcm_dpo/margin": 179.55380249023438,
"fcm_dpo/q_t": 0.38520491123199463,
"grad_norm": 30.17645263671875,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.23838657140731812,
"logits/rejected": -0.23660916090011597,
"logps/chosen": -293.7915344238281,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -512.555419921875,
"loss": 1.0304,
"margin_dpo/margin_mean": 179.55380249023438,
"margin_dpo/margin_std": 223.5946044921875,
"step": 279
},
{
"KL/chosen_KL_mean": -248.6533203125,
"KL/mean": -343.9190673828125,
"KL/rejected_KL_mean": -439.184814453125,
"KL/std": 181.25201416015625,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.002845579292625189,
"fcm_dpo/delta": -0.14617213606834412,
"fcm_dpo/margin": 190.531494140625,
"fcm_dpo/q_t": 0.3750941753387451,
"grad_norm": 27.4671688079834,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.271923303604126,
"logits/rejected": -0.2730262279510498,
"logps/chosen": -307.450439453125,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -537.8037109375,
"loss": 0.9844,
"margin_dpo/margin_mean": 190.53150939941406,
"margin_dpo/margin_std": 191.6612548828125,
"step": 280
},
{
"KL/chosen_KL_mean": -226.9546356201172,
"KL/mean": -309.49078369140625,
"KL/rejected_KL_mean": -392.0268859863281,
"KL/std": 174.78067016601562,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0028265006840229034,
"fcm_dpo/delta": -0.06817743927240372,
"fcm_dpo/margin": 165.07223510742188,
"fcm_dpo/q_t": 0.39088958501815796,
"grad_norm": 26.444442749023438,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.3203830122947693,
"logits/rejected": -0.2967671751976013,
"logps/chosen": -282.44317626953125,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -472.9094543457031,
"loss": 1.0406,
"margin_dpo/margin_mean": 165.07223510742188,
"margin_dpo/margin_std": 188.06488037109375,
"step": 281
},
{
"KL/chosen_KL_mean": -251.03106689453125,
"KL/mean": -316.7037658691406,
"KL/rejected_KL_mean": -382.37646484375,
"KL/std": 178.154541015625,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.0028289747424423695,
"fcm_dpo/delta": 0.028688903898000717,
"fcm_dpo/margin": 131.34539794921875,
"fcm_dpo/q_t": 0.41740673780441284,
"grad_norm": 27.60743522644043,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.3705713748931885,
"logits/rejected": -0.34871238470077515,
"logps/chosen": -324.1011962890625,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -477.7274169921875,
"loss": 1.1241,
"margin_dpo/margin_mean": 131.34539794921875,
"margin_dpo/margin_std": 205.223876953125,
"step": 282
},
{
"KL/chosen_KL_mean": -265.2757263183594,
"KL/mean": -361.2529296875,
"KL/rejected_KL_mean": -457.23016357421875,
"KL/std": 214.1881103515625,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.002800577785819769,
"fcm_dpo/delta": -0.14126329123973846,
"fcm_dpo/margin": 191.95440673828125,
"fcm_dpo/q_t": 0.37666594982147217,
"grad_norm": 34.12551498413086,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.3212432265281677,
"logits/rejected": -0.32373398542404175,
"logps/chosen": -327.1741638183594,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -554.2166748046875,
"loss": 0.9907,
"margin_dpo/margin_mean": 191.9544219970703,
"margin_dpo/margin_std": 196.35537719726562,
"step": 283
},
{
"KL/chosen_KL_mean": -247.26083374023438,
"KL/mean": -337.656982421875,
"KL/rejected_KL_mean": -428.0531005859375,
"KL/std": 198.7021484375,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.0027607777155935764,
"fcm_dpo/delta": -0.10182726383209229,
"fcm_dpo/margin": 180.79226684570312,
"fcm_dpo/q_t": 0.38608792424201965,
"grad_norm": 41.67982864379883,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.3618922829627991,
"logits/rejected": -0.3465798795223236,
"logps/chosen": -305.6963806152344,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -521.5223999023438,
"loss": 1.0107,
"margin_dpo/margin_mean": 180.79226684570312,
"margin_dpo/margin_std": 191.53631591796875,
"step": 284
},
{
"KL/chosen_KL_mean": -280.42071533203125,
"KL/mean": -361.66033935546875,
"KL/rejected_KL_mean": -442.8999938964844,
"KL/std": 175.34622192382812,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.002754541113972664,
"fcm_dpo/delta": -0.048833686858415604,
"fcm_dpo/margin": 162.47930908203125,
"fcm_dpo/q_t": 0.3953213095664978,
"grad_norm": 26.926042556762695,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.3326127529144287,
"logits/rejected": -0.31620490550994873,
"logps/chosen": -346.65289306640625,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -542.02685546875,
"loss": 1.0516,
"margin_dpo/margin_mean": 162.47930908203125,
"margin_dpo/margin_std": 184.34237670898438,
"step": 285
},
{
"KL/chosen_KL_mean": -277.4112548828125,
"KL/mean": -369.2188720703125,
"KL/rejected_KL_mean": -461.0264892578125,
"KL/std": 199.40582275390625,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.0027244749944657087,
"fcm_dpo/delta": -0.10277135670185089,
"fcm_dpo/margin": 183.615234375,
"fcm_dpo/q_t": 0.38644248247146606,
"grad_norm": 28.425804138183594,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.3559093475341797,
"logits/rejected": -0.3347648084163666,
"logps/chosen": -350.36224365234375,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -549.6149291992188,
"loss": 1.0286,
"margin_dpo/margin_mean": 183.615234375,
"margin_dpo/margin_std": 217.96173095703125,
"step": 286
},
{
"KL/chosen_KL_mean": -261.5997314453125,
"KL/mean": -335.04229736328125,
"KL/rejected_KL_mean": -408.4848327636719,
"KL/std": 173.95217895507812,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.002716212999075651,
"fcm_dpo/delta": 0.0010463353246450424,
"fcm_dpo/margin": 146.88510131835938,
"fcm_dpo/q_t": 0.4084421694278717,
"grad_norm": 26.484556198120117,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.3276909589767456,
"logits/rejected": -0.30225396156311035,
"logps/chosen": -323.140869140625,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -486.180908203125,
"loss": 1.0875,
"margin_dpo/margin_mean": 146.88510131835938,
"margin_dpo/margin_std": 189.57159423828125,
"step": 287
},
{
"KL/chosen_KL_mean": -250.09185791015625,
"KL/mean": -342.5625,
"KL/rejected_KL_mean": -435.0331726074219,
"KL/std": 174.55943298339844,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.0026842262595891953,
"fcm_dpo/delta": -0.09998185932636261,
"fcm_dpo/margin": 184.9413299560547,
"fcm_dpo/q_t": 0.38463830947875977,
"grad_norm": 21.34337043762207,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.3534676432609558,
"logits/rejected": -0.3342832326889038,
"logps/chosen": -306.7530517578125,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -522.368896484375,
"loss": 1.0087,
"margin_dpo/margin_mean": 184.94134521484375,
"margin_dpo/margin_std": 177.844970703125,
"step": 288
},
{
"KL/chosen_KL_mean": -247.0579833984375,
"KL/mean": -350.28240966796875,
"KL/rejected_KL_mean": -453.50677490234375,
"KL/std": 198.4517822265625,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.002661726903170347,
"fcm_dpo/delta": -0.15359723567962646,
"fcm_dpo/margin": 206.44882202148438,
"fcm_dpo/q_t": 0.3754338324069977,
"grad_norm": 26.822708129882812,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.2866683602333069,
"logits/rejected": -0.28309381008148193,
"logps/chosen": -292.28839111328125,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -541.1494140625,
"loss": 0.9846,
"margin_dpo/margin_mean": 206.44879150390625,
"margin_dpo/margin_std": 213.51644897460938,
"step": 289
},
{
"KL/chosen_KL_mean": -268.5120849609375,
"KL/mean": -371.7317810058594,
"KL/rejected_KL_mean": -474.95147705078125,
"KL/std": 210.44107055664062,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.002616926096379757,
"fcm_dpo/delta": -0.14428411424160004,
"fcm_dpo/margin": 206.43942260742188,
"fcm_dpo/q_t": 0.3804370164871216,
"grad_norm": 24.522186279296875,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.2609245181083679,
"logits/rejected": -0.2813086211681366,
"logps/chosen": -323.98358154296875,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -591.6600341796875,
"loss": 1.0094,
"margin_dpo/margin_mean": 206.43942260742188,
"margin_dpo/margin_std": 247.92044067382812,
"step": 290
},
{
"KL/chosen_KL_mean": -230.06326293945312,
"KL/mean": -323.6925048828125,
"KL/rejected_KL_mean": -417.32177734375,
"KL/std": 181.36814880371094,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.0025994423776865005,
"fcm_dpo/delta": -0.08899101614952087,
"fcm_dpo/margin": 187.25848388671875,
"fcm_dpo/q_t": 0.3880588114261627,
"grad_norm": 31.68715476989746,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.3303547501564026,
"logits/rejected": -0.31436848640441895,
"logps/chosen": -293.3236083984375,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -506.61883544921875,
"loss": 1.0307,
"margin_dpo/margin_mean": 187.25848388671875,
"margin_dpo/margin_std": 211.5198974609375,
"step": 291
},
{
"KL/chosen_KL_mean": -261.43878173828125,
"KL/mean": -362.51861572265625,
"KL/rejected_KL_mean": -463.598388671875,
"KL/std": 217.88809204101562,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0025680752005428076,
"fcm_dpo/delta": -0.12225315719842911,
"fcm_dpo/margin": 202.15960693359375,
"fcm_dpo/q_t": 0.382857084274292,
"grad_norm": 29.966228485107422,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.3206707537174225,
"logits/rejected": -0.3061414361000061,
"logps/chosen": -315.3572998046875,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -553.559814453125,
"loss": 1.0308,
"margin_dpo/margin_mean": 202.15960693359375,
"margin_dpo/margin_std": 255.03884887695312,
"step": 292
},
{
"KL/chosen_KL_mean": -266.5294189453125,
"KL/mean": -346.7402038574219,
"KL/rejected_KL_mean": -426.95098876953125,
"KL/std": 211.74429321289062,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.002555608283728361,
"fcm_dpo/delta": -0.010212033987045288,
"fcm_dpo/margin": 160.42156982421875,
"fcm_dpo/q_t": 0.40869617462158203,
"grad_norm": 45.290653228759766,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.3368123769760132,
"logits/rejected": -0.32695645093917847,
"logps/chosen": -326.90545654296875,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -504.8034362792969,
"loss": 1.1009,
"margin_dpo/margin_mean": 160.4215850830078,
"margin_dpo/margin_std": 237.15798950195312,
"step": 293
},
{
"KL/chosen_KL_mean": -252.91036987304688,
"KL/mean": -340.8631591796875,
"KL/rejected_KL_mean": -428.81591796875,
"KL/std": 192.0465087890625,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.0025399108417332172,
"fcm_dpo/delta": -0.047990452498197556,
"fcm_dpo/margin": 175.9055633544922,
"fcm_dpo/q_t": 0.400339812040329,
"grad_norm": 29.165407180786133,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.3062417805194855,
"logits/rejected": -0.3089578151702881,
"logps/chosen": -300.9979248046875,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -510.7129211425781,
"loss": 1.0685,
"margin_dpo/margin_mean": 175.9055633544922,
"margin_dpo/margin_std": 233.61227416992188,
"step": 294
},
{
"KL/chosen_KL_mean": -301.30224609375,
"KL/mean": -397.0389404296875,
"KL/rejected_KL_mean": -492.775634765625,
"KL/std": 212.322265625,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.002525617368519306,
"fcm_dpo/delta": -0.08562206476926804,
"fcm_dpo/margin": 191.47335815429688,
"fcm_dpo/q_t": 0.3909651041030884,
"grad_norm": 27.463563919067383,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.2896770238876343,
"logits/rejected": -0.290955126285553,
"logps/chosen": -351.2268981933594,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -580.23193359375,
"loss": 1.0469,
"margin_dpo/margin_mean": 191.47335815429688,
"margin_dpo/margin_std": 241.90957641601562,
"step": 295
},
{
"KL/chosen_KL_mean": -359.9180603027344,
"KL/mean": -426.7522277832031,
"KL/rejected_KL_mean": -493.5863952636719,
"KL/std": 202.1461181640625,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.002509201178327203,
"fcm_dpo/delta": -0.03729523718357086,
"fcm_dpo/margin": 133.6683349609375,
"fcm_dpo/q_t": 0.4235140383243561,
"grad_norm": 45.111167907714844,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.3604152202606201,
"logits/rejected": -0.3418993353843689,
"logps/chosen": -425.4093017578125,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -586.6754760742188,
"loss": 1.1758,
"margin_dpo/margin_mean": 133.6683349609375,
"margin_dpo/margin_std": 252.33218383789062,
"step": 296
},
{
"KL/chosen_KL_mean": -313.9520568847656,
"KL/mean": -391.02191162109375,
"KL/rejected_KL_mean": -468.09173583984375,
"KL/std": 195.4499969482422,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.002508984412997961,
"fcm_dpo/delta": 0.013493766076862812,
"fcm_dpo/margin": 154.13967895507812,
"fcm_dpo/q_t": 0.41029807925224304,
"grad_norm": 49.82147979736328,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.3424445390701294,
"logits/rejected": -0.3403598368167877,
"logps/chosen": -370.42901611328125,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -563.2302856445312,
"loss": 1.0935,
"margin_dpo/margin_mean": 154.13967895507812,
"margin_dpo/margin_std": 194.82879638671875,
"step": 297
},
{
"KL/chosen_KL_mean": -310.0024719238281,
"KL/mean": -410.1289978027344,
"KL/rejected_KL_mean": -510.2554931640625,
"KL/std": 235.77732849121094,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.0024994537234306335,
"fcm_dpo/delta": -0.10309357941150665,
"fcm_dpo/margin": 200.2530517578125,
"fcm_dpo/q_t": 0.3903202712535858,
"grad_norm": 27.238431930541992,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.3699042797088623,
"logits/rejected": -0.3843574523925781,
"logps/chosen": -377.32763671875,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -626.917724609375,
"loss": 1.0457,
"margin_dpo/margin_mean": 200.25302124023438,
"margin_dpo/margin_std": 266.128662109375,
"step": 298
},
{
"KL/chosen_KL_mean": -259.36639404296875,
"KL/mean": -347.03094482421875,
"KL/rejected_KL_mean": -434.695556640625,
"KL/std": 185.60708618164062,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.0024865521118044853,
"fcm_dpo/delta": -0.03695205599069595,
"fcm_dpo/margin": 175.32911682128906,
"fcm_dpo/q_t": 0.40157145261764526,
"grad_norm": 34.25696563720703,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.36456483602523804,
"logits/rejected": -0.36391931772232056,
"logps/chosen": -308.3284912109375,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -519.0238037109375,
"loss": 1.0759,
"margin_dpo/margin_mean": 175.3291015625,
"margin_dpo/margin_std": 231.766357421875,
"step": 299
},
{
"KL/chosen_KL_mean": -315.7066650390625,
"KL/mean": -422.8812255859375,
"KL/rejected_KL_mean": -530.0557861328125,
"KL/std": 226.07566833496094,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0024577926378697157,
"fcm_dpo/delta": -0.1301524043083191,
"fcm_dpo/margin": 214.34909057617188,
"fcm_dpo/q_t": 0.3810431957244873,
"grad_norm": 35.657649993896484,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.35023725032806396,
"logits/rejected": -0.34927883744239807,
"logps/chosen": -374.7803955078125,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -626.022216796875,
"loss": 1.0246,
"margin_dpo/margin_mean": 214.34910583496094,
"margin_dpo/margin_std": 262.06231689453125,
"step": 300
},
{
"KL/chosen_KL_mean": -261.52374267578125,
"KL/mean": -361.13104248046875,
"KL/rejected_KL_mean": -460.73834228515625,
"KL/std": 200.12277221679688,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.002426966093480587,
"fcm_dpo/delta": -0.0857255756855011,
"fcm_dpo/margin": 199.21456909179688,
"fcm_dpo/q_t": 0.3913128077983856,
"grad_norm": 29.40239906311035,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.38150107860565186,
"logits/rejected": -0.3794858753681183,
"logps/chosen": -318.77313232421875,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -553.0919189453125,
"loss": 1.0345,
"margin_dpo/margin_mean": 199.21456909179688,
"margin_dpo/margin_std": 237.46759033203125,
"step": 301
},
{
"KL/chosen_KL_mean": -216.02481079101562,
"KL/mean": -303.01495361328125,
"KL/rejected_KL_mean": -390.005126953125,
"KL/std": 180.0056915283203,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.0024160263128578663,
"fcm_dpo/delta": -0.020986871793866158,
"fcm_dpo/margin": 173.98028564453125,
"fcm_dpo/q_t": 0.4036945402622223,
"grad_norm": 19.06124496459961,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.3539729118347168,
"logits/rejected": -0.3578333854675293,
"logps/chosen": -267.2228088378906,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -487.2314758300781,
"loss": 1.0715,
"margin_dpo/margin_mean": 173.98028564453125,
"margin_dpo/margin_std": 212.90484619140625,
"step": 302
},
{
"KL/chosen_KL_mean": -238.58770751953125,
"KL/mean": -316.30780029296875,
"KL/rejected_KL_mean": -394.02789306640625,
"KL/std": 194.1252899169922,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.002420827280730009,
"fcm_dpo/delta": 0.024122366681694984,
"fcm_dpo/margin": 155.44015502929688,
"fcm_dpo/q_t": 0.4143924117088318,
"grad_norm": 24.42957305908203,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.37989115715026855,
"logits/rejected": -0.3611317574977875,
"logps/chosen": -305.3016357421875,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -480.97332763671875,
"loss": 1.1184,
"margin_dpo/margin_mean": 155.440185546875,
"margin_dpo/margin_std": 229.31715393066406,
"step": 303
},
{
"KL/chosen_KL_mean": -208.25198364257812,
"KL/mean": -296.2002258300781,
"KL/rejected_KL_mean": -384.1484375,
"KL/std": 177.62567138671875,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.002425071783363819,
"fcm_dpo/delta": -0.027229398488998413,
"fcm_dpo/margin": 175.89645385742188,
"fcm_dpo/q_t": 0.39837926626205444,
"grad_norm": 29.601119995117188,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.3989083766937256,
"logits/rejected": -0.3849946856498718,
"logps/chosen": -280.20269775390625,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -474.6204833984375,
"loss": 1.0409,
"margin_dpo/margin_mean": 175.89645385742188,
"margin_dpo/margin_std": 156.81117248535156,
"step": 304
},
{
"KL/chosen_KL_mean": -231.21951293945312,
"KL/mean": -307.9289855957031,
"KL/rejected_KL_mean": -384.6384582519531,
"KL/std": 201.5638885498047,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.002417425625026226,
"fcm_dpo/delta": 0.02943410724401474,
"fcm_dpo/margin": 153.41891479492188,
"fcm_dpo/q_t": 0.41627079248428345,
"grad_norm": 27.138019561767578,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.41401392221450806,
"logits/rejected": -0.39217787981033325,
"logps/chosen": -298.0147705078125,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -477.39306640625,
"loss": 1.1181,
"margin_dpo/margin_mean": 153.41891479492188,
"margin_dpo/margin_std": 221.77337646484375,
"step": 305
},
{
"KL/chosen_KL_mean": -229.625244140625,
"KL/mean": -305.220947265625,
"KL/rejected_KL_mean": -380.8166198730469,
"KL/std": 177.791015625,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.0024337819777429104,
"fcm_dpo/delta": 0.03258597105741501,
"fcm_dpo/margin": 151.19137573242188,
"fcm_dpo/q_t": 0.4141908884048462,
"grad_norm": 26.584671020507812,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.35081660747528076,
"logits/rejected": -0.3195233941078186,
"logps/chosen": -299.30914306640625,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -465.9757995605469,
"loss": 1.0999,
"margin_dpo/margin_mean": 151.19139099121094,
"margin_dpo/margin_std": 187.10317993164062,
"step": 306
},
{
"KL/chosen_KL_mean": -215.2943572998047,
"KL/mean": -296.1317138671875,
"KL/rejected_KL_mean": -376.96905517578125,
"KL/std": 165.89390563964844,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0024325139820575714,
"fcm_dpo/delta": 0.00681762769818306,
"fcm_dpo/margin": 161.6746826171875,
"fcm_dpo/q_t": 0.4085942506790161,
"grad_norm": 25.906414031982422,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.3687888979911804,
"logits/rejected": -0.34732580184936523,
"logps/chosen": -285.45977783203125,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -463.94134521484375,
"loss": 1.0872,
"margin_dpo/margin_mean": 161.6746826171875,
"margin_dpo/margin_std": 200.76708984375,
"step": 307
},
{
"KL/chosen_KL_mean": -225.435546875,
"KL/mean": -316.33099365234375,
"KL/rejected_KL_mean": -407.2264404296875,
"KL/std": 194.58316040039062,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.0024283677339553833,
"fcm_dpo/delta": -0.04238416999578476,
"fcm_dpo/margin": 181.79086303710938,
"fcm_dpo/q_t": 0.3963702917098999,
"grad_norm": 29.603750228881836,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.34480029344558716,
"logits/rejected": -0.3308371305465698,
"logps/chosen": -280.6805419921875,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -486.59869384765625,
"loss": 1.0408,
"margin_dpo/margin_mean": 181.79086303710938,
"margin_dpo/margin_std": 184.86737060546875,
"step": 308
},
{
"KL/chosen_KL_mean": -229.71560668945312,
"KL/mean": -321.12689208984375,
"KL/rejected_KL_mean": -412.5382080078125,
"KL/std": 200.42385864257812,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.0024233111180365086,
"fcm_dpo/delta": -0.04414871707558632,
"fcm_dpo/margin": 182.82260131835938,
"fcm_dpo/q_t": 0.3967331647872925,
"grad_norm": 40.203651428222656,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.30514609813690186,
"logits/rejected": -0.2906920909881592,
"logps/chosen": -278.6746826171875,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -494.8789367675781,
"loss": 1.0527,
"margin_dpo/margin_mean": 182.82260131835938,
"margin_dpo/margin_std": 206.30902099609375,
"step": 309
},
{
"KL/chosen_KL_mean": -269.08026123046875,
"KL/mean": -356.98431396484375,
"KL/rejected_KL_mean": -444.88836669921875,
"KL/std": 182.12594604492188,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.00241499999538064,
"fcm_dpo/delta": -0.02522626519203186,
"fcm_dpo/margin": 175.80810546875,
"fcm_dpo/q_t": 0.3999808430671692,
"grad_norm": 25.993465423583984,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.37713727355003357,
"logits/rejected": -0.3597584366798401,
"logps/chosen": -331.8220520019531,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -524.8186645507812,
"loss": 1.0501,
"margin_dpo/margin_mean": 175.80810546875,
"margin_dpo/margin_std": 173.99118041992188,
"step": 310
},
{
"KL/chosen_KL_mean": -291.9866943359375,
"KL/mean": -389.12078857421875,
"KL/rejected_KL_mean": -486.2549133300781,
"KL/std": 237.48434448242188,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.0023961286060512066,
"fcm_dpo/delta": -0.06706373393535614,
"fcm_dpo/margin": 194.26821899414062,
"fcm_dpo/q_t": 0.39537882804870605,
"grad_norm": 25.610626220703125,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.33371835947036743,
"logits/rejected": -0.3037863075733185,
"logps/chosen": -345.0146484375,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -563.693115234375,
"loss": 1.0552,
"margin_dpo/margin_mean": 194.26821899414062,
"margin_dpo/margin_std": 249.4111328125,
"step": 311
},
{
"KL/chosen_KL_mean": -284.6394348144531,
"KL/mean": -373.61962890625,
"KL/rejected_KL_mean": -462.599853515625,
"KL/std": 207.23092651367188,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.002392760245129466,
"fcm_dpo/delta": -0.0265023373067379,
"fcm_dpo/margin": 177.96038818359375,
"fcm_dpo/q_t": 0.40469247102737427,
"grad_norm": 26.777978897094727,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.29555267095565796,
"logits/rejected": -0.2810285985469818,
"logps/chosen": -334.0316467285156,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -538.3926391601562,
"loss": 1.0824,
"margin_dpo/margin_mean": 177.96038818359375,
"margin_dpo/margin_std": 244.01686096191406,
"step": 312
},
{
"KL/chosen_KL_mean": -274.753662109375,
"KL/mean": -365.9559326171875,
"KL/rejected_KL_mean": -457.1582336425781,
"KL/std": 222.33108520507812,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.0023880554363131523,
"fcm_dpo/delta": -0.03676465526223183,
"fcm_dpo/margin": 182.40457153320312,
"fcm_dpo/q_t": 0.4009496569633484,
"grad_norm": 25.21873664855957,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.35764968395233154,
"logits/rejected": -0.3531670570373535,
"logps/chosen": -324.90643310546875,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -543.564453125,
"loss": 1.0789,
"margin_dpo/margin_mean": 182.40457153320312,
"margin_dpo/margin_std": 242.63485717773438,
"step": 313
},
{
"KL/chosen_KL_mean": -293.580810546875,
"KL/mean": -380.2122802734375,
"KL/rejected_KL_mean": -466.84375,
"KL/std": 214.81533813476562,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0023697358556091785,
"fcm_dpo/delta": -0.011065401136875153,
"fcm_dpo/margin": 173.26296997070312,
"fcm_dpo/q_t": 0.4087506830692291,
"grad_norm": 21.553348541259766,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.3941684067249298,
"logits/rejected": -0.39253175258636475,
"logps/chosen": -350.818359375,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -564.4403076171875,
"loss": 1.1109,
"margin_dpo/margin_mean": 173.26296997070312,
"margin_dpo/margin_std": 267.32855224609375,
"step": 314
},
{
"KL/chosen_KL_mean": -272.3924865722656,
"KL/mean": -351.9873962402344,
"KL/rejected_KL_mean": -431.582275390625,
"KL/std": 185.17767333984375,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.0023773042485117912,
"fcm_dpo/delta": 0.02197282202541828,
"fcm_dpo/margin": 159.18983459472656,
"fcm_dpo/q_t": 0.4126739203929901,
"grad_norm": 25.598684310913086,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.27878469228744507,
"logits/rejected": -0.28102076053619385,
"logps/chosen": -321.46209716796875,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -517.26318359375,
"loss": 1.0996,
"margin_dpo/margin_mean": 159.18984985351562,
"margin_dpo/margin_std": 206.75869750976562,
"step": 315
},
{
"KL/chosen_KL_mean": -262.9434509277344,
"KL/mean": -372.22076416015625,
"KL/rejected_KL_mean": -481.4980773925781,
"KL/std": 218.68289184570312,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.0023532514460384846,
"fcm_dpo/delta": -0.11787203699350357,
"fcm_dpo/margin": 218.55462646484375,
"fcm_dpo/q_t": 0.38381052017211914,
"grad_norm": 26.231597900390625,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.35521891713142395,
"logits/rejected": -0.35822099447250366,
"logps/chosen": -317.2041931152344,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -582.779541015625,
"loss": 1.0101,
"margin_dpo/margin_mean": 218.55462646484375,
"margin_dpo/margin_std": 240.36514282226562,
"step": 316
},
{
"KL/chosen_KL_mean": -265.1885681152344,
"KL/mean": -386.59716796875,
"KL/rejected_KL_mean": -508.0057373046875,
"KL/std": 205.4729461669922,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.0023275299463421106,
"fcm_dpo/delta": -0.1697898656129837,
"fcm_dpo/margin": 242.81719970703125,
"fcm_dpo/q_t": 0.37069547176361084,
"grad_norm": 29.441675186157227,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.361335813999176,
"logits/rejected": -0.34446266293525696,
"logps/chosen": -321.28277587890625,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -608.704833984375,
"loss": 0.9674,
"margin_dpo/margin_mean": 242.81719970703125,
"margin_dpo/margin_std": 227.80020141601562,
"step": 317
},
{
"KL/chosen_KL_mean": -276.64666748046875,
"KL/mean": -368.34423828125,
"KL/rejected_KL_mean": -460.04180908203125,
"KL/std": 210.08200073242188,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0023121268022805452,
"fcm_dpo/delta": -0.024627672508358955,
"fcm_dpo/margin": 183.39511108398438,
"fcm_dpo/q_t": 0.40299540758132935,
"grad_norm": 28.099123001098633,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.4194433093070984,
"logits/rejected": -0.38650479912757874,
"logps/chosen": -341.2923583984375,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -542.8060302734375,
"loss": 1.0716,
"margin_dpo/margin_mean": 183.3951416015625,
"margin_dpo/margin_std": 228.771728515625,
"step": 318
},
{
"KL/chosen_KL_mean": -248.31747436523438,
"KL/mean": -355.7877197265625,
"KL/rejected_KL_mean": -463.2579345703125,
"KL/std": 224.69723510742188,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0022909443359822035,
"fcm_dpo/delta": -0.09473305940628052,
"fcm_dpo/margin": 214.94046020507812,
"fcm_dpo/q_t": 0.38672956824302673,
"grad_norm": 27.515636444091797,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.365750253200531,
"logits/rejected": -0.37738245725631714,
"logps/chosen": -297.70123291015625,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -577.1644287109375,
"loss": 1.0225,
"margin_dpo/margin_mean": 214.94046020507812,
"margin_dpo/margin_std": 239.97752380371094,
"step": 319
},
{
"KL/chosen_KL_mean": -263.43206787109375,
"KL/mean": -381.2987365722656,
"KL/rejected_KL_mean": -499.1654052734375,
"KL/std": 234.49795532226562,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0022584707476198673,
"fcm_dpo/delta": -0.13631628453731537,
"fcm_dpo/margin": 235.73333740234375,
"fcm_dpo/q_t": 0.3790479302406311,
"grad_norm": 35.816165924072266,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.35946518182754517,
"logits/rejected": -0.36113548278808594,
"logps/chosen": -322.93695068359375,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -596.83251953125,
"loss": 0.9952,
"margin_dpo/margin_mean": 235.73333740234375,
"margin_dpo/margin_std": 250.2997589111328,
"step": 320
},
{
"KL/chosen_KL_mean": -316.8851318359375,
"KL/mean": -419.2160339355469,
"KL/rejected_KL_mean": -521.546875,
"KL/std": 239.943115234375,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.002237518085166812,
"fcm_dpo/delta": -0.059808555990457535,
"fcm_dpo/margin": 204.66180419921875,
"fcm_dpo/q_t": 0.39591050148010254,
"grad_norm": 24.665813446044922,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.3208162784576416,
"logits/rejected": -0.3067053556442261,
"logps/chosen": -378.433837890625,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -613.18798828125,
"loss": 1.0571,
"margin_dpo/margin_mean": 204.6617889404297,
"margin_dpo/margin_std": 253.84730529785156,
"step": 321
},
{
"KL/chosen_KL_mean": -280.55859375,
"KL/mean": -392.18109130859375,
"KL/rejected_KL_mean": -503.8035583496094,
"KL/std": 217.13014221191406,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.0022206236608326435,
"fcm_dpo/delta": -0.09844300150871277,
"fcm_dpo/margin": 223.24496459960938,
"fcm_dpo/q_t": 0.3849971294403076,
"grad_norm": 25.391277313232422,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.33665627241134644,
"logits/rejected": -0.33168351650238037,
"logps/chosen": -337.8486328125,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -599.553466796875,
"loss": 1.0101,
"margin_dpo/margin_mean": 223.24496459960938,
"margin_dpo/margin_std": 225.46527099609375,
"step": 322
},
{
"KL/chosen_KL_mean": -303.197509765625,
"KL/mean": -397.31976318359375,
"KL/rejected_KL_mean": -491.4420166015625,
"KL/std": 215.2032470703125,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.0022140834480524063,
"fcm_dpo/delta": -0.017185624688863754,
"fcm_dpo/margin": 188.2445068359375,
"fcm_dpo/q_t": 0.4033932685852051,
"grad_norm": 35.27677536010742,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.3552204370498657,
"logits/rejected": -0.3479047417640686,
"logps/chosen": -354.4314880371094,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -566.50390625,
"loss": 1.0792,
"margin_dpo/margin_mean": 188.2445068359375,
"margin_dpo/margin_std": 236.12969970703125,
"step": 323
},
{
"KL/chosen_KL_mean": -351.04327392578125,
"KL/mean": -444.7715759277344,
"KL/rejected_KL_mean": -538.4998779296875,
"KL/std": 236.59896850585938,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.0022205086424946785,
"fcm_dpo/delta": -0.01711501181125641,
"fcm_dpo/margin": 187.4566192626953,
"fcm_dpo/q_t": 0.4070439636707306,
"grad_norm": 41.267127990722656,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.3706352710723877,
"logits/rejected": -0.35114365816116333,
"logps/chosen": -416.1784362792969,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -624.9774169921875,
"loss": 1.1032,
"margin_dpo/margin_mean": 187.4566192626953,
"margin_dpo/margin_std": 269.1782531738281,
"step": 324
},
{
"KL/chosen_KL_mean": -282.0893249511719,
"KL/mean": -358.64837646484375,
"KL/rejected_KL_mean": -435.2073974609375,
"KL/std": 205.12814331054688,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.0022224104031920433,
"fcm_dpo/delta": 0.06069522351026535,
"fcm_dpo/margin": 153.11805725097656,
"fcm_dpo/q_t": 0.4203560948371887,
"grad_norm": 36.374202728271484,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.4578849673271179,
"logits/rejected": -0.43039295077323914,
"logps/chosen": -338.304931640625,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -505.2933044433594,
"loss": 1.1272,
"margin_dpo/margin_mean": 153.11807250976562,
"margin_dpo/margin_std": 203.81394958496094,
"step": 325
},
{
"KL/chosen_KL_mean": -320.332763671875,
"KL/mean": -397.1983642578125,
"KL/rejected_KL_mean": -474.06396484375,
"KL/std": 202.8357696533203,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0022304444573819637,
"fcm_dpo/delta": 0.0580889992415905,
"fcm_dpo/margin": 153.73118591308594,
"fcm_dpo/q_t": 0.41890084743499756,
"grad_norm": 45.461204528808594,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.44517433643341064,
"logits/rejected": -0.4201740622520447,
"logps/chosen": -393.0577087402344,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -553.9107666015625,
"loss": 1.1188,
"margin_dpo/margin_mean": 153.731201171875,
"margin_dpo/margin_std": 192.40139770507812,
"step": 326
},
{
"KL/chosen_KL_mean": -286.7748107910156,
"KL/mean": -405.4556884765625,
"KL/rejected_KL_mean": -524.1365356445312,
"KL/std": 219.92962646484375,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0022127358242869377,
"fcm_dpo/delta": -0.1288842260837555,
"fcm_dpo/margin": 237.36172485351562,
"fcm_dpo/q_t": 0.3784708082675934,
"grad_norm": 38.16813659667969,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.4000493884086609,
"logits/rejected": -0.38037049770355225,
"logps/chosen": -355.90924072265625,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -636.0703125,
"loss": 0.9955,
"margin_dpo/margin_mean": 237.36172485351562,
"margin_dpo/margin_std": 241.5782470703125,
"step": 327
},
{
"KL/chosen_KL_mean": -305.38720703125,
"KL/mean": -413.01141357421875,
"KL/rejected_KL_mean": -520.6356201171875,
"KL/std": 244.02554321289062,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0021971408277750015,
"fcm_dpo/delta": -0.07468114793300629,
"fcm_dpo/margin": 215.24847412109375,
"fcm_dpo/q_t": 0.39337801933288574,
"grad_norm": 28.978845596313477,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.369107723236084,
"logits/rejected": -0.367361843585968,
"logps/chosen": -365.07440185546875,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -611.4906616210938,
"loss": 1.0456,
"margin_dpo/margin_mean": 215.24847412109375,
"margin_dpo/margin_std": 267.3666687011719,
"step": 328
},
{
"KL/chosen_KL_mean": -340.37298583984375,
"KL/mean": -448.5474853515625,
"KL/rejected_KL_mean": -556.7220458984375,
"KL/std": 253.79428100585938,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0021742871031165123,
"fcm_dpo/delta": -0.07264669984579086,
"fcm_dpo/margin": 216.34902954101562,
"fcm_dpo/q_t": 0.39304083585739136,
"grad_norm": 33.51917266845703,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.37974807620048523,
"logits/rejected": -0.3639347553253174,
"logps/chosen": -405.61920166015625,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -657.4197387695312,
"loss": 1.0604,
"margin_dpo/margin_mean": 216.34902954101562,
"margin_dpo/margin_std": 278.5408630371094,
"step": 329
},
{
"KL/chosen_KL_mean": -288.67608642578125,
"KL/mean": -402.93505859375,
"KL/rejected_KL_mean": -517.1940307617188,
"KL/std": 241.49658203125,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0021610576659440994,
"fcm_dpo/delta": -0.09621290862560272,
"fcm_dpo/margin": 228.5179443359375,
"fcm_dpo/q_t": 0.38732674717903137,
"grad_norm": 28.922462463378906,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.36659538745880127,
"logits/rejected": -0.3661782145500183,
"logps/chosen": -335.6744384765625,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -604.0708618164062,
"loss": 1.0234,
"margin_dpo/margin_mean": 228.5179443359375,
"margin_dpo/margin_std": 258.7063903808594,
"step": 330
},
{
"KL/chosen_KL_mean": -310.08087158203125,
"KL/mean": -423.1429443359375,
"KL/rejected_KL_mean": -536.2050170898438,
"KL/std": 198.00454711914062,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.002141648903489113,
"fcm_dpo/delta": -0.08637881278991699,
"fcm_dpo/margin": 226.1241455078125,
"fcm_dpo/q_t": 0.3862583339214325,
"grad_norm": 28.123804092407227,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.4031466245651245,
"logits/rejected": -0.395566463470459,
"logps/chosen": -360.6051025390625,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -625.220458984375,
"loss": 0.9994,
"margin_dpo/margin_mean": 226.1241455078125,
"margin_dpo/margin_std": 195.76531982421875,
"step": 331
},
{
"KL/chosen_KL_mean": -316.3252258300781,
"KL/mean": -394.60198974609375,
"KL/rejected_KL_mean": -472.87872314453125,
"KL/std": 199.60614013671875,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.002148597501218319,
"fcm_dpo/delta": 0.06467652320861816,
"fcm_dpo/margin": 156.5535125732422,
"fcm_dpo/q_t": 0.42228198051452637,
"grad_norm": 25.59779930114746,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.3974687159061432,
"logits/rejected": -0.37797918915748596,
"logps/chosen": -365.5054931640625,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -549.3638916015625,
"loss": 1.1305,
"margin_dpo/margin_mean": 156.55352783203125,
"margin_dpo/margin_std": 220.91140747070312,
"step": 332
},
{
"KL/chosen_KL_mean": -325.0455627441406,
"KL/mean": -431.7874450683594,
"KL/rejected_KL_mean": -538.529296875,
"KL/std": 234.52767944335938,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.0021342344116419554,
"fcm_dpo/delta": -0.057633012533187866,
"fcm_dpo/margin": 213.48370361328125,
"fcm_dpo/q_t": 0.3971046805381775,
"grad_norm": 30.5307674407959,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.4062424302101135,
"logits/rejected": -0.3987647593021393,
"logps/chosen": -388.80133056640625,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -633.5733642578125,
"loss": 1.0699,
"margin_dpo/margin_mean": 213.48370361328125,
"margin_dpo/margin_std": 282.26739501953125,
"step": 333
},
{
"KL/chosen_KL_mean": -314.6381530761719,
"KL/mean": -410.3465270996094,
"KL/rejected_KL_mean": -506.054931640625,
"KL/std": 256.6167907714844,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.0021365054417401552,
"fcm_dpo/delta": -0.00915764831006527,
"fcm_dpo/margin": 191.416748046875,
"fcm_dpo/q_t": 0.4075648784637451,
"grad_norm": 27.895448684692383,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.4134640693664551,
"logits/rejected": -0.40898776054382324,
"logps/chosen": -381.617919921875,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -601.371826171875,
"loss": 1.1071,
"margin_dpo/margin_mean": 191.416748046875,
"margin_dpo/margin_std": 287.8053283691406,
"step": 334
},
{
"KL/chosen_KL_mean": -340.8347473144531,
"KL/mean": -420.40362548828125,
"KL/rejected_KL_mean": -499.97247314453125,
"KL/std": 239.92379760742188,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.00215163454413414,
"fcm_dpo/delta": 0.058172550052404404,
"fcm_dpo/margin": 159.1376953125,
"fcm_dpo/q_t": 0.4224546253681183,
"grad_norm": 31.30469512939453,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.47105371952056885,
"logits/rejected": -0.44997382164001465,
"logps/chosen": -403.3772277832031,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -587.5902099609375,
"loss": 1.147,
"margin_dpo/margin_mean": 159.1376953125,
"margin_dpo/margin_std": 256.5185241699219,
"step": 335
},
{
"KL/chosen_KL_mean": -337.67694091796875,
"KL/mean": -456.9693603515625,
"KL/rejected_KL_mean": -576.26171875,
"KL/std": 266.134033203125,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0021344278939068317,
"fcm_dpo/delta": -0.1120191439986229,
"fcm_dpo/margin": 238.58480834960938,
"fcm_dpo/q_t": 0.3868195414543152,
"grad_norm": 38.22102737426758,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.4278082847595215,
"logits/rejected": -0.4280624985694885,
"logps/chosen": -392.20806884765625,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -680.666015625,
"loss": 1.0426,
"margin_dpo/margin_mean": 238.58480834960938,
"margin_dpo/margin_std": 310.658447265625,
"step": 336
},
{
"KL/chosen_KL_mean": -302.09124755859375,
"KL/mean": -415.0406494140625,
"KL/rejected_KL_mean": -527.9901123046875,
"KL/std": 218.13992309570312,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.0021110186353325844,
"fcm_dpo/delta": -0.0788479745388031,
"fcm_dpo/margin": 225.89886474609375,
"fcm_dpo/q_t": 0.39037024974823,
"grad_norm": 39.11850357055664,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.4359098970890045,
"logits/rejected": -0.4225747585296631,
"logps/chosen": -367.21990966796875,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -629.7171020507812,
"loss": 1.0344,
"margin_dpo/margin_mean": 225.89886474609375,
"margin_dpo/margin_std": 259.23974609375,
"step": 337
},
{
"KL/chosen_KL_mean": -288.96722412109375,
"KL/mean": -404.56036376953125,
"KL/rejected_KL_mean": -520.1534423828125,
"KL/std": 224.17669677734375,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.002094029448926449,
"fcm_dpo/delta": -0.08624735474586487,
"fcm_dpo/margin": 231.18624877929688,
"fcm_dpo/q_t": 0.3876437246799469,
"grad_norm": 39.132667541503906,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.4282386302947998,
"logits/rejected": -0.42397069931030273,
"logps/chosen": -347.3899230957031,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -609.2219848632812,
"loss": 1.016,
"margin_dpo/margin_mean": 231.18624877929688,
"margin_dpo/margin_std": 235.46348571777344,
"step": 338
},
{
"KL/chosen_KL_mean": -302.4131774902344,
"KL/mean": -405.36090087890625,
"KL/rejected_KL_mean": -508.3086242675781,
"KL/std": 225.501708984375,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.00209475914016366,
"fcm_dpo/delta": -0.032657869160175323,
"fcm_dpo/margin": 205.89544677734375,
"fcm_dpo/q_t": 0.4000750184059143,
"grad_norm": 39.0589485168457,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.4808998703956604,
"logits/rejected": -0.4646437168121338,
"logps/chosen": -362.40850830078125,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -612.219482421875,
"loss": 1.0614,
"margin_dpo/margin_mean": 205.89544677734375,
"margin_dpo/margin_std": 227.13043212890625,
"step": 339
},
{
"KL/chosen_KL_mean": -306.40838623046875,
"KL/mean": -403.55328369140625,
"KL/rejected_KL_mean": -500.6982116699219,
"KL/std": 214.56753540039062,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0020795511081814766,
"fcm_dpo/delta": -0.004235394299030304,
"fcm_dpo/margin": 194.28976440429688,
"fcm_dpo/q_t": 0.40789303183555603,
"grad_norm": 32.46416091918945,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.42205601930618286,
"logits/rejected": -0.4012700319290161,
"logps/chosen": -359.2386474609375,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -573.805419921875,
"loss": 1.0937,
"margin_dpo/margin_mean": 194.28977966308594,
"margin_dpo/margin_std": 265.2552185058594,
"step": 340
},
{
"KL/chosen_KL_mean": -299.121826171875,
"KL/mean": -406.19964599609375,
"KL/rejected_KL_mean": -513.2774658203125,
"KL/std": 231.76580810546875,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.0020710937678813934,
"fcm_dpo/delta": -0.044832952320575714,
"fcm_dpo/margin": 214.15560913085938,
"fcm_dpo/q_t": 0.398419588804245,
"grad_norm": 32.579837799072266,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.42187175154685974,
"logits/rejected": -0.4231615364551544,
"logps/chosen": -347.0216369628906,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -615.0873413085938,
"loss": 1.0562,
"margin_dpo/margin_mean": 214.15560913085938,
"margin_dpo/margin_std": 249.23519897460938,
"step": 341
},
{
"KL/chosen_KL_mean": -285.2144470214844,
"KL/mean": -383.55755615234375,
"KL/rejected_KL_mean": -481.900634765625,
"KL/std": 211.0296173095703,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0020756039302796125,
"fcm_dpo/delta": -0.008514024317264557,
"fcm_dpo/margin": 196.68621826171875,
"fcm_dpo/q_t": 0.40535086393356323,
"grad_norm": 29.37921905517578,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.41295289993286133,
"logits/rejected": -0.3835510313510895,
"logps/chosen": -357.2110900878906,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -574.490234375,
"loss": 1.0793,
"margin_dpo/margin_mean": 196.68621826171875,
"margin_dpo/margin_std": 243.93203735351562,
"step": 342
},
{
"KL/chosen_KL_mean": -275.50653076171875,
"KL/mean": -393.09033203125,
"KL/rejected_KL_mean": -510.6741027832031,
"KL/std": 219.71334838867188,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.0020575113594532013,
"fcm_dpo/delta": -0.08598469197750092,
"fcm_dpo/margin": 235.1676025390625,
"fcm_dpo/q_t": 0.38554584980010986,
"grad_norm": 25.824504852294922,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.4432937502861023,
"logits/rejected": -0.4439564347267151,
"logps/chosen": -329.91217041015625,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -621.715576171875,
"loss": 0.9987,
"margin_dpo/margin_mean": 235.16761779785156,
"margin_dpo/margin_std": 193.387451171875,
"step": 343
},
{
"KL/chosen_KL_mean": -278.17584228515625,
"KL/mean": -382.2356262207031,
"KL/rejected_KL_mean": -486.29541015625,
"KL/std": 239.05442810058594,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.00205832626670599,
"fcm_dpo/delta": -0.029630951583385468,
"fcm_dpo/margin": 208.11956787109375,
"fcm_dpo/q_t": 0.40320706367492676,
"grad_norm": 28.157310485839844,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.421103835105896,
"logits/rejected": -0.42124348878860474,
"logps/chosen": -332.1405029296875,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -576.9188232421875,
"loss": 1.066,
"margin_dpo/margin_mean": 208.11956787109375,
"margin_dpo/margin_std": 241.80572509765625,
"step": 344
},
{
"KL/chosen_KL_mean": -327.5186767578125,
"KL/mean": -443.9566650390625,
"KL/rejected_KL_mean": -560.3946533203125,
"KL/std": 230.4110107421875,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.0020382110960781574,
"fcm_dpo/delta": -0.0764290988445282,
"fcm_dpo/margin": 232.87599182128906,
"fcm_dpo/q_t": 0.3901875615119934,
"grad_norm": 24.22003173828125,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.43897050619125366,
"logits/rejected": -0.4347718358039856,
"logps/chosen": -389.2043762207031,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -659.8851318359375,
"loss": 1.0293,
"margin_dpo/margin_mean": 232.87600708007812,
"margin_dpo/margin_std": 254.49136352539062,
"step": 345
},
{
"KL/chosen_KL_mean": -339.9031677246094,
"KL/mean": -448.6493835449219,
"KL/rejected_KL_mean": -557.3956298828125,
"KL/std": 233.10879516601562,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.0020257416181266308,
"fcm_dpo/delta": -0.04150301218032837,
"fcm_dpo/margin": 217.49244689941406,
"fcm_dpo/q_t": 0.3968901038169861,
"grad_norm": 27.246360778808594,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.3788645267486572,
"logits/rejected": -0.3643970489501953,
"logps/chosen": -395.159423828125,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -634.8109130859375,
"loss": 1.0528,
"margin_dpo/margin_mean": 217.49244689941406,
"margin_dpo/margin_std": 247.9464111328125,
"step": 346
},
{
"KL/chosen_KL_mean": -344.7647705078125,
"KL/mean": -448.3440246582031,
"KL/rejected_KL_mean": -551.9232177734375,
"KL/std": 247.40902709960938,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.002020738087594509,
"fcm_dpo/delta": -0.019013745710253716,
"fcm_dpo/margin": 207.15847778320312,
"fcm_dpo/q_t": 0.4043384790420532,
"grad_norm": 29.887693405151367,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.3656623959541321,
"logits/rejected": -0.35582268238067627,
"logps/chosen": -402.3310241699219,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -644.2783203125,
"loss": 1.0832,
"margin_dpo/margin_mean": 207.15847778320312,
"margin_dpo/margin_std": 276.2279052734375,
"step": 347
},
{
"KL/chosen_KL_mean": -311.39959716796875,
"KL/mean": -401.85382080078125,
"KL/rejected_KL_mean": -492.3080749511719,
"KL/std": 215.942626953125,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.002018729457631707,
"fcm_dpo/delta": 0.03512571379542351,
"fcm_dpo/margin": 180.908447265625,
"fcm_dpo/q_t": 0.4162946939468384,
"grad_norm": 29.292268753051758,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.3954634666442871,
"logits/rejected": -0.3925415873527527,
"logps/chosen": -367.7173156738281,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -581.4464111328125,
"loss": 1.1176,
"margin_dpo/margin_mean": 180.90846252441406,
"margin_dpo/margin_std": 251.56272888183594,
"step": 348
},
{
"KL/chosen_KL_mean": -302.69281005859375,
"KL/mean": -409.18768310546875,
"KL/rejected_KL_mean": -515.6825561523438,
"KL/std": 244.12184143066406,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.002026638016104698,
"fcm_dpo/delta": -0.03247387707233429,
"fcm_dpo/margin": 212.98977661132812,
"fcm_dpo/q_t": 0.40324437618255615,
"grad_norm": 24.500629425048828,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.37826380133628845,
"logits/rejected": -0.37416696548461914,
"logps/chosen": -360.71832275390625,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -613.187744140625,
"loss": 1.0867,
"margin_dpo/margin_mean": 212.98977661132812,
"margin_dpo/margin_std": 299.3753662109375,
"step": 349
},
{
"KL/chosen_KL_mean": -322.9959716796875,
"KL/mean": -425.35693359375,
"KL/rejected_KL_mean": -527.7179565429688,
"KL/std": 237.5716552734375,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.002013745252043009,
"fcm_dpo/delta": -0.012782763689756393,
"fcm_dpo/margin": 204.7219696044922,
"fcm_dpo/q_t": 0.40567225217819214,
"grad_norm": 32.45606994628906,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.4236910343170166,
"logits/rejected": -0.4241228699684143,
"logps/chosen": -387.32647705078125,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -617.589599609375,
"loss": 1.1016,
"margin_dpo/margin_mean": 204.72198486328125,
"margin_dpo/margin_std": 298.5418701171875,
"step": 350
},
{
"KL/chosen_KL_mean": -301.8697509765625,
"KL/mean": -426.64111328125,
"KL/rejected_KL_mean": -551.4124755859375,
"KL/std": 277.66021728515625,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0019994243048131466,
"fcm_dpo/delta": -0.10174670815467834,
"fcm_dpo/margin": 249.542724609375,
"fcm_dpo/q_t": 0.39052557945251465,
"grad_norm": 26.109012603759766,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.4383613169193268,
"logits/rejected": -0.434881329536438,
"logps/chosen": -362.5418701171875,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -652.9779052734375,
"loss": 1.047,
"margin_dpo/margin_mean": 249.542724609375,
"margin_dpo/margin_std": 331.46405029296875,
"step": 351
},
{
"KL/chosen_KL_mean": -353.83587646484375,
"KL/mean": -431.2773132324219,
"KL/rejected_KL_mean": -508.7187194824219,
"KL/std": 258.7840576171875,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.002007380360737443,
"fcm_dpo/delta": 0.09052670747041702,
"fcm_dpo/margin": 154.8828887939453,
"fcm_dpo/q_t": 0.4296043813228607,
"grad_norm": 35.03353500366211,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.4138913154602051,
"logits/rejected": -0.381538987159729,
"logps/chosen": -424.7793273925781,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -585.3607177734375,
"loss": 1.1807,
"margin_dpo/margin_mean": 154.8828887939453,
"margin_dpo/margin_std": 290.50494384765625,
"step": 352
},
{
"KL/chosen_KL_mean": -340.01251220703125,
"KL/mean": -443.328857421875,
"KL/rejected_KL_mean": -546.6452026367188,
"KL/std": 257.87725830078125,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.002007407834753394,
"fcm_dpo/delta": -0.015510857105255127,
"fcm_dpo/margin": 206.63265991210938,
"fcm_dpo/q_t": 0.40632784366607666,
"grad_norm": 29.06142234802246,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.46683937311172485,
"logits/rejected": -0.4540403485298157,
"logps/chosen": -398.4078674316406,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -626.980712890625,
"loss": 1.0954,
"margin_dpo/margin_mean": 206.63265991210938,
"margin_dpo/margin_std": 289.2551574707031,
"step": 353
},
{
"KL/chosen_KL_mean": -282.2817687988281,
"KL/mean": -397.5120544433594,
"KL/rejected_KL_mean": -512.7423095703125,
"KL/std": 236.1700439453125,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.0020015356130898,
"fcm_dpo/delta": -0.062798872590065,
"fcm_dpo/margin": 230.4605712890625,
"fcm_dpo/q_t": 0.39260250329971313,
"grad_norm": 30.49873161315918,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.3843461275100708,
"logits/rejected": -0.36748528480529785,
"logps/chosen": -342.08477783203125,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -601.4998779296875,
"loss": 1.0285,
"margin_dpo/margin_mean": 230.4605712890625,
"margin_dpo/margin_std": 233.89678955078125,
"step": 354
},
{
"KL/chosen_KL_mean": -279.9733581542969,
"KL/mean": -393.3092041015625,
"KL/rejected_KL_mean": -506.6451110839844,
"KL/std": 233.89459228515625,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.001989755779504776,
"fcm_dpo/delta": -0.05232168734073639,
"fcm_dpo/margin": 226.67178344726562,
"fcm_dpo/q_t": 0.39414530992507935,
"grad_norm": 42.037132263183594,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.4780992865562439,
"logits/rejected": -0.45962148904800415,
"logps/chosen": -334.10186767578125,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -589.0511474609375,
"loss": 1.0331,
"margin_dpo/margin_mean": 226.67176818847656,
"margin_dpo/margin_std": 225.0481414794922,
"step": 355
},
{
"KL/chosen_KL_mean": -339.6640319824219,
"KL/mean": -402.67962646484375,
"KL/rejected_KL_mean": -465.6952819824219,
"KL/std": 238.7618865966797,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.001991572557017207,
"fcm_dpo/delta": 0.020430708304047585,
"fcm_dpo/margin": 126.03125,
"fcm_dpo/q_t": 0.44326043128967285,
"grad_norm": 35.44585418701172,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.43858081102371216,
"logits/rejected": -0.4161129593849182,
"logps/chosen": -404.33782958984375,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -541.5945434570312,
"loss": 1.2348,
"margin_dpo/margin_mean": 126.03125762939453,
"margin_dpo/margin_std": 294.35003662109375,
"step": 356
},
{
"KL/chosen_KL_mean": -307.77178955078125,
"KL/mean": -404.3070068359375,
"KL/rejected_KL_mean": -500.8422546386719,
"KL/std": 227.6900634765625,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.001992112258449197,
"fcm_dpo/delta": 0.015565093606710434,
"fcm_dpo/margin": 193.07044982910156,
"fcm_dpo/q_t": 0.40995845198631287,
"grad_norm": 27.028295516967773,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.4511488080024719,
"logits/rejected": -0.43502819538116455,
"logps/chosen": -360.49761962890625,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -587.6834106445312,
"loss": 1.0897,
"margin_dpo/margin_mean": 193.07044982910156,
"margin_dpo/margin_std": 232.12307739257812,
"step": 357
},
{
"KL/chosen_KL_mean": -285.88421630859375,
"KL/mean": -376.27392578125,
"KL/rejected_KL_mean": -466.66363525390625,
"KL/std": 219.6377716064453,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0020017188508063555,
"fcm_dpo/delta": 0.0388420931994915,
"fcm_dpo/margin": 180.77944946289062,
"fcm_dpo/q_t": 0.4163302183151245,
"grad_norm": 24.81241798400879,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.4996035397052765,
"logits/rejected": -0.4798714816570282,
"logps/chosen": -349.08966064453125,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -555.0369873046875,
"loss": 1.1092,
"margin_dpo/margin_mean": 180.77944946289062,
"margin_dpo/margin_std": 236.979248046875,
"step": 358
},
{
"KL/chosen_KL_mean": -329.12579345703125,
"KL/mean": -436.353271484375,
"KL/rejected_KL_mean": -543.5807495117188,
"KL/std": 229.46609497070312,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0019993209280073643,
"fcm_dpo/delta": -0.02941155806183815,
"fcm_dpo/margin": 214.4549560546875,
"fcm_dpo/q_t": 0.4025813341140747,
"grad_norm": 51.26544952392578,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.4144907593727112,
"logits/rejected": -0.3954973518848419,
"logps/chosen": -385.4959716796875,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -625.7545166015625,
"loss": 1.076,
"margin_dpo/margin_mean": 214.4549560546875,
"margin_dpo/margin_std": 278.1531982421875,
"step": 359
},
{
"KL/chosen_KL_mean": -320.970703125,
"KL/mean": -405.2179870605469,
"KL/rejected_KL_mean": -489.4652404785156,
"KL/std": 195.44993591308594,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.002009341726079583,
"fcm_dpo/delta": 0.06243222951889038,
"fcm_dpo/margin": 168.49453735351562,
"fcm_dpo/q_t": 0.4197266101837158,
"grad_norm": 39.978214263916016,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.41467490792274475,
"logits/rejected": -0.39842575788497925,
"logps/chosen": -372.43109130859375,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -559.30419921875,
"loss": 1.1129,
"margin_dpo/margin_mean": 168.49453735351562,
"margin_dpo/margin_std": 194.20201110839844,
"step": 360
},
{
"KL/chosen_KL_mean": -342.26776123046875,
"KL/mean": -424.4127197265625,
"KL/rejected_KL_mean": -506.5577087402344,
"KL/std": 229.03176879882812,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0020234398543834686,
"fcm_dpo/delta": 0.06860539317131042,
"fcm_dpo/margin": 164.2899627685547,
"fcm_dpo/q_t": 0.42270296812057495,
"grad_norm": 31.659528732299805,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.46579843759536743,
"logits/rejected": -0.45978784561157227,
"logps/chosen": -396.13726806640625,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -597.3270263671875,
"loss": 1.1397,
"margin_dpo/margin_mean": 164.2899627685547,
"margin_dpo/margin_std": 245.25772094726562,
"step": 361
},
{
"KL/chosen_KL_mean": -297.3023376464844,
"KL/mean": -432.37384033203125,
"KL/rejected_KL_mean": -567.4454345703125,
"KL/std": 242.36074829101562,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0019996415358036757,
"fcm_dpo/delta": -0.14447136223316193,
"fcm_dpo/margin": 270.14306640625,
"fcm_dpo/q_t": 0.3760761320590973,
"grad_norm": 30.424165725708008,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.47015565633773804,
"logits/rejected": -0.47379571199417114,
"logps/chosen": -355.94140625,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -673.02734375,
"loss": 0.9747,
"margin_dpo/margin_mean": 270.14306640625,
"margin_dpo/margin_std": 252.4774169921875,
"step": 362
},
{
"KL/chosen_KL_mean": -270.51788330078125,
"KL/mean": -407.674560546875,
"KL/rejected_KL_mean": -544.8311767578125,
"KL/std": 241.01730346679688,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0019766553305089474,
"fcm_dpo/delta": -0.1460692584514618,
"fcm_dpo/margin": 274.31329345703125,
"fcm_dpo/q_t": 0.3755940794944763,
"grad_norm": 27.271324157714844,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.4525992274284363,
"logits/rejected": -0.43701645731925964,
"logps/chosen": -315.0762939453125,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -619.526123046875,
"loss": 0.9773,
"margin_dpo/margin_mean": 274.31329345703125,
"margin_dpo/margin_std": 263.22528076171875,
"step": 363
},
{
"KL/chosen_KL_mean": -289.7828369140625,
"KL/mean": -389.1034240722656,
"KL/rejected_KL_mean": -488.4240417480469,
"KL/std": 230.4542236328125,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0019719074480235577,
"fcm_dpo/delta": 0.008304569870233536,
"fcm_dpo/margin": 198.64120483398438,
"fcm_dpo/q_t": 0.40914398431777954,
"grad_norm": 25.84320640563965,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.4640185236930847,
"logits/rejected": -0.46951600909233093,
"logps/chosen": -338.67742919921875,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -579.81982421875,
"loss": 1.0964,
"margin_dpo/margin_mean": 198.64120483398438,
"margin_dpo/margin_std": 262.0460205078125,
"step": 364
},
{
"KL/chosen_KL_mean": -278.07470703125,
"KL/mean": -384.00201416015625,
"KL/rejected_KL_mean": -489.9293518066406,
"KL/std": 242.47592163085938,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0019668107852339745,
"fcm_dpo/delta": -0.01703200675547123,
"fcm_dpo/margin": 211.85464477539062,
"fcm_dpo/q_t": 0.40431296825408936,
"grad_norm": 24.915437698364258,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.4698528051376343,
"logits/rejected": -0.4782125949859619,
"logps/chosen": -329.56744384765625,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -582.6309814453125,
"loss": 1.0661,
"margin_dpo/margin_mean": 211.8546600341797,
"margin_dpo/margin_std": 245.96029663085938,
"step": 365
},
{
"KL/chosen_KL_mean": -254.35284423828125,
"KL/mean": -363.2435302734375,
"KL/rejected_KL_mean": -472.13421630859375,
"KL/std": 230.414306640625,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.001956676598638296,
"fcm_dpo/delta": -0.02698526531457901,
"fcm_dpo/margin": 217.7813720703125,
"fcm_dpo/q_t": 0.40227261185646057,
"grad_norm": 23.5860652923584,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.44011300802230835,
"logits/rejected": -0.4551384449005127,
"logps/chosen": -299.07342529296875,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -555.4446411132812,
"loss": 1.0726,
"margin_dpo/margin_mean": 217.7813720703125,
"margin_dpo/margin_std": 272.1701354980469,
"step": 366
},
{
"KL/chosen_KL_mean": -265.6231689453125,
"KL/mean": -356.1582336425781,
"KL/rejected_KL_mean": -446.69329833984375,
"KL/std": 205.83560180664062,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0019680976402014494,
"fcm_dpo/delta": 0.04436366260051727,
"fcm_dpo/margin": 181.07012939453125,
"fcm_dpo/q_t": 0.4157208800315857,
"grad_norm": 19.829448699951172,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.4592265188694,
"logits/rejected": -0.4390316903591156,
"logps/chosen": -324.0285949707031,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -523.444580078125,
"loss": 1.1049,
"margin_dpo/margin_mean": 181.07012939453125,
"margin_dpo/margin_std": 216.0451202392578,
"step": 367
},
{
"KL/chosen_KL_mean": -253.53057861328125,
"KL/mean": -390.93402099609375,
"KL/rejected_KL_mean": -528.3375244140625,
"KL/std": 229.09591674804688,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0019493200816214085,
"fcm_dpo/delta": -0.13935965299606323,
"fcm_dpo/margin": 274.8069152832031,
"fcm_dpo/q_t": 0.3731291890144348,
"grad_norm": 39.253082275390625,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.46787190437316895,
"logits/rejected": -0.47481292486190796,
"logps/chosen": -297.98309326171875,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -626.8927612304688,
"loss": 0.9579,
"margin_dpo/margin_mean": 274.8069152832031,
"margin_dpo/margin_std": 203.40640258789062,
"step": 368
},
{
"KL/chosen_KL_mean": -335.7747497558594,
"KL/mean": -411.5692138671875,
"KL/rejected_KL_mean": -487.3636474609375,
"KL/std": 234.81884765625,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0019605052657425404,
"fcm_dpo/delta": 0.10399104654788971,
"fcm_dpo/margin": 151.58892822265625,
"fcm_dpo/q_t": 0.4309307336807251,
"grad_norm": 35.00344467163086,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.5002185702323914,
"logits/rejected": -0.4782012701034546,
"logps/chosen": -407.15625,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -578.6594848632812,
"loss": 1.1739,
"margin_dpo/margin_mean": 151.58892822265625,
"margin_dpo/margin_std": 259.8010559082031,
"step": 369
},
{
"KL/chosen_KL_mean": -357.89031982421875,
"KL/mean": -425.114990234375,
"KL/rejected_KL_mean": -492.33966064453125,
"KL/std": 241.58839416503906,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.0019837317522615194,
"fcm_dpo/delta": 0.1349019706249237,
"fcm_dpo/margin": 134.4493408203125,
"fcm_dpo/q_t": 0.4381292462348938,
"grad_norm": 32.33375549316406,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.5110623836517334,
"logits/rejected": -0.5016048550605774,
"logps/chosen": -429.4978332519531,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -589.5994873046875,
"loss": 1.2033,
"margin_dpo/margin_mean": 134.4493408203125,
"margin_dpo/margin_std": 263.70660400390625,
"step": 370
},
{
"KL/chosen_KL_mean": -349.0395202636719,
"KL/mean": -454.69659423828125,
"KL/rejected_KL_mean": -560.3536987304688,
"KL/std": 257.37115478515625,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0019879695028066635,
"fcm_dpo/delta": -0.020561737939715385,
"fcm_dpo/margin": 211.31417846679688,
"fcm_dpo/q_t": 0.40404176712036133,
"grad_norm": 22.81478500366211,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.5306122303009033,
"logits/rejected": -0.5140925645828247,
"logps/chosen": -418.4540100097656,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -659.52587890625,
"loss": 1.0828,
"margin_dpo/margin_mean": 211.31417846679688,
"margin_dpo/margin_std": 280.86492919921875,
"step": 371
},
{
"KL/chosen_KL_mean": -343.93743896484375,
"KL/mean": -468.98333740234375,
"KL/rejected_KL_mean": -594.0292358398438,
"KL/std": 296.0002136230469,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.0019684848375618458,
"fcm_dpo/delta": -0.09475220739841461,
"fcm_dpo/margin": 250.091796875,
"fcm_dpo/q_t": 0.39228248596191406,
"grad_norm": 26.199745178222656,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.522529661655426,
"logits/rejected": -0.49960562586784363,
"logps/chosen": -405.75543212890625,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -672.5687255859375,
"loss": 1.0493,
"margin_dpo/margin_mean": 250.09181213378906,
"margin_dpo/margin_std": 338.2940673828125,
"step": 372
},
{
"KL/chosen_KL_mean": -372.826171875,
"KL/mean": -492.62713623046875,
"KL/rejected_KL_mean": -612.4281005859375,
"KL/std": 283.1507873535156,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.001959893386811018,
"fcm_dpo/delta": -0.07133302092552185,
"fcm_dpo/margin": 239.6019287109375,
"fcm_dpo/q_t": 0.3923265337944031,
"grad_norm": 32.04021072387695,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.5104029774665833,
"logits/rejected": -0.48797744512557983,
"logps/chosen": -437.04327392578125,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -698.3876953125,
"loss": 1.0425,
"margin_dpo/margin_mean": 239.6019287109375,
"margin_dpo/margin_std": 276.7124328613281,
"step": 373
},
{
"KL/chosen_KL_mean": -337.2335205078125,
"KL/mean": -444.0258483886719,
"KL/rejected_KL_mean": -550.8182373046875,
"KL/std": 301.60516357421875,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.001944505493156612,
"fcm_dpo/delta": -0.01592355966567993,
"fcm_dpo/margin": 213.58465576171875,
"fcm_dpo/q_t": 0.4093152582645416,
"grad_norm": 35.01904296875,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.5831848382949829,
"logits/rejected": -0.563025951385498,
"logps/chosen": -407.8836975097656,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -644.4583740234375,
"loss": 1.1086,
"margin_dpo/margin_mean": 213.5846710205078,
"margin_dpo/margin_std": 333.40771484375,
"step": 374
},
{
"KL/chosen_KL_mean": -351.09576416015625,
"KL/mean": -455.89691162109375,
"KL/rejected_KL_mean": -560.697998046875,
"KL/std": 252.77169799804688,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.00194136006757617,
"fcm_dpo/delta": -0.0074718669056892395,
"fcm_dpo/margin": 209.60220336914062,
"fcm_dpo/q_t": 0.4071958065032959,
"grad_norm": 37.43479537963867,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.5338048934936523,
"logits/rejected": -0.531802773475647,
"logps/chosen": -411.176025390625,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -649.6363525390625,
"loss": 1.1005,
"margin_dpo/margin_mean": 209.6022186279297,
"margin_dpo/margin_std": 296.93292236328125,
"step": 375
},
{
"KL/chosen_KL_mean": -351.7929992675781,
"KL/mean": -482.906494140625,
"KL/rejected_KL_mean": -614.02001953125,
"KL/std": 278.260009765625,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0019302696455270052,
"fcm_dpo/delta": -0.10899202525615692,
"fcm_dpo/margin": 262.2270202636719,
"fcm_dpo/q_t": 0.38526269793510437,
"grad_norm": 26.96346092224121,
"learning_rate": 2.5e-07,
"logits/chosen": -0.5299756526947021,
"logits/rejected": -0.5187512040138245,
"logps/chosen": -414.45330810546875,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -719.546630859375,
"loss": 1.03,
"margin_dpo/margin_mean": 262.2270202636719,
"margin_dpo/margin_std": 318.892822265625,
"step": 376
},
{
"KL/chosen_KL_mean": -351.45703125,
"KL/mean": -473.6996765136719,
"KL/rejected_KL_mean": -595.9423828125,
"KL/std": 293.1138000488281,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.001917961984872818,
"fcm_dpo/delta": -0.07053986191749573,
"fcm_dpo/margin": 244.4853515625,
"fcm_dpo/q_t": 0.39490044116973877,
"grad_norm": 32.56324005126953,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.4774741530418396,
"logits/rejected": -0.4759942889213562,
"logps/chosen": -405.9357604980469,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -694.645751953125,
"loss": 1.0543,
"margin_dpo/margin_mean": 244.4853515625,
"margin_dpo/margin_std": 313.40069580078125,
"step": 377
},
{
"KL/chosen_KL_mean": -332.24822998046875,
"KL/mean": -464.07470703125,
"KL/rejected_KL_mean": -595.9012451171875,
"KL/std": 259.45196533203125,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.001897640759125352,
"fcm_dpo/delta": -0.10289917886257172,
"fcm_dpo/margin": 263.6529541015625,
"fcm_dpo/q_t": 0.38391435146331787,
"grad_norm": 40.62555694580078,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.5073676109313965,
"logits/rejected": -0.5229573249816895,
"logps/chosen": -377.2687683105469,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -683.9481811523438,
"loss": 1.0141,
"margin_dpo/margin_mean": 263.6529541015625,
"margin_dpo/margin_std": 282.84375,
"step": 378
},
{
"KL/chosen_KL_mean": -369.21722412109375,
"KL/mean": -490.90283203125,
"KL/rejected_KL_mean": -612.58837890625,
"KL/std": 263.14385986328125,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.0018807162996381521,
"fcm_dpo/delta": -0.059339869767427444,
"fcm_dpo/margin": 243.37112426757812,
"fcm_dpo/q_t": 0.3961556851863861,
"grad_norm": 41.72347640991211,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.5051323771476746,
"logits/rejected": -0.5261910557746887,
"logps/chosen": -422.39935302734375,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -726.8885498046875,
"loss": 1.0571,
"margin_dpo/margin_mean": 243.3711395263672,
"margin_dpo/margin_std": 304.7427978515625,
"step": 379
},
{
"KL/chosen_KL_mean": -356.805419921875,
"KL/mean": -492.5494384765625,
"KL/rejected_KL_mean": -628.29345703125,
"KL/std": 293.99835205078125,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.001867371378466487,
"fcm_dpo/delta": -0.10970066487789154,
"fcm_dpo/margin": 271.48809814453125,
"fcm_dpo/q_t": 0.3830481171607971,
"grad_norm": 29.717735290527344,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.45388370752334595,
"logits/rejected": -0.480474054813385,
"logps/chosen": -408.158447265625,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -732.4851684570312,
"loss": 1.0195,
"margin_dpo/margin_mean": 271.4880676269531,
"margin_dpo/margin_std": 310.1554260253906,
"step": 380
},
{
"KL/chosen_KL_mean": -361.4216613769531,
"KL/mean": -462.06500244140625,
"KL/rejected_KL_mean": -562.7083129882812,
"KL/std": 248.3142852783203,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.001862676115706563,
"fcm_dpo/delta": 0.02552897296845913,
"fcm_dpo/margin": 201.28662109375,
"fcm_dpo/q_t": 0.4147588610649109,
"grad_norm": 27.22147560119629,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.4980400800704956,
"logits/rejected": -0.4761255383491516,
"logps/chosen": -419.2247314453125,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -641.927734375,
"loss": 1.1335,
"margin_dpo/margin_mean": 201.28662109375,
"margin_dpo/margin_std": 328.384033203125,
"step": 381
},
{
"KL/chosen_KL_mean": -360.59393310546875,
"KL/mean": -480.1690673828125,
"KL/rejected_KL_mean": -599.7442626953125,
"KL/std": 239.404296875,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0018593824934214354,
"fcm_dpo/delta": -0.045690521597862244,
"fcm_dpo/margin": 239.15036010742188,
"fcm_dpo/q_t": 0.3968258202075958,
"grad_norm": 37.34941101074219,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.5110651254653931,
"logits/rejected": -0.5134783983230591,
"logps/chosen": -426.61419677734375,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -710.4544677734375,
"loss": 1.0446,
"margin_dpo/margin_mean": 239.15036010742188,
"margin_dpo/margin_std": 259.5975646972656,
"step": 382
},
{
"KL/chosen_KL_mean": -368.52093505859375,
"KL/mean": -489.22088623046875,
"KL/rejected_KL_mean": -609.9208984375,
"KL/std": 261.40411376953125,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0018501915037631989,
"fcm_dpo/delta": -0.04771873354911804,
"fcm_dpo/margin": 241.3999481201172,
"fcm_dpo/q_t": 0.39832523465156555,
"grad_norm": 27.045928955078125,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.49125421047210693,
"logits/rejected": -0.4994109869003296,
"logps/chosen": -418.91241455078125,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -703.63671875,
"loss": 1.0802,
"margin_dpo/margin_mean": 241.39993286132812,
"margin_dpo/margin_std": 340.89404296875,
"step": 383
},
{
"KL/chosen_KL_mean": -365.1363525390625,
"KL/mean": -463.38427734375,
"KL/rejected_KL_mean": -561.6322021484375,
"KL/std": 249.699462890625,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.0018579517491161823,
"fcm_dpo/delta": 0.0352373830974102,
"fcm_dpo/margin": 196.4958038330078,
"fcm_dpo/q_t": 0.4140353202819824,
"grad_norm": 22.924989700317383,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.5382051467895508,
"logits/rejected": -0.529058575630188,
"logps/chosen": -417.1824645996094,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -647.39306640625,
"loss": 1.1015,
"margin_dpo/margin_mean": 196.4958038330078,
"margin_dpo/margin_std": 234.65902709960938,
"step": 384
},
{
"KL/chosen_KL_mean": -334.7351379394531,
"KL/mean": -454.896240234375,
"KL/rejected_KL_mean": -575.0573120117188,
"KL/std": 213.41387939453125,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0018485202454030514,
"fcm_dpo/delta": -0.04525205120444298,
"fcm_dpo/margin": 240.32217407226562,
"fcm_dpo/q_t": 0.3952626585960388,
"grad_norm": 33.12544250488281,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.531487762928009,
"logits/rejected": -0.4987449645996094,
"logps/chosen": -400.28729248046875,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -652.88525390625,
"loss": 1.0361,
"margin_dpo/margin_mean": 240.32217407226562,
"margin_dpo/margin_std": 236.79721069335938,
"step": 385
},
{
"KL/chosen_KL_mean": -352.7705078125,
"KL/mean": -480.95391845703125,
"KL/rejected_KL_mean": -609.1373291015625,
"KL/std": 269.5135498046875,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0018375307554379106,
"fcm_dpo/delta": -0.07277083396911621,
"fcm_dpo/margin": 256.3668212890625,
"fcm_dpo/q_t": 0.39270809292793274,
"grad_norm": 27.93240737915039,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.5182179808616638,
"logits/rejected": -0.510960578918457,
"logps/chosen": -410.9923400878906,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -701.4647216796875,
"loss": 1.0448,
"margin_dpo/margin_mean": 256.3668212890625,
"margin_dpo/margin_std": 312.1371154785156,
"step": 386
},
{
"KL/chosen_KL_mean": -390.4794006347656,
"KL/mean": -495.60986328125,
"KL/rejected_KL_mean": -600.7403564453125,
"KL/std": 248.39511108398438,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0018315855413675308,
"fcm_dpo/delta": 0.015043869614601135,
"fcm_dpo/margin": 210.26095581054688,
"fcm_dpo/q_t": 0.4094206690788269,
"grad_norm": 41.249908447265625,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.6175644397735596,
"logits/rejected": -0.5995555520057678,
"logps/chosen": -456.89886474609375,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -692.9094848632812,
"loss": 1.0887,
"margin_dpo/margin_mean": 210.26095581054688,
"margin_dpo/margin_std": 248.58956909179688,
"step": 387
},
{
"KL/chosen_KL_mean": -377.8406677246094,
"KL/mean": -515.5494384765625,
"KL/rejected_KL_mean": -653.2581787109375,
"KL/std": 286.0020751953125,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.0018191644921898842,
"fcm_dpo/delta": -0.10386500507593155,
"fcm_dpo/margin": 275.41748046875,
"fcm_dpo/q_t": 0.3879193663597107,
"grad_norm": 30.190383911132812,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.49368101358413696,
"logits/rejected": -0.5035091042518616,
"logps/chosen": -427.9701232910156,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -757.6912231445312,
"loss": 1.0263,
"margin_dpo/margin_mean": 275.41748046875,
"margin_dpo/margin_std": 328.736328125,
"step": 388
},
{
"KL/chosen_KL_mean": -415.5103454589844,
"KL/mean": -533.9041748046875,
"KL/rejected_KL_mean": -652.2980346679688,
"KL/std": 284.1199951171875,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.001811205642297864,
"fcm_dpo/delta": -0.029575031250715256,
"fcm_dpo/margin": 236.78768920898438,
"fcm_dpo/q_t": 0.40235158801078796,
"grad_norm": 33.09611892700195,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.5412299633026123,
"logits/rejected": -0.5279366970062256,
"logps/chosen": -473.41693115234375,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -730.2125854492188,
"loss": 1.0798,
"margin_dpo/margin_mean": 236.78770446777344,
"margin_dpo/margin_std": 318.11456298828125,
"step": 389
},
{
"KL/chosen_KL_mean": -415.3945007324219,
"KL/mean": -532.8773193359375,
"KL/rejected_KL_mean": -650.360107421875,
"KL/std": 292.0013427734375,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.001804722473025322,
"fcm_dpo/delta": -0.024779455736279488,
"fcm_dpo/margin": 234.965576171875,
"fcm_dpo/q_t": 0.40768325328826904,
"grad_norm": 31.104320526123047,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.521344780921936,
"logits/rejected": -0.5126668810844421,
"logps/chosen": -464.62042236328125,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -735.8883056640625,
"loss": 1.1072,
"margin_dpo/margin_mean": 234.965576171875,
"margin_dpo/margin_std": 377.9402160644531,
"step": 390
},
{
"KL/chosen_KL_mean": -402.7524719238281,
"KL/mean": -475.3613586425781,
"KL/rejected_KL_mean": -547.97021484375,
"KL/std": 272.24322509765625,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0018261673394590616,
"fcm_dpo/delta": 0.13658833503723145,
"fcm_dpo/margin": 145.21774291992188,
"fcm_dpo/q_t": 0.43881237506866455,
"grad_norm": 44.82551193237305,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.5299597978591919,
"logits/rejected": -0.5175088047981262,
"logps/chosen": -467.0821228027344,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -634.7084350585938,
"loss": 1.1992,
"margin_dpo/margin_mean": 145.2177276611328,
"margin_dpo/margin_std": 272.92193603515625,
"step": 391
},
{
"KL/chosen_KL_mean": -344.9163513183594,
"KL/mean": -484.80401611328125,
"KL/rejected_KL_mean": -624.6917114257812,
"KL/std": 282.89739990234375,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0018200704362243414,
"fcm_dpo/delta": -0.11199337244033813,
"fcm_dpo/margin": 279.7752990722656,
"fcm_dpo/q_t": 0.383048415184021,
"grad_norm": 29.697546005249023,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.5169392824172974,
"logits/rejected": -0.5098834037780762,
"logps/chosen": -398.42034912109375,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -727.0375366210938,
"loss": 0.9991,
"margin_dpo/margin_mean": 279.77532958984375,
"margin_dpo/margin_std": 275.7098083496094,
"step": 392
},
{
"KL/chosen_KL_mean": -343.71734619140625,
"KL/mean": -456.7199401855469,
"KL/rejected_KL_mean": -569.7225341796875,
"KL/std": 284.97540283203125,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0018092221580445766,
"fcm_dpo/delta": -0.009102771058678627,
"fcm_dpo/margin": 226.00515747070312,
"fcm_dpo/q_t": 0.41020962595939636,
"grad_norm": 47.57746124267578,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.5029352307319641,
"logits/rejected": -0.504021406173706,
"logps/chosen": -390.1912841796875,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -641.69140625,
"loss": 1.106,
"margin_dpo/margin_mean": 226.00515747070312,
"margin_dpo/margin_std": 352.2773742675781,
"step": 393
},
{
"KL/chosen_KL_mean": -391.92578125,
"KL/mean": -508.56024169921875,
"KL/rejected_KL_mean": -625.1947021484375,
"KL/std": 295.7176818847656,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.0018048053607344627,
"fcm_dpo/delta": -0.02154848352074623,
"fcm_dpo/margin": 233.2689208984375,
"fcm_dpo/q_t": 0.4036337435245514,
"grad_norm": 27.146669387817383,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.5260942578315735,
"logits/rejected": -0.524019181728363,
"logps/chosen": -444.83734130859375,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -716.017333984375,
"loss": 1.0749,
"margin_dpo/margin_mean": 233.2689208984375,
"margin_dpo/margin_std": 294.8382568359375,
"step": 394
},
{
"KL/chosen_KL_mean": -382.04046630859375,
"KL/mean": -507.2308654785156,
"KL/rejected_KL_mean": -632.4212646484375,
"KL/std": 293.729736328125,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.001797014381736517,
"fcm_dpo/delta": -0.051209770143032074,
"fcm_dpo/margin": 250.38079833984375,
"fcm_dpo/q_t": 0.39935189485549927,
"grad_norm": 38.09134292602539,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.5735211372375488,
"logits/rejected": -0.5697096586227417,
"logps/chosen": -444.5865478515625,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -716.203857421875,
"loss": 1.0749,
"margin_dpo/margin_mean": 250.38082885742188,
"margin_dpo/margin_std": 346.9578857421875,
"step": 395
},
{
"KL/chosen_KL_mean": -390.10418701171875,
"KL/mean": -504.8359375,
"KL/rejected_KL_mean": -619.5677490234375,
"KL/std": 280.2290344238281,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.00179403240326792,
"fcm_dpo/delta": -0.01193785946816206,
"fcm_dpo/margin": 229.46353149414062,
"fcm_dpo/q_t": 0.4044559597969055,
"grad_norm": 26.76568031311035,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.5892349481582642,
"logits/rejected": -0.5674149394035339,
"logps/chosen": -459.10009765625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -708.21435546875,
"loss": 1.0683,
"margin_dpo/margin_mean": 229.4635467529297,
"margin_dpo/margin_std": 264.5689392089844,
"step": 396
},
{
"KL/chosen_KL_mean": -362.35675048828125,
"KL/mean": -515.5724487304688,
"KL/rejected_KL_mean": -668.7880859375,
"KL/std": 288.4814453125,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0017716542351990938,
"fcm_dpo/delta": -0.14731790125370026,
"fcm_dpo/margin": 306.431396484375,
"fcm_dpo/q_t": 0.3749983310699463,
"grad_norm": 41.37638854980469,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.5667599439620972,
"logits/rejected": -0.5667216181755066,
"logps/chosen": -423.6339111328125,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -771.9042358398438,
"loss": 0.9805,
"margin_dpo/margin_mean": 306.4313659667969,
"margin_dpo/margin_std": 299.587646484375,
"step": 397
},
{
"KL/chosen_KL_mean": -423.58306884765625,
"KL/mean": -554.8473510742188,
"KL/rejected_KL_mean": -686.1116943359375,
"KL/std": 320.2618713378906,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0017602917505428195,
"fcm_dpo/delta": -0.0635901689529419,
"fcm_dpo/margin": 262.528564453125,
"fcm_dpo/q_t": 0.3985213041305542,
"grad_norm": 32.223777770996094,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.6214488744735718,
"logits/rejected": -0.6171579957008362,
"logps/chosen": -491.734619140625,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -794.63525390625,
"loss": 1.0673,
"margin_dpo/margin_mean": 262.528564453125,
"margin_dpo/margin_std": 365.87701416015625,
"step": 398
},
{
"KL/chosen_KL_mean": -380.365478515625,
"KL/mean": -508.25189208984375,
"KL/rejected_KL_mean": -636.1383056640625,
"KL/std": 293.50164794921875,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0017485737334936857,
"fcm_dpo/delta": -0.048456672579050064,
"fcm_dpo/margin": 255.77288818359375,
"fcm_dpo/q_t": 0.40013986825942993,
"grad_norm": 38.44267272949219,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.5989868640899658,
"logits/rejected": -0.5554165840148926,
"logps/chosen": -441.2552795410156,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -714.1038818359375,
"loss": 1.08,
"margin_dpo/margin_mean": 255.77285766601562,
"margin_dpo/margin_std": 361.84649658203125,
"step": 399
},
{
"KL/chosen_KL_mean": -368.0055236816406,
"KL/mean": -543.0501098632812,
"KL/rejected_KL_mean": -718.0947265625,
"KL/std": 286.2700500488281,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.0017187241464853287,
"fcm_dpo/delta": -0.20846638083457947,
"fcm_dpo/margin": 350.08917236328125,
"fcm_dpo/q_t": 0.3612917959690094,
"grad_norm": 26.391523361206055,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.5936537981033325,
"logits/rejected": -0.5919471383094788,
"logps/chosen": -431.64910888671875,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -823.347412109375,
"loss": 0.9419,
"margin_dpo/margin_mean": 350.08917236328125,
"margin_dpo/margin_std": 319.36767578125,
"step": 400
},
{
"KL/chosen_KL_mean": -435.6270446777344,
"KL/mean": -533.0802612304688,
"KL/rejected_KL_mean": -630.533447265625,
"KL/std": 307.6553039550781,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.0017190496437251568,
"fcm_dpo/delta": 0.06607242673635483,
"fcm_dpo/margin": 194.9063720703125,
"fcm_dpo/q_t": 0.426999032497406,
"grad_norm": 35.03347396850586,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.591301441192627,
"logits/rejected": -0.5788199305534363,
"logps/chosen": -492.79010009765625,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -714.325927734375,
"loss": 1.1903,
"margin_dpo/margin_mean": 194.9063720703125,
"margin_dpo/margin_std": 405.22808837890625,
"step": 401
},
{
"KL/chosen_KL_mean": -349.45294189453125,
"KL/mean": -544.983642578125,
"KL/rejected_KL_mean": -740.514404296875,
"KL/std": 371.795654296875,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.001689492492005229,
"fcm_dpo/delta": -0.26964718103408813,
"fcm_dpo/margin": 391.06146240234375,
"fcm_dpo/q_t": 0.3538634181022644,
"grad_norm": 27.208879470825195,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.6506165266036987,
"logits/rejected": -0.6465029120445251,
"logps/chosen": -400.1933288574219,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -821.5604248046875,
"loss": 0.917,
"margin_dpo/margin_mean": 391.06146240234375,
"margin_dpo/margin_std": 385.4092712402344,
"step": 402
},
{
"KL/chosen_KL_mean": -376.1459655761719,
"KL/mean": -522.2297973632812,
"KL/rejected_KL_mean": -668.3135375976562,
"KL/std": 303.7095031738281,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0016665069852024317,
"fcm_dpo/delta": -0.08910196274518967,
"fcm_dpo/margin": 292.1676330566406,
"fcm_dpo/q_t": 0.388282835483551,
"grad_norm": 30.522640228271484,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.6762701869010925,
"logits/rejected": -0.6811692714691162,
"logps/chosen": -423.29327392578125,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -745.5802001953125,
"loss": 1.0256,
"margin_dpo/margin_mean": 292.1676025390625,
"margin_dpo/margin_std": 321.37841796875,
"step": 403
},
{
"KL/chosen_KL_mean": -453.879638671875,
"KL/mean": -607.360107421875,
"KL/rejected_KL_mean": -760.840576171875,
"KL/std": 370.8985595703125,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.0016497753094881773,
"fcm_dpo/delta": -0.1091829165816307,
"fcm_dpo/margin": 306.96087646484375,
"fcm_dpo/q_t": 0.3911134600639343,
"grad_norm": 31.827224731445312,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.6803830862045288,
"logits/rejected": -0.6808692216873169,
"logps/chosen": -501.75494384765625,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -837.9955444335938,
"loss": 1.081,
"margin_dpo/margin_mean": 306.96087646484375,
"margin_dpo/margin_std": 484.5469970703125,
"step": 404
},
{
"KL/chosen_KL_mean": -506.3373718261719,
"KL/mean": -623.0357666015625,
"KL/rejected_KL_mean": -739.7340698242188,
"KL/std": 341.96575927734375,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.0016373193357139826,
"fcm_dpo/delta": -0.09679268300533295,
"fcm_dpo/margin": 233.39666748046875,
"fcm_dpo/q_t": 0.4155503511428833,
"grad_norm": 44.04417419433594,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.6529548168182373,
"logits/rejected": -0.6453539729118347,
"logps/chosen": -571.500244140625,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -826.9208984375,
"loss": 1.1604,
"margin_dpo/margin_mean": 233.3966827392578,
"margin_dpo/margin_std": 420.79632568359375,
"step": 405
},
{
"KL/chosen_KL_mean": -504.4444580078125,
"KL/mean": -670.9434814453125,
"KL/rejected_KL_mean": -837.4425048828125,
"KL/std": 409.6977233886719,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0016175368800759315,
"fcm_dpo/delta": -0.14244601130485535,
"fcm_dpo/margin": 332.998046875,
"fcm_dpo/q_t": 0.38683146238327026,
"grad_norm": 31.830657958984375,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.7117749452590942,
"logits/rejected": -0.7342991232872009,
"logps/chosen": -554.1852416992188,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -929.5211181640625,
"loss": 1.041,
"margin_dpo/margin_mean": 332.998046875,
"margin_dpo/margin_std": 462.95513916015625,
"step": 406
},
{
"KL/chosen_KL_mean": -487.5089111328125,
"KL/mean": -582.2493896484375,
"KL/rejected_KL_mean": -676.9898681640625,
"KL/std": 341.89263916015625,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0016208180459216237,
"fcm_dpo/delta": 0.0940016433596611,
"fcm_dpo/margin": 189.48098754882812,
"fcm_dpo/q_t": 0.42886239290237427,
"grad_norm": 40.08480453491211,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.7205852270126343,
"logits/rejected": -0.6887321472167969,
"logps/chosen": -543.839599609375,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -754.501953125,
"loss": 1.2139,
"margin_dpo/margin_mean": 189.48097229003906,
"margin_dpo/margin_std": 426.8612060546875,
"step": 407
},
{
"KL/chosen_KL_mean": -488.0704345703125,
"KL/mean": -594.4813232421875,
"KL/rejected_KL_mean": -700.8922729492188,
"KL/std": 310.1326904296875,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0016301569994539022,
"fcm_dpo/delta": 0.05386827886104584,
"fcm_dpo/margin": 212.82177734375,
"fcm_dpo/q_t": 0.4206947088241577,
"grad_norm": 41.006221771240234,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.6842703819274902,
"logits/rejected": -0.6612120866775513,
"logps/chosen": -557.8597412109375,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -790.9891967773438,
"loss": 1.1492,
"margin_dpo/margin_mean": 212.82179260253906,
"margin_dpo/margin_std": 353.79248046875,
"step": 408
},
{
"KL/chosen_KL_mean": -453.1000671386719,
"KL/mean": -564.703125,
"KL/rejected_KL_mean": -676.30615234375,
"KL/std": 381.3478088378906,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0016393279656767845,
"fcm_dpo/delta": 0.03426938131451607,
"fcm_dpo/margin": 223.20606994628906,
"fcm_dpo/q_t": 0.41481828689575195,
"grad_norm": 59.193416595458984,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.6385992765426636,
"logits/rejected": -0.6249116063117981,
"logps/chosen": -520.41748046875,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -761.21044921875,
"loss": 1.1496,
"margin_dpo/margin_mean": 223.20608520507812,
"margin_dpo/margin_std": 382.4623107910156,
"step": 409
},
{
"KL/chosen_KL_mean": -416.6119384765625,
"KL/mean": -551.2454833984375,
"KL/rejected_KL_mean": -685.8790893554688,
"KL/std": 340.37841796875,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.0016306547913700342,
"fcm_dpo/delta": -0.039969172328710556,
"fcm_dpo/margin": 269.26708984375,
"fcm_dpo/q_t": 0.39879608154296875,
"grad_norm": 33.50337219238281,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.6794567108154297,
"logits/rejected": -0.6865516901016235,
"logps/chosen": -468.0772705078125,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -769.0780639648438,
"loss": 1.0792,
"margin_dpo/margin_mean": 269.26708984375,
"margin_dpo/margin_std": 368.21893310546875,
"step": 410
},
{
"KL/chosen_KL_mean": -423.6837158203125,
"KL/mean": -541.8031616210938,
"KL/rejected_KL_mean": -659.922607421875,
"KL/std": 336.0458984375,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0016164029948413372,
"fcm_dpo/delta": -0.07978048175573349,
"fcm_dpo/margin": 236.23883056640625,
"fcm_dpo/q_t": 0.4166368842124939,
"grad_norm": 45.770408630371094,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.6459128260612488,
"logits/rejected": -0.6529507637023926,
"logps/chosen": -475.9909973144531,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -740.6175537109375,
"loss": 1.1373,
"margin_dpo/margin_mean": 236.23883056640625,
"margin_dpo/margin_std": 390.1459655761719,
"step": 411
},
{
"KL/chosen_KL_mean": -404.8624267578125,
"KL/mean": -531.0133056640625,
"KL/rejected_KL_mean": -657.1642456054688,
"KL/std": 315.80413818359375,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0016159769147634506,
"fcm_dpo/delta": -0.007876180112361908,
"fcm_dpo/margin": 252.3018341064453,
"fcm_dpo/q_t": 0.40613073110580444,
"grad_norm": 43.324363708496094,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.6817220449447632,
"logits/rejected": -0.7102159261703491,
"logps/chosen": -458.00653076171875,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -757.2250366210938,
"loss": 1.0908,
"margin_dpo/margin_mean": 252.3018341064453,
"margin_dpo/margin_std": 339.39776611328125,
"step": 412
},
{
"KL/chosen_KL_mean": -445.6668701171875,
"KL/mean": -565.676025390625,
"KL/rejected_KL_mean": -685.6851806640625,
"KL/std": 343.90045166015625,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.0016158015932887793,
"fcm_dpo/delta": 0.012414616532623768,
"fcm_dpo/margin": 240.018310546875,
"fcm_dpo/q_t": 0.4103829562664032,
"grad_norm": 48.77497482299805,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.581852912902832,
"logits/rejected": -0.5829395055770874,
"logps/chosen": -507.24884033203125,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -785.1585693359375,
"loss": 1.1251,
"margin_dpo/margin_mean": 240.01829528808594,
"margin_dpo/margin_std": 382.1087646484375,
"step": 413
},
{
"KL/chosen_KL_mean": -410.1985778808594,
"KL/mean": -545.940673828125,
"KL/rejected_KL_mean": -681.6827392578125,
"KL/std": 313.23284912109375,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.0016170567832887173,
"fcm_dpo/delta": -0.04011237248778343,
"fcm_dpo/margin": 271.4841613769531,
"fcm_dpo/q_t": 0.3984266519546509,
"grad_norm": 41.59056091308594,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.5111421346664429,
"logits/rejected": -0.5018640160560608,
"logps/chosen": -456.830078125,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -769.3292236328125,
"loss": 1.0743,
"margin_dpo/margin_mean": 271.4841613769531,
"margin_dpo/margin_std": 356.08306884765625,
"step": 414
},
{
"KL/chosen_KL_mean": -406.1727294921875,
"KL/mean": -505.6904296875,
"KL/rejected_KL_mean": -605.2080688476562,
"KL/std": 278.98065185546875,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0016174819320440292,
"fcm_dpo/delta": 0.07935836911201477,
"fcm_dpo/margin": 199.03533935546875,
"fcm_dpo/q_t": 0.4250248372554779,
"grad_norm": 32.30403518676758,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.5725187063217163,
"logits/rejected": -0.5640978813171387,
"logps/chosen": -484.791015625,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -705.685546875,
"loss": 1.1467,
"margin_dpo/margin_mean": 199.0353240966797,
"margin_dpo/margin_std": 299.36395263671875,
"step": 415
},
{
"KL/chosen_KL_mean": -368.3544616699219,
"KL/mean": -525.67529296875,
"KL/rejected_KL_mean": -682.99609375,
"KL/std": 330.7052001953125,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.001612334861420095,
"fcm_dpo/delta": -0.11002575606107712,
"fcm_dpo/margin": 314.6416015625,
"fcm_dpo/q_t": 0.38389724493026733,
"grad_norm": 30.73332405090332,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.5767337083816528,
"logits/rejected": -0.577792763710022,
"logps/chosen": -426.63360595703125,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -773.5648193359375,
"loss": 1.011,
"margin_dpo/margin_mean": 314.6416015625,
"margin_dpo/margin_std": 339.9158020019531,
"step": 416
},
{
"KL/chosen_KL_mean": -364.384765625,
"KL/mean": -508.7856140136719,
"KL/rejected_KL_mean": -653.1865234375,
"KL/std": 298.44720458984375,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.001599437789991498,
"fcm_dpo/delta": -0.06335951387882233,
"fcm_dpo/margin": 288.8017272949219,
"fcm_dpo/q_t": 0.391997754573822,
"grad_norm": 47.862770080566406,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.5380935668945312,
"logits/rejected": -0.5061866641044617,
"logps/chosen": -414.58349609375,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -721.3383178710938,
"loss": 1.0336,
"margin_dpo/margin_mean": 288.8017272949219,
"margin_dpo/margin_std": 306.3311462402344,
"step": 417
},
{
"KL/chosen_KL_mean": -390.33111572265625,
"KL/mean": -528.9058837890625,
"KL/rejected_KL_mean": -667.4805908203125,
"KL/std": 329.7533874511719,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.001596780028194189,
"fcm_dpo/delta": -0.04398631677031517,
"fcm_dpo/margin": 277.1495361328125,
"fcm_dpo/q_t": 0.4014670252799988,
"grad_norm": 24.271198272705078,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.6195993423461914,
"logits/rejected": -0.5918015241622925,
"logps/chosen": -472.3095703125,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -772.172119140625,
"loss": 1.0803,
"margin_dpo/margin_mean": 277.1495361328125,
"margin_dpo/margin_std": 378.1399841308594,
"step": 418
},
{
"KL/chosen_KL_mean": -359.47064208984375,
"KL/mean": -516.6668701171875,
"KL/rejected_KL_mean": -673.8631591796875,
"KL/std": 296.4808349609375,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0015776680083945394,
"fcm_dpo/delta": -0.09841260313987732,
"fcm_dpo/margin": 314.3925476074219,
"fcm_dpo/q_t": 0.38518738746643066,
"grad_norm": 39.38912582397461,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.5384039282798767,
"logits/rejected": -0.5316938161849976,
"logps/chosen": -412.4192810058594,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -765.4462890625,
"loss": 1.0206,
"margin_dpo/margin_mean": 314.39251708984375,
"margin_dpo/margin_std": 347.848876953125,
"step": 419
},
{
"KL/chosen_KL_mean": -473.1541748046875,
"KL/mean": -576.36474609375,
"KL/rejected_KL_mean": -679.5751953125,
"KL/std": 321.3580322265625,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.0015831240452826023,
"fcm_dpo/delta": 0.0743355080485344,
"fcm_dpo/margin": 206.4210968017578,
"fcm_dpo/q_t": 0.4267392158508301,
"grad_norm": 55.58370590209961,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.5968104600906372,
"logits/rejected": -0.5501741170883179,
"logps/chosen": -550.924072265625,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -748.8951416015625,
"loss": 1.1855,
"margin_dpo/margin_mean": 206.42111206054688,
"margin_dpo/margin_std": 412.8056640625,
"step": 420
},
{
"KL/chosen_KL_mean": -396.0514831542969,
"KL/mean": -511.7779235839844,
"KL/rejected_KL_mean": -627.5043334960938,
"KL/std": 307.17010498046875,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0015912302769720554,
"fcm_dpo/delta": 0.03213408216834068,
"fcm_dpo/margin": 231.45286560058594,
"fcm_dpo/q_t": 0.41308170557022095,
"grad_norm": 37.79745101928711,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.6047526001930237,
"logits/rejected": -0.5907719135284424,
"logps/chosen": -449.8173522949219,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -716.7857666015625,
"loss": 1.1119,
"margin_dpo/margin_mean": 231.45286560058594,
"margin_dpo/margin_std": 305.76007080078125,
"step": 421
},
{
"KL/chosen_KL_mean": -444.47344970703125,
"KL/mean": -577.7868041992188,
"KL/rejected_KL_mean": -711.10009765625,
"KL/std": 318.5824890136719,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0015860656276345253,
"fcm_dpo/delta": -0.02341364696621895,
"fcm_dpo/margin": 266.6266784667969,
"fcm_dpo/q_t": 0.4029381275177002,
"grad_norm": 37.30603790283203,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.6128122806549072,
"logits/rejected": -0.6130838394165039,
"logps/chosen": -513.1072387695312,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -798.963623046875,
"loss": 1.09,
"margin_dpo/margin_mean": 266.62664794921875,
"margin_dpo/margin_std": 376.72772216796875,
"step": 422
},
{
"KL/chosen_KL_mean": -442.69097900390625,
"KL/mean": -593.7772216796875,
"KL/rejected_KL_mean": -744.863525390625,
"KL/std": 316.8233947753906,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0015762131661176682,
"fcm_dpo/delta": -0.07819212973117828,
"fcm_dpo/margin": 302.17254638671875,
"fcm_dpo/q_t": 0.39091968536376953,
"grad_norm": 35.244747161865234,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.6149768829345703,
"logits/rejected": -0.6239042282104492,
"logps/chosen": -497.6812744140625,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -831.1700439453125,
"loss": 1.0466,
"margin_dpo/margin_mean": 302.17254638671875,
"margin_dpo/margin_std": 374.8946533203125,
"step": 423
},
{
"KL/chosen_KL_mean": -408.0943908691406,
"KL/mean": -545.8341064453125,
"KL/rejected_KL_mean": -683.57373046875,
"KL/std": 307.42901611328125,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0015672901645302773,
"fcm_dpo/delta": -0.03262971714138985,
"fcm_dpo/margin": 275.4793395996094,
"fcm_dpo/q_t": 0.40227210521698,
"grad_norm": 37.73208236694336,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.6178228855133057,
"logits/rejected": -0.5884814262390137,
"logps/chosen": -464.1063232421875,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -750.052734375,
"loss": 1.073,
"margin_dpo/margin_mean": 275.4793395996094,
"margin_dpo/margin_std": 357.9488525390625,
"step": 424
},
{
"KL/chosen_KL_mean": -495.09136962890625,
"KL/mean": -594.1044921875,
"KL/rejected_KL_mean": -693.11767578125,
"KL/std": 313.45208740234375,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.001581181539222598,
"fcm_dpo/delta": 0.08807256817817688,
"fcm_dpo/margin": 198.02630615234375,
"fcm_dpo/q_t": 0.4265795946121216,
"grad_norm": 40.72890090942383,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.6312949657440186,
"logits/rejected": -0.6337833404541016,
"logps/chosen": -541.9603271484375,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -789.0431518554688,
"loss": 1.1785,
"margin_dpo/margin_mean": 198.02630615234375,
"margin_dpo/margin_std": 365.4080505371094,
"step": 425
},
{
"KL/chosen_KL_mean": -447.80145263671875,
"KL/mean": -569.6341552734375,
"KL/rejected_KL_mean": -691.466796875,
"KL/std": 296.7207946777344,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0015872271033003926,
"fcm_dpo/delta": 0.013328127562999725,
"fcm_dpo/margin": 243.6654052734375,
"fcm_dpo/q_t": 0.40938708186149597,
"grad_norm": 39.436275482177734,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.6700231432914734,
"logits/rejected": -0.6314303874969482,
"logps/chosen": -524.385009765625,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -772.7333984375,
"loss": 1.0934,
"margin_dpo/margin_mean": 243.6654052734375,
"margin_dpo/margin_std": 301.1048583984375,
"step": 426
},
{
"KL/chosen_KL_mean": -404.7652587890625,
"KL/mean": -502.8582458496094,
"KL/rejected_KL_mean": -600.9512939453125,
"KL/std": 270.48968505859375,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.0015933769755065441,
"fcm_dpo/delta": 0.08883827924728394,
"fcm_dpo/margin": 196.18600463867188,
"fcm_dpo/q_t": 0.4279603958129883,
"grad_norm": 38.65625,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.6598665714263916,
"logits/rejected": -0.6343759894371033,
"logps/chosen": -469.619140625,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -679.5172729492188,
"loss": 1.1568,
"margin_dpo/margin_mean": 196.18600463867188,
"margin_dpo/margin_std": 310.1150817871094,
"step": 427
},
{
"KL/chosen_KL_mean": -541.8001708984375,
"KL/mean": -685.3818359375,
"KL/rejected_KL_mean": -828.9635009765625,
"KL/std": 373.19793701171875,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.0015891847433522344,
"fcm_dpo/delta": -0.05788760259747505,
"fcm_dpo/margin": 287.1632080078125,
"fcm_dpo/q_t": 0.3983868360519409,
"grad_norm": 42.709285736083984,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.6648718118667603,
"logits/rejected": -0.6649109125137329,
"logps/chosen": -604.4368896484375,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -932.2452392578125,
"loss": 1.0954,
"margin_dpo/margin_mean": 287.1632080078125,
"margin_dpo/margin_std": 445.24249267578125,
"step": 428
},
{
"KL/chosen_KL_mean": -558.6399536132812,
"KL/mean": -670.6189575195312,
"KL/rejected_KL_mean": -782.5979614257812,
"KL/std": 353.8562927246094,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.001599712180905044,
"fcm_dpo/delta": 0.04193819314241409,
"fcm_dpo/margin": 223.9580078125,
"fcm_dpo/q_t": 0.4191283583641052,
"grad_norm": 42.31879425048828,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.7320696711540222,
"logits/rejected": -0.7067664861679077,
"logps/chosen": -639.8739624023438,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -874.3928833007812,
"loss": 1.1686,
"margin_dpo/margin_mean": 223.9580078125,
"margin_dpo/margin_std": 418.2169494628906,
"step": 429
},
{
"KL/chosen_KL_mean": -485.84906005859375,
"KL/mean": -585.9974365234375,
"KL/rejected_KL_mean": -686.145751953125,
"KL/std": 333.52752685546875,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.0016081253997981548,
"fcm_dpo/delta": 0.07896231859922409,
"fcm_dpo/margin": 200.2966766357422,
"fcm_dpo/q_t": 0.4268134832382202,
"grad_norm": 45.6950569152832,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.6843982934951782,
"logits/rejected": -0.6812521815299988,
"logps/chosen": -546.7694091796875,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -790.5685424804688,
"loss": 1.1699,
"margin_dpo/margin_mean": 200.29666137695312,
"margin_dpo/margin_std": 362.1824035644531,
"step": 430
},
{
"KL/chosen_KL_mean": -476.85858154296875,
"KL/mean": -639.10986328125,
"KL/rejected_KL_mean": -801.361083984375,
"KL/std": 353.5206298828125,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.001593923894688487,
"fcm_dpo/delta": -0.12039424479007721,
"fcm_dpo/margin": 324.5025329589844,
"fcm_dpo/q_t": 0.3837839663028717,
"grad_norm": 37.95612716674805,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.7060000896453857,
"logits/rejected": -0.7101027965545654,
"logps/chosen": -534.2073364257812,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -894.2013549804688,
"loss": 1.0314,
"margin_dpo/margin_mean": 324.5025329589844,
"margin_dpo/margin_std": 400.6416015625,
"step": 431
},
{
"KL/chosen_KL_mean": -433.40106201171875,
"KL/mean": -608.9974975585938,
"KL/rejected_KL_mean": -784.593994140625,
"KL/std": 369.924560546875,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.0015768797602504492,
"fcm_dpo/delta": -0.15814030170440674,
"fcm_dpo/margin": 351.19293212890625,
"fcm_dpo/q_t": 0.37685060501098633,
"grad_norm": 34.167293548583984,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.6776980757713318,
"logits/rejected": -0.6659233570098877,
"logps/chosen": -504.475830078125,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -883.1734619140625,
"loss": 0.9869,
"margin_dpo/margin_mean": 351.19293212890625,
"margin_dpo/margin_std": 373.4866943359375,
"step": 432
},
{
"KL/chosen_KL_mean": -559.2772827148438,
"KL/mean": -664.7277221679688,
"KL/rejected_KL_mean": -770.1781616210938,
"KL/std": 378.35809326171875,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0015741533134132624,
"fcm_dpo/delta": 0.06906940788030624,
"fcm_dpo/margin": 210.90089416503906,
"fcm_dpo/q_t": 0.4204699993133545,
"grad_norm": 33.000389099121094,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.7252877950668335,
"logits/rejected": -0.7109028697013855,
"logps/chosen": -617.5504760742188,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -866.1290283203125,
"loss": 1.1957,
"margin_dpo/margin_mean": 210.90089416503906,
"margin_dpo/margin_std": 443.9112548828125,
"step": 433
},
{
"KL/chosen_KL_mean": -496.6466369628906,
"KL/mean": -609.65185546875,
"KL/rejected_KL_mean": -722.6571044921875,
"KL/std": 331.86968994140625,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0015789458993822336,
"fcm_dpo/delta": 0.04392336681485176,
"fcm_dpo/margin": 226.01043701171875,
"fcm_dpo/q_t": 0.4241020679473877,
"grad_norm": 29.258861541748047,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.7415832281112671,
"logits/rejected": -0.7311919927597046,
"logps/chosen": -558.620361328125,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -801.1557006835938,
"loss": 1.1536,
"margin_dpo/margin_mean": 226.01043701171875,
"margin_dpo/margin_std": 413.2788391113281,
"step": 434
},
{
"KL/chosen_KL_mean": -455.0494689941406,
"KL/mean": -597.1414184570312,
"KL/rejected_KL_mean": -739.2333374023438,
"KL/std": 330.17828369140625,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.0015741032548248768,
"fcm_dpo/delta": -0.048613518476486206,
"fcm_dpo/margin": 284.1838684082031,
"fcm_dpo/q_t": 0.39784517884254456,
"grad_norm": 33.527957916259766,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.8012436628341675,
"logits/rejected": -0.784874677658081,
"logps/chosen": -506.551513671875,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -826.8002319335938,
"loss": 1.065,
"margin_dpo/margin_mean": 284.183837890625,
"margin_dpo/margin_std": 358.4755859375,
"step": 435
},
{
"KL/chosen_KL_mean": -454.6264343261719,
"KL/mean": -576.6045532226562,
"KL/rejected_KL_mean": -698.5827026367188,
"KL/std": 299.61407470703125,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0015738060465082526,
"fcm_dpo/delta": 0.016265233978629112,
"fcm_dpo/margin": 243.956298828125,
"fcm_dpo/q_t": 0.4141663908958435,
"grad_norm": 39.384552001953125,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.702154815196991,
"logits/rejected": -0.6535923480987549,
"logps/chosen": -526.0301513671875,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -781.3104248046875,
"loss": 1.1158,
"margin_dpo/margin_mean": 243.956298828125,
"margin_dpo/margin_std": 367.78271484375,
"step": 436
},
{
"KL/chosen_KL_mean": -440.7288513183594,
"KL/mean": -567.1207885742188,
"KL/rejected_KL_mean": -693.5126953125,
"KL/std": 293.3643798828125,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0015777763910591602,
"fcm_dpo/delta": 0.0011859128717333078,
"fcm_dpo/margin": 252.7838897705078,
"fcm_dpo/q_t": 0.40734195709228516,
"grad_norm": 29.819622039794922,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.7542532086372375,
"logits/rejected": -0.729715883731842,
"logps/chosen": -505.4731140136719,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -775.5562744140625,
"loss": 1.0905,
"margin_dpo/margin_mean": 252.78387451171875,
"margin_dpo/margin_std": 329.2221984863281,
"step": 437
},
{
"KL/chosen_KL_mean": -469.1627502441406,
"KL/mean": -613.673095703125,
"KL/rejected_KL_mean": -758.1834716796875,
"KL/std": 323.0591735839844,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0015675770118832588,
"fcm_dpo/delta": -0.054624516516923904,
"fcm_dpo/margin": 289.0207214355469,
"fcm_dpo/q_t": 0.39459216594696045,
"grad_norm": 32.62807846069336,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.7667056322097778,
"logits/rejected": -0.7360678911209106,
"logps/chosen": -528.181396484375,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -841.26025390625,
"loss": 1.05,
"margin_dpo/margin_mean": 289.0207214355469,
"margin_dpo/margin_std": 334.79864501953125,
"step": 438
},
{
"KL/chosen_KL_mean": -487.53643798828125,
"KL/mean": -603.5372314453125,
"KL/rejected_KL_mean": -719.5379638671875,
"KL/std": 336.40850830078125,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.001555339666083455,
"fcm_dpo/delta": -0.08725128322839737,
"fcm_dpo/margin": 232.0015411376953,
"fcm_dpo/q_t": 0.41888073086738586,
"grad_norm": 40.83659744262695,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.757964015007019,
"logits/rejected": -0.7572835683822632,
"logps/chosen": -541.3204956054688,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -803.5234375,
"loss": 1.1478,
"margin_dpo/margin_mean": 232.00155639648438,
"margin_dpo/margin_std": 380.4766845703125,
"step": 439
},
{
"KL/chosen_KL_mean": -491.5701904296875,
"KL/mean": -634.1220703125,
"KL/rejected_KL_mean": -776.6740112304688,
"KL/std": 404.11834716796875,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0015516489511355758,
"fcm_dpo/delta": -0.043332286179065704,
"fcm_dpo/margin": 285.1037902832031,
"fcm_dpo/q_t": 0.4026166796684265,
"grad_norm": 44.514137268066406,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.773739218711853,
"logits/rejected": -0.7669236660003662,
"logps/chosen": -570.1369018554688,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -873.1717529296875,
"loss": 1.0713,
"margin_dpo/margin_mean": 285.103759765625,
"margin_dpo/margin_std": 386.797119140625,
"step": 440
},
{
"KL/chosen_KL_mean": -591.2857666015625,
"KL/mean": -716.7557373046875,
"KL/rejected_KL_mean": -842.2255859375,
"KL/std": 407.85955810546875,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.001553539652377367,
"fcm_dpo/delta": 0.010096218436956406,
"fcm_dpo/margin": 250.93984985351562,
"fcm_dpo/q_t": 0.4177462160587311,
"grad_norm": 40.89066696166992,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.7674189805984497,
"logits/rejected": -0.7517750263214111,
"logps/chosen": -652.1102294921875,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -938.6964111328125,
"loss": 1.1743,
"margin_dpo/margin_mean": 250.93988037109375,
"margin_dpo/margin_std": 502.06768798828125,
"step": 441
},
{
"KL/chosen_KL_mean": -503.0433349609375,
"KL/mean": -659.7222900390625,
"KL/rejected_KL_mean": -816.4011840820312,
"KL/std": 401.5861511230469,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0015460660215467215,
"fcm_dpo/delta": -0.08682595193386078,
"fcm_dpo/margin": 313.3577880859375,
"fcm_dpo/q_t": 0.3956022262573242,
"grad_norm": 36.8726806640625,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.7465405464172363,
"logits/rejected": -0.7388708591461182,
"logps/chosen": -550.0545654296875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -892.9404296875,
"loss": 1.0721,
"margin_dpo/margin_mean": 313.3577880859375,
"margin_dpo/margin_std": 448.83795166015625,
"step": 442
},
{
"KL/chosen_KL_mean": -538.9011840820312,
"KL/mean": -627.173095703125,
"KL/rejected_KL_mean": -715.4451904296875,
"KL/std": 372.1048583984375,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0015383637510240078,
"fcm_dpo/delta": 0.010316350497305393,
"fcm_dpo/margin": 176.54397583007812,
"fcm_dpo/q_t": 0.43916434049606323,
"grad_norm": 39.52730178833008,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.733747124671936,
"logits/rejected": -0.710411548614502,
"logps/chosen": -610.1741943359375,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -802.1251220703125,
"loss": 1.239,
"margin_dpo/margin_mean": 176.54397583007812,
"margin_dpo/margin_std": 431.0137634277344,
"step": 443
},
{
"KL/chosen_KL_mean": -545.4714965820312,
"KL/mean": -706.712890625,
"KL/rejected_KL_mean": -867.954345703125,
"KL/std": 410.585693359375,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0015250996220856905,
"fcm_dpo/delta": -0.09439031034708023,
"fcm_dpo/margin": 322.4828796386719,
"fcm_dpo/q_t": 0.39275673031806946,
"grad_norm": 35.95709991455078,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.6589124202728271,
"logits/rejected": -0.6668537259101868,
"logps/chosen": -602.6851806640625,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -965.209228515625,
"loss": 1.059,
"margin_dpo/margin_mean": 322.48291015625,
"margin_dpo/margin_std": 454.73382568359375,
"step": 444
},
{
"KL/chosen_KL_mean": -464.7125549316406,
"KL/mean": -615.153076171875,
"KL/rejected_KL_mean": -765.593505859375,
"KL/std": 321.0918273925781,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0015192454447969794,
"fcm_dpo/delta": -0.058439984917640686,
"fcm_dpo/margin": 300.8808898925781,
"fcm_dpo/q_t": 0.39640486240386963,
"grad_norm": 32.19375228881836,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.6859538555145264,
"logits/rejected": -0.6752911806106567,
"logps/chosen": -532.0123291015625,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -858.276123046875,
"loss": 1.0589,
"margin_dpo/margin_mean": 300.88092041015625,
"margin_dpo/margin_std": 378.2691650390625,
"step": 445
},
{
"KL/chosen_KL_mean": -440.65826416015625,
"KL/mean": -595.5625,
"KL/rejected_KL_mean": -750.4666748046875,
"KL/std": 347.827880859375,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.0015123614575713873,
"fcm_dpo/delta": -0.07043258845806122,
"fcm_dpo/margin": 309.8084716796875,
"fcm_dpo/q_t": 0.39075666666030884,
"grad_norm": 36.1616096496582,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.7100153565406799,
"logits/rejected": -0.7190457582473755,
"logps/chosen": -499.7567138671875,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -851.7308349609375,
"loss": 1.0343,
"margin_dpo/margin_mean": 309.8084716796875,
"margin_dpo/margin_std": 329.4242248535156,
"step": 446
},
{
"KL/chosen_KL_mean": -485.10504150390625,
"KL/mean": -651.3726806640625,
"KL/rejected_KL_mean": -817.6403198242188,
"KL/std": 411.0166015625,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0014915233477950096,
"fcm_dpo/delta": -0.09860781580209732,
"fcm_dpo/margin": 332.5352783203125,
"fcm_dpo/q_t": 0.39083898067474365,
"grad_norm": 33.39097595214844,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.7963074445724487,
"logits/rejected": -0.8120626211166382,
"logps/chosen": -541.1803588867188,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -916.3350830078125,
"loss": 1.0444,
"margin_dpo/margin_mean": 332.5352783203125,
"margin_dpo/margin_std": 443.71478271484375,
"step": 447
},
{
"KL/chosen_KL_mean": -549.9185791015625,
"KL/mean": -715.212646484375,
"KL/rejected_KL_mean": -880.5067138671875,
"KL/std": 387.88446044921875,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.0014809533022344112,
"fcm_dpo/delta": -0.09178226441144943,
"fcm_dpo/margin": 330.588134765625,
"fcm_dpo/q_t": 0.3909049928188324,
"grad_norm": 37.11544418334961,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.7098146677017212,
"logits/rejected": -0.7185966372489929,
"logps/chosen": -609.9224243164062,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -982.7713623046875,
"loss": 1.0476,
"margin_dpo/margin_mean": 330.588134765625,
"margin_dpo/margin_std": 435.3096923828125,
"step": 448
},
{
"KL/chosen_KL_mean": -565.3760986328125,
"KL/mean": -737.2021484375,
"KL/rejected_KL_mean": -909.028076171875,
"KL/std": 458.106689453125,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.001463989494368434,
"fcm_dpo/delta": -0.10579749941825867,
"fcm_dpo/margin": 343.6519470214844,
"fcm_dpo/q_t": 0.3931967616081238,
"grad_norm": 33.413673400878906,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.7576150894165039,
"logits/rejected": -0.746334433555603,
"logps/chosen": -632.9027099609375,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -997.625,
"loss": 1.0698,
"margin_dpo/margin_mean": 343.6519775390625,
"margin_dpo/margin_std": 530.8461303710938,
"step": 449
},
{
"KL/chosen_KL_mean": -459.6373596191406,
"KL/mean": -645.7894897460938,
"KL/rejected_KL_mean": -831.9415893554688,
"KL/std": 388.321044921875,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0014468804001808167,
"fcm_dpo/delta": -0.14240342378616333,
"fcm_dpo/margin": 372.30426025390625,
"fcm_dpo/q_t": 0.37629586458206177,
"grad_norm": 43.87886428833008,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.8462213277816772,
"logits/rejected": -0.8595950603485107,
"logps/chosen": -516.7454833984375,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -934.696533203125,
"loss": 0.9856,
"margin_dpo/margin_mean": 372.30426025390625,
"margin_dpo/margin_std": 375.06005859375,
"step": 450
},
{
"KL/chosen_KL_mean": -655.2659912109375,
"KL/mean": -786.9049072265625,
"KL/rejected_KL_mean": -918.5439453125,
"KL/std": 500.51873779296875,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0014441582607105374,
"fcm_dpo/delta": 0.02004837803542614,
"fcm_dpo/margin": 263.2779541015625,
"fcm_dpo/q_t": 0.4101364016532898,
"grad_norm": 50.99306106567383,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.9174356460571289,
"logits/rejected": -0.8886551856994629,
"logps/chosen": -713.7348022460938,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -991.4733276367188,
"loss": 1.2051,
"margin_dpo/margin_mean": 263.2779541015625,
"margin_dpo/margin_std": 591.6530151367188,
"step": 451
},
{
"KL/chosen_KL_mean": -478.00341796875,
"KL/mean": -634.8177490234375,
"KL/rejected_KL_mean": -791.6319580078125,
"KL/std": 377.42657470703125,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.001435277285054326,
"fcm_dpo/delta": -0.05143965408205986,
"fcm_dpo/margin": 313.6285400390625,
"fcm_dpo/q_t": 0.3956111669540405,
"grad_norm": 30.187389373779297,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.8042168617248535,
"logits/rejected": -0.8225170373916626,
"logps/chosen": -528.3939819335938,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -889.4034423828125,
"loss": 1.0508,
"margin_dpo/margin_mean": 313.6285400390625,
"margin_dpo/margin_std": 363.00054931640625,
"step": 452
},
{
"KL/chosen_KL_mean": -546.2333984375,
"KL/mean": -708.6800537109375,
"KL/rejected_KL_mean": -871.1268310546875,
"KL/std": 392.1535949707031,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0014257467119023204,
"fcm_dpo/delta": -0.06497231125831604,
"fcm_dpo/margin": 324.8934326171875,
"fcm_dpo/q_t": 0.3990872800350189,
"grad_norm": 34.372772216796875,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.8440671563148499,
"logits/rejected": -0.8263705968856812,
"logps/chosen": -603.9482421875,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -953.334228515625,
"loss": 1.0778,
"margin_dpo/margin_mean": 324.8934631347656,
"margin_dpo/margin_std": 473.4591064453125,
"step": 453
},
{
"KL/chosen_KL_mean": -651.534912109375,
"KL/mean": -822.3595581054688,
"KL/rejected_KL_mean": -993.18408203125,
"KL/std": 444.50738525390625,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0014141410356387496,
"fcm_dpo/delta": -0.08552803844213486,
"fcm_dpo/margin": 341.649169921875,
"fcm_dpo/q_t": 0.3941487669944763,
"grad_norm": 39.139339447021484,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.8544209003448486,
"logits/rejected": -0.8400380611419678,
"logps/chosen": -712.4805908203125,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -1078.1348876953125,
"loss": 1.0725,
"margin_dpo/margin_mean": 341.649169921875,
"margin_dpo/margin_std": 507.084228515625,
"step": 454
},
{
"KL/chosen_KL_mean": -564.0460205078125,
"KL/mean": -756.771484375,
"KL/rejected_KL_mean": -949.4969482421875,
"KL/std": 434.53131103515625,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0014031952014192939,
"fcm_dpo/delta": -0.14472892880439758,
"fcm_dpo/margin": 385.450927734375,
"fcm_dpo/q_t": 0.38020026683807373,
"grad_norm": 33.70637130737305,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.8295320272445679,
"logits/rejected": -0.8505617380142212,
"logps/chosen": -608.9327392578125,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -1064.79833984375,
"loss": 1.0249,
"margin_dpo/margin_mean": 385.450927734375,
"margin_dpo/margin_std": 487.4344482421875,
"step": 455
},
{
"KL/chosen_KL_mean": -542.165771484375,
"KL/mean": -737.29248046875,
"KL/rejected_KL_mean": -932.4191284179688,
"KL/std": 445.7962341308594,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0013802102766931057,
"fcm_dpo/delta": -0.14236783981323242,
"fcm_dpo/margin": 390.25335693359375,
"fcm_dpo/q_t": 0.37905341386795044,
"grad_norm": 34.32986831665039,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.8049008846282959,
"logits/rejected": -0.829048752784729,
"logps/chosen": -599.2025756835938,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -1037.636962890625,
"loss": 0.9897,
"margin_dpo/margin_mean": 390.25335693359375,
"margin_dpo/margin_std": 416.21002197265625,
"step": 456
},
{
"KL/chosen_KL_mean": -505.49066162109375,
"KL/mean": -710.5162963867188,
"KL/rejected_KL_mean": -915.5419921875,
"KL/std": 429.17242431640625,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.001355804386548698,
"fcm_dpo/delta": -0.1606726348400116,
"fcm_dpo/margin": 410.0513000488281,
"fcm_dpo/q_t": 0.37336650490760803,
"grad_norm": 50.008827209472656,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.8348103761672974,
"logits/rejected": -0.8382035493850708,
"logps/chosen": -559.733154296875,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -1000.6514892578125,
"loss": 0.9782,
"margin_dpo/margin_mean": 410.0513000488281,
"margin_dpo/margin_std": 415.6295166015625,
"step": 457
},
{
"KL/chosen_KL_mean": -571.1600341796875,
"KL/mean": -721.54638671875,
"KL/rejected_KL_mean": -871.9327392578125,
"KL/std": 375.446044921875,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0013467208482325077,
"fcm_dpo/delta": -0.005669653415679932,
"fcm_dpo/margin": 300.77264404296875,
"fcm_dpo/q_t": 0.4096975326538086,
"grad_norm": 33.263877868652344,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.8518056273460388,
"logits/rejected": -0.840267539024353,
"logps/chosen": -626.56884765625,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -969.615966796875,
"loss": 1.0987,
"margin_dpo/margin_mean": 300.77264404296875,
"margin_dpo/margin_std": 419.0684814453125,
"step": 458
},
{
"KL/chosen_KL_mean": -596.946533203125,
"KL/mean": -784.0278930664062,
"KL/rejected_KL_mean": -971.109130859375,
"KL/std": 449.78326416015625,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0013373238034546375,
"fcm_dpo/delta": -0.10349556058645248,
"fcm_dpo/margin": 374.16259765625,
"fcm_dpo/q_t": 0.3898807168006897,
"grad_norm": 37.392234802246094,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.8045568466186523,
"logits/rejected": -0.8199119567871094,
"logps/chosen": -643.5040283203125,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -1057.2777099609375,
"loss": 1.0568,
"margin_dpo/margin_mean": 374.16259765625,
"margin_dpo/margin_std": 516.4439086914062,
"step": 459
},
{
"KL/chosen_KL_mean": -642.6575317382812,
"KL/mean": -851.060791015625,
"KL/rejected_KL_mean": -1059.4639892578125,
"KL/std": 428.037109375,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0013223644345998764,
"fcm_dpo/delta": -0.1554030030965805,
"fcm_dpo/margin": 416.8063049316406,
"fcm_dpo/q_t": 0.37675726413726807,
"grad_norm": 36.663360595703125,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.8338453769683838,
"logits/rejected": -0.8649412393569946,
"logps/chosen": -694.29248046875,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -1163.583251953125,
"loss": 0.9875,
"margin_dpo/margin_mean": 416.80633544921875,
"margin_dpo/margin_std": 453.399658203125,
"step": 460
},
{
"KL/chosen_KL_mean": -657.5977172851562,
"KL/mean": -817.214599609375,
"KL/rejected_KL_mean": -976.83154296875,
"KL/std": 429.95391845703125,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0013168307486921549,
"fcm_dpo/delta": -0.020892973989248276,
"fcm_dpo/margin": 319.23382568359375,
"fcm_dpo/q_t": 0.40674418210983276,
"grad_norm": 33.73992919921875,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.858330249786377,
"logits/rejected": -0.8671220541000366,
"logps/chosen": -712.7796630859375,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -1063.3084716796875,
"loss": 1.0947,
"margin_dpo/margin_mean": 319.2338562011719,
"margin_dpo/margin_std": 468.2879333496094,
"step": 461
},
{
"KL/chosen_KL_mean": -621.3289794921875,
"KL/mean": -758.6487426757812,
"KL/rejected_KL_mean": -895.9684448242188,
"KL/std": 408.9393005371094,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.001316360547207296,
"fcm_dpo/delta": 0.03917883336544037,
"fcm_dpo/margin": 274.63946533203125,
"fcm_dpo/q_t": 0.4196895658969879,
"grad_norm": 41.01453399658203,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.9267274141311646,
"logits/rejected": -0.9083724617958069,
"logps/chosen": -691.2569580078125,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -974.8095703125,
"loss": 1.1402,
"margin_dpo/margin_mean": 274.63946533203125,
"margin_dpo/margin_std": 454.64776611328125,
"step": 462
},
{
"KL/chosen_KL_mean": -614.42333984375,
"KL/mean": -786.9886474609375,
"KL/rejected_KL_mean": -959.553955078125,
"KL/std": 419.7926940917969,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.001315572764724493,
"fcm_dpo/delta": -0.05530533939599991,
"fcm_dpo/margin": 345.13055419921875,
"fcm_dpo/q_t": 0.39796507358551025,
"grad_norm": 32.41996765136719,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.8691219687461853,
"logits/rejected": -0.8700894117355347,
"logps/chosen": -669.69775390625,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -1048.578857421875,
"loss": 1.0662,
"margin_dpo/margin_mean": 345.13055419921875,
"margin_dpo/margin_std": 457.62176513671875,
"step": 463
},
{
"KL/chosen_KL_mean": -600.5834350585938,
"KL/mean": -850.0731811523438,
"KL/rejected_KL_mean": -1099.56298828125,
"KL/std": 488.45526123046875,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0012829885818064213,
"fcm_dpo/delta": -0.24955548346042633,
"fcm_dpo/margin": 498.9794921875,
"fcm_dpo/q_t": 0.3563224673271179,
"grad_norm": 42.92744827270508,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.864883303642273,
"logits/rejected": -0.9228291511535645,
"logps/chosen": -651.4957275390625,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -1202.05224609375,
"loss": 0.9282,
"margin_dpo/margin_mean": 498.9794921875,
"margin_dpo/margin_std": 475.25860595703125,
"step": 464
},
{
"KL/chosen_KL_mean": -598.503662109375,
"KL/mean": -802.850341796875,
"KL/rejected_KL_mean": -1007.1970825195312,
"KL/std": 488.08880615234375,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.001269632950425148,
"fcm_dpo/delta": -0.12196629494428635,
"fcm_dpo/margin": 408.6933898925781,
"fcm_dpo/q_t": 0.37997591495513916,
"grad_norm": 30.996089935302734,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.851067841053009,
"logits/rejected": -0.871573805809021,
"logps/chosen": -658.6204833984375,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -1121.14306640625,
"loss": 1.0264,
"margin_dpo/margin_mean": 408.6933898925781,
"margin_dpo/margin_std": 498.4493408203125,
"step": 465
},
{
"KL/chosen_KL_mean": -651.2394409179688,
"KL/mean": -829.18798828125,
"KL/rejected_KL_mean": -1007.1365966796875,
"KL/std": 461.8182067871094,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0012555132852867246,
"fcm_dpo/delta": -0.04844123125076294,
"fcm_dpo/margin": 355.89715576171875,
"fcm_dpo/q_t": 0.39813193678855896,
"grad_norm": 34.06509017944336,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.8493870496749878,
"logits/rejected": -0.8454769849777222,
"logps/chosen": -704.1603393554688,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -1097.4520263671875,
"loss": 1.074,
"margin_dpo/margin_mean": 355.89715576171875,
"margin_dpo/margin_std": 472.82452392578125,
"step": 466
},
{
"KL/chosen_KL_mean": -826.9212646484375,
"KL/mean": -1013.4310302734375,
"KL/rejected_KL_mean": -1199.940673828125,
"KL/std": 584.6832275390625,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0012465352192521095,
"fcm_dpo/delta": -0.06744933128356934,
"fcm_dpo/margin": 373.0193786621094,
"fcm_dpo/q_t": 0.39987167716026306,
"grad_norm": 53.18272018432617,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.9681617021560669,
"logits/rejected": -0.9566558599472046,
"logps/chosen": -905.6370849609375,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1302.8009033203125,
"loss": 1.1327,
"margin_dpo/margin_mean": 373.0194091796875,
"margin_dpo/margin_std": 667.3912963867188,
"step": 467
},
{
"KL/chosen_KL_mean": -637.194091796875,
"KL/mean": -866.535400390625,
"KL/rejected_KL_mean": -1095.876708984375,
"KL/std": 503.98370361328125,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0012332710903137922,
"fcm_dpo/delta": -0.17055264115333557,
"fcm_dpo/margin": 458.6826477050781,
"fcm_dpo/q_t": 0.37581923604011536,
"grad_norm": 52.71803283691406,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.8924639225006104,
"logits/rejected": -0.9000778794288635,
"logps/chosen": -707.1294555664062,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -1196.905517578125,
"loss": 0.9999,
"margin_dpo/margin_mean": 458.6826171875,
"margin_dpo/margin_std": 557.9136962890625,
"step": 468
},
{
"KL/chosen_KL_mean": -674.091796875,
"KL/mean": -917.2705078125,
"KL/rejected_KL_mean": -1160.44921875,
"KL/std": 503.095947265625,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0012071155942976475,
"fcm_dpo/delta": -0.1936628669500351,
"fcm_dpo/margin": 486.35748291015625,
"fcm_dpo/q_t": 0.3694315552711487,
"grad_norm": 36.40598678588867,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.8922240734100342,
"logits/rejected": -0.9052385091781616,
"logps/chosen": -742.2164916992188,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -1265.235595703125,
"loss": 0.978,
"margin_dpo/margin_mean": 486.35748291015625,
"margin_dpo/margin_std": 537.0926513671875,
"step": 469
},
{
"KL/chosen_KL_mean": -584.605224609375,
"KL/mean": -782.6975708007812,
"KL/rejected_KL_mean": -980.7899169921875,
"KL/std": 488.2651672363281,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0012003884185105562,
"fcm_dpo/delta": -0.07743942737579346,
"fcm_dpo/margin": 396.18463134765625,
"fcm_dpo/q_t": 0.392379492521286,
"grad_norm": 29.941707611083984,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.8477715253829956,
"logits/rejected": -0.8522796630859375,
"logps/chosen": -628.397216796875,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -1063.4927978515625,
"loss": 1.0593,
"margin_dpo/margin_mean": 396.18463134765625,
"margin_dpo/margin_std": 526.9808349609375,
"step": 470
},
{
"KL/chosen_KL_mean": -723.69287109375,
"KL/mean": -888.711669921875,
"KL/rejected_KL_mean": -1053.73046875,
"KL/std": 517.3129272460938,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.00119293003808707,
"fcm_dpo/delta": 0.006166655570268631,
"fcm_dpo/margin": 330.03759765625,
"fcm_dpo/q_t": 0.4141054153442383,
"grad_norm": 44.29683303833008,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.9283441305160522,
"logits/rejected": -0.9297612905502319,
"logps/chosen": -787.0324096679688,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -1137.3409423828125,
"loss": 1.1301,
"margin_dpo/margin_mean": 330.0375671386719,
"margin_dpo/margin_std": 555.3355712890625,
"step": 471
},
{
"KL/chosen_KL_mean": -694.7491455078125,
"KL/mean": -900.8193359375,
"KL/rejected_KL_mean": -1106.8895263671875,
"KL/std": 589.9913940429688,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.001185831381008029,
"fcm_dpo/delta": -0.0912611111998558,
"fcm_dpo/margin": 412.1404113769531,
"fcm_dpo/q_t": 0.39623403549194336,
"grad_norm": 34.139041900634766,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.9295023679733276,
"logits/rejected": -0.9044417142868042,
"logps/chosen": -778.415283203125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -1224.0987548828125,
"loss": 1.0802,
"margin_dpo/margin_mean": 412.140380859375,
"margin_dpo/margin_std": 622.5784912109375,
"step": 472
},
{
"KL/chosen_KL_mean": -844.9259033203125,
"KL/mean": -950.9039306640625,
"KL/rejected_KL_mean": -1056.8818359375,
"KL/std": 620.759521484375,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0011900889221578836,
"fcm_dpo/delta": 0.04035104438662529,
"fcm_dpo/margin": 211.95591735839844,
"fcm_dpo/q_t": 0.44867947697639465,
"grad_norm": 131.445068359375,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.8819540739059448,
"logits/rejected": -0.851367712020874,
"logps/chosen": -908.4229125976562,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -1138.0283203125,
"loss": 1.3351,
"margin_dpo/margin_mean": 211.9559326171875,
"margin_dpo/margin_std": 784.6436767578125,
"step": 473
},
{
"KL/chosen_KL_mean": -676.4264526367188,
"KL/mean": -852.287109375,
"KL/rejected_KL_mean": -1028.147705078125,
"KL/std": 525.454833984375,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.0011939804535359144,
"fcm_dpo/delta": -0.021219126880168915,
"fcm_dpo/margin": 351.7213439941406,
"fcm_dpo/q_t": 0.4053725600242615,
"grad_norm": 47.71774673461914,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.8572415113449097,
"logits/rejected": -0.8637101650238037,
"logps/chosen": -729.0383911132812,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -1118.2281494140625,
"loss": 1.1248,
"margin_dpo/margin_mean": 351.7213439941406,
"margin_dpo/margin_std": 567.389892578125,
"step": 474
},
{
"KL/chosen_KL_mean": -492.31280517578125,
"KL/mean": -697.96044921875,
"KL/rejected_KL_mean": -903.608154296875,
"KL/std": 407.37347412109375,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0011786803370341659,
"fcm_dpo/delta": -0.0870213657617569,
"fcm_dpo/margin": 411.29534912109375,
"fcm_dpo/q_t": 0.38825133442878723,
"grad_norm": 56.91255569458008,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.8645976781845093,
"logits/rejected": -0.8880435228347778,
"logps/chosen": -534.8079833984375,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -993.6710815429688,
"loss": 1.0216,
"margin_dpo/margin_mean": 411.29534912109375,
"margin_dpo/margin_std": 438.8066711425781,
"step": 475
},
{
"KL/chosen_KL_mean": -581.162109375,
"KL/mean": -763.0509033203125,
"KL/rejected_KL_mean": -944.939697265625,
"KL/std": 481.3084411621094,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0011740537593141198,
"fcm_dpo/delta": -0.027722399681806564,
"fcm_dpo/margin": 363.7776184082031,
"fcm_dpo/q_t": 0.40231823921203613,
"grad_norm": 102.54533386230469,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.8250603675842285,
"logits/rejected": -0.8420618772506714,
"logps/chosen": -624.1114501953125,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -1018.64990234375,
"loss": 1.0709,
"margin_dpo/margin_mean": 363.7776184082031,
"margin_dpo/margin_std": 458.6659240722656,
"step": 476
},
{
"KL/chosen_KL_mean": -576.864990234375,
"KL/mean": -748.666015625,
"KL/rejected_KL_mean": -920.467041015625,
"KL/std": 453.67510986328125,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0011754365405067801,
"fcm_dpo/delta": -0.004048405215144157,
"fcm_dpo/margin": 343.6020812988281,
"fcm_dpo/q_t": 0.407484769821167,
"grad_norm": 50.94183349609375,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.8757802248001099,
"logits/rejected": -0.8439153432846069,
"logps/chosen": -647.6376342773438,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -996.6044311523438,
"loss": 1.1156,
"margin_dpo/margin_mean": 343.6020812988281,
"margin_dpo/margin_std": 538.941650390625,
"step": 477
},
{
"KL/chosen_KL_mean": -512.1103515625,
"KL/mean": -712.9010009765625,
"KL/rejected_KL_mean": -913.691650390625,
"KL/std": 417.81170654296875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.0011684978380799294,
"fcm_dpo/delta": -0.07088005542755127,
"fcm_dpo/margin": 401.581298828125,
"fcm_dpo/q_t": 0.39256751537323,
"grad_norm": 44.561859130859375,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.7855898141860962,
"logits/rejected": -0.8061913251876831,
"logps/chosen": -553.5509033203125,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -999.0535888671875,
"loss": 1.0413,
"margin_dpo/margin_mean": 401.581298828125,
"margin_dpo/margin_std": 472.5474853515625,
"step": 478
},
{
"KL/chosen_KL_mean": -679.7722778320312,
"KL/mean": -867.8972778320312,
"KL/rejected_KL_mean": -1056.022216796875,
"KL/std": 525.4520874023438,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0011651396052911878,
"fcm_dpo/delta": -0.03945866972208023,
"fcm_dpo/margin": 376.24993896484375,
"fcm_dpo/q_t": 0.4034544825553894,
"grad_norm": 36.7021484375,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.8868274092674255,
"logits/rejected": -0.9125785827636719,
"logps/chosen": -733.68017578125,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -1151.1385498046875,
"loss": 1.1051,
"margin_dpo/margin_mean": 376.24993896484375,
"margin_dpo/margin_std": 603.150146484375,
"step": 479
},
{
"KL/chosen_KL_mean": -800.5087890625,
"KL/mean": -963.208740234375,
"KL/rejected_KL_mean": -1125.90869140625,
"KL/std": 507.62554931640625,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0011562576983124018,
"fcm_dpo/delta": -0.07623150199651718,
"fcm_dpo/margin": 325.4000244140625,
"fcm_dpo/q_t": 0.4170445203781128,
"grad_norm": 49.098358154296875,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.9548571109771729,
"logits/rejected": -0.9503375291824341,
"logps/chosen": -859.1915283203125,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1208.8411865234375,
"loss": 1.1562,
"margin_dpo/margin_mean": 325.4000244140625,
"margin_dpo/margin_std": 569.0355224609375,
"step": 480
},
{
"KL/chosen_KL_mean": -698.514404296875,
"KL/mean": -934.3134765625,
"KL/rejected_KL_mean": -1170.112548828125,
"KL/std": 530.9942016601562,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.001140213687904179,
"fcm_dpo/delta": -0.14147771894931793,
"fcm_dpo/margin": 471.5982360839844,
"fcm_dpo/q_t": 0.3806016147136688,
"grad_norm": 40.23090362548828,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.994126558303833,
"logits/rejected": -1.0202418565750122,
"logps/chosen": -753.4786376953125,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -1262.532958984375,
"loss": 1.0124,
"margin_dpo/margin_mean": 471.5982666015625,
"margin_dpo/margin_std": 567.219482421875,
"step": 481
},
{
"KL/chosen_KL_mean": -790.0252685546875,
"KL/mean": -897.4144897460938,
"KL/rejected_KL_mean": -1004.8038330078125,
"KL/std": 556.745849609375,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.001139242434874177,
"fcm_dpo/delta": 0.06273117661476135,
"fcm_dpo/margin": 214.77850341796875,
"fcm_dpo/q_t": 0.4425661563873291,
"grad_norm": 51.694149017333984,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.9190036058425903,
"logits/rejected": -0.90464186668396,
"logps/chosen": -857.5787353515625,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -1092.393310546875,
"loss": 1.2784,
"margin_dpo/margin_mean": 214.7785186767578,
"margin_dpo/margin_std": 644.366943359375,
"step": 482
},
{
"KL/chosen_KL_mean": -724.80859375,
"KL/mean": -970.60693359375,
"KL/rejected_KL_mean": -1216.4053955078125,
"KL/std": 547.3455810546875,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0011287643574178219,
"fcm_dpo/delta": -0.15940281748771667,
"fcm_dpo/margin": 491.59686279296875,
"fcm_dpo/q_t": 0.3793754577636719,
"grad_norm": 40.69145202636719,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.9518593549728394,
"logits/rejected": -0.9543057680130005,
"logps/chosen": -788.1383666992188,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1312.1923828125,
"loss": 1.011,
"margin_dpo/margin_mean": 491.5968322753906,
"margin_dpo/margin_std": 612.5662841796875,
"step": 483
},
{
"KL/chosen_KL_mean": -609.849609375,
"KL/mean": -838.743408203125,
"KL/rejected_KL_mean": -1067.6370849609375,
"KL/std": 533.2965087890625,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0011179596185684204,
"fcm_dpo/delta": -0.1146901473402977,
"fcm_dpo/margin": 457.7874755859375,
"fcm_dpo/q_t": 0.3862311840057373,
"grad_norm": 33.894779205322266,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -1.0216343402862549,
"logits/rejected": -1.041937232017517,
"logps/chosen": -668.98779296875,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -1152.008544921875,
"loss": 1.024,
"margin_dpo/margin_mean": 457.7874755859375,
"margin_dpo/margin_std": 545.956298828125,
"step": 484
},
{
"KL/chosen_KL_mean": -657.9251708984375,
"KL/mean": -873.3860473632812,
"KL/rejected_KL_mean": -1088.846923828125,
"KL/std": 551.51806640625,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0011059865355491638,
"fcm_dpo/delta": -0.07843407988548279,
"fcm_dpo/margin": 430.9217529296875,
"fcm_dpo/q_t": 0.395374596118927,
"grad_norm": 30.792177200317383,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.9116197824478149,
"logits/rejected": -0.9275870323181152,
"logps/chosen": -716.7747802734375,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -1192.2109375,
"loss": 1.0703,
"margin_dpo/margin_mean": 430.9217224121094,
"margin_dpo/margin_std": 620.1131591796875,
"step": 485
},
{
"KL/chosen_KL_mean": -730.796142578125,
"KL/mean": -973.44775390625,
"KL/rejected_KL_mean": -1216.099365234375,
"KL/std": 588.2183837890625,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.001091314246878028,
"fcm_dpo/delta": -0.1332990676164627,
"fcm_dpo/margin": 485.30328369140625,
"fcm_dpo/q_t": 0.38558250665664673,
"grad_norm": 42.85213851928711,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -1.025226354598999,
"logits/rejected": -1.0336995124816895,
"logps/chosen": -786.0557861328125,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -1308.23876953125,
"loss": 1.0581,
"margin_dpo/margin_mean": 485.30328369140625,
"margin_dpo/margin_std": 714.6268310546875,
"step": 486
},
{
"KL/chosen_KL_mean": -738.9359130859375,
"KL/mean": -917.86376953125,
"KL/rejected_KL_mean": -1096.791748046875,
"KL/std": 529.3869018554688,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0010912182042375207,
"fcm_dpo/delta": 0.009562283754348755,
"fcm_dpo/margin": 357.8558349609375,
"fcm_dpo/q_t": 0.40953782200813293,
"grad_norm": 35.427616119384766,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -1.0364776849746704,
"logits/rejected": -1.0385382175445557,
"logps/chosen": -791.9991455078125,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -1189.2105712890625,
"loss": 1.1157,
"margin_dpo/margin_mean": 357.85589599609375,
"margin_dpo/margin_std": 545.7218017578125,
"step": 487
},
{
"KL/chosen_KL_mean": -610.1397705078125,
"KL/mean": -795.30224609375,
"KL/rejected_KL_mean": -980.4647216796875,
"KL/std": 488.50335693359375,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0010911128483712673,
"fcm_dpo/delta": -0.00425250269472599,
"fcm_dpo/margin": 370.32489013671875,
"fcm_dpo/q_t": 0.4075971841812134,
"grad_norm": 37.004825592041016,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.9542009830474854,
"logits/rejected": -0.9616404175758362,
"logps/chosen": -662.3679809570312,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -1064.4713134765625,
"loss": 1.0826,
"margin_dpo/margin_mean": 370.32489013671875,
"margin_dpo/margin_std": 467.35565185546875,
"step": 488
},
{
"KL/chosen_KL_mean": -572.46044921875,
"KL/mean": -741.5594482421875,
"KL/rejected_KL_mean": -910.658447265625,
"KL/std": 456.1171875,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0010937793413177133,
"fcm_dpo/delta": 0.030445974320173264,
"fcm_dpo/margin": 338.1979675292969,
"fcm_dpo/q_t": 0.41659069061279297,
"grad_norm": 35.718448638916016,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.9860827922821045,
"logits/rejected": -0.9753962755203247,
"logps/chosen": -628.4500732421875,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -990.0565185546875,
"loss": 1.1177,
"margin_dpo/margin_mean": 338.19793701171875,
"margin_dpo/margin_std": 492.15753173828125,
"step": 489
},
{
"KL/chosen_KL_mean": -665.18701171875,
"KL/mean": -841.7398071289062,
"KL/rejected_KL_mean": -1018.2926025390625,
"KL/std": 565.1947021484375,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0010953254532068968,
"fcm_dpo/delta": 0.013375891372561455,
"fcm_dpo/margin": 353.1055908203125,
"fcm_dpo/q_t": 0.4144512116909027,
"grad_norm": 36.92960739135742,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.9983633756637573,
"logits/rejected": -1.038588523864746,
"logps/chosen": -717.5534057617188,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -1128.70166015625,
"loss": 1.1514,
"margin_dpo/margin_mean": 353.1055603027344,
"margin_dpo/margin_std": 646.734130859375,
"step": 490
},
{
"KL/chosen_KL_mean": -616.6781616210938,
"KL/mean": -748.5814208984375,
"KL/rejected_KL_mean": -880.484619140625,
"KL/std": 459.8958740234375,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.0011040510144084692,
"fcm_dpo/delta": 0.11021701991558075,
"fcm_dpo/margin": 263.8064880371094,
"fcm_dpo/q_t": 0.43390926718711853,
"grad_norm": 33.199649810791016,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.9711790084838867,
"logits/rejected": -0.9616298675537109,
"logps/chosen": -676.79443359375,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -953.7574462890625,
"loss": 1.1829,
"margin_dpo/margin_mean": 263.8064880371094,
"margin_dpo/margin_std": 483.6207580566406,
"step": 491
},
{
"KL/chosen_KL_mean": -635.3643798828125,
"KL/mean": -769.837890625,
"KL/rejected_KL_mean": -904.3115234375,
"KL/std": 484.90753173828125,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0011131614446640015,
"fcm_dpo/delta": 0.10221239924430847,
"fcm_dpo/margin": 268.947021484375,
"fcm_dpo/q_t": 0.43284350633621216,
"grad_norm": 43.230899810791016,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -1.0389471054077148,
"logits/rejected": -1.0215075016021729,
"logps/chosen": -689.3585815429688,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -976.9710693359375,
"loss": 1.2,
"margin_dpo/margin_mean": 268.947021484375,
"margin_dpo/margin_std": 554.927978515625,
"step": 492
},
{
"KL/chosen_KL_mean": -699.6456298828125,
"KL/mean": -847.1690063476562,
"KL/rejected_KL_mean": -994.6923828125,
"KL/std": 572.62060546875,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0011133572552353144,
"fcm_dpo/delta": -0.03688516467809677,
"fcm_dpo/margin": 295.0467224121094,
"fcm_dpo/q_t": 0.42351529002189636,
"grad_norm": 34.844478607177734,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -1.1079821586608887,
"logits/rejected": -1.0783543586730957,
"logps/chosen": -775.1428833007812,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -1082.015380859375,
"loss": 1.1584,
"margin_dpo/margin_mean": 295.04669189453125,
"margin_dpo/margin_std": 489.8016357421875,
"step": 493
},
{
"KL/chosen_KL_mean": -565.2142944335938,
"KL/mean": -797.517578125,
"KL/rejected_KL_mean": -1029.821044921875,
"KL/std": 503.0618896484375,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.0011037448421120644,
"fcm_dpo/delta": -0.11577919125556946,
"fcm_dpo/margin": 464.60662841796875,
"fcm_dpo/q_t": 0.38258957862854004,
"grad_norm": 38.89155960083008,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.9311180114746094,
"logits/rejected": -0.9682860970497131,
"logps/chosen": -606.5736083984375,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -1115.912353515625,
"loss": 1.0082,
"margin_dpo/margin_mean": 464.606689453125,
"margin_dpo/margin_std": 502.99725341796875,
"step": 494
},
{
"KL/chosen_KL_mean": -633.6134033203125,
"KL/mean": -830.8568725585938,
"KL/rejected_KL_mean": -1028.1004638671875,
"KL/std": 498.0575866699219,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0010981929954141378,
"fcm_dpo/delta": -0.033951401710510254,
"fcm_dpo/margin": 394.487060546875,
"fcm_dpo/q_t": 0.40414437651634216,
"grad_norm": 42.01950454711914,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -1.0234172344207764,
"logits/rejected": -1.0299253463745117,
"logps/chosen": -697.1484375,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -1119.52490234375,
"loss": 1.0924,
"margin_dpo/margin_mean": 394.487060546875,
"margin_dpo/margin_std": 591.7906494140625,
"step": 495
},
{
"KL/chosen_KL_mean": -781.1953735351562,
"KL/mean": -882.2847900390625,
"KL/rejected_KL_mean": -983.3742065429688,
"KL/std": 428.71868896484375,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0011013501789420843,
"fcm_dpo/delta": 0.08242875337600708,
"fcm_dpo/margin": 202.17886352539062,
"fcm_dpo/q_t": 0.44761157035827637,
"grad_norm": 73.75658416748047,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -1.0234228372573853,
"logits/rejected": -0.9919195175170898,
"logps/chosen": -853.787353515625,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -1067.70361328125,
"loss": 1.2357,
"margin_dpo/margin_mean": 202.17886352539062,
"margin_dpo/margin_std": 450.7599792480469,
"step": 496
},
{
"KL/chosen_KL_mean": -759.360595703125,
"KL/mean": -857.68359375,
"KL/rejected_KL_mean": -956.0064697265625,
"KL/std": 512.5221557617188,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0011091852793470025,
"fcm_dpo/delta": 0.029760103672742844,
"fcm_dpo/margin": 196.64585876464844,
"fcm_dpo/q_t": 0.4496675729751587,
"grad_norm": 32.76177978515625,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.9546867609024048,
"logits/rejected": -0.935001015663147,
"logps/chosen": -817.95458984375,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -1032.294921875,
"loss": 1.2508,
"margin_dpo/margin_mean": 196.6458740234375,
"margin_dpo/margin_std": 478.19500732421875,
"step": 497
},
{
"KL/chosen_KL_mean": -722.9497680664062,
"KL/mean": -899.7124633789062,
"KL/rejected_KL_mean": -1076.47509765625,
"KL/std": 522.7940063476562,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0011089190375059843,
"fcm_dpo/delta": 0.008106292225420475,
"fcm_dpo/margin": 353.52532958984375,
"fcm_dpo/q_t": 0.4107200503349304,
"grad_norm": 31.680055618286133,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.9777836799621582,
"logits/rejected": -0.9642585515975952,
"logps/chosen": -794.1553955078125,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -1160.43310546875,
"loss": 1.1266,
"margin_dpo/margin_mean": 353.5253601074219,
"margin_dpo/margin_std": 576.6744384765625,
"step": 498
},
{
"KL/chosen_KL_mean": -631.90966796875,
"KL/mean": -870.5003051757812,
"KL/rejected_KL_mean": -1109.091064453125,
"KL/std": 562.7127685546875,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0010997585486620665,
"fcm_dpo/delta": -0.1281210333108902,
"fcm_dpo/margin": 477.1813049316406,
"fcm_dpo/q_t": 0.38377606868743896,
"grad_norm": 32.53940963745117,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.988601565361023,
"logits/rejected": -1.0279262065887451,
"logps/chosen": -683.1648559570312,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -1210.169677734375,
"loss": 1.0295,
"margin_dpo/margin_mean": 477.1813049316406,
"margin_dpo/margin_std": 616.3780517578125,
"step": 499
},
{
"KL/chosen_KL_mean": -733.3824462890625,
"KL/mean": -922.1387329101562,
"KL/rejected_KL_mean": -1110.89501953125,
"KL/std": 483.62493896484375,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0010943380184471607,
"fcm_dpo/delta": -0.013418500311672688,
"fcm_dpo/margin": 377.5125427246094,
"fcm_dpo/q_t": 0.4065711498260498,
"grad_norm": 32.50249099731445,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.9592075347900391,
"logits/rejected": -0.9868057370185852,
"logps/chosen": -790.409912109375,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -1204.8291015625,
"loss": 1.1156,
"margin_dpo/margin_mean": 377.5125732421875,
"margin_dpo/margin_std": 596.400634765625,
"step": 500
},
{
"KL/chosen_KL_mean": -652.4368286132812,
"KL/mean": -851.3330688476562,
"KL/rejected_KL_mean": -1050.229248046875,
"KL/std": 501.94122314453125,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.0010930509306490421,
"fcm_dpo/delta": -0.03560645133256912,
"fcm_dpo/margin": 397.7924499511719,
"fcm_dpo/q_t": 0.40338021516799927,
"grad_norm": 31.65900993347168,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.9449106454849243,
"logits/rejected": -0.9296088218688965,
"logps/chosen": -706.79638671875,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -1130.385986328125,
"loss": 1.1051,
"margin_dpo/margin_mean": 397.79241943359375,
"margin_dpo/margin_std": 624.0454711914062,
"step": 501
},
{
"KL/chosen_KL_mean": -590.9495849609375,
"KL/mean": -798.0103149414062,
"KL/rejected_KL_mean": -1005.0711059570312,
"KL/std": 499.19720458984375,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0010861887130886316,
"fcm_dpo/delta": -0.050974659621715546,
"fcm_dpo/margin": 414.1214599609375,
"fcm_dpo/q_t": 0.401166707277298,
"grad_norm": 28.097309112548828,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.8737450838088989,
"logits/rejected": -0.8553862571716309,
"logps/chosen": -658.550048828125,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -1088.019775390625,
"loss": 1.0626,
"margin_dpo/margin_mean": 414.1214599609375,
"margin_dpo/margin_std": 551.1231689453125,
"step": 502
},
{
"KL/chosen_KL_mean": -591.0940551757812,
"KL/mean": -777.262939453125,
"KL/rejected_KL_mean": -963.431884765625,
"KL/std": 456.4488220214844,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0010829106904566288,
"fcm_dpo/delta": -0.003399871289730072,
"fcm_dpo/margin": 372.33782958984375,
"fcm_dpo/q_t": 0.4094755947589874,
"grad_norm": 30.805957794189453,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.8509905338287354,
"logits/rejected": -0.8399511575698853,
"logps/chosen": -646.1724853515625,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -1045.937255859375,
"loss": 1.0941,
"margin_dpo/margin_mean": 372.33782958984375,
"margin_dpo/margin_std": 513.9088134765625,
"step": 503
},
{
"KL/chosen_KL_mean": -612.4645385742188,
"KL/mean": -844.0081176757812,
"KL/rejected_KL_mean": -1075.5516357421875,
"KL/std": 534.1448974609375,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.0010781993623822927,
"fcm_dpo/delta": -0.10178365558385849,
"fcm_dpo/margin": 463.0871887207031,
"fcm_dpo/q_t": 0.38906794786453247,
"grad_norm": 28.29525375366211,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.9321706295013428,
"logits/rejected": -0.9592180252075195,
"logps/chosen": -672.4302978515625,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -1179.313720703125,
"loss": 1.0388,
"margin_dpo/margin_mean": 463.087158203125,
"margin_dpo/margin_std": 572.2286987304688,
"step": 504
},
{
"KL/chosen_KL_mean": -688.0867919921875,
"KL/mean": -798.6611938476562,
"KL/rejected_KL_mean": -909.235595703125,
"KL/std": 483.4139099121094,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0010871076956391335,
"fcm_dpo/delta": 0.1614731252193451,
"fcm_dpo/margin": 221.14871215820312,
"fcm_dpo/q_t": 0.44385814666748047,
"grad_norm": 40.906707763671875,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.9417963027954102,
"logits/rejected": -0.9116191267967224,
"logps/chosen": -764.24169921875,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -997.8209228515625,
"loss": 1.2195,
"margin_dpo/margin_mean": 221.14871215820312,
"margin_dpo/margin_std": 449.42291259765625,
"step": 505
},
{
"KL/chosen_KL_mean": -552.0079345703125,
"KL/mean": -739.86669921875,
"KL/rejected_KL_mean": -927.7255249023438,
"KL/std": 480.6181640625,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.001087966375052929,
"fcm_dpo/delta": -0.009178068488836288,
"fcm_dpo/margin": 375.71759033203125,
"fcm_dpo/q_t": 0.40685611963272095,
"grad_norm": 41.05369567871094,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.9096465110778809,
"logits/rejected": -0.9178716540336609,
"logps/chosen": -600.9683837890625,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -1006.1405639648438,
"loss": 1.0943,
"margin_dpo/margin_mean": 375.71759033203125,
"margin_dpo/margin_std": 518.1092529296875,
"step": 506
},
{
"KL/chosen_KL_mean": -654.116943359375,
"KL/mean": -816.158447265625,
"KL/rejected_KL_mean": -978.199951171875,
"KL/std": 556.29931640625,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0010937817860394716,
"fcm_dpo/delta": 0.04635169357061386,
"fcm_dpo/margin": 324.0830383300781,
"fcm_dpo/q_t": 0.4228259325027466,
"grad_norm": 30.889352798461914,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.957282543182373,
"logits/rejected": -0.95253586769104,
"logps/chosen": -720.9183959960938,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -1073.5728759765625,
"loss": 1.1562,
"margin_dpo/margin_mean": 324.08306884765625,
"margin_dpo/margin_std": 591.4808959960938,
"step": 507
},
{
"KL/chosen_KL_mean": -661.949951171875,
"KL/mean": -807.3236083984375,
"KL/rejected_KL_mean": -952.697265625,
"KL/std": 519.9277954101562,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0011030698660761118,
"fcm_dpo/delta": 0.08043861389160156,
"fcm_dpo/margin": 290.74737548828125,
"fcm_dpo/q_t": 0.4288504719734192,
"grad_norm": 35.57503890991211,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.9091905355453491,
"logits/rejected": -0.9017305374145508,
"logps/chosen": -733.25341796875,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -1048.3248291015625,
"loss": 1.1934,
"margin_dpo/margin_mean": 290.74737548828125,
"margin_dpo/margin_std": 599.302734375,
"step": 508
},
{
"KL/chosen_KL_mean": -501.7856750488281,
"KL/mean": -661.9292602539062,
"KL/rejected_KL_mean": -822.0728149414062,
"KL/std": 382.68975830078125,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0011099616531282663,
"fcm_dpo/delta": 0.04505918547511101,
"fcm_dpo/margin": 320.2871398925781,
"fcm_dpo/q_t": 0.4187580645084381,
"grad_norm": 29.257396697998047,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.814157247543335,
"logits/rejected": -0.7995736598968506,
"logps/chosen": -565.6046142578125,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -905.3292236328125,
"loss": 1.1258,
"margin_dpo/margin_mean": 320.287109375,
"margin_dpo/margin_std": 466.1398010253906,
"step": 509
},
{
"KL/chosen_KL_mean": -626.7471923828125,
"KL/mean": -820.874755859375,
"KL/rejected_KL_mean": -1015.002197265625,
"KL/std": 441.17962646484375,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0011074226349592209,
"fcm_dpo/delta": -0.030622189864516258,
"fcm_dpo/margin": 388.2550354003906,
"fcm_dpo/q_t": 0.3993905484676361,
"grad_norm": 29.603723526000977,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.8510131239891052,
"logits/rejected": -0.8649104237556458,
"logps/chosen": -678.6256103515625,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1117.767333984375,
"loss": 1.0558,
"margin_dpo/margin_mean": 388.25506591796875,
"margin_dpo/margin_std": 431.11676025390625,
"step": 510
},
{
"KL/chosen_KL_mean": -593.8864135742188,
"KL/mean": -752.0758056640625,
"KL/rejected_KL_mean": -910.26513671875,
"KL/std": 475.2417907714844,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.0011101202107965946,
"fcm_dpo/delta": 0.04963508993387222,
"fcm_dpo/margin": 316.3787841796875,
"fcm_dpo/q_t": 0.4193497896194458,
"grad_norm": 29.786012649536133,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.9209908246994019,
"logits/rejected": -0.9174632430076599,
"logps/chosen": -654.12451171875,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -1003.1219482421875,
"loss": 1.1544,
"margin_dpo/margin_mean": 316.3787536621094,
"margin_dpo/margin_std": 560.738037109375,
"step": 511
},
{
"KL/chosen_KL_mean": -489.36114501953125,
"KL/mean": -704.290283203125,
"KL/rejected_KL_mean": -919.219482421875,
"KL/std": 439.2633056640625,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.001103471266105771,
"fcm_dpo/delta": -0.07630272209644318,
"fcm_dpo/margin": 429.8582763671875,
"fcm_dpo/q_t": 0.3897845149040222,
"grad_norm": 37.09547805786133,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.8986982107162476,
"logits/rejected": -0.9108966588973999,
"logps/chosen": -544.2666625976562,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -1001.0953369140625,
"loss": 1.0214,
"margin_dpo/margin_mean": 429.8583068847656,
"margin_dpo/margin_std": 438.10321044921875,
"step": 512
},
{
"KL/chosen_KL_mean": -636.2532958984375,
"KL/mean": -780.3316650390625,
"KL/rejected_KL_mean": -924.4100341796875,
"KL/std": 423.2544250488281,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.00109992700163275,
"fcm_dpo/delta": -0.025476014241576195,
"fcm_dpo/margin": 288.15679931640625,
"fcm_dpo/q_t": 0.4271167516708374,
"grad_norm": 44.68075942993164,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.9700770378112793,
"logits/rejected": -0.9396867752075195,
"logps/chosen": -701.169677734375,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -1000.4725341796875,
"loss": 1.1665,
"margin_dpo/margin_mean": 288.15679931640625,
"margin_dpo/margin_std": 489.13134765625,
"step": 513
},
{
"KL/chosen_KL_mean": -640.1827392578125,
"KL/mean": -787.67919921875,
"KL/rejected_KL_mean": -935.1756591796875,
"KL/std": 454.5134582519531,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.00110641831997782,
"fcm_dpo/delta": 0.07481080293655396,
"fcm_dpo/margin": 294.99298095703125,
"fcm_dpo/q_t": 0.42692023515701294,
"grad_norm": 36.836143493652344,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.9246986508369446,
"logits/rejected": -0.904691755771637,
"logps/chosen": -714.412353515625,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -1014.1212158203125,
"loss": 1.1608,
"margin_dpo/margin_mean": 294.99298095703125,
"margin_dpo/margin_std": 514.4725341796875,
"step": 514
},
{
"KL/chosen_KL_mean": -525.1871337890625,
"KL/mean": -651.1346435546875,
"KL/rejected_KL_mean": -777.08203125,
"KL/std": 396.79437255859375,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0011181033914908767,
"fcm_dpo/delta": 0.12005515396595001,
"fcm_dpo/margin": 251.8948974609375,
"fcm_dpo/q_t": 0.4334961175918579,
"grad_norm": 34.989688873291016,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.8561180830001831,
"logits/rejected": -0.8611321449279785,
"logps/chosen": -575.5887451171875,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -864.1798095703125,
"loss": 1.1867,
"margin_dpo/margin_mean": 251.89492797851562,
"margin_dpo/margin_std": 460.27935791015625,
"step": 515
},
{
"KL/chosen_KL_mean": -596.61669921875,
"KL/mean": -770.6539306640625,
"KL/rejected_KL_mean": -944.6910400390625,
"KL/std": 452.91241455078125,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.0011240649037063122,
"fcm_dpo/delta": 0.00880388356745243,
"fcm_dpo/margin": 348.07440185546875,
"fcm_dpo/q_t": 0.408348023891449,
"grad_norm": 32.15882110595703,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.9794058799743652,
"logits/rejected": -0.9973136186599731,
"logps/chosen": -661.2662353515625,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -1056.4134521484375,
"loss": 1.0971,
"margin_dpo/margin_mean": 348.0743713378906,
"margin_dpo/margin_std": 459.40338134765625,
"step": 516
},
{
"KL/chosen_KL_mean": -583.4486083984375,
"KL/mean": -800.865478515625,
"KL/rejected_KL_mean": -1018.2823486328125,
"KL/std": 497.50537109375,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0011140817077830434,
"fcm_dpo/delta": -0.0866817757487297,
"fcm_dpo/margin": 434.8336486816406,
"fcm_dpo/q_t": 0.39268559217453003,
"grad_norm": 31.913042068481445,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.93073570728302,
"logits/rejected": -0.9230749607086182,
"logps/chosen": -644.3621826171875,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1107.365478515625,
"loss": 1.0456,
"margin_dpo/margin_mean": 434.83367919921875,
"margin_dpo/margin_std": 564.2802124023438,
"step": 517
},
{
"KL/chosen_KL_mean": -531.2779541015625,
"KL/mean": -708.3522338867188,
"KL/rejected_KL_mean": -885.426513671875,
"KL/std": 404.1721496582031,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.0011119421105831861,
"fcm_dpo/delta": 0.006271369755268097,
"fcm_dpo/margin": 354.1485900878906,
"fcm_dpo/q_t": 0.40764957666397095,
"grad_norm": 31.667274475097656,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.9522829055786133,
"logits/rejected": -0.9628884792327881,
"logps/chosen": -588.7338256835938,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -970.7392578125,
"loss": 1.0874,
"margin_dpo/margin_mean": 354.1485900878906,
"margin_dpo/margin_std": 437.1719970703125,
"step": 518
},
{
"KL/chosen_KL_mean": -544.94873046875,
"KL/mean": -669.02685546875,
"KL/rejected_KL_mean": -793.10498046875,
"KL/std": 379.0975646972656,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0011145803146064281,
"fcm_dpo/delta": 0.018153877928853035,
"fcm_dpo/margin": 248.15621948242188,
"fcm_dpo/q_t": 0.4351785182952881,
"grad_norm": 26.59686851501465,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.9192925691604614,
"logits/rejected": -0.8974975347518921,
"logps/chosen": -619.0120849609375,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -897.5491943359375,
"loss": 1.192,
"margin_dpo/margin_mean": 248.15621948242188,
"margin_dpo/margin_std": 452.4583435058594,
"step": 519
},
{
"KL/chosen_KL_mean": -566.6231689453125,
"KL/mean": -728.2750854492188,
"KL/rejected_KL_mean": -889.9268798828125,
"KL/std": 431.6119689941406,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0011171854566782713,
"fcm_dpo/delta": 0.039489611983299255,
"fcm_dpo/margin": 323.3037109375,
"fcm_dpo/q_t": 0.41814154386520386,
"grad_norm": 34.39997100830078,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.8914647102355957,
"logits/rejected": -0.8902548551559448,
"logps/chosen": -636.9230346679688,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -989.9082641601562,
"loss": 1.1248,
"margin_dpo/margin_mean": 323.3037109375,
"margin_dpo/margin_std": 469.2210693359375,
"step": 520
},
{
"KL/chosen_KL_mean": -525.2251586914062,
"KL/mean": -698.4849243164062,
"KL/rejected_KL_mean": -871.74462890625,
"KL/std": 449.91253662109375,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.0011244425550103188,
"fcm_dpo/delta": 0.010120447725057602,
"fcm_dpo/margin": 346.51953125,
"fcm_dpo/q_t": 0.4106011986732483,
"grad_norm": 25.767515182495117,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.9617199897766113,
"logits/rejected": -0.9598067402839661,
"logps/chosen": -583.3680419921875,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -955.0252685546875,
"loss": 1.1012,
"margin_dpo/margin_mean": 346.51953125,
"margin_dpo/margin_std": 464.26416015625,
"step": 521
},
{
"KL/chosen_KL_mean": -578.2764892578125,
"KL/mean": -737.2076416015625,
"KL/rejected_KL_mean": -896.1387939453125,
"KL/std": 469.0858154296875,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.001124875620007515,
"fcm_dpo/delta": 0.04321271926164627,
"fcm_dpo/margin": 317.8622131347656,
"fcm_dpo/q_t": 0.4179668128490448,
"grad_norm": 27.95970916748047,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.9487807750701904,
"logits/rejected": -0.9443739652633667,
"logps/chosen": -624.82421875,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -962.1526489257812,
"loss": 1.1318,
"margin_dpo/margin_mean": 317.86224365234375,
"margin_dpo/margin_std": 489.20440673828125,
"step": 522
},
{
"KL/chosen_KL_mean": -616.9910888671875,
"KL/mean": -790.3553466796875,
"KL/rejected_KL_mean": -963.7196044921875,
"KL/std": 531.9453125,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0011264176573604345,
"fcm_dpo/delta": 0.00962541438639164,
"fcm_dpo/margin": 346.72845458984375,
"fcm_dpo/q_t": 0.4115051031112671,
"grad_norm": 49.35715103149414,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.9954909086227417,
"logits/rejected": -0.9879505634307861,
"logps/chosen": -678.7607421875,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1047.48095703125,
"loss": 1.1189,
"margin_dpo/margin_mean": 346.72845458984375,
"margin_dpo/margin_std": 540.300537109375,
"step": 523
},
{
"KL/chosen_KL_mean": -613.022705078125,
"KL/mean": -792.3504638671875,
"KL/rejected_KL_mean": -971.6781005859375,
"KL/std": 508.4144287109375,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.001124032773077488,
"fcm_dpo/delta": -0.00345630943775177,
"fcm_dpo/margin": 358.6553649902344,
"fcm_dpo/q_t": 0.4057971239089966,
"grad_norm": 34.90718078613281,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -1.0312086343765259,
"logits/rejected": -1.0126185417175293,
"logps/chosen": -691.0947875976562,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -1052.9801025390625,
"loss": 1.1073,
"margin_dpo/margin_mean": 358.6553649902344,
"margin_dpo/margin_std": 517.4310302734375,
"step": 524
},
{
"KL/chosen_KL_mean": -596.0318603515625,
"KL/mean": -821.548828125,
"KL/rejected_KL_mean": -1047.0657958984375,
"KL/std": 515.7371826171875,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0011153742671012878,
"fcm_dpo/delta": -0.10615138709545135,
"fcm_dpo/margin": 451.0339660644531,
"fcm_dpo/q_t": 0.38702392578125,
"grad_norm": 34.25413131713867,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.9680420160293579,
"logits/rejected": -1.0015491247177124,
"logps/chosen": -646.8596801757812,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1147.11865234375,
"loss": 1.026,
"margin_dpo/margin_mean": 451.0339660644531,
"margin_dpo/margin_std": 533.57666015625,
"step": 525
},
{
"KL/chosen_KL_mean": -641.201904296875,
"KL/mean": -872.0587158203125,
"KL/rejected_KL_mean": -1102.91552734375,
"KL/std": 532.249267578125,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.0011071430053561926,
"fcm_dpo/delta": -0.1140209287405014,
"fcm_dpo/margin": 461.7136535644531,
"fcm_dpo/q_t": 0.383984237909317,
"grad_norm": 31.422950744628906,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.9887993335723877,
"logits/rejected": -0.9786882400512695,
"logps/chosen": -704.369140625,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1189.224853515625,
"loss": 1.0411,
"margin_dpo/margin_mean": 461.713623046875,
"margin_dpo/margin_std": 597.789794921875,
"step": 526
},
{
"KL/chosen_KL_mean": -675.8800048828125,
"KL/mean": -825.571533203125,
"KL/rejected_KL_mean": -975.262939453125,
"KL/std": 547.1878662109375,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0011063042329624295,
"fcm_dpo/delta": 0.06996496021747589,
"fcm_dpo/margin": 299.3829650878906,
"fcm_dpo/q_t": 0.4228494465351105,
"grad_norm": 37.20889663696289,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -1.1690475940704346,
"logits/rejected": -1.126373529434204,
"logps/chosen": -747.0086669921875,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1053.60546875,
"loss": 1.1431,
"margin_dpo/margin_mean": 299.3829650878906,
"margin_dpo/margin_std": 453.30816650390625,
"step": 527
},
{
"KL/chosen_KL_mean": -692.7294921875,
"KL/mean": -880.4638671875,
"KL/rejected_KL_mean": -1068.1982421875,
"KL/std": 529.0624389648438,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0011081206612288952,
"fcm_dpo/delta": -0.016417687758803368,
"fcm_dpo/margin": 375.46881103515625,
"fcm_dpo/q_t": 0.4085617959499359,
"grad_norm": 37.37120056152344,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -1.0335596799850464,
"logits/rejected": -1.0141860246658325,
"logps/chosen": -760.8189697265625,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1162.1083984375,
"loss": 1.1177,
"margin_dpo/margin_mean": 375.46881103515625,
"margin_dpo/margin_std": 612.314697265625,
"step": 528
},
{
"KL/chosen_KL_mean": -838.7152099609375,
"KL/mean": -968.595458984375,
"KL/rejected_KL_mean": -1098.4757080078125,
"KL/std": 483.984130859375,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.001110037206672132,
"fcm_dpo/delta": 0.017718147486448288,
"fcm_dpo/margin": 259.760498046875,
"fcm_dpo/q_t": 0.4340352416038513,
"grad_norm": 38.99552536010742,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.1005570888519287,
"logits/rejected": -1.1074516773223877,
"logps/chosen": -894.2901611328125,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1187.684814453125,
"loss": 1.2326,
"margin_dpo/margin_mean": 259.760498046875,
"margin_dpo/margin_std": 621.6509399414062,
"step": 529
},
{
"KL/chosen_KL_mean": -708.62939453125,
"KL/mean": -925.3953857421875,
"KL/rejected_KL_mean": -1142.161376953125,
"KL/std": 603.607666015625,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0011045248247683048,
"fcm_dpo/delta": -0.0807407796382904,
"fcm_dpo/margin": 433.53204345703125,
"fcm_dpo/q_t": 0.39779338240623474,
"grad_norm": 48.64304733276367,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.9852283000946045,
"logits/rejected": -0.9839862585067749,
"logps/chosen": -756.2308349609375,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1229.4459228515625,
"loss": 1.1157,
"margin_dpo/margin_mean": 433.53204345703125,
"margin_dpo/margin_std": 764.8116455078125,
"step": 530
},
{
"KL/chosen_KL_mean": -784.3384399414062,
"KL/mean": -969.3201904296875,
"KL/rejected_KL_mean": -1154.302001953125,
"KL/std": 622.963623046875,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.0011020496021956205,
"fcm_dpo/delta": -0.007951600477099419,
"fcm_dpo/margin": 369.9634704589844,
"fcm_dpo/q_t": 0.4096531271934509,
"grad_norm": 47.655757904052734,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.1287320852279663,
"logits/rejected": -1.113837480545044,
"logps/chosen": -839.58447265625,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1224.907958984375,
"loss": 1.1196,
"margin_dpo/margin_mean": 369.9634704589844,
"margin_dpo/margin_std": 602.9121704101562,
"step": 531
},
{
"KL/chosen_KL_mean": -803.9302368164062,
"KL/mean": -986.0310668945312,
"KL/rejected_KL_mean": -1168.1319580078125,
"KL/std": 576.912353515625,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0010961306979879737,
"fcm_dpo/delta": 0.0002321675419807434,
"fcm_dpo/margin": 364.2017822265625,
"fcm_dpo/q_t": 0.4125855565071106,
"grad_norm": 69.55883026123047,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -1.0360071659088135,
"logits/rejected": -1.0059716701507568,
"logps/chosen": -874.2162475585938,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1254.7232666015625,
"loss": 1.1623,
"margin_dpo/margin_mean": 364.2017517089844,
"margin_dpo/margin_std": 691.018310546875,
"step": 532
},
{
"KL/chosen_KL_mean": -728.8594360351562,
"KL/mean": -868.2634887695312,
"KL/rejected_KL_mean": -1007.6676025390625,
"KL/std": 510.1298522949219,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0010999096557497978,
"fcm_dpo/delta": -0.006392236333340406,
"fcm_dpo/margin": 278.80816650390625,
"fcm_dpo/q_t": 0.43125709891319275,
"grad_norm": 32.62123489379883,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -1.0549545288085938,
"logits/rejected": -1.0321242809295654,
"logps/chosen": -793.7145385742188,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1084.255615234375,
"loss": 1.2058,
"margin_dpo/margin_mean": 278.80816650390625,
"margin_dpo/margin_std": 602.3246459960938,
"step": 533
},
{
"KL/chosen_KL_mean": -744.268798828125,
"KL/mean": -940.6812744140625,
"KL/rejected_KL_mean": -1137.09375,
"KL/std": 581.1688232421875,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.001092701917514205,
"fcm_dpo/delta": -0.030640922486782074,
"fcm_dpo/margin": 392.8249206542969,
"fcm_dpo/q_t": 0.4050602912902832,
"grad_norm": 32.57998275756836,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -1.0226364135742188,
"logits/rejected": -1.0192043781280518,
"logps/chosen": -804.38818359375,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1215.63720703125,
"loss": 1.0993,
"margin_dpo/margin_mean": 392.824951171875,
"margin_dpo/margin_std": 582.9520874023438,
"step": 534
},
{
"KL/chosen_KL_mean": -627.1175537109375,
"KL/mean": -852.0411987304688,
"KL/rejected_KL_mean": -1076.96484375,
"KL/std": 558.7318115234375,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.0010921892244368792,
"fcm_dpo/delta": -0.09375444799661636,
"fcm_dpo/margin": 449.84735107421875,
"fcm_dpo/q_t": 0.3906930983066559,
"grad_norm": 33.22504806518555,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.9372042417526245,
"logits/rejected": -0.939862847328186,
"logps/chosen": -681.44775390625,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1173.2724609375,
"loss": 1.0486,
"margin_dpo/margin_mean": 449.84735107421875,
"margin_dpo/margin_std": 577.599609375,
"step": 535
},
{
"KL/chosen_KL_mean": -568.00146484375,
"KL/mean": -827.193603515625,
"KL/rejected_KL_mean": -1086.3857421875,
"KL/std": 608.2098388671875,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0010737399570643902,
"fcm_dpo/delta": -0.16100189089775085,
"fcm_dpo/margin": 518.3843383789062,
"fcm_dpo/q_t": 0.3829028010368347,
"grad_norm": 56.74153137207031,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.9405168294906616,
"logits/rejected": -0.9727465510368347,
"logps/chosen": -615.08203125,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1175.483642578125,
"loss": 1.0262,
"margin_dpo/margin_mean": 518.3843383789062,
"margin_dpo/margin_std": 723.623046875,
"step": 536
},
{
"KL/chosen_KL_mean": -583.8455810546875,
"KL/mean": -774.345703125,
"KL/rejected_KL_mean": -964.8458251953125,
"KL/std": 504.427490234375,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0010713583324104548,
"fcm_dpo/delta": -0.008746057748794556,
"fcm_dpo/margin": 381.0003662109375,
"fcm_dpo/q_t": 0.4074852764606476,
"grad_norm": 44.69304275512695,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.9550840854644775,
"logits/rejected": -0.9326896667480469,
"logps/chosen": -641.593017578125,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -1035.2841796875,
"loss": 1.1209,
"margin_dpo/margin_mean": 381.0003356933594,
"margin_dpo/margin_std": 604.8282470703125,
"step": 537
},
{
"KL/chosen_KL_mean": -731.8800048828125,
"KL/mean": -923.4337158203125,
"KL/rejected_KL_mean": -1114.9873046875,
"KL/std": 585.79248046875,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0010648441966623068,
"fcm_dpo/delta": -0.008301455527544022,
"fcm_dpo/margin": 383.10736083984375,
"fcm_dpo/q_t": 0.4077230989933014,
"grad_norm": 41.17292785644531,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.9506703019142151,
"logits/rejected": -0.9313552379608154,
"logps/chosen": -798.2959594726562,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1199.2154541015625,
"loss": 1.1046,
"margin_dpo/margin_mean": 383.1073913574219,
"margin_dpo/margin_std": 566.696044921875,
"step": 538
},
{
"KL/chosen_KL_mean": -605.9473876953125,
"KL/mean": -802.4550170898438,
"KL/rejected_KL_mean": -998.962646484375,
"KL/std": 455.50811767578125,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.001066508935764432,
"fcm_dpo/delta": -0.01959817484021187,
"fcm_dpo/margin": 393.01519775390625,
"fcm_dpo/q_t": 0.40525415539741516,
"grad_norm": 32.76362228393555,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.982710599899292,
"logits/rejected": -0.9756343364715576,
"logps/chosen": -664.4402465820312,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -1090.816650390625,
"loss": 1.08,
"margin_dpo/margin_mean": 393.0152282714844,
"margin_dpo/margin_std": 511.8883972167969,
"step": 539
},
{
"KL/chosen_KL_mean": -607.8150024414062,
"KL/mean": -850.8222045898438,
"KL/rejected_KL_mean": -1093.829345703125,
"KL/std": 560.0186767578125,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0010573212057352066,
"fcm_dpo/delta": -0.11679581552743912,
"fcm_dpo/margin": 486.01434326171875,
"fcm_dpo/q_t": 0.38948220014572144,
"grad_norm": 36.21888732910156,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.9439930319786072,
"logits/rejected": -0.96489417552948,
"logps/chosen": -671.2975463867188,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1210.25927734375,
"loss": 1.0536,
"margin_dpo/margin_mean": 486.0143127441406,
"margin_dpo/margin_std": 680.3463134765625,
"step": 540
},
{
"KL/chosen_KL_mean": -727.278564453125,
"KL/mean": -857.4025268554688,
"KL/rejected_KL_mean": -987.5263671875,
"KL/std": 491.0820007324219,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.001053128275088966,
"fcm_dpo/delta": 0.007406635209918022,
"fcm_dpo/margin": 260.24786376953125,
"fcm_dpo/q_t": 0.43850696086883545,
"grad_norm": 51.57310104370117,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -1.0330259799957275,
"logits/rejected": -1.0214934349060059,
"logps/chosen": -805.5589599609375,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1090.800048828125,
"loss": 1.2217,
"margin_dpo/margin_mean": 260.24786376953125,
"margin_dpo/margin_std": 586.0404052734375,
"step": 541
},
{
"KL/chosen_KL_mean": -627.02099609375,
"KL/mean": -874.4854125976562,
"KL/rejected_KL_mean": -1121.94970703125,
"KL/std": 565.48876953125,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.0010395846329629421,
"fcm_dpo/delta": -0.11870333552360535,
"fcm_dpo/margin": 494.9288024902344,
"fcm_dpo/q_t": 0.3869064450263977,
"grad_norm": 46.05047607421875,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -1.0332136154174805,
"logits/rejected": -1.0657650232315063,
"logps/chosen": -684.5059814453125,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1218.4248046875,
"loss": 1.0441,
"margin_dpo/margin_mean": 494.9288024902344,
"margin_dpo/margin_std": 656.42431640625,
"step": 542
},
{
"KL/chosen_KL_mean": -702.0233764648438,
"KL/mean": -875.46875,
"KL/rejected_KL_mean": -1048.9141845703125,
"KL/std": 650.103271484375,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0010423632338643074,
"fcm_dpo/delta": 0.03910157456994057,
"fcm_dpo/margin": 346.890869140625,
"fcm_dpo/q_t": 0.4295872449874878,
"grad_norm": 56.74467468261719,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.9723141193389893,
"logits/rejected": -1.0032031536102295,
"logps/chosen": -762.640869140625,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1154.51318359375,
"loss": 1.199,
"margin_dpo/margin_mean": 346.890869140625,
"margin_dpo/margin_std": 787.779541015625,
"step": 543
},
{
"KL/chosen_KL_mean": -699.0665893554688,
"KL/mean": -904.1417236328125,
"KL/rejected_KL_mean": -1109.217041015625,
"KL/std": 525.4854125976562,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0010421974584460258,
"fcm_dpo/delta": -0.028073592111468315,
"fcm_dpo/margin": 410.15032958984375,
"fcm_dpo/q_t": 0.4034753441810608,
"grad_norm": 39.14487838745117,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.9627532958984375,
"logits/rejected": -0.9823678135871887,
"logps/chosen": -758.7088623046875,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1210.171630859375,
"loss": 1.0915,
"margin_dpo/margin_mean": 410.15032958984375,
"margin_dpo/margin_std": 594.836669921875,
"step": 544
},
{
"KL/chosen_KL_mean": -665.3682861328125,
"KL/mean": -904.0440063476562,
"KL/rejected_KL_mean": -1142.7197265625,
"KL/std": 538.8303833007812,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0010349673684686422,
"fcm_dpo/delta": -0.0963834896683693,
"fcm_dpo/margin": 477.3514709472656,
"fcm_dpo/q_t": 0.38999858498573303,
"grad_norm": 32.36417007446289,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.8844515085220337,
"logits/rejected": -0.9089665412902832,
"logps/chosen": -733.016845703125,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1238.6278076171875,
"loss": 1.0415,
"margin_dpo/margin_mean": 477.3514404296875,
"margin_dpo/margin_std": 607.8944091796875,
"step": 545
},
{
"KL/chosen_KL_mean": -620.0447998046875,
"KL/mean": -773.5802001953125,
"KL/rejected_KL_mean": -927.1156005859375,
"KL/std": 466.5025634765625,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.001038281712681055,
"fcm_dpo/delta": 0.0824858769774437,
"fcm_dpo/margin": 307.0708923339844,
"fcm_dpo/q_t": 0.4243466258049011,
"grad_norm": 31.921030044555664,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.9386703372001648,
"logits/rejected": -0.9219088554382324,
"logps/chosen": -670.7890014648438,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -1008.9818725585938,
"loss": 1.1562,
"margin_dpo/margin_mean": 307.0708923339844,
"margin_dpo/margin_std": 493.91357421875,
"step": 546
},
{
"KL/chosen_KL_mean": -614.9042358398438,
"KL/mean": -828.0643310546875,
"KL/rejected_KL_mean": -1041.2242431640625,
"KL/std": 532.6348266601562,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.001038061804138124,
"fcm_dpo/delta": -0.04350613057613373,
"fcm_dpo/margin": 426.320068359375,
"fcm_dpo/q_t": 0.4003497362136841,
"grad_norm": 49.95761489868164,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.9308812618255615,
"logits/rejected": -0.9381631016731262,
"logps/chosen": -688.5919189453125,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -1131.985595703125,
"loss": 1.0764,
"margin_dpo/margin_mean": 426.320068359375,
"margin_dpo/margin_std": 586.7791748046875,
"step": 547
},
{
"KL/chosen_KL_mean": -657.8882446289062,
"KL/mean": -843.9827270507812,
"KL/rejected_KL_mean": -1030.0771484375,
"KL/std": 555.6254272460938,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0010367643553763628,
"fcm_dpo/delta": 0.014406869187951088,
"fcm_dpo/margin": 372.18896484375,
"fcm_dpo/q_t": 0.4164394736289978,
"grad_norm": 30.012165069580078,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -1.041344165802002,
"logits/rejected": -1.0381653308868408,
"logps/chosen": -723.1345825195312,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1124.1953125,
"loss": 1.1255,
"margin_dpo/margin_mean": 372.18896484375,
"margin_dpo/margin_std": 620.2734375,
"step": 548
},
{
"KL/chosen_KL_mean": -680.7687377929688,
"KL/mean": -830.600341796875,
"KL/rejected_KL_mean": -980.431884765625,
"KL/std": 459.7933349609375,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0010372500400990248,
"fcm_dpo/delta": -0.002494156127795577,
"fcm_dpo/margin": 299.6632080078125,
"fcm_dpo/q_t": 0.429217129945755,
"grad_norm": 53.25551223754883,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -0.9812500476837158,
"logits/rejected": -0.9636787176132202,
"logps/chosen": -729.9810791015625,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1054.34228515625,
"loss": 1.1786,
"margin_dpo/margin_mean": 299.6632385253906,
"margin_dpo/margin_std": 550.6836547851562,
"step": 549
},
{
"KL/chosen_KL_mean": -688.97412109375,
"KL/mean": -860.2440185546875,
"KL/rejected_KL_mean": -1031.513916015625,
"KL/std": 518.7420043945312,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0010400728788226843,
"fcm_dpo/delta": 0.04453900456428528,
"fcm_dpo/margin": 342.53973388671875,
"fcm_dpo/q_t": 0.41772040724754333,
"grad_norm": 38.28087615966797,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.9451702833175659,
"logits/rejected": -0.9619972705841064,
"logps/chosen": -745.7811279296875,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1126.6396484375,
"loss": 1.1281,
"margin_dpo/margin_mean": 342.5397644042969,
"margin_dpo/margin_std": 515.8134155273438,
"step": 550
},
{
"KL/chosen_KL_mean": -614.6915283203125,
"KL/mean": -889.161376953125,
"KL/rejected_KL_mean": -1163.6312255859375,
"KL/std": 555.507568359375,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0010279393754899502,
"fcm_dpo/delta": -0.1691897213459015,
"fcm_dpo/margin": 548.9395751953125,
"fcm_dpo/q_t": 0.3716570734977722,
"grad_norm": 40.599369049072266,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.935494065284729,
"logits/rejected": -0.987216055393219,
"logps/chosen": -673.7978515625,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1275.303955078125,
"loss": 0.9718,
"margin_dpo/margin_mean": 548.939697265625,
"margin_dpo/margin_std": 550.9382934570312,
"step": 551
},
{
"KL/chosen_KL_mean": -578.4395751953125,
"KL/mean": -906.6759643554688,
"KL/rejected_KL_mean": -1234.912353515625,
"KL/std": 645.207763671875,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.000999854295514524,
"fcm_dpo/delta": -0.2669294476509094,
"fcm_dpo/margin": 656.4727783203125,
"fcm_dpo/q_t": 0.3599150478839874,
"grad_norm": 48.74257278442383,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.9877578020095825,
"logits/rejected": -1.0204415321350098,
"logps/chosen": -640.794189453125,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1339.474365234375,
"loss": 0.9475,
"margin_dpo/margin_mean": 656.4727783203125,
"margin_dpo/margin_std": 730.9444580078125,
"step": 552
},
{
"KL/chosen_KL_mean": -671.1646728515625,
"KL/mean": -830.6427001953125,
"KL/rejected_KL_mean": -990.1207275390625,
"KL/std": 539.9063110351562,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0010021075140684843,
"fcm_dpo/delta": 0.08171496540307999,
"fcm_dpo/margin": 318.9561462402344,
"fcm_dpo/q_t": 0.4295930862426758,
"grad_norm": 27.265270233154297,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.9643408060073853,
"logits/rejected": -0.9552521109580994,
"logps/chosen": -739.4234619140625,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1088.2178955078125,
"loss": 1.168,
"margin_dpo/margin_mean": 318.9561462402344,
"margin_dpo/margin_std": 580.3341064453125,
"step": 553
},
{
"KL/chosen_KL_mean": -698.4722900390625,
"KL/mean": -916.677978515625,
"KL/rejected_KL_mean": -1134.8837890625,
"KL/std": 573.2615966796875,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0010056591127067804,
"fcm_dpo/delta": -0.04003161936998367,
"fcm_dpo/margin": 436.4113464355469,
"fcm_dpo/q_t": 0.40393197536468506,
"grad_norm": 90.09304809570312,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.945639967918396,
"logits/rejected": -0.9097083806991577,
"logps/chosen": -766.4199829101562,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1224.66650390625,
"loss": 1.1214,
"margin_dpo/margin_mean": 436.411376953125,
"margin_dpo/margin_std": 732.8826293945312,
"step": 554
},
{
"KL/chosen_KL_mean": -696.947509765625,
"KL/mean": -960.1400146484375,
"KL/rejected_KL_mean": -1223.3326416015625,
"KL/std": 601.5198974609375,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0009933036053553224,
"fcm_dpo/delta": -0.12606467306613922,
"fcm_dpo/margin": 526.385009765625,
"fcm_dpo/q_t": 0.3890535533428192,
"grad_norm": 34.906185150146484,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -1.0455199480056763,
"logits/rejected": -1.097933053970337,
"logps/chosen": -750.2780151367188,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1331.81201171875,
"loss": 1.0568,
"margin_dpo/margin_mean": 526.385009765625,
"margin_dpo/margin_std": 771.40576171875,
"step": 555
},
{
"KL/chosen_KL_mean": -692.2669677734375,
"KL/mean": -844.3529052734375,
"KL/rejected_KL_mean": -996.4388427734375,
"KL/std": 467.8826599121094,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.000995925161987543,
"fcm_dpo/delta": 0.0985875129699707,
"fcm_dpo/margin": 304.17181396484375,
"fcm_dpo/q_t": 0.42986422777175903,
"grad_norm": 33.946044921875,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -1.021904468536377,
"logits/rejected": -1.0080738067626953,
"logps/chosen": -750.9114990234375,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1097.779296875,
"loss": 1.1616,
"margin_dpo/margin_mean": 304.17181396484375,
"margin_dpo/margin_std": 485.27044677734375,
"step": 556
},
{
"KL/chosen_KL_mean": -720.950439453125,
"KL/mean": -956.94775390625,
"KL/rejected_KL_mean": -1192.9451904296875,
"KL/std": 639.4453735351562,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.000991870416328311,
"fcm_dpo/delta": -0.07001565396785736,
"fcm_dpo/margin": 471.9947204589844,
"fcm_dpo/q_t": 0.39792197942733765,
"grad_norm": 49.5215950012207,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -1.0259175300598145,
"logits/rejected": -1.060760259628296,
"logps/chosen": -788.7911376953125,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1302.884765625,
"loss": 1.0874,
"margin_dpo/margin_mean": 471.9947509765625,
"margin_dpo/margin_std": 715.3715209960938,
"step": 557
},
{
"KL/chosen_KL_mean": -663.1494750976562,
"KL/mean": -974.2451171875,
"KL/rejected_KL_mean": -1285.3406982421875,
"KL/std": 596.8526611328125,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0009766137227416039,
"fcm_dpo/delta": -0.21403643488883972,
"fcm_dpo/margin": 622.1912841796875,
"fcm_dpo/q_t": 0.3626197576522827,
"grad_norm": 29.86164093017578,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -1.0814244747161865,
"logits/rejected": -1.0883204936981201,
"logps/chosen": -725.5177001953125,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1387.501708984375,
"loss": 0.963,
"margin_dpo/margin_mean": 622.1912841796875,
"margin_dpo/margin_std": 662.1346435546875,
"step": 558
},
{
"KL/chosen_KL_mean": -754.6362915039062,
"KL/mean": -1022.7030029296875,
"KL/rejected_KL_mean": -1290.7696533203125,
"KL/std": 619.6690673828125,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0009626256651245058,
"fcm_dpo/delta": -0.11912199854850769,
"fcm_dpo/margin": 536.13330078125,
"fcm_dpo/q_t": 0.3861456513404846,
"grad_norm": 31.161109924316406,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.1892856359481812,
"logits/rejected": -1.1905286312103271,
"logps/chosen": -815.388671875,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1384.2119140625,
"loss": 1.0321,
"margin_dpo/margin_mean": 536.13330078125,
"margin_dpo/margin_std": 686.558349609375,
"step": 559
},
{
"KL/chosen_KL_mean": -694.3114013671875,
"KL/mean": -896.381591796875,
"KL/rejected_KL_mean": -1098.4517822265625,
"KL/std": 594.5814208984375,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0009576280135661364,
"fcm_dpo/delta": 0.012987758964300156,
"fcm_dpo/margin": 404.14044189453125,
"fcm_dpo/q_t": 0.4146135449409485,
"grad_norm": 38.00310134887695,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.9253185987472534,
"logits/rejected": -0.9051507711410522,
"logps/chosen": -752.4151611328125,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1178.4429931640625,
"loss": 1.1291,
"margin_dpo/margin_mean": 404.14044189453125,
"margin_dpo/margin_std": 662.3407592773438,
"step": 560
},
{
"KL/chosen_KL_mean": -819.3272705078125,
"KL/mean": -979.8409423828125,
"KL/rejected_KL_mean": -1140.3546142578125,
"KL/std": 548.7021484375,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0009685677941888571,
"fcm_dpo/delta": 0.09021740406751633,
"fcm_dpo/margin": 321.0272521972656,
"fcm_dpo/q_t": 0.4277527630329132,
"grad_norm": 40.23988723754883,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.1046230792999268,
"logits/rejected": -1.1246148347854614,
"logps/chosen": -870.2401733398438,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1239.423095703125,
"loss": 1.2007,
"margin_dpo/margin_mean": 321.02728271484375,
"margin_dpo/margin_std": 687.259521484375,
"step": 561
},
{
"KL/chosen_KL_mean": -745.23193359375,
"KL/mean": -967.99609375,
"KL/rejected_KL_mean": -1190.76025390625,
"KL/std": 601.22607421875,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.0009640766074880958,
"fcm_dpo/delta": -0.030630581080913544,
"fcm_dpo/margin": 445.52838134765625,
"fcm_dpo/q_t": 0.404424250125885,
"grad_norm": 46.153045654296875,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -1.0498061180114746,
"logits/rejected": -1.0524308681488037,
"logps/chosen": -804.6962890625,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1287.302978515625,
"loss": 1.0927,
"margin_dpo/margin_mean": 445.5283508300781,
"margin_dpo/margin_std": 656.296630859375,
"step": 562
},
{
"KL/chosen_KL_mean": -846.0560913085938,
"KL/mean": -1012.888916015625,
"KL/rejected_KL_mean": -1179.7216796875,
"KL/std": 638.2447509765625,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.0009634558809921145,
"fcm_dpo/delta": -0.05767164006829262,
"fcm_dpo/margin": 333.6656494140625,
"fcm_dpo/q_t": 0.42173728346824646,
"grad_norm": 55.212547302246094,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -1.0601184368133545,
"logits/rejected": -1.059585690498352,
"logps/chosen": -906.664306640625,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1274.2894287109375,
"loss": 1.2261,
"margin_dpo/margin_mean": 333.6656494140625,
"margin_dpo/margin_std": 764.817138671875,
"step": 563
},
{
"KL/chosen_KL_mean": -765.0787353515625,
"KL/mean": -982.6279296875,
"KL/rejected_KL_mean": -1200.17724609375,
"KL/std": 581.216064453125,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0009602903737686574,
"fcm_dpo/delta": -0.0182206854224205,
"fcm_dpo/margin": 435.0984191894531,
"fcm_dpo/q_t": 0.4047633409500122,
"grad_norm": 36.67222595214844,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -1.1164246797561646,
"logits/rejected": -1.107216477394104,
"logps/chosen": -839.3470458984375,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1293.415283203125,
"loss": 1.0771,
"margin_dpo/margin_mean": 435.0984191894531,
"margin_dpo/margin_std": 550.7127685546875,
"step": 564
},
{
"KL/chosen_KL_mean": -791.003173828125,
"KL/mean": -1022.08740234375,
"KL/rejected_KL_mean": -1253.171630859375,
"KL/std": 678.6461181640625,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.000955872586928308,
"fcm_dpo/delta": -0.04271770268678665,
"fcm_dpo/margin": 462.16839599609375,
"fcm_dpo/q_t": 0.4068571925163269,
"grad_norm": 51.650264739990234,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -1.0443668365478516,
"logits/rejected": -1.038867712020874,
"logps/chosen": -860.0230712890625,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1338.9505615234375,
"loss": 1.1256,
"margin_dpo/margin_mean": 462.16839599609375,
"margin_dpo/margin_std": 813.2290649414062,
"step": 565
},
{
"KL/chosen_KL_mean": -779.5007934570312,
"KL/mean": -1033.7266845703125,
"KL/rejected_KL_mean": -1287.95263671875,
"KL/std": 692.0765380859375,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0009514681296423078,
"fcm_dpo/delta": -0.08593890815973282,
"fcm_dpo/margin": 508.45184326171875,
"fcm_dpo/q_t": 0.39394253492355347,
"grad_norm": 33.00722885131836,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -1.0474693775177002,
"logits/rejected": -1.0704981088638306,
"logps/chosen": -846.046142578125,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1391.822021484375,
"loss": 1.0916,
"margin_dpo/margin_mean": 508.4518127441406,
"margin_dpo/margin_std": 790.054931640625,
"step": 566
},
{
"KL/chosen_KL_mean": -752.8310546875,
"KL/mean": -941.5557861328125,
"KL/rejected_KL_mean": -1130.2806396484375,
"KL/std": 474.6645202636719,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0009493097895756364,
"fcm_dpo/delta": 0.042454224079847336,
"fcm_dpo/margin": 377.4496154785156,
"fcm_dpo/q_t": 0.416124165058136,
"grad_norm": 37.01401138305664,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -1.1754206418991089,
"logits/rejected": -1.1703362464904785,
"logps/chosen": -805.6893310546875,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1215.651611328125,
"loss": 1.1146,
"margin_dpo/margin_mean": 377.4496154785156,
"margin_dpo/margin_std": 508.71038818359375,
"step": 567
},
{
"KL/chosen_KL_mean": -732.2120361328125,
"KL/mean": -992.5909423828125,
"KL/rejected_KL_mean": -1252.9698486328125,
"KL/std": 562.322265625,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0009439606801606715,
"fcm_dpo/delta": -0.09389245510101318,
"fcm_dpo/margin": 520.7578735351562,
"fcm_dpo/q_t": 0.38713839650154114,
"grad_norm": 28.33488655090332,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -1.0602999925613403,
"logits/rejected": -1.0991747379302979,
"logps/chosen": -777.4044189453125,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1342.062255859375,
"loss": 1.0249,
"margin_dpo/margin_mean": 520.7578125,
"margin_dpo/margin_std": 590.0891723632812,
"step": 568
},
{
"KL/chosen_KL_mean": -826.0765380859375,
"KL/mean": -1036.68115234375,
"KL/rejected_KL_mean": -1247.285888671875,
"KL/std": 716.984130859375,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.000945072213653475,
"fcm_dpo/delta": 0.0015114173293113708,
"fcm_dpo/margin": 421.209228515625,
"fcm_dpo/q_t": 0.4105045795440674,
"grad_norm": 47.668739318847656,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -1.1122533082962036,
"logits/rejected": -1.1140843629837036,
"logps/chosen": -889.797119140625,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1326.3890380859375,
"loss": 1.1478,
"margin_dpo/margin_mean": 421.2092590332031,
"margin_dpo/margin_std": 756.34326171875,
"step": 569
},
{
"KL/chosen_KL_mean": -775.0736083984375,
"KL/mean": -970.983154296875,
"KL/rejected_KL_mean": -1166.8927001953125,
"KL/std": 585.8502197265625,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0009430573554709554,
"fcm_dpo/delta": 0.031048066914081573,
"fcm_dpo/margin": 391.8191223144531,
"fcm_dpo/q_t": 0.4179254174232483,
"grad_norm": 31.495166778564453,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.1752269268035889,
"logits/rejected": -1.1550638675689697,
"logps/chosen": -836.6881103515625,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1249.03466796875,
"loss": 1.1347,
"margin_dpo/margin_mean": 391.819091796875,
"margin_dpo/margin_std": 633.9605102539062,
"step": 570
},
{
"KL/chosen_KL_mean": -785.480224609375,
"KL/mean": -1001.910888671875,
"KL/rejected_KL_mean": -1218.341552734375,
"KL/std": 558.670166015625,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0009431964717805386,
"fcm_dpo/delta": -0.008509235456585884,
"fcm_dpo/margin": 432.861328125,
"fcm_dpo/q_t": 0.40848302841186523,
"grad_norm": 44.37576675415039,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -1.1002910137176514,
"logits/rejected": -1.1096596717834473,
"logps/chosen": -838.5343017578125,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1309.678466796875,
"loss": 1.1233,
"margin_dpo/margin_mean": 432.861328125,
"margin_dpo/margin_std": 713.20458984375,
"step": 571
},
{
"KL/chosen_KL_mean": -822.61279296875,
"KL/mean": -1061.8223876953125,
"KL/rejected_KL_mean": -1301.031982421875,
"KL/std": 679.26806640625,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.0009392797946929932,
"fcm_dpo/delta": -0.05064094066619873,
"fcm_dpo/margin": 478.4192810058594,
"fcm_dpo/q_t": 0.40520572662353516,
"grad_norm": 29.390687942504883,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.128666877746582,
"logits/rejected": -1.1588270664215088,
"logps/chosen": -871.0720825195312,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1384.589111328125,
"loss": 1.1148,
"margin_dpo/margin_mean": 478.41925048828125,
"margin_dpo/margin_std": 823.0018310546875,
"step": 572
},
{
"KL/chosen_KL_mean": -781.2991943359375,
"KL/mean": -1001.1578979492188,
"KL/rejected_KL_mean": -1221.0164794921875,
"KL/std": 596.3881225585938,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0009392928332090378,
"fcm_dpo/delta": -0.013323244638741016,
"fcm_dpo/margin": 439.71728515625,
"fcm_dpo/q_t": 0.40707600116729736,
"grad_norm": 29.680734634399414,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -1.162482738494873,
"logits/rejected": -1.1760870218276978,
"logps/chosen": -832.921875,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1306.3414306640625,
"loss": 1.0857,
"margin_dpo/margin_mean": 439.71728515625,
"margin_dpo/margin_std": 595.1873779296875,
"step": 573
},
{
"KL/chosen_KL_mean": -724.2657470703125,
"KL/mean": -953.5592041015625,
"KL/rejected_KL_mean": -1182.8525390625,
"KL/std": 532.4700927734375,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0009368173778057098,
"fcm_dpo/delta": -0.030269186943769455,
"fcm_dpo/margin": 458.5869140625,
"fcm_dpo/q_t": 0.40228691697120667,
"grad_norm": 29.51076889038086,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -1.1240806579589844,
"logits/rejected": -1.145003318786621,
"logps/chosen": -775.3101806640625,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1275.658935546875,
"loss": 1.0787,
"margin_dpo/margin_mean": 458.5869140625,
"margin_dpo/margin_std": 613.4527587890625,
"step": 574
},
{
"KL/chosen_KL_mean": -811.00634765625,
"KL/mean": -1024.9534912109375,
"KL/rejected_KL_mean": -1238.9005126953125,
"KL/std": 559.50830078125,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0009320209501311183,
"fcm_dpo/delta": 0.000727042555809021,
"fcm_dpo/margin": 427.89422607421875,
"fcm_dpo/q_t": 0.4095437228679657,
"grad_norm": 42.62068557739258,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -1.046234130859375,
"logits/rejected": -1.0514554977416992,
"logps/chosen": -882.7965087890625,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1334.2867431640625,
"loss": 1.1328,
"margin_dpo/margin_mean": 427.89422607421875,
"margin_dpo/margin_std": 713.029541015625,
"step": 575
},
{
"KL/chosen_KL_mean": -782.13525390625,
"KL/mean": -943.5439453125,
"KL/rejected_KL_mean": -1104.95263671875,
"KL/std": 511.6619567871094,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0009402718860656023,
"fcm_dpo/delta": 0.09798312187194824,
"fcm_dpo/margin": 322.81732177734375,
"fcm_dpo/q_t": 0.429963618516922,
"grad_norm": 39.150245666503906,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -1.1449861526489258,
"logits/rejected": -1.1737542152404785,
"logps/chosen": -836.3982543945312,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1205.7069091796875,
"loss": 1.1796,
"margin_dpo/margin_mean": 322.8173828125,
"margin_dpo/margin_std": 582.9664306640625,
"step": 576
},
{
"KL/chosen_KL_mean": -690.1397094726562,
"KL/mean": -899.7841186523438,
"KL/rejected_KL_mean": -1109.428466796875,
"KL/std": 570.5064086914062,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0009443428134545684,
"fcm_dpo/delta": 0.004122039768844843,
"fcm_dpo/margin": 419.2888488769531,
"fcm_dpo/q_t": 0.4105992913246155,
"grad_norm": 30.886186599731445,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.102234125137329,
"logits/rejected": -1.082305669784546,
"logps/chosen": -752.0494384765625,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1193.499267578125,
"loss": 1.1191,
"margin_dpo/margin_mean": 419.28887939453125,
"margin_dpo/margin_std": 626.2908325195312,
"step": 577
},
{
"KL/chosen_KL_mean": -685.677978515625,
"KL/mean": -919.28759765625,
"KL/rejected_KL_mean": -1152.897216796875,
"KL/std": 582.769287109375,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0009395558736287057,
"fcm_dpo/delta": -0.04024638235569,
"fcm_dpo/margin": 467.21929931640625,
"fcm_dpo/q_t": 0.39982476830482483,
"grad_norm": 37.12548065185547,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.9650574922561646,
"logits/rejected": -0.9971319437026978,
"logps/chosen": -734.941650390625,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1236.33349609375,
"loss": 1.0655,
"margin_dpo/margin_mean": 467.21929931640625,
"margin_dpo/margin_std": 579.5484619140625,
"step": 578
},
{
"KL/chosen_KL_mean": -784.9281005859375,
"KL/mean": -929.6224365234375,
"KL/rejected_KL_mean": -1074.316650390625,
"KL/std": 574.9615478515625,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0009488767245784402,
"fcm_dpo/delta": 0.1272491067647934,
"fcm_dpo/margin": 289.38848876953125,
"fcm_dpo/q_t": 0.4369036555290222,
"grad_norm": 43.24967956542969,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -1.0656390190124512,
"logits/rejected": -1.0647817850112915,
"logps/chosen": -843.81396484375,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1169.104248046875,
"loss": 1.2174,
"margin_dpo/margin_mean": 289.3885192871094,
"margin_dpo/margin_std": 634.266845703125,
"step": 579
},
{
"KL/chosen_KL_mean": -582.00390625,
"KL/mean": -842.2623901367188,
"KL/rejected_KL_mean": -1102.5208740234375,
"KL/std": 623.9799194335938,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0009446794865652919,
"fcm_dpo/delta": -0.09435415267944336,
"fcm_dpo/margin": 520.5169677734375,
"fcm_dpo/q_t": 0.39273136854171753,
"grad_norm": 27.779617309570312,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8903079032897949,
"logits/rejected": -0.9164822101593018,
"logps/chosen": -630.7107543945312,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1184.279296875,
"loss": 1.049,
"margin_dpo/margin_mean": 520.5169677734375,
"margin_dpo/margin_std": 708.4932861328125,
"step": 580
},
{
"KL/chosen_KL_mean": -791.675048828125,
"KL/mean": -977.7584228515625,
"KL/rejected_KL_mean": -1163.841796875,
"KL/std": 612.6318359375,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0009482804453000426,
"fcm_dpo/delta": 0.04787049442529678,
"fcm_dpo/margin": 372.1667175292969,
"fcm_dpo/q_t": 0.4224420487880707,
"grad_norm": 41.1692008972168,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -1.1497280597686768,
"logits/rejected": -1.1572540998458862,
"logps/chosen": -853.8973999023438,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1255.577392578125,
"loss": 1.154,
"margin_dpo/margin_mean": 372.166748046875,
"margin_dpo/margin_std": 670.7994995117188,
"step": 581
},
{
"KL/chosen_KL_mean": -701.05224609375,
"KL/mean": -898.874755859375,
"KL/rejected_KL_mean": -1096.6973876953125,
"KL/std": 523.3825073242188,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0009447854827158153,
"fcm_dpo/delta": -0.08350235223770142,
"fcm_dpo/margin": 395.6450500488281,
"fcm_dpo/q_t": 0.4139753580093384,
"grad_norm": 35.411434173583984,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -1.082298994064331,
"logits/rejected": -1.106623649597168,
"logps/chosen": -759.2808837890625,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1206.7669677734375,
"loss": 1.1245,
"margin_dpo/margin_mean": 395.6450500488281,
"margin_dpo/margin_std": 574.4161376953125,
"step": 582
},
{
"KL/chosen_KL_mean": -788.7952880859375,
"KL/mean": -915.5237426757812,
"KL/rejected_KL_mean": -1042.252197265625,
"KL/std": 566.1787109375,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0009400502895005047,
"fcm_dpo/delta": -0.008407074958086014,
"fcm_dpo/margin": 253.45693969726562,
"fcm_dpo/q_t": 0.44672808051109314,
"grad_norm": 57.46125411987305,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.9853731393814087,
"logits/rejected": -0.978585422039032,
"logps/chosen": -845.6581420898438,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1130.6561279296875,
"loss": 1.2656,
"margin_dpo/margin_mean": 253.45692443847656,
"margin_dpo/margin_std": 682.3837890625,
"step": 583
},
{
"KL/chosen_KL_mean": -670.3692626953125,
"KL/mean": -927.6574096679688,
"KL/rejected_KL_mean": -1184.945556640625,
"KL/std": 594.9502563476562,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.000934588722884655,
"fcm_dpo/delta": -0.0828862339258194,
"fcm_dpo/margin": 514.5762939453125,
"fcm_dpo/q_t": 0.39284205436706543,
"grad_norm": 29.62038230895996,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -1.0637917518615723,
"logits/rejected": -1.0736515522003174,
"logps/chosen": -727.2699584960938,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1282.5816650390625,
"loss": 1.0451,
"margin_dpo/margin_mean": 514.5762939453125,
"margin_dpo/margin_std": 644.3516845703125,
"step": 584
},
{
"KL/chosen_KL_mean": -772.5379638671875,
"KL/mean": -1022.5894775390625,
"KL/rejected_KL_mean": -1272.6409912109375,
"KL/std": 647.20556640625,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0009266703855246305,
"fcm_dpo/delta": -0.06506932526826859,
"fcm_dpo/margin": 500.10296630859375,
"fcm_dpo/q_t": 0.3975642919540405,
"grad_norm": 34.582088470458984,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.089249849319458,
"logits/rejected": -1.112525224685669,
"logps/chosen": -833.511962890625,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1356.810546875,
"loss": 1.0897,
"margin_dpo/margin_mean": 500.10296630859375,
"margin_dpo/margin_std": 768.308349609375,
"step": 585
},
{
"KL/chosen_KL_mean": -807.0784912109375,
"KL/mean": -980.6554565429688,
"KL/rejected_KL_mean": -1154.232421875,
"KL/std": 573.5162963867188,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0009305774001404643,
"fcm_dpo/delta": 0.07824563980102539,
"fcm_dpo/margin": 347.15399169921875,
"fcm_dpo/q_t": 0.42543265223503113,
"grad_norm": 33.32202911376953,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.1124446392059326,
"logits/rejected": -1.137439250946045,
"logps/chosen": -863.93408203125,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1246.03515625,
"loss": 1.1743,
"margin_dpo/margin_mean": 347.15399169921875,
"margin_dpo/margin_std": 641.89892578125,
"step": 586
},
{
"KL/chosen_KL_mean": -590.2689208984375,
"KL/mean": -805.8556518554688,
"KL/rejected_KL_mean": -1021.4425048828125,
"KL/std": 601.960205078125,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0009339781245216727,
"fcm_dpo/delta": -0.002780054695904255,
"fcm_dpo/margin": 431.173583984375,
"fcm_dpo/q_t": 0.4075835347175598,
"grad_norm": 36.960052490234375,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -1.1249654293060303,
"logits/rejected": -1.1489168405532837,
"logps/chosen": -634.96044921875,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1104.06640625,
"loss": 1.0941,
"margin_dpo/margin_mean": 431.1735534667969,
"margin_dpo/margin_std": 591.1593017578125,
"step": 587
},
{
"KL/chosen_KL_mean": -746.4495849609375,
"KL/mean": -930.6257934570312,
"KL/rejected_KL_mean": -1114.802001953125,
"KL/std": 514.0170288085938,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0009352130000479519,
"fcm_dpo/delta": 0.05642148479819298,
"fcm_dpo/margin": 368.352294921875,
"fcm_dpo/q_t": 0.4200473427772522,
"grad_norm": 30.264469146728516,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.071812391281128,
"logits/rejected": -1.0980116128921509,
"logps/chosen": -796.7445678710938,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1222.171875,
"loss": 1.1337,
"margin_dpo/margin_mean": 368.3523254394531,
"margin_dpo/margin_std": 540.8074340820312,
"step": 588
},
{
"KL/chosen_KL_mean": -765.505615234375,
"KL/mean": -1002.3455810546875,
"KL/rejected_KL_mean": -1239.1854248046875,
"KL/std": 586.9735717773438,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0009358683601021767,
"fcm_dpo/delta": -0.04427178204059601,
"fcm_dpo/margin": 473.67987060546875,
"fcm_dpo/q_t": 0.3995531499385834,
"grad_norm": 30.180952072143555,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -1.046541690826416,
"logits/rejected": -1.0619137287139893,
"logps/chosen": -825.435546875,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1350.8408203125,
"loss": 1.071,
"margin_dpo/margin_mean": 473.67987060546875,
"margin_dpo/margin_std": 629.702392578125,
"step": 589
},
{
"KL/chosen_KL_mean": -648.0178833007812,
"KL/mean": -879.0958862304688,
"KL/rejected_KL_mean": -1110.173828125,
"KL/std": 540.598388671875,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.0009314791532233357,
"fcm_dpo/delta": -0.031209638342261314,
"fcm_dpo/margin": 462.15606689453125,
"fcm_dpo/q_t": 0.40161752700805664,
"grad_norm": 23.975587844848633,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -1.0303199291229248,
"logits/rejected": -1.0523946285247803,
"logps/chosen": -703.82763671875,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1216.23681640625,
"loss": 1.0681,
"margin_dpo/margin_mean": 462.1560363769531,
"margin_dpo/margin_std": 576.578125,
"step": 590
},
{
"KL/chosen_KL_mean": -737.061767578125,
"KL/mean": -956.1275024414062,
"KL/rejected_KL_mean": -1175.1932373046875,
"KL/std": 550.1436157226562,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0009308913140557706,
"fcm_dpo/delta": -0.008016519248485565,
"fcm_dpo/margin": 438.1313781738281,
"fcm_dpo/q_t": 0.4057263135910034,
"grad_norm": 30.565818786621094,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -1.1124975681304932,
"logits/rejected": -1.0994548797607422,
"logps/chosen": -793.3024291992188,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1258.9794921875,
"loss": 1.0962,
"margin_dpo/margin_mean": 438.13140869140625,
"margin_dpo/margin_std": 615.4335327148438,
"step": 591
},
{
"KL/chosen_KL_mean": -743.1383666992188,
"KL/mean": -941.9227905273438,
"KL/rejected_KL_mean": -1140.707275390625,
"KL/std": 541.6781616210938,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0009322206024080515,
"fcm_dpo/delta": 0.029935326427221298,
"fcm_dpo/margin": 397.5688171386719,
"fcm_dpo/q_t": 0.416223406791687,
"grad_norm": 30.50063705444336,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.1092453002929688,
"logits/rejected": -1.1124801635742188,
"logps/chosen": -791.07861328125,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1216.440185546875,
"loss": 1.1635,
"margin_dpo/margin_mean": 397.56884765625,
"margin_dpo/margin_std": 738.2605590820312,
"step": 592
},
{
"KL/chosen_KL_mean": -737.1370849609375,
"KL/mean": -894.179931640625,
"KL/rejected_KL_mean": -1051.2227783203125,
"KL/std": 630.5286865234375,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0009341684053651989,
"fcm_dpo/delta": 0.006865399423986673,
"fcm_dpo/margin": 314.085693359375,
"fcm_dpo/q_t": 0.4349074959754944,
"grad_norm": 45.077449798583984,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.925495982170105,
"logits/rejected": -0.9140543937683105,
"logps/chosen": -785.827880859375,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1112.130859375,
"loss": 1.2123,
"margin_dpo/margin_mean": 314.085693359375,
"margin_dpo/margin_std": 687.1311645507812,
"step": 593
},
{
"KL/chosen_KL_mean": -696.455810546875,
"KL/mean": -892.4623413085938,
"KL/rejected_KL_mean": -1088.468994140625,
"KL/std": 599.05810546875,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0009395517408847809,
"fcm_dpo/delta": 0.03171183168888092,
"fcm_dpo/margin": 392.01312255859375,
"fcm_dpo/q_t": 0.4168938100337982,
"grad_norm": 32.01091003417969,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.0287201404571533,
"logits/rejected": -1.0127062797546387,
"logps/chosen": -751.3906860351562,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1174.568603515625,
"loss": 1.1387,
"margin_dpo/margin_mean": 392.01312255859375,
"margin_dpo/margin_std": 633.80126953125,
"step": 594
},
{
"KL/chosen_KL_mean": -645.5115356445312,
"KL/mean": -869.5858154296875,
"KL/rejected_KL_mean": -1093.66015625,
"KL/std": 564.3441772460938,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0009344806894659996,
"fcm_dpo/delta": -0.01929367519915104,
"fcm_dpo/margin": 448.14862060546875,
"fcm_dpo/q_t": 0.40738070011138916,
"grad_norm": 33.268638610839844,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -1.0355620384216309,
"logits/rejected": -1.0576274394989014,
"logps/chosen": -694.9319458007812,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1174.2874755859375,
"loss": 1.0873,
"margin_dpo/margin_mean": 448.14862060546875,
"margin_dpo/margin_std": 625.08984375,
"step": 595
},
{
"KL/chosen_KL_mean": -755.9296875,
"KL/mean": -896.432373046875,
"KL/rejected_KL_mean": -1036.93505859375,
"KL/std": 544.8298950195312,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.000945114647038281,
"fcm_dpo/delta": 0.1361963152885437,
"fcm_dpo/margin": 281.0052795410156,
"fcm_dpo/q_t": 0.4416961669921875,
"grad_norm": 70.67866516113281,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -1.043994665145874,
"logits/rejected": -1.005753517150879,
"logps/chosen": -815.7213745117188,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1117.026123046875,
"loss": 1.2272,
"margin_dpo/margin_mean": 281.00531005859375,
"margin_dpo/margin_std": 656.0474853515625,
"step": 596
},
{
"KL/chosen_KL_mean": -699.6539306640625,
"KL/mean": -960.3472290039062,
"KL/rejected_KL_mean": -1221.04052734375,
"KL/std": 631.4920043945312,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0009366714511997998,
"fcm_dpo/delta": -0.09220761060714722,
"fcm_dpo/margin": 521.3866577148438,
"fcm_dpo/q_t": 0.39339327812194824,
"grad_norm": 30.50818634033203,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.9620223045349121,
"logits/rejected": -1.0084235668182373,
"logps/chosen": -756.9146728515625,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1321.7342529296875,
"loss": 1.0648,
"margin_dpo/margin_mean": 521.3866577148438,
"margin_dpo/margin_std": 711.2468872070312,
"step": 597
},
{
"KL/chosen_KL_mean": -694.2019653320312,
"KL/mean": -900.156494140625,
"KL/rejected_KL_mean": -1106.111083984375,
"KL/std": 543.1419677734375,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0009368107421323657,
"fcm_dpo/delta": 0.013848438858985901,
"fcm_dpo/margin": 411.9090881347656,
"fcm_dpo/q_t": 0.41072171926498413,
"grad_norm": 36.10586166381836,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -1.0843559503555298,
"logits/rejected": -1.0926883220672607,
"logps/chosen": -746.720458984375,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1195.554931640625,
"loss": 1.1177,
"margin_dpo/margin_mean": 411.9090881347656,
"margin_dpo/margin_std": 603.3158569335938,
"step": 598
},
{
"KL/chosen_KL_mean": -711.0474853515625,
"KL/mean": -883.8363037109375,
"KL/rejected_KL_mean": -1056.625244140625,
"KL/std": 532.8595581054688,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0009362338460050523,
"fcm_dpo/delta": -0.048169832676649094,
"fcm_dpo/margin": 345.5777587890625,
"fcm_dpo/q_t": 0.4225807189941406,
"grad_norm": 32.4377326965332,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.9888404011726379,
"logits/rejected": -0.994670569896698,
"logps/chosen": -760.85009765625,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1139.603759765625,
"loss": 1.1487,
"margin_dpo/margin_mean": 345.5777587890625,
"margin_dpo/margin_std": 510.5614318847656,
"step": 599
},
{
"KL/chosen_KL_mean": -777.1749267578125,
"KL/mean": -945.5758056640625,
"KL/rejected_KL_mean": -1113.9766845703125,
"KL/std": 559.109619140625,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.000941394129768014,
"fcm_dpo/delta": 0.0843147411942482,
"fcm_dpo/margin": 336.8017272949219,
"fcm_dpo/q_t": 0.42964982986450195,
"grad_norm": 36.367210388183594,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -1.1235439777374268,
"logits/rejected": -1.1162173748016357,
"logps/chosen": -843.6098022460938,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1199.43310546875,
"loss": 1.1856,
"margin_dpo/margin_mean": 336.8017272949219,
"margin_dpo/margin_std": 669.5869140625,
"step": 600
},
{
"KL/chosen_KL_mean": -782.239501953125,
"KL/mean": -993.9614868164062,
"KL/rejected_KL_mean": -1205.683349609375,
"KL/std": 575.0701904296875,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0009437098633497953,
"fcm_dpo/delta": 0.00038953591138124466,
"fcm_dpo/margin": 423.4439697265625,
"fcm_dpo/q_t": 0.4073420763015747,
"grad_norm": 30.614376068115234,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -1.0543596744537354,
"logits/rejected": -1.0704107284545898,
"logps/chosen": -841.373046875,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1300.3743896484375,
"loss": 1.0983,
"margin_dpo/margin_mean": 423.4439697265625,
"margin_dpo/margin_std": 586.927001953125,
"step": 601
},
{
"KL/chosen_KL_mean": -553.6896362304688,
"KL/mean": -824.164794921875,
"KL/rejected_KL_mean": -1094.64013671875,
"KL/std": 556.9711303710938,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0009391128551214933,
"fcm_dpo/delta": -0.11084318161010742,
"fcm_dpo/margin": 540.950439453125,
"fcm_dpo/q_t": 0.3849470615386963,
"grad_norm": 61.494728088378906,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -1.071298360824585,
"logits/rejected": -1.1071069240570068,
"logps/chosen": -602.2831420898438,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -1182.30859375,
"loss": 1.014,
"margin_dpo/margin_mean": 540.950439453125,
"margin_dpo/margin_std": 597.431396484375,
"step": 602
},
{
"KL/chosen_KL_mean": -677.567138671875,
"KL/mean": -920.722900390625,
"KL/rejected_KL_mean": -1163.8787841796875,
"KL/std": 601.8591918945312,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0009299052762798965,
"fcm_dpo/delta": -0.05344250053167343,
"fcm_dpo/margin": 486.3115234375,
"fcm_dpo/q_t": 0.40060174465179443,
"grad_norm": 30.66020393371582,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -1.1096835136413574,
"logits/rejected": -1.0989437103271484,
"logps/chosen": -747.9818115234375,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1264.204345703125,
"loss": 1.0749,
"margin_dpo/margin_mean": 486.3115234375,
"margin_dpo/margin_std": 690.6925048828125,
"step": 603
},
{
"KL/chosen_KL_mean": -667.291015625,
"KL/mean": -937.2022705078125,
"KL/rejected_KL_mean": -1207.113525390625,
"KL/std": 592.4906616210938,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0009219404309988022,
"fcm_dpo/delta": -0.1001720055937767,
"fcm_dpo/margin": 539.822509765625,
"fcm_dpo/q_t": 0.3912475109100342,
"grad_norm": 30.202251434326172,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -1.0218074321746826,
"logits/rejected": -1.0635687112808228,
"logps/chosen": -713.7491455078125,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1298.968017578125,
"loss": 1.0509,
"margin_dpo/margin_mean": 539.822509765625,
"margin_dpo/margin_std": 732.5137939453125,
"step": 604
},
{
"KL/chosen_KL_mean": -696.3240966796875,
"KL/mean": -882.376708984375,
"KL/rejected_KL_mean": -1068.42919921875,
"KL/std": 532.2725219726562,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0009225456742569804,
"fcm_dpo/delta": 0.05772203952074051,
"fcm_dpo/margin": 372.1051025390625,
"fcm_dpo/q_t": 0.4235140085220337,
"grad_norm": 36.44785690307617,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -1.0042846202850342,
"logits/rejected": -1.0125752687454224,
"logps/chosen": -762.573486328125,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1170.734130859375,
"loss": 1.148,
"margin_dpo/margin_mean": 372.1051025390625,
"margin_dpo/margin_std": 632.8191528320312,
"step": 605
},
{
"KL/chosen_KL_mean": -676.556396484375,
"KL/mean": -889.0665283203125,
"KL/rejected_KL_mean": -1101.5765380859375,
"KL/std": 537.904296875,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0009270627633668482,
"fcm_dpo/delta": 0.005940597504377365,
"fcm_dpo/margin": 425.02008056640625,
"fcm_dpo/q_t": 0.4101853370666504,
"grad_norm": 31.26183319091797,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -1.0010316371917725,
"logits/rejected": -1.0146284103393555,
"logps/chosen": -731.3755493164062,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1199.947998046875,
"loss": 1.0951,
"margin_dpo/margin_mean": 425.02008056640625,
"margin_dpo/margin_std": 566.8992919921875,
"step": 606
},
{
"KL/chosen_KL_mean": -745.302978515625,
"KL/mean": -913.755615234375,
"KL/rejected_KL_mean": -1082.208251953125,
"KL/std": 564.3204345703125,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0009324230486527085,
"fcm_dpo/delta": 0.08710216730833054,
"fcm_dpo/margin": 336.9051818847656,
"fcm_dpo/q_t": 0.4282468557357788,
"grad_norm": 29.58710479736328,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.9958839416503906,
"logits/rejected": -0.9777606129646301,
"logps/chosen": -803.3870239257812,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1161.9853515625,
"loss": 1.1734,
"margin_dpo/margin_mean": 336.9051513671875,
"margin_dpo/margin_std": 613.3471069335938,
"step": 607
},
{
"KL/chosen_KL_mean": -618.9171142578125,
"KL/mean": -831.6253662109375,
"KL/rejected_KL_mean": -1044.3336181640625,
"KL/std": 493.6427001953125,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.000934786512516439,
"fcm_dpo/delta": 0.002314794808626175,
"fcm_dpo/margin": 425.41650390625,
"fcm_dpo/q_t": 0.4072082042694092,
"grad_norm": 28.977453231811523,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.0519757270812988,
"logits/rejected": -1.0653681755065918,
"logps/chosen": -676.367919921875,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1139.10693359375,
"loss": 1.0788,
"margin_dpo/margin_mean": 425.41650390625,
"margin_dpo/margin_std": 498.175048828125,
"step": 608
},
{
"KL/chosen_KL_mean": -661.7661743164062,
"KL/mean": -925.5433349609375,
"KL/rejected_KL_mean": -1189.320556640625,
"KL/std": 666.0048828125,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0009277096251025796,
"fcm_dpo/delta": -0.09163244068622589,
"fcm_dpo/margin": 527.5543823242188,
"fcm_dpo/q_t": 0.3955400884151459,
"grad_norm": 30.367385864257812,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.1196882724761963,
"logits/rejected": -1.1408710479736328,
"logps/chosen": -720.571533203125,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1278.136474609375,
"loss": 1.0618,
"margin_dpo/margin_mean": 527.5543823242188,
"margin_dpo/margin_std": 781.7075805664062,
"step": 609
},
{
"KL/chosen_KL_mean": -647.3359375,
"KL/mean": -821.218505859375,
"KL/rejected_KL_mean": -995.10107421875,
"KL/std": 509.16314697265625,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0009309325832873583,
"fcm_dpo/delta": 0.07747611403465271,
"fcm_dpo/margin": 347.76507568359375,
"fcm_dpo/q_t": 0.42488405108451843,
"grad_norm": 41.63359451293945,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -1.0002460479736328,
"logits/rejected": -0.9726261496543884,
"logps/chosen": -713.031005859375,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1078.5064697265625,
"loss": 1.1582,
"margin_dpo/margin_mean": 347.76507568359375,
"margin_dpo/margin_std": 584.4713745117188,
"step": 610
},
{
"KL/chosen_KL_mean": -680.7236328125,
"KL/mean": -954.4203491210938,
"KL/rejected_KL_mean": -1228.1170654296875,
"KL/std": 674.031494140625,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0009249848080798984,
"fcm_dpo/delta": -0.1090783104300499,
"fcm_dpo/margin": 547.3934326171875,
"fcm_dpo/q_t": 0.3915501832962036,
"grad_norm": 40.06733703613281,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -1.0733530521392822,
"logits/rejected": -1.1131242513656616,
"logps/chosen": -733.3231201171875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1314.447998046875,
"loss": 1.057,
"margin_dpo/margin_mean": 547.3934326171875,
"margin_dpo/margin_std": 783.890869140625,
"step": 611
},
{
"KL/chosen_KL_mean": -730.0038452148438,
"KL/mean": -944.6229248046875,
"KL/rejected_KL_mean": -1159.2420654296875,
"KL/std": 537.7095336914062,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0009237851481884718,
"fcm_dpo/delta": 0.003468703478574753,
"fcm_dpo/margin": 429.2381896972656,
"fcm_dpo/q_t": 0.40873080492019653,
"grad_norm": 30.787771224975586,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -1.054150104522705,
"logits/rejected": -1.0440592765808105,
"logps/chosen": -789.3275756835938,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1247.554443359375,
"loss": 1.0915,
"margin_dpo/margin_mean": 429.2381591796875,
"margin_dpo/margin_std": 562.5599975585938,
"step": 612
},
{
"KL/chosen_KL_mean": -648.699951171875,
"KL/mean": -902.3504028320312,
"KL/rejected_KL_mean": -1156.0009765625,
"KL/std": 597.5882568359375,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0009168373653665185,
"fcm_dpo/delta": -0.066818006336689,
"fcm_dpo/margin": 507.301025390625,
"fcm_dpo/q_t": 0.39431166648864746,
"grad_norm": 34.06965255737305,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -1.0495436191558838,
"logits/rejected": -1.0977256298065186,
"logps/chosen": -708.4298706054688,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1261.1083984375,
"loss": 1.048,
"margin_dpo/margin_mean": 507.301025390625,
"margin_dpo/margin_std": 619.2185668945312,
"step": 613
},
{
"KL/chosen_KL_mean": -726.5540771484375,
"KL/mean": -973.306640625,
"KL/rejected_KL_mean": -1220.0592041015625,
"KL/std": 595.4200439453125,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0009136309381574392,
"fcm_dpo/delta": -0.05203912407159805,
"fcm_dpo/margin": 493.5052185058594,
"fcm_dpo/q_t": 0.3989720642566681,
"grad_norm": 43.53131866455078,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -1.0377488136291504,
"logits/rejected": -1.1064826250076294,
"logps/chosen": -779.4930419921875,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1324.738525390625,
"loss": 1.0802,
"margin_dpo/margin_mean": 493.50518798828125,
"margin_dpo/margin_std": 707.6263427734375,
"step": 614
},
{
"KL/chosen_KL_mean": -656.21240234375,
"KL/mean": -860.7072143554688,
"KL/rejected_KL_mean": -1065.2021484375,
"KL/std": 597.1307373046875,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0009152768179774284,
"fcm_dpo/delta": 0.025946544483304024,
"fcm_dpo/margin": 408.9896545410156,
"fcm_dpo/q_t": 0.4139822721481323,
"grad_norm": 33.06850814819336,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.9963966608047485,
"logits/rejected": -1.0074797868728638,
"logps/chosen": -722.0296630859375,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1160.379638671875,
"loss": 1.1254,
"margin_dpo/margin_mean": 408.9896545410156,
"margin_dpo/margin_std": 637.387451171875,
"step": 615
},
{
"KL/chosen_KL_mean": -777.4338989257812,
"KL/mean": -951.5509033203125,
"KL/rejected_KL_mean": -1125.66796875,
"KL/std": 516.37451171875,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0009200784843415022,
"fcm_dpo/delta": 0.08084507286548615,
"fcm_dpo/margin": 348.23406982421875,
"fcm_dpo/q_t": 0.4268506169319153,
"grad_norm": 27.89613151550293,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.0936863422393799,
"logits/rejected": -1.075373649597168,
"logps/chosen": -842.5667724609375,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1200.3685302734375,
"loss": 1.1618,
"margin_dpo/margin_mean": 348.2341003417969,
"margin_dpo/margin_std": 590.9950561523438,
"step": 616
},
{
"KL/chosen_KL_mean": -742.44384765625,
"KL/mean": -881.6417846679688,
"KL/rejected_KL_mean": -1020.8397216796875,
"KL/std": 495.73095703125,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0009322729893028736,
"fcm_dpo/delta": 0.1421297788619995,
"fcm_dpo/margin": 278.3958740234375,
"fcm_dpo/q_t": 0.4408647119998932,
"grad_norm": 45.14849853515625,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.9931391477584839,
"logits/rejected": -0.9588996171951294,
"logps/chosen": -805.4493408203125,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1085.0738525390625,
"loss": 1.2146,
"margin_dpo/margin_mean": 278.3958740234375,
"margin_dpo/margin_std": 587.977294921875,
"step": 617
},
{
"KL/chosen_KL_mean": -761.5391845703125,
"KL/mean": -1049.184814453125,
"KL/rejected_KL_mean": -1336.83056640625,
"KL/std": 683.9238891601562,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0009282865794375539,
"fcm_dpo/delta": -0.1377088725566864,
"fcm_dpo/margin": 575.2913818359375,
"fcm_dpo/q_t": 0.3874184489250183,
"grad_norm": 37.7491455078125,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.0628851652145386,
"logits/rejected": -1.1012842655181885,
"logps/chosen": -828.6405639648438,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1428.98388671875,
"loss": 1.0682,
"margin_dpo/margin_mean": 575.2913818359375,
"margin_dpo/margin_std": 880.9173583984375,
"step": 618
},
{
"KL/chosen_KL_mean": -778.9534301757812,
"KL/mean": -977.70556640625,
"KL/rejected_KL_mean": -1176.457763671875,
"KL/std": 610.49609375,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0009244297398254275,
"fcm_dpo/delta": 0.03313371166586876,
"fcm_dpo/margin": 397.50433349609375,
"fcm_dpo/q_t": 0.4229835271835327,
"grad_norm": 41.30840301513672,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.0145196914672852,
"logits/rejected": -1.0216684341430664,
"logps/chosen": -834.931640625,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1269.64306640625,
"loss": 1.1829,
"margin_dpo/margin_mean": 397.5043029785156,
"margin_dpo/margin_std": 825.1723022460938,
"step": 619
},
{
"KL/chosen_KL_mean": -700.730224609375,
"KL/mean": -897.155517578125,
"KL/rejected_KL_mean": -1093.5806884765625,
"KL/std": 533.931640625,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0009257015772163868,
"fcm_dpo/delta": 0.03692508488893509,
"fcm_dpo/margin": 392.8504638671875,
"fcm_dpo/q_t": 0.4179452955722809,
"grad_norm": 41.25801467895508,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -1.0765249729156494,
"logits/rejected": -1.0782887935638428,
"logps/chosen": -760.5277099609375,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1171.991455078125,
"loss": 1.1321,
"margin_dpo/margin_mean": 392.8504943847656,
"margin_dpo/margin_std": 619.072998046875,
"step": 620
},
{
"KL/chosen_KL_mean": -700.159912109375,
"KL/mean": -985.0526123046875,
"KL/rejected_KL_mean": -1269.9453125,
"KL/std": 644.7921142578125,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.000923096900805831,
"fcm_dpo/delta": -0.12948347628116608,
"fcm_dpo/margin": 569.7854614257812,
"fcm_dpo/q_t": 0.3818792700767517,
"grad_norm": 36.94768142700195,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.092390775680542,
"logits/rejected": -1.13364839553833,
"logps/chosen": -754.0936279296875,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1358.31494140625,
"loss": 1.027,
"margin_dpo/margin_mean": 569.7854614257812,
"margin_dpo/margin_std": 705.4525146484375,
"step": 621
},
{
"KL/chosen_KL_mean": -697.857421875,
"KL/mean": -898.1604614257812,
"KL/rejected_KL_mean": -1098.463623046875,
"KL/std": 515.55908203125,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0009166207164525986,
"fcm_dpo/delta": 0.0332704596221447,
"fcm_dpo/margin": 400.6061096191406,
"fcm_dpo/q_t": 0.4171668291091919,
"grad_norm": 37.14033889770508,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.9720423221588135,
"logits/rejected": -0.9517272710800171,
"logps/chosen": -758.1432495117188,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1183.9822998046875,
"loss": 1.1303,
"margin_dpo/margin_mean": 400.6060791015625,
"margin_dpo/margin_std": 636.8636474609375,
"step": 622
},
{
"KL/chosen_KL_mean": -729.516845703125,
"KL/mean": -890.4384765625,
"KL/rejected_KL_mean": -1051.3599853515625,
"KL/std": 516.193359375,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0009268617141060531,
"fcm_dpo/delta": 0.10320872068405151,
"fcm_dpo/margin": 321.8431701660156,
"fcm_dpo/q_t": 0.43382978439331055,
"grad_norm": 35.06966018676758,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -1.096697211265564,
"logits/rejected": -1.092256784439087,
"logps/chosen": -793.673828125,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1136.443115234375,
"loss": 1.1852,
"margin_dpo/margin_mean": 321.8431396484375,
"margin_dpo/margin_std": 620.9134521484375,
"step": 623
},
{
"KL/chosen_KL_mean": -732.138916015625,
"KL/mean": -979.6350708007812,
"KL/rejected_KL_mean": -1227.131103515625,
"KL/std": 541.4876708984375,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0009254544274881482,
"fcm_dpo/delta": -0.05943404883146286,
"fcm_dpo/margin": 494.9923400878906,
"fcm_dpo/q_t": 0.39315974712371826,
"grad_norm": 33.594234466552734,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.132512092590332,
"logits/rejected": -1.1252646446228027,
"logps/chosen": -804.0575561523438,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1324.26318359375,
"loss": 1.0492,
"margin_dpo/margin_mean": 494.9923400878906,
"margin_dpo/margin_std": 585.3521118164062,
"step": 624
},
{
"KL/chosen_KL_mean": -680.2445068359375,
"KL/mean": -972.3754272460938,
"KL/rejected_KL_mean": -1264.50634765625,
"KL/std": 626.7379150390625,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0009159696055576205,
"fcm_dpo/delta": -0.13877010345458984,
"fcm_dpo/margin": 584.261962890625,
"fcm_dpo/q_t": 0.3786303400993347,
"grad_norm": 48.92192459106445,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -1.0401864051818848,
"logits/rejected": -1.0676482915878296,
"logps/chosen": -738.5865478515625,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1350.5968017578125,
"loss": 0.9865,
"margin_dpo/margin_mean": 584.261962890625,
"margin_dpo/margin_std": 589.0737915039062,
"step": 625
},
{
"KL/chosen_KL_mean": -864.771484375,
"KL/mean": -1021.89208984375,
"KL/rejected_KL_mean": -1179.0126953125,
"KL/std": 683.1890869140625,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0009178274776786566,
"fcm_dpo/delta": 0.1132790595293045,
"fcm_dpo/margin": 314.2411193847656,
"fcm_dpo/q_t": 0.4319329857826233,
"grad_norm": 38.202056884765625,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -1.0161819458007812,
"logits/rejected": -1.007856011390686,
"logps/chosen": -939.8840942382812,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1278.201416015625,
"loss": 1.2125,
"margin_dpo/margin_mean": 314.2411193847656,
"margin_dpo/margin_std": 675.1224365234375,
"step": 626
},
{
"KL/chosen_KL_mean": -591.8564453125,
"KL/mean": -881.7455444335938,
"KL/rejected_KL_mean": -1171.6346435546875,
"KL/std": 698.69775390625,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0009149962570518255,
"fcm_dpo/delta": -0.13404129445552826,
"fcm_dpo/margin": 579.7781372070312,
"fcm_dpo/q_t": 0.3895081877708435,
"grad_norm": 31.1433162689209,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.9388647079467773,
"logits/rejected": -1.0119527578353882,
"logps/chosen": -639.599609375,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1278.38916015625,
"loss": 1.0424,
"margin_dpo/margin_mean": 579.7781372070312,
"margin_dpo/margin_std": 799.517578125,
"step": 627
},
{
"KL/chosen_KL_mean": -759.9505615234375,
"KL/mean": -988.3452758789062,
"KL/rejected_KL_mean": -1216.739990234375,
"KL/std": 587.220458984375,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0009073036490008235,
"fcm_dpo/delta": -0.014781368896365166,
"fcm_dpo/margin": 456.78936767578125,
"fcm_dpo/q_t": 0.40726301074028015,
"grad_norm": 34.24974060058594,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -1.096536636352539,
"logits/rejected": -1.1101562976837158,
"logps/chosen": -820.1334838867188,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1318.294677734375,
"loss": 1.1024,
"margin_dpo/margin_mean": 456.78936767578125,
"margin_dpo/margin_std": 680.5447998046875,
"step": 628
},
{
"KL/chosen_KL_mean": -751.41064453125,
"KL/mean": -967.8169555664062,
"KL/rejected_KL_mean": -1184.2232666015625,
"KL/std": 572.2176513671875,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0009084055200219154,
"fcm_dpo/delta": 0.006959038320928812,
"fcm_dpo/margin": 432.81256103515625,
"fcm_dpo/q_t": 0.40856897830963135,
"grad_norm": 38.99527359008789,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.0798592567443848,
"logits/rejected": -1.0886130332946777,
"logps/chosen": -815.6241455078125,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1275.876953125,
"loss": 1.0948,
"margin_dpo/margin_mean": 432.81256103515625,
"margin_dpo/margin_std": 571.304443359375,
"step": 629
},
{
"KL/chosen_KL_mean": -677.60888671875,
"KL/mean": -813.6373291015625,
"KL/rejected_KL_mean": -949.665771484375,
"KL/std": 625.6536254882812,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0009094603592529893,
"fcm_dpo/delta": 0.0273385439068079,
"fcm_dpo/margin": 272.05682373046875,
"fcm_dpo/q_t": 0.4511290192604065,
"grad_norm": 44.24210739135742,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -1.0548908710479736,
"logits/rejected": -1.0271886587142944,
"logps/chosen": -736.89990234375,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -1033.2640380859375,
"loss": 1.2546,
"margin_dpo/margin_mean": 272.0568542480469,
"margin_dpo/margin_std": 752.2532958984375,
"step": 630
},
{
"KL/chosen_KL_mean": -818.7833862304688,
"KL/mean": -1021.9134521484375,
"KL/rejected_KL_mean": -1225.04345703125,
"KL/std": 593.9693603515625,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0009135643485933542,
"fcm_dpo/delta": 0.029337339103221893,
"fcm_dpo/margin": 406.260009765625,
"fcm_dpo/q_t": 0.4151837229728699,
"grad_norm": 36.602779388427734,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.141347885131836,
"logits/rejected": -1.1473594903945923,
"logps/chosen": -878.2369995117188,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1305.9949951171875,
"loss": 1.14,
"margin_dpo/margin_mean": 406.2600402832031,
"margin_dpo/margin_std": 679.403564453125,
"step": 631
},
{
"KL/chosen_KL_mean": -730.484130859375,
"KL/mean": -936.6009521484375,
"KL/rejected_KL_mean": -1142.7177734375,
"KL/std": 546.6138305664062,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0009125604410655797,
"fcm_dpo/delta": 0.024038948118686676,
"fcm_dpo/margin": 412.233642578125,
"fcm_dpo/q_t": 0.4137340188026428,
"grad_norm": 39.51885986328125,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.1271458864212036,
"logits/rejected": -1.126634955406189,
"logps/chosen": -791.835693359375,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1228.8779296875,
"loss": 1.1307,
"margin_dpo/margin_mean": 412.233642578125,
"margin_dpo/margin_std": 653.526611328125,
"step": 632
},
{
"KL/chosen_KL_mean": -793.9225463867188,
"KL/mean": -951.2642822265625,
"KL/rejected_KL_mean": -1108.6060791015625,
"KL/std": 532.9287109375,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.0009164665825664997,
"fcm_dpo/delta": 0.019403798505663872,
"fcm_dpo/margin": 314.6834716796875,
"fcm_dpo/q_t": 0.4337080717086792,
"grad_norm": 44.8255615234375,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.1348984241485596,
"logits/rejected": -1.147522211074829,
"logps/chosen": -851.20068359375,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1200.1900634765625,
"loss": 1.1933,
"margin_dpo/margin_mean": 314.6835021972656,
"margin_dpo/margin_std": 608.6378784179688,
"step": 633
},
{
"KL/chosen_KL_mean": -881.3724365234375,
"KL/mean": -1016.420654296875,
"KL/rejected_KL_mean": -1151.4688720703125,
"KL/std": 623.2406005859375,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0009277792414650321,
"fcm_dpo/delta": 0.15131084620952606,
"fcm_dpo/margin": 270.09649658203125,
"fcm_dpo/q_t": 0.4442192018032074,
"grad_norm": 36.3792724609375,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.985339879989624,
"logits/rejected": -0.9769416451454163,
"logps/chosen": -947.991455078125,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1258.594482421875,
"loss": 1.2225,
"margin_dpo/margin_mean": 270.09649658203125,
"margin_dpo/margin_std": 597.4962768554688,
"step": 634
},
{
"KL/chosen_KL_mean": -775.334716796875,
"KL/mean": -957.4782104492188,
"KL/rejected_KL_mean": -1139.62158203125,
"KL/std": 646.2401733398438,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0009354540961794555,
"fcm_dpo/delta": 0.060258638113737106,
"fcm_dpo/margin": 364.28692626953125,
"fcm_dpo/q_t": 0.4247502088546753,
"grad_norm": 43.95642852783203,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.0711841583251953,
"logits/rejected": -1.0444178581237793,
"logps/chosen": -849.290283203125,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1222.1220703125,
"loss": 1.163,
"margin_dpo/margin_mean": 364.28692626953125,
"margin_dpo/margin_std": 672.04248046875,
"step": 635
},
{
"KL/chosen_KL_mean": -732.2950439453125,
"KL/mean": -943.854248046875,
"KL/rejected_KL_mean": -1155.413330078125,
"KL/std": 642.1853637695312,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0009403472067788243,
"fcm_dpo/delta": 0.0016737580299377441,
"fcm_dpo/margin": 423.1182861328125,
"fcm_dpo/q_t": 0.41292804479599,
"grad_norm": 25.051197052001953,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.9883960485458374,
"logits/rejected": -0.9851752519607544,
"logps/chosen": -791.9239501953125,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1237.3922119140625,
"loss": 1.1096,
"margin_dpo/margin_mean": 423.1182861328125,
"margin_dpo/margin_std": 657.2255859375,
"step": 636
},
{
"KL/chosen_KL_mean": -731.9287719726562,
"KL/mean": -977.4234619140625,
"KL/rejected_KL_mean": -1222.9180908203125,
"KL/std": 657.0423583984375,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0009306943975389004,
"fcm_dpo/delta": -0.05869361758232117,
"fcm_dpo/margin": 490.98931884765625,
"fcm_dpo/q_t": 0.39948800206184387,
"grad_norm": 32.110877990722656,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.1167852878570557,
"logits/rejected": -1.1629787683486938,
"logps/chosen": -781.5814208984375,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1321.3232421875,
"loss": 1.0728,
"margin_dpo/margin_mean": 490.98931884765625,
"margin_dpo/margin_std": 690.1351318359375,
"step": 637
},
{
"KL/chosen_KL_mean": -686.4628295898438,
"KL/mean": -884.8165283203125,
"KL/rejected_KL_mean": -1083.1702880859375,
"KL/std": 587.4907836914062,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0009277926292270422,
"fcm_dpo/delta": -0.08301420509815216,
"fcm_dpo/margin": 396.7074279785156,
"fcm_dpo/q_t": 0.4109494686126709,
"grad_norm": 36.82447052001953,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.9534268379211426,
"logits/rejected": -0.9300310611724854,
"logps/chosen": -744.6195068359375,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1162.4716796875,
"loss": 1.1481,
"margin_dpo/margin_mean": 396.7074279785156,
"margin_dpo/margin_std": 668.0907592773438,
"step": 638
},
{
"KL/chosen_KL_mean": -987.62060546875,
"KL/mean": -1060.539306640625,
"KL/rejected_KL_mean": -1133.458251953125,
"KL/std": 582.9657592773438,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0009239416103810072,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 145.8376007080078,
"fcm_dpo/q_t": 0.46903330087661743,
"grad_norm": 139.1773681640625,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.9747638702392578,
"logits/rejected": -0.9038654565811157,
"logps/chosen": -1059.94384765625,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1207.733154296875,
"loss": 1.3733,
"margin_dpo/margin_mean": 145.83761596679688,
"margin_dpo/margin_std": 759.8252563476562,
"step": 639
},
{
"KL/chosen_KL_mean": -737.2076416015625,
"KL/mean": -1026.8175048828125,
"KL/rejected_KL_mean": -1316.42724609375,
"KL/std": 669.41357421875,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.00091610784875229,
"fcm_dpo/delta": -0.13407810032367706,
"fcm_dpo/margin": 579.2196044921875,
"fcm_dpo/q_t": 0.3861461579799652,
"grad_norm": 52.59541320800781,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.9671276807785034,
"logits/rejected": -0.9929375648498535,
"logps/chosen": -793.342041015625,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1425.02734375,
"loss": 1.0344,
"margin_dpo/margin_mean": 579.2196044921875,
"margin_dpo/margin_std": 785.578369140625,
"step": 640
},
{
"KL/chosen_KL_mean": -859.6257934570312,
"KL/mean": -1046.6640625,
"KL/rejected_KL_mean": -1233.7021484375,
"KL/std": 573.8028564453125,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0009146465454250574,
"fcm_dpo/delta": 0.05887487903237343,
"fcm_dpo/margin": 374.076416015625,
"fcm_dpo/q_t": 0.42490917444229126,
"grad_norm": 37.22019958496094,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -1.0567216873168945,
"logits/rejected": -1.0576525926589966,
"logps/chosen": -924.6226806640625,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1320.694580078125,
"loss": 1.1778,
"margin_dpo/margin_mean": 374.076416015625,
"margin_dpo/margin_std": 746.7939453125,
"step": 641
},
{
"KL/chosen_KL_mean": -806.0206298828125,
"KL/mean": -1029.8985595703125,
"KL/rejected_KL_mean": -1253.7764892578125,
"KL/std": 721.0260620117188,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0009186153765767813,
"fcm_dpo/delta": -0.011798828840255737,
"fcm_dpo/margin": 447.75579833984375,
"fcm_dpo/q_t": 0.41545358300209045,
"grad_norm": 39.59940719604492,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -1.001420021057129,
"logits/rejected": -1.0281758308410645,
"logps/chosen": -871.7098999023438,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1364.0185546875,
"loss": 1.1498,
"margin_dpo/margin_mean": 447.75579833984375,
"margin_dpo/margin_std": 873.0943603515625,
"step": 642
},
{
"KL/chosen_KL_mean": -727.7049560546875,
"KL/mean": -922.38037109375,
"KL/rejected_KL_mean": -1117.055908203125,
"KL/std": 554.1806640625,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0009108081576414406,
"fcm_dpo/delta": -0.05616462975740433,
"fcm_dpo/margin": 389.350830078125,
"fcm_dpo/q_t": 0.41762280464172363,
"grad_norm": 45.18101119995117,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -1.1256649494171143,
"logits/rejected": -1.1269869804382324,
"logps/chosen": -779.6549072265625,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1204.524169921875,
"loss": 1.1243,
"margin_dpo/margin_mean": 389.3507995605469,
"margin_dpo/margin_std": 536.47412109375,
"step": 643
},
{
"KL/chosen_KL_mean": -711.8359375,
"KL/mean": -895.3848876953125,
"KL/rejected_KL_mean": -1078.9339599609375,
"KL/std": 636.5994873046875,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0009149383986368775,
"fcm_dpo/delta": 0.06524142622947693,
"fcm_dpo/margin": 367.0979919433594,
"fcm_dpo/q_t": 0.42591458559036255,
"grad_norm": 48.07936096191406,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -1.0160619020462036,
"logits/rejected": -1.021895170211792,
"logps/chosen": -770.8535766601562,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1166.070556640625,
"loss": 1.1859,
"margin_dpo/margin_mean": 367.0979919433594,
"margin_dpo/margin_std": 747.0296020507812,
"step": 644
},
{
"KL/chosen_KL_mean": -738.4013061523438,
"KL/mean": -835.6104125976562,
"KL/rejected_KL_mean": -932.8195190429688,
"KL/std": 500.1334228515625,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0009230460273101926,
"fcm_dpo/delta": 0.068226657807827,
"fcm_dpo/margin": 194.41815185546875,
"fcm_dpo/q_t": 0.4604693055152893,
"grad_norm": 92.3852310180664,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.9825325608253479,
"logits/rejected": -0.969984769821167,
"logps/chosen": -794.27734375,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -1030.600341796875,
"loss": 1.3114,
"margin_dpo/margin_mean": 194.41818237304688,
"margin_dpo/margin_std": 666.4583740234375,
"step": 645
},
{
"KL/chosen_KL_mean": -734.58056640625,
"KL/mean": -900.8929443359375,
"KL/rejected_KL_mean": -1067.205322265625,
"KL/std": 515.5413818359375,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0009276814525946975,
"fcm_dpo/delta": 0.09289085865020752,
"fcm_dpo/margin": 332.624755859375,
"fcm_dpo/q_t": 0.43122583627700806,
"grad_norm": 54.124847412109375,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -1.0459076166152954,
"logits/rejected": -1.022136926651001,
"logps/chosen": -795.8563232421875,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1144.711181640625,
"loss": 1.184,
"margin_dpo/margin_mean": 332.624755859375,
"margin_dpo/margin_std": 628.484130859375,
"step": 646
},
{
"KL/chosen_KL_mean": -620.7327880859375,
"KL/mean": -821.4715576171875,
"KL/rejected_KL_mean": -1022.2103271484375,
"KL/std": 562.453125,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0009334119968116283,
"fcm_dpo/delta": 0.025739368051290512,
"fcm_dpo/margin": 401.47760009765625,
"fcm_dpo/q_t": 0.4135919511318207,
"grad_norm": 33.10821533203125,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.9425860047340393,
"logits/rejected": -0.9679901599884033,
"logps/chosen": -675.585205078125,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1115.729736328125,
"loss": 1.1112,
"margin_dpo/margin_mean": 401.47760009765625,
"margin_dpo/margin_std": 560.4625244140625,
"step": 647
},
{
"KL/chosen_KL_mean": -693.855224609375,
"KL/mean": -988.9381103515625,
"KL/rejected_KL_mean": -1284.02099609375,
"KL/std": 669.7720947265625,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.000923770247027278,
"fcm_dpo/delta": -0.14923778176307678,
"fcm_dpo/margin": 590.1658935546875,
"fcm_dpo/q_t": 0.3826466500759125,
"grad_norm": 28.995315551757812,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.0780737400054932,
"logits/rejected": -1.1365818977355957,
"logps/chosen": -748.026611328125,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1382.73388671875,
"loss": 1.024,
"margin_dpo/margin_mean": 590.1658935546875,
"margin_dpo/margin_std": 785.5125122070312,
"step": 648
},
{
"KL/chosen_KL_mean": -715.6250610351562,
"KL/mean": -850.495361328125,
"KL/rejected_KL_mean": -985.3656005859375,
"KL/std": 527.7951049804688,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0009227098198607564,
"fcm_dpo/delta": 0.05385739728808403,
"fcm_dpo/margin": 269.74053955078125,
"fcm_dpo/q_t": 0.44609135389328003,
"grad_norm": 33.012882232666016,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.1264615058898926,
"logits/rejected": -1.1146023273468018,
"logps/chosen": -778.1054077148438,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -1065.44287109375,
"loss": 1.2419,
"margin_dpo/margin_mean": 269.7405700683594,
"margin_dpo/margin_std": 652.0167236328125,
"step": 649
},
{
"KL/chosen_KL_mean": -752.7981567382812,
"KL/mean": -973.55517578125,
"KL/rejected_KL_mean": -1194.312255859375,
"KL/std": 645.759765625,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0009236993500962853,
"fcm_dpo/delta": -0.008039167150855064,
"fcm_dpo/margin": 441.51409912109375,
"fcm_dpo/q_t": 0.4098489284515381,
"grad_norm": 28.0744571685791,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -1.0201561450958252,
"logits/rejected": -1.0400092601776123,
"logps/chosen": -808.8909912109375,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1292.5771484375,
"loss": 1.1285,
"margin_dpo/margin_mean": 441.51409912109375,
"margin_dpo/margin_std": 749.44873046875,
"step": 650
},
{
"KL/chosen_KL_mean": -520.7056884765625,
"KL/mean": -799.834228515625,
"KL/rejected_KL_mean": -1078.9627685546875,
"KL/std": 597.411865234375,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0009163864306174219,
"fcm_dpo/delta": -0.11453430354595184,
"fcm_dpo/margin": 558.2569580078125,
"fcm_dpo/q_t": 0.3837953507900238,
"grad_norm": 45.06791687011719,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -1.0805871486663818,
"logits/rejected": -1.1315158605575562,
"logps/chosen": -564.1311645507812,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1178.920654296875,
"loss": 1.0074,
"margin_dpo/margin_mean": 558.2569580078125,
"margin_dpo/margin_std": 605.560791015625,
"step": 651
},
{
"KL/chosen_KL_mean": -649.9320678710938,
"KL/mean": -872.9180908203125,
"KL/rejected_KL_mean": -1095.904052734375,
"KL/std": 631.821044921875,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.000913501949980855,
"fcm_dpo/delta": -0.007549758069217205,
"fcm_dpo/margin": 445.9720153808594,
"fcm_dpo/q_t": 0.4097236096858978,
"grad_norm": 41.35665512084961,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -1.05128812789917,
"logits/rejected": -1.0744919776916504,
"logps/chosen": -712.5089111328125,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1207.671875,
"loss": 1.1206,
"margin_dpo/margin_mean": 445.9720153808594,
"margin_dpo/margin_std": 735.88134765625,
"step": 652
},
{
"KL/chosen_KL_mean": -779.2682495117188,
"KL/mean": -977.2479858398438,
"KL/rejected_KL_mean": -1175.2276611328125,
"KL/std": 638.5052490234375,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0009161117486655712,
"fcm_dpo/delta": 0.0379294790327549,
"fcm_dpo/margin": 395.95941162109375,
"fcm_dpo/q_t": 0.41951340436935425,
"grad_norm": 39.457916259765625,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.0988178253173828,
"logits/rejected": -1.118626594543457,
"logps/chosen": -840.3812255859375,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1278.477294921875,
"loss": 1.1425,
"margin_dpo/margin_mean": 395.95941162109375,
"margin_dpo/margin_std": 670.7147827148438,
"step": 653
},
{
"KL/chosen_KL_mean": -704.796875,
"KL/mean": -902.2587890625,
"KL/rejected_KL_mean": -1099.720703125,
"KL/std": 529.0486450195312,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.000919223006349057,
"fcm_dpo/delta": 0.03766302019357681,
"fcm_dpo/margin": 394.923828125,
"fcm_dpo/q_t": 0.41876959800720215,
"grad_norm": 32.23603820800781,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -1.0824410915374756,
"logits/rejected": -1.1002042293548584,
"logps/chosen": -766.5250244140625,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1198.4945068359375,
"loss": 1.1299,
"margin_dpo/margin_mean": 394.923828125,
"margin_dpo/margin_std": 613.5523681640625,
"step": 654
},
{
"KL/chosen_KL_mean": -666.8828125,
"KL/mean": -870.272705078125,
"KL/rejected_KL_mean": -1073.66259765625,
"KL/std": 549.2578735351562,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0009220571955665946,
"fcm_dpo/delta": 0.025397051125764847,
"fcm_dpo/margin": 406.7799072265625,
"fcm_dpo/q_t": 0.41584494709968567,
"grad_norm": 34.27344512939453,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -1.0440378189086914,
"logits/rejected": -1.0873972177505493,
"logps/chosen": -716.4595947265625,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1171.9544677734375,
"loss": 1.1277,
"margin_dpo/margin_mean": 406.7799072265625,
"margin_dpo/margin_std": 646.5740966796875,
"step": 655
},
{
"KL/chosen_KL_mean": -702.065673828125,
"KL/mean": -1036.923828125,
"KL/rejected_KL_mean": -1371.781982421875,
"KL/std": 697.924072265625,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0009094408596865833,
"fcm_dpo/delta": -0.2153792828321457,
"fcm_dpo/margin": 669.7164306640625,
"fcm_dpo/q_t": 0.3670673072338104,
"grad_norm": 81.25033569335938,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.9668943881988525,
"logits/rejected": -1.0353293418884277,
"logps/chosen": -754.6151123046875,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1485.456787109375,
"loss": 0.9595,
"margin_dpo/margin_mean": 669.7164306640625,
"margin_dpo/margin_std": 723.3271484375,
"step": 656
},
{
"KL/chosen_KL_mean": -684.0799560546875,
"KL/mean": -962.5208129882812,
"KL/rejected_KL_mean": -1240.961669921875,
"KL/std": 694.053466796875,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0008950576884672046,
"fcm_dpo/delta": -0.10112009942531586,
"fcm_dpo/margin": 556.8817138671875,
"fcm_dpo/q_t": 0.38936832547187805,
"grad_norm": 43.923797607421875,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.0988727807998657,
"logits/rejected": -1.1430981159210205,
"logps/chosen": -730.780517578125,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1338.87646484375,
"loss": 1.0394,
"margin_dpo/margin_mean": 556.8817138671875,
"margin_dpo/margin_std": 717.0012817382812,
"step": 657
},
{
"KL/chosen_KL_mean": -721.3847045898438,
"KL/mean": -944.3277587890625,
"KL/rejected_KL_mean": -1167.27099609375,
"KL/std": 536.6946411132812,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0008913222118280828,
"fcm_dpo/delta": 0.002288389950990677,
"fcm_dpo/margin": 445.8861389160156,
"fcm_dpo/q_t": 0.4090738594532013,
"grad_norm": 33.75960159301758,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -1.0495469570159912,
"logits/rejected": -1.0703763961791992,
"logps/chosen": -782.3428955078125,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1263.21044921875,
"loss": 1.1046,
"margin_dpo/margin_mean": 445.8861083984375,
"margin_dpo/margin_std": 629.4903564453125,
"step": 658
},
{
"KL/chosen_KL_mean": -631.7986450195312,
"KL/mean": -848.4490966796875,
"KL/rejected_KL_mean": -1065.099609375,
"KL/std": 545.1578369140625,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0008934010984376073,
"fcm_dpo/delta": 0.013058962300419807,
"fcm_dpo/margin": 433.3009033203125,
"fcm_dpo/q_t": 0.41191983222961426,
"grad_norm": 32.50177001953125,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.9998725056648254,
"logits/rejected": -0.9671860933303833,
"logps/chosen": -708.5416259765625,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1152.570556640625,
"loss": 1.1012,
"margin_dpo/margin_mean": 433.3009338378906,
"margin_dpo/margin_std": 592.5736083984375,
"step": 659
},
{
"KL/chosen_KL_mean": -696.3098754882812,
"KL/mean": -962.7720947265625,
"KL/rejected_KL_mean": -1229.234375,
"KL/std": 621.4789428710938,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0008908716263249516,
"fcm_dpo/delta": -0.07654912024736404,
"fcm_dpo/margin": 532.9244995117188,
"fcm_dpo/q_t": 0.3918866515159607,
"grad_norm": 45.116607666015625,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.0789703130722046,
"logits/rejected": -1.089571237564087,
"logps/chosen": -755.3577880859375,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1305.1944580078125,
"loss": 1.0467,
"margin_dpo/margin_mean": 532.9244995117188,
"margin_dpo/margin_std": 662.8816528320312,
"step": 660
},
{
"KL/chosen_KL_mean": -641.9539794921875,
"KL/mean": -880.4549560546875,
"KL/rejected_KL_mean": -1118.9559326171875,
"KL/std": 687.37255859375,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.000888321315869689,
"fcm_dpo/delta": -0.02433537133038044,
"fcm_dpo/margin": 477.001953125,
"fcm_dpo/q_t": 0.40677112340927124,
"grad_norm": 43.87881851196289,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.1158676147460938,
"logits/rejected": -1.1237622499465942,
"logps/chosen": -692.6279296875,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1204.961669921875,
"loss": 1.0893,
"margin_dpo/margin_mean": 477.001953125,
"margin_dpo/margin_std": 703.0227661132812,
"step": 661
},
{
"KL/chosen_KL_mean": -701.1622314453125,
"KL/mean": -907.7703857421875,
"KL/rejected_KL_mean": -1114.378662109375,
"KL/std": 595.3218994140625,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0008879530942067504,
"fcm_dpo/delta": 0.03370830416679382,
"fcm_dpo/margin": 413.2164306640625,
"fcm_dpo/q_t": 0.4180566072463989,
"grad_norm": 28.001462936401367,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -1.0258715152740479,
"logits/rejected": -1.023505449295044,
"logps/chosen": -770.4232788085938,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1203.4345703125,
"loss": 1.1412,
"margin_dpo/margin_mean": 413.2164306640625,
"margin_dpo/margin_std": 703.9146728515625,
"step": 662
},
{
"KL/chosen_KL_mean": -683.2754516601562,
"KL/mean": -903.65966796875,
"KL/rejected_KL_mean": -1124.0439453125,
"KL/std": 640.8971557617188,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.0008887812728062272,
"fcm_dpo/delta": 0.008383674547076225,
"fcm_dpo/margin": 440.7684020996094,
"fcm_dpo/q_t": 0.41332346200942993,
"grad_norm": 30.94547462463379,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -1.0513726472854614,
"logits/rejected": -1.0720903873443604,
"logps/chosen": -748.1544189453125,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1237.96923828125,
"loss": 1.1309,
"margin_dpo/margin_mean": 440.7684020996094,
"margin_dpo/margin_std": 747.7581787109375,
"step": 663
},
{
"KL/chosen_KL_mean": -748.7786254882812,
"KL/mean": -1007.6143798828125,
"KL/rejected_KL_mean": -1266.4501953125,
"KL/std": 679.8283081054688,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0008842787938192487,
"fcm_dpo/delta": -0.05937542766332626,
"fcm_dpo/margin": 517.6715698242188,
"fcm_dpo/q_t": 0.39915400743484497,
"grad_norm": 30.055925369262695,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -1.0716259479522705,
"logits/rejected": -1.1003488302230835,
"logps/chosen": -809.6671142578125,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1371.971923828125,
"loss": 1.0709,
"margin_dpo/margin_mean": 517.6715698242188,
"margin_dpo/margin_std": 723.2990112304688,
"step": 664
},
{
"KL/chosen_KL_mean": -657.037109375,
"KL/mean": -874.8280029296875,
"KL/rejected_KL_mean": -1092.618896484375,
"KL/std": 550.8089599609375,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0008855801424942911,
"fcm_dpo/delta": 0.014537844806909561,
"fcm_dpo/margin": 435.581787109375,
"fcm_dpo/q_t": 0.41070112586021423,
"grad_norm": 56.73260498046875,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.1636441946029663,
"logits/rejected": -1.1826890707015991,
"logps/chosen": -717.6011962890625,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1177.427734375,
"loss": 1.0941,
"margin_dpo/margin_mean": 435.581787109375,
"margin_dpo/margin_std": 550.626708984375,
"step": 665
},
{
"KL/chosen_KL_mean": -698.910400390625,
"KL/mean": -909.08203125,
"KL/rejected_KL_mean": -1119.2535400390625,
"KL/std": 539.8720092773438,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0008871962781995535,
"fcm_dpo/delta": 0.027589332312345505,
"fcm_dpo/margin": 420.3431396484375,
"fcm_dpo/q_t": 0.415420264005661,
"grad_norm": 30.555706024169922,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.9608547687530518,
"logits/rejected": -0.9632211923599243,
"logps/chosen": -763.3303833007812,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1215.145263671875,
"loss": 1.1199,
"margin_dpo/margin_mean": 420.3431396484375,
"margin_dpo/margin_std": 612.9646606445312,
"step": 666
},
{
"KL/chosen_KL_mean": -784.1095581054688,
"KL/mean": -1053.423828125,
"KL/rejected_KL_mean": -1322.738037109375,
"KL/std": 630.7745361328125,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0008815985638648272,
"fcm_dpo/delta": -0.07696354389190674,
"fcm_dpo/margin": 538.6284790039062,
"fcm_dpo/q_t": 0.3935006856918335,
"grad_norm": 44.02839660644531,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.9979052543640137,
"logits/rejected": -0.9953716993331909,
"logps/chosen": -853.3865966796875,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1410.573486328125,
"loss": 1.0497,
"margin_dpo/margin_mean": 538.6284790039062,
"margin_dpo/margin_std": 688.3106689453125,
"step": 667
},
{
"KL/chosen_KL_mean": -829.9423217773438,
"KL/mean": -1009.3814697265625,
"KL/rejected_KL_mean": -1188.820556640625,
"KL/std": 652.5074462890625,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0008891418110579252,
"fcm_dpo/delta": 0.0817713588476181,
"fcm_dpo/margin": 358.8783264160156,
"fcm_dpo/q_t": 0.4309133291244507,
"grad_norm": 50.39697265625,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -1.052807092666626,
"logits/rejected": -1.0501337051391602,
"logps/chosen": -902.5463256835938,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1292.5596923828125,
"loss": 1.2006,
"margin_dpo/margin_mean": 358.8783264160156,
"margin_dpo/margin_std": 768.2645263671875,
"step": 668
},
{
"KL/chosen_KL_mean": -619.5028076171875,
"KL/mean": -885.7294311523438,
"KL/rejected_KL_mean": -1151.9560546875,
"KL/std": 607.2215576171875,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0008850685553625226,
"fcm_dpo/delta": -0.07294955104589462,
"fcm_dpo/margin": 532.453369140625,
"fcm_dpo/q_t": 0.39315131306648254,
"grad_norm": 24.650182723999023,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -1.0345218181610107,
"logits/rejected": -1.041704773902893,
"logps/chosen": -665.6192016601562,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1229.8804931640625,
"loss": 1.054,
"margin_dpo/margin_mean": 532.453369140625,
"margin_dpo/margin_std": 684.6708984375,
"step": 669
},
{
"KL/chosen_KL_mean": -620.4657592773438,
"KL/mean": -882.550048828125,
"KL/rejected_KL_mean": -1144.6343994140625,
"KL/std": 581.8966674804688,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0008802937809377909,
"fcm_dpo/delta": -0.06295409053564072,
"fcm_dpo/margin": 524.1685791015625,
"fcm_dpo/q_t": 0.3964523673057556,
"grad_norm": 27.264894485473633,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.9793117642402649,
"logits/rejected": -0.9908572435379028,
"logps/chosen": -682.8115234375,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1241.574951171875,
"loss": 1.0678,
"margin_dpo/margin_mean": 524.1685791015625,
"margin_dpo/margin_std": 703.9255981445312,
"step": 670
},
{
"KL/chosen_KL_mean": -780.59716796875,
"KL/mean": -994.5520629882812,
"KL/rejected_KL_mean": -1208.5069580078125,
"KL/std": 590.3243408203125,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.000880036735907197,
"fcm_dpo/delta": 0.02367694117128849,
"fcm_dpo/margin": 427.90972900390625,
"fcm_dpo/q_t": 0.413568913936615,
"grad_norm": 35.296512603759766,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.0900222063064575,
"logits/rejected": -1.1154989004135132,
"logps/chosen": -828.5972900390625,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1292.3262939453125,
"loss": 1.1384,
"margin_dpo/margin_mean": 427.90972900390625,
"margin_dpo/margin_std": 720.406005859375,
"step": 671
},
{
"KL/chosen_KL_mean": -845.4471435546875,
"KL/mean": -1062.668701171875,
"KL/rejected_KL_mean": -1279.8902587890625,
"KL/std": 706.8720703125,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0008793792221695185,
"fcm_dpo/delta": 0.01829908788204193,
"fcm_dpo/margin": 434.44317626953125,
"fcm_dpo/q_t": 0.4161950945854187,
"grad_norm": 38.76650619506836,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.1482672691345215,
"logits/rejected": -1.1567609310150146,
"logps/chosen": -904.0303955078125,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1373.030517578125,
"loss": 1.1456,
"margin_dpo/margin_mean": 434.4431457519531,
"margin_dpo/margin_std": 770.93408203125,
"step": 672
},
{
"KL/chosen_KL_mean": -725.902587890625,
"KL/mean": -928.3306274414062,
"KL/rejected_KL_mean": -1130.758544921875,
"KL/std": 572.4488525390625,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0008829334983602166,
"fcm_dpo/delta": 0.04332459345459938,
"fcm_dpo/margin": 404.85595703125,
"fcm_dpo/q_t": 0.41951626539230347,
"grad_norm": 26.817668914794922,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -1.091918706893921,
"logits/rejected": -1.0944968461990356,
"logps/chosen": -772.6258544921875,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1216.0548095703125,
"loss": 1.1323,
"margin_dpo/margin_mean": 404.85595703125,
"margin_dpo/margin_std": 645.776611328125,
"step": 673
},
{
"KL/chosen_KL_mean": -647.2359008789062,
"KL/mean": -898.0692138671875,
"KL/rejected_KL_mean": -1148.902587890625,
"KL/std": 574.1219482421875,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0008830685401335359,
"fcm_dpo/delta": -0.044042401015758514,
"fcm_dpo/margin": 501.666748046875,
"fcm_dpo/q_t": 0.4016549587249756,
"grad_norm": 40.91920852661133,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.9943381547927856,
"logits/rejected": -1.0083208084106445,
"logps/chosen": -692.681396484375,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1218.948486328125,
"loss": 1.075,
"margin_dpo/margin_mean": 501.666748046875,
"margin_dpo/margin_std": 687.1994018554688,
"step": 674
},
{
"KL/chosen_KL_mean": -708.86376953125,
"KL/mean": -968.4271240234375,
"KL/rejected_KL_mean": -1227.990478515625,
"KL/std": 640.6890258789062,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0008735453011468053,
"fcm_dpo/delta": -0.05547531321644783,
"fcm_dpo/margin": 519.1265869140625,
"fcm_dpo/q_t": 0.3998996317386627,
"grad_norm": 34.21153259277344,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.0351924896240234,
"logits/rejected": -1.0452386140823364,
"logps/chosen": -753.0400390625,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1302.0823974609375,
"loss": 1.07,
"margin_dpo/margin_mean": 519.1265869140625,
"margin_dpo/margin_std": 684.087646484375,
"step": 675
},
{
"KL/chosen_KL_mean": -776.6356201171875,
"KL/mean": -1018.5604858398438,
"KL/rejected_KL_mean": -1260.4853515625,
"KL/std": 617.2457885742188,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0008749394328333437,
"fcm_dpo/delta": -0.023866407573223114,
"fcm_dpo/margin": 483.8497619628906,
"fcm_dpo/q_t": 0.40422454476356506,
"grad_norm": 28.231639862060547,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -1.0005314350128174,
"logits/rejected": -1.0063731670379639,
"logps/chosen": -848.0341186523438,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1348.8441162109375,
"loss": 1.0782,
"margin_dpo/margin_mean": 483.8497314453125,
"margin_dpo/margin_std": 629.48583984375,
"step": 676
},
{
"KL/chosen_KL_mean": -752.8453979492188,
"KL/mean": -1003.5128173828125,
"KL/rejected_KL_mean": -1254.1802978515625,
"KL/std": 636.5200805664062,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.000870502321049571,
"fcm_dpo/delta": -0.037262238562107086,
"fcm_dpo/margin": 501.3348693847656,
"fcm_dpo/q_t": 0.4067116677761078,
"grad_norm": 33.513336181640625,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.0794928073883057,
"logits/rejected": -1.0712941884994507,
"logps/chosen": -809.372802734375,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1332.40673828125,
"loss": 1.0906,
"margin_dpo/margin_mean": 501.3348693847656,
"margin_dpo/margin_std": 767.4678344726562,
"step": 677
},
{
"KL/chosen_KL_mean": -626.833251953125,
"KL/mean": -919.983642578125,
"KL/rejected_KL_mean": -1213.134033203125,
"KL/std": 699.7562255859375,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0008654722478240728,
"fcm_dpo/delta": -0.1102285087108612,
"fcm_dpo/margin": 586.30078125,
"fcm_dpo/q_t": 0.3892865777015686,
"grad_norm": 35.17583465576172,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -1.0084434747695923,
"logits/rejected": -1.0442242622375488,
"logps/chosen": -672.9677734375,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1293.73876953125,
"loss": 1.0515,
"margin_dpo/margin_mean": 586.30078125,
"margin_dpo/margin_std": 820.827392578125,
"step": 678
},
{
"KL/chosen_KL_mean": -740.3648681640625,
"KL/mean": -935.5812377929688,
"KL/rejected_KL_mean": -1130.797607421875,
"KL/std": 556.906005859375,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.000864343426655978,
"fcm_dpo/delta": 0.06362688541412354,
"fcm_dpo/margin": 390.4327087402344,
"fcm_dpo/q_t": 0.4237492084503174,
"grad_norm": 33.905616760253906,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -1.0304598808288574,
"logits/rejected": -1.0171773433685303,
"logps/chosen": -790.6597900390625,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1207.395751953125,
"loss": 1.1458,
"margin_dpo/margin_mean": 390.4327087402344,
"margin_dpo/margin_std": 631.3106079101562,
"step": 679
},
{
"KL/chosen_KL_mean": -745.6943359375,
"KL/mean": -1011.821533203125,
"KL/rejected_KL_mean": -1277.94873046875,
"KL/std": 722.32958984375,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0008604623144492507,
"fcm_dpo/delta": -0.059656865894794464,
"fcm_dpo/margin": 532.2544555664062,
"fcm_dpo/q_t": 0.39625048637390137,
"grad_norm": 34.24609375,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -1.0160003900527954,
"logits/rejected": -1.0428314208984375,
"logps/chosen": -822.6099853515625,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1390.33349609375,
"loss": 1.0828,
"margin_dpo/margin_mean": 532.2544555664062,
"margin_dpo/margin_std": 783.269287109375,
"step": 680
},
{
"KL/chosen_KL_mean": -750.117919921875,
"KL/mean": -952.0081787109375,
"KL/rejected_KL_mean": -1153.8985595703125,
"KL/std": 590.4039306640625,
"epoch": 1.0,
"fcm_dpo/beta": 0.0008671648683957756,
"fcm_dpo/delta": 0.05011419206857681,
"fcm_dpo/margin": 403.78057861328125,
"fcm_dpo/q_t": 0.4205058813095093,
"grad_norm": 33.01895523071289,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -1.0952690839767456,
"logits/rejected": -1.1073505878448486,
"logps/chosen": -811.0751953125,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1242.45654296875,
"loss": 1.1452,
"margin_dpo/margin_mean": 403.7805480957031,
"margin_dpo/margin_std": 639.1034545898438,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.0870012780118095,
"train_runtime": 1735.4876,
"train_samples_per_second": 25.121,
"train_steps_per_second": 0.392
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink