W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-1
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC e58d4d9296 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-1
Source: Original Platform
2026-05-26 11:47:18 +08:00

15707 lines
589 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.52816009521484,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.20802307128906,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": 0.016103744506835938,
"KL/mean": 0.019720733165740967,
"KL/rejected_KL_mean": 0.023336410522460938,
"KL/std": 0.25345832109451294,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.007235169410705566,
"fcm_dpo/q_t": 0.5001815557479858,
"grad_norm": 70.94044494628906,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.48162591457366943,
"logits/rejected": -0.4422367215156555,
"logps/chosen": -60.965492248535156,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.64926147460938,
"loss": 1.3874,
"margin_dpo/margin_mean": -0.007234543561935425,
"margin_dpo/margin_std": 0.3761873245239258,
"step": 3
},
{
"KL/chosen_KL_mean": 0.007844924926757812,
"KL/mean": 0.011015832424163818,
"KL/rejected_KL_mean": 0.014186859130859375,
"KL/std": 0.27283304929733276,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.006340086460113525,
"fcm_dpo/q_t": 0.5001586675643921,
"grad_norm": 71.89739990234375,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.4688315987586975,
"logits/rejected": -0.4412044882774353,
"logps/chosen": -56.759864807128906,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.63291931152344,
"loss": 1.3873,
"margin_dpo/margin_mean": -0.00634080171585083,
"margin_dpo/margin_std": 0.4030381143093109,
"step": 4
},
{
"KL/chosen_KL_mean": 0.0037784576416015625,
"KL/mean": 0.03290402889251709,
"KL/rejected_KL_mean": 0.06203460693359375,
"KL/std": 0.3190087080001831,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.05825453996658325,
"fcm_dpo/q_t": 0.5014559030532837,
"grad_norm": 90.05491638183594,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.5145796537399292,
"logits/rejected": -0.47089338302612305,
"logps/chosen": -53.85559844970703,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.08715057373047,
"loss": 1.3926,
"margin_dpo/margin_mean": -0.05825403332710266,
"margin_dpo/margin_std": 0.4133654236793518,
"step": 5
},
{
"KL/chosen_KL_mean": -0.008159637451171875,
"KL/mean": -0.009616628289222717,
"KL/rejected_KL_mean": -0.011074066162109375,
"KL/std": 0.2630811631679535,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.0029135048389434814,
"fcm_dpo/q_t": 0.49992650747299194,
"grad_norm": 91.91436004638672,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.5270358920097351,
"logits/rejected": -0.48671644926071167,
"logps/chosen": -63.01564407348633,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.65641784667969,
"loss": 1.3864,
"margin_dpo/margin_mean": 0.002912759780883789,
"margin_dpo/margin_std": 0.37860625982284546,
"step": 6
},
{
"KL/chosen_KL_mean": 0.018707275390625,
"KL/mean": 0.022922277450561523,
"KL/rejected_KL_mean": 0.027141571044921875,
"KL/std": 0.28703397512435913,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008430540561676025,
"fcm_dpo/q_t": 0.5002114176750183,
"grad_norm": 82.6740493774414,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.498199999332428,
"logits/rejected": -0.4643540382385254,
"logps/chosen": -57.75611114501953,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.89344787597656,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.008429855108261108,
"margin_dpo/margin_std": 0.42599087953567505,
"step": 7
},
{
"KL/chosen_KL_mean": -0.012296676635742188,
"KL/mean": 0.0035399198532104492,
"KL/rejected_KL_mean": 0.019378662109375,
"KL/std": 0.2982986569404602,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.031677067279815674,
"fcm_dpo/q_t": 0.5007911920547485,
"grad_norm": 79.09713745117188,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.5000207424163818,
"logits/rejected": -0.4732367694377899,
"logps/chosen": -58.72833251953125,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.29204559326172,
"loss": 1.3899,
"margin_dpo/margin_mean": -0.03167632222175598,
"margin_dpo/margin_std": 0.404108464717865,
"step": 8
},
{
"KL/chosen_KL_mean": 0.021236419677734375,
"KL/mean": 0.0014689713716506958,
"KL/rejected_KL_mean": -0.018299102783203125,
"KL/std": 0.270100474357605,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.039538174867630005,
"fcm_dpo/q_t": 0.4990116357803345,
"grad_norm": 84.92678833007812,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.4937548339366913,
"logits/rejected": -0.4469431936740875,
"logps/chosen": -69.84561157226562,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.62096405029297,
"loss": 1.3827,
"margin_dpo/margin_mean": 0.03953742980957031,
"margin_dpo/margin_std": 0.35995495319366455,
"step": 9
},
{
"KL/chosen_KL_mean": 0.003574371337890625,
"KL/mean": -0.0032436102628707886,
"KL/rejected_KL_mean": -0.010059356689453125,
"KL/std": 0.25196051597595215,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.013635605573654175,
"fcm_dpo/q_t": 0.49965864419937134,
"grad_norm": 70.92430114746094,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.5009649991989136,
"logits/rejected": -0.4579939842224121,
"logps/chosen": -48.35411071777344,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.38212585449219,
"loss": 1.3853,
"margin_dpo/margin_mean": 0.013636231422424316,
"margin_dpo/margin_std": 0.36122623085975647,
"step": 10
},
{
"KL/chosen_KL_mean": 0.005367279052734375,
"KL/mean": -0.006761014461517334,
"KL/rejected_KL_mean": -0.018890380859375,
"KL/std": 0.24701017141342163,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.024251699447631836,
"fcm_dpo/q_t": 0.49939417839050293,
"grad_norm": 68.93864440917969,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.45688319206237793,
"logits/rejected": -0.4317648708820343,
"logps/chosen": -53.01148986816406,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.79927062988281,
"loss": 1.3842,
"margin_dpo/margin_mean": 0.024251878261566162,
"margin_dpo/margin_std": 0.3476247489452362,
"step": 11
},
{
"KL/chosen_KL_mean": -0.05808830261230469,
"KL/mean": -0.053139448165893555,
"KL/rejected_KL_mean": -0.048183441162109375,
"KL/std": 0.2798803448677063,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.009897112846374512,
"fcm_dpo/q_t": 0.5002454519271851,
"grad_norm": 89.550048828125,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.5406717658042908,
"logits/rejected": -0.5045890808105469,
"logps/chosen": -61.863525390625,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.90645599365234,
"loss": 1.3877,
"margin_dpo/margin_mean": -0.009897887706756592,
"margin_dpo/margin_std": 0.40574026107788086,
"step": 12
},
{
"KL/chosen_KL_mean": 0.030796051025390625,
"KL/mean": 0.014434963464736938,
"KL/rejected_KL_mean": -0.00193023681640625,
"KL/std": 0.294820100069046,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.032728374004364014,
"fcm_dpo/q_t": 0.49918168783187866,
"grad_norm": 79.24481201171875,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.4753603935241699,
"logits/rejected": -0.44688206911087036,
"logps/chosen": -64.22956085205078,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.20501708984375,
"loss": 1.3835,
"margin_dpo/margin_mean": 0.03272801637649536,
"margin_dpo/margin_std": 0.4066898226737976,
"step": 13
},
{
"KL/chosen_KL_mean": -0.05864906311035156,
"KL/mean": -0.018393635749816895,
"KL/rejected_KL_mean": 0.021862030029296875,
"KL/std": 0.29221126437187195,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0805087685585022,
"fcm_dpo/q_t": 0.5020102858543396,
"grad_norm": 85.43807983398438,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.4688841998577118,
"logits/rejected": -0.4292323887348175,
"logps/chosen": -58.168861389160156,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.02522277832031,
"loss": 1.3949,
"margin_dpo/margin_mean": -0.08050811290740967,
"margin_dpo/margin_std": 0.4493297040462494,
"step": 14
},
{
"KL/chosen_KL_mean": 0.013751983642578125,
"KL/mean": -0.004344776272773743,
"KL/rejected_KL_mean": -0.022441864013671875,
"KL/std": 0.2262694537639618,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.036195188760757446,
"fcm_dpo/q_t": 0.49909502267837524,
"grad_norm": 64.15103912353516,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.5063309669494629,
"logits/rejected": -0.48831403255462646,
"logps/chosen": -56.953155517578125,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.83108520507812,
"loss": 1.3829,
"margin_dpo/margin_mean": 0.036195307970047,
"margin_dpo/margin_std": 0.31367939710617065,
"step": 15
},
{
"KL/chosen_KL_mean": -0.02686309814453125,
"KL/mean": -0.023277834057807922,
"KL/rejected_KL_mean": -0.01969146728515625,
"KL/std": 0.28893446922302246,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.007169216871261597,
"fcm_dpo/q_t": 0.5001786947250366,
"grad_norm": 83.99118041992188,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5097619891166687,
"logits/rejected": -0.46720778942108154,
"logps/chosen": -61.766754150390625,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.38916015625,
"loss": 1.3874,
"margin_dpo/margin_mean": -0.007169753313064575,
"margin_dpo/margin_std": 0.4101220369338989,
"step": 16
},
{
"KL/chosen_KL_mean": 0.011844635009765625,
"KL/mean": -0.012153714895248413,
"KL/rejected_KL_mean": -0.03615570068359375,
"KL/std": 0.26074981689453125,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.047999799251556396,
"fcm_dpo/q_t": 0.4987994134426117,
"grad_norm": 78.72090148925781,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.5130709409713745,
"logits/rejected": -0.47745299339294434,
"logps/chosen": -67.698486328125,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.41481018066406,
"loss": 1.3818,
"margin_dpo/margin_mean": 0.04799976944923401,
"margin_dpo/margin_std": 0.34855252504348755,
"step": 17
},
{
"KL/chosen_KL_mean": -0.01177215576171875,
"KL/mean": -0.058126211166381836,
"KL/rejected_KL_mean": -0.10448455810546875,
"KL/std": 0.24388915300369263,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.092709481716156,
"fcm_dpo/q_t": 0.49768269062042236,
"grad_norm": 81.854248046875,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.49614471197128296,
"logits/rejected": -0.4409538507461548,
"logps/chosen": -47.75126266479492,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.57676696777344,
"loss": 1.3773,
"margin_dpo/margin_mean": 0.09270939230918884,
"margin_dpo/margin_std": 0.3233555257320404,
"step": 18
},
{
"KL/chosen_KL_mean": 0.007350921630859375,
"KL/mean": -0.03270593285560608,
"KL/rejected_KL_mean": -0.07275772094726562,
"KL/std": 0.22124189138412476,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08010363578796387,
"fcm_dpo/q_t": 0.4979983866214752,
"grad_norm": 73.39251708984375,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.5021491050720215,
"logits/rejected": -0.4532221555709839,
"logps/chosen": -70.19801330566406,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.83035278320312,
"loss": 1.3786,
"margin_dpo/margin_mean": 0.08010467886924744,
"margin_dpo/margin_std": 0.34056466817855835,
"step": 19
},
{
"KL/chosen_KL_mean": 0.010915756225585938,
"KL/mean": -0.03122629225254059,
"KL/rejected_KL_mean": -0.07336044311523438,
"KL/std": 0.26908406615257263,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08428379893302917,
"fcm_dpo/q_t": 0.4978940486907959,
"grad_norm": 73.79589080810547,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5578715801239014,
"logits/rejected": -0.5020963549613953,
"logps/chosen": -50.792327880859375,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.89671325683594,
"loss": 1.3783,
"margin_dpo/margin_mean": 0.08428427577018738,
"margin_dpo/margin_std": 0.3846767842769623,
"step": 20
},
{
"KL/chosen_KL_mean": 0.016414642333984375,
"KL/mean": -0.05401710420846939,
"KL/rejected_KL_mean": -0.12445068359375,
"KL/std": 0.31130141019821167,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.14086376130580902,
"fcm_dpo/q_t": 0.49647945165634155,
"grad_norm": 77.40738677978516,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.4897395968437195,
"logits/rejected": -0.4655163288116455,
"logps/chosen": -50.046600341796875,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.99324035644531,
"loss": 1.3726,
"margin_dpo/margin_mean": 0.14086447656154633,
"margin_dpo/margin_std": 0.3929217755794525,
"step": 21
},
{
"KL/chosen_KL_mean": 0.07324600219726562,
"KL/mean": -0.054158031940460205,
"KL/rejected_KL_mean": -0.18156051635742188,
"KL/std": 0.3565206825733185,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.2548089623451233,
"fcm_dpo/q_t": 0.4936355948448181,
"grad_norm": 83.94691467285156,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.4774931073188782,
"logits/rejected": -0.43337398767471313,
"logps/chosen": -58.98439025878906,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.68623352050781,
"loss": 1.3617,
"margin_dpo/margin_mean": 0.2548080384731293,
"margin_dpo/margin_std": 0.523194432258606,
"step": 22
},
{
"KL/chosen_KL_mean": 0.026395797729492188,
"KL/mean": -0.0561884343624115,
"KL/rejected_KL_mean": -0.13876724243164062,
"KL/std": 0.30823904275894165,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.1651589274406433,
"fcm_dpo/q_t": 0.49587440490722656,
"grad_norm": 80.6090087890625,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.5116769671440125,
"logits/rejected": -0.4903678894042969,
"logps/chosen": -60.051300048828125,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.2783203125,
"loss": 1.3703,
"margin_dpo/margin_mean": 0.16515937447547913,
"margin_dpo/margin_std": 0.4328920841217041,
"step": 23
},
{
"KL/chosen_KL_mean": 0.053745269775390625,
"KL/mean": -0.060146935284137726,
"KL/rejected_KL_mean": -0.174041748046875,
"KL/std": 0.277465283870697,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.22778569161891937,
"fcm_dpo/q_t": 0.49430739879608154,
"grad_norm": 82.69357299804688,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5428138375282288,
"logits/rejected": -0.525646448135376,
"logps/chosen": -44.23728942871094,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.29925537109375,
"loss": 1.364,
"margin_dpo/margin_mean": 0.22778579592704773,
"margin_dpo/margin_std": 0.36717647314071655,
"step": 24
},
{
"KL/chosen_KL_mean": 0.017560958862304688,
"KL/mean": -0.10330188274383545,
"KL/rejected_KL_mean": -0.22416305541992188,
"KL/std": 0.36914387345314026,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24172252416610718,
"fcm_dpo/q_t": 0.4939602315425873,
"grad_norm": 73.93270111083984,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5116393566131592,
"logits/rejected": -0.48214203119277954,
"logps/chosen": -52.519493103027344,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.56635284423828,
"loss": 1.3628,
"margin_dpo/margin_mean": 0.24172285199165344,
"margin_dpo/margin_std": 0.4416242837905884,
"step": 25
},
{
"KL/chosen_KL_mean": 0.13255691528320312,
"KL/mean": -0.08234043419361115,
"KL/rejected_KL_mean": -0.2972412109375,
"KL/std": 0.4482305645942688,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.42980238795280457,
"fcm_dpo/q_t": 0.48926833271980286,
"grad_norm": 86.75181579589844,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5427736043930054,
"logits/rejected": -0.5113131999969482,
"logps/chosen": -53.790252685546875,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.65695190429688,
"loss": 1.3445,
"margin_dpo/margin_mean": 0.42980268597602844,
"margin_dpo/margin_std": 0.5426281690597534,
"step": 26
},
{
"KL/chosen_KL_mean": 0.12249946594238281,
"KL/mean": -0.1331784576177597,
"KL/rejected_KL_mean": -0.3888587951660156,
"KL/std": 0.4570690989494324,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5113544464111328,
"fcm_dpo/q_t": 0.4872331917285919,
"grad_norm": 93.51718139648438,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5056776404380798,
"logits/rejected": -0.4691791832447052,
"logps/chosen": -42.776031494140625,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.1130599975586,
"loss": 1.3366,
"margin_dpo/margin_mean": 0.5113544464111328,
"margin_dpo/margin_std": 0.5590527057647705,
"step": 27
},
{
"KL/chosen_KL_mean": 0.03354072570800781,
"KL/mean": -0.11374713480472565,
"KL/rejected_KL_mean": -0.2610359191894531,
"KL/std": 0.41249018907546997,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.29457351565361023,
"fcm_dpo/q_t": 0.49264103174209595,
"grad_norm": 75.92487335205078,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.49452316761016846,
"logits/rejected": -0.4381081163883209,
"logps/chosen": -60.5229606628418,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.66215515136719,
"loss": 1.3579,
"margin_dpo/margin_mean": 0.29457390308380127,
"margin_dpo/margin_std": 0.5741066336631775,
"step": 28
},
{
"KL/chosen_KL_mean": 0.06393051147460938,
"KL/mean": -0.18335245549678802,
"KL/rejected_KL_mean": -0.430633544921875,
"KL/std": 0.4628360867500305,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4945622384548187,
"fcm_dpo/q_t": 0.4876513183116913,
"grad_norm": 91.02857971191406,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5597679615020752,
"logits/rejected": -0.5140695571899414,
"logps/chosen": -57.74385452270508,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.82498168945312,
"loss": 1.3383,
"margin_dpo/margin_mean": 0.4945625960826874,
"margin_dpo/margin_std": 0.5654176473617554,
"step": 29
},
{
"KL/chosen_KL_mean": 0.1397686004638672,
"KL/mean": -0.20081256330013275,
"KL/rejected_KL_mean": -0.5413970947265625,
"KL/std": 0.5910484194755554,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6811658143997192,
"fcm_dpo/q_t": 0.48299551010131836,
"grad_norm": 87.69357299804688,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.4750695824623108,
"logits/rejected": -0.4435637593269348,
"logps/chosen": -52.43760299682617,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.03060150146484,
"loss": 1.3203,
"margin_dpo/margin_mean": 0.6811659336090088,
"margin_dpo/margin_std": 0.6095134019851685,
"step": 30
},
{
"KL/chosen_KL_mean": 0.1102294921875,
"KL/mean": -0.13631178438663483,
"KL/rejected_KL_mean": -0.3828544616699219,
"KL/std": 0.5659123063087463,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.49308571219444275,
"fcm_dpo/q_t": 0.4877015948295593,
"grad_norm": 67.67135620117188,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.5170990228652954,
"logits/rejected": -0.4736338257789612,
"logps/chosen": -63.696693420410156,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.27685546875,
"loss": 1.3388,
"margin_dpo/margin_mean": 0.49308544397354126,
"margin_dpo/margin_std": 0.6761853694915771,
"step": 31
},
{
"KL/chosen_KL_mean": 0.21260833740234375,
"KL/mean": -0.19022028148174286,
"KL/rejected_KL_mean": -0.5930519104003906,
"KL/std": 0.7161545753479004,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.8056564331054688,
"fcm_dpo/q_t": 0.47991523146629333,
"grad_norm": 83.33721160888672,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.504935085773468,
"logits/rejected": -0.46319282054901123,
"logps/chosen": -62.52691650390625,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.91055297851562,
"loss": 1.3091,
"margin_dpo/margin_mean": 0.8056566715240479,
"margin_dpo/margin_std": 0.8240557909011841,
"step": 32
},
{
"KL/chosen_KL_mean": 0.09860420227050781,
"KL/mean": -0.18287688493728638,
"KL/rejected_KL_mean": -0.4643592834472656,
"KL/std": 0.5310133099555969,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5629608035087585,
"fcm_dpo/q_t": 0.48594412207603455,
"grad_norm": 72.46797180175781,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.5104891657829285,
"logits/rejected": -0.4849121570587158,
"logps/chosen": -53.16236877441406,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.34949493408203,
"loss": 1.3317,
"margin_dpo/margin_mean": 0.5629609823226929,
"margin_dpo/margin_std": 0.5864159464836121,
"step": 33
},
{
"KL/chosen_KL_mean": 0.04570198059082031,
"KL/mean": -0.32518571615219116,
"KL/rejected_KL_mean": -0.696075439453125,
"KL/std": 0.7097070813179016,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7417768239974976,
"fcm_dpo/q_t": 0.48150891065597534,
"grad_norm": 77.95159912109375,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.4825457036495209,
"logits/rejected": -0.4650344252586365,
"logps/chosen": -50.77162551879883,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.617919921875,
"loss": 1.3155,
"margin_dpo/margin_mean": 0.7417754530906677,
"margin_dpo/margin_std": 0.8860844969749451,
"step": 34
},
{
"KL/chosen_KL_mean": 0.15726661682128906,
"KL/mean": -0.44504478573799133,
"KL/rejected_KL_mean": -1.0473518371582031,
"KL/std": 0.9632636308670044,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2046229839324951,
"fcm_dpo/q_t": 0.47004756331443787,
"grad_norm": 83.60721588134766,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.5268377065658569,
"logits/rejected": -0.49007338285446167,
"logps/chosen": -50.867225646972656,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.87178039550781,
"loss": 1.2721,
"margin_dpo/margin_mean": 1.204622745513916,
"margin_dpo/margin_std": 1.0357599258422852,
"step": 35
},
{
"KL/chosen_KL_mean": 0.02988433837890625,
"KL/mean": -0.5372347831726074,
"KL/rejected_KL_mean": -1.1043548583984375,
"KL/std": 1.094662070274353,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1342408657073975,
"fcm_dpo/q_t": 0.47180622816085815,
"grad_norm": 72.68156433105469,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5595717430114746,
"logits/rejected": -0.523262619972229,
"logps/chosen": -51.96160888671875,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.14498138427734,
"loss": 1.2802,
"margin_dpo/margin_mean": 1.1342405080795288,
"margin_dpo/margin_std": 1.2309517860412598,
"step": 36
},
{
"KL/chosen_KL_mean": 0.023458480834960938,
"KL/mean": -0.4883181154727936,
"KL/rejected_KL_mean": -1.0000991821289062,
"KL/std": 1.0986820459365845,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0235590934753418,
"fcm_dpo/q_t": 0.4746357798576355,
"grad_norm": 61.56761932373047,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5261737108230591,
"logits/rejected": -0.4837297797203064,
"logps/chosen": -62.78364944458008,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.89517211914062,
"loss": 1.2918,
"margin_dpo/margin_mean": 1.0235590934753418,
"margin_dpo/margin_std": 1.4123997688293457,
"step": 37
},
{
"KL/chosen_KL_mean": 0.1512908935546875,
"KL/mean": -0.5362459421157837,
"KL/rejected_KL_mean": -1.2237815856933594,
"KL/std": 1.3535866737365723,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.3750734329223633,
"fcm_dpo/q_t": 0.4660550653934479,
"grad_norm": 69.93470001220703,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5330358743667603,
"logits/rejected": -0.5002619028091431,
"logps/chosen": -48.239227294921875,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.13622283935547,
"loss": 1.2601,
"margin_dpo/margin_mean": 1.3750736713409424,
"margin_dpo/margin_std": 1.628260850906372,
"step": 38
},
{
"KL/chosen_KL_mean": 0.08315658569335938,
"KL/mean": -0.7190086841583252,
"KL/rejected_KL_mean": -1.5211677551269531,
"KL/std": 1.2815454006195068,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.6043219566345215,
"fcm_dpo/q_t": 0.46017903089523315,
"grad_norm": 74.16421508789062,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5571820735931396,
"logits/rejected": -0.5172927379608154,
"logps/chosen": -50.66731643676758,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.09068298339844,
"loss": 1.2371,
"margin_dpo/margin_mean": 1.6043217182159424,
"margin_dpo/margin_std": 1.3714519739151,
"step": 39
},
{
"KL/chosen_KL_mean": 0.18222999572753906,
"KL/mean": -0.5860625505447388,
"KL/rejected_KL_mean": -1.3543624877929688,
"KL/std": 1.4266393184661865,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5365939140319824,
"fcm_dpo/q_t": 0.46209070086479187,
"grad_norm": 60.399505615234375,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5290929079055786,
"logits/rejected": -0.4998553693294525,
"logps/chosen": -57.80283737182617,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.65443420410156,
"loss": 1.246,
"margin_dpo/margin_mean": 1.5365946292877197,
"margin_dpo/margin_std": 1.694861650466919,
"step": 40
},
{
"KL/chosen_KL_mean": 0.00180816650390625,
"KL/mean": -0.9451515078544617,
"KL/rejected_KL_mean": -1.8921089172363281,
"KL/std": 1.8964862823486328,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.8939173221588135,
"fcm_dpo/q_t": 0.4534837007522583,
"grad_norm": 68.17810821533203,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.5442007184028625,
"logits/rejected": -0.5080921649932861,
"logps/chosen": -62.694007873535156,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.91563415527344,
"loss": 1.217,
"margin_dpo/margin_mean": 1.8939182758331299,
"margin_dpo/margin_std": 2.07293701171875,
"step": 41
},
{
"KL/chosen_KL_mean": 0.20568084716796875,
"KL/mean": -1.066096544265747,
"KL/rejected_KL_mean": -2.337871551513672,
"KL/std": 2.129526138305664,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.12230241298675537,
"fcm_dpo/delta": 0.12432946264743805,
"fcm_dpo/margin": 2.5435502529144287,
"fcm_dpo/q_t": 0.43291333317756653,
"grad_norm": 92.2998046875,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5354119539260864,
"logits/rejected": -0.4888347387313843,
"logps/chosen": -58.760746002197266,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.24624633789062,
"loss": 1.119,
"margin_dpo/margin_mean": 2.5435495376586914,
"margin_dpo/margin_std": 2.3769054412841797,
"step": 42
},
{
"KL/chosen_KL_mean": 0.5492343902587891,
"KL/mean": -0.6893892288208008,
"KL/rejected_KL_mean": -1.9280166625976562,
"KL/std": 1.8261648416519165,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.14197269082069397,
"fcm_dpo/delta": 0.07470498234033585,
"fcm_dpo/margin": 2.477250099182129,
"fcm_dpo/q_t": 0.42087799310684204,
"grad_norm": 93.16883087158203,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5261520147323608,
"logits/rejected": -0.5005497932434082,
"logps/chosen": -53.60676193237305,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.408203125,
"loss": 1.0844,
"margin_dpo/margin_mean": 2.477250099182129,
"margin_dpo/margin_std": 1.9302922487258911,
"step": 43
},
{
"KL/chosen_KL_mean": 0.22637557983398438,
"KL/mean": -1.1899750232696533,
"KL/rejected_KL_mean": -2.6063270568847656,
"KL/std": 2.267411231994629,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.14421430230140686,
"fcm_dpo/delta": -0.01492190733551979,
"fcm_dpo/margin": 2.8327009677886963,
"fcm_dpo/q_t": 0.40067583322525024,
"grad_norm": 101.48235321044922,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4636678099632263,
"logits/rejected": -0.44394707679748535,
"logps/chosen": -49.85212326049805,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.39009094238281,
"loss": 1.0429,
"margin_dpo/margin_mean": 2.8327014446258545,
"margin_dpo/margin_std": 2.2093987464904785,
"step": 44
},
{
"KL/chosen_KL_mean": 0.19671249389648438,
"KL/mean": -0.9757533073425293,
"KL/rejected_KL_mean": -2.1482200622558594,
"KL/std": 2.1102242469787598,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.15825220942497253,
"fcm_dpo/delta": 0.041776977479457855,
"fcm_dpo/margin": 2.344935417175293,
"fcm_dpo/q_t": 0.4162585139274597,
"grad_norm": 85.96018981933594,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.5120048522949219,
"logits/rejected": -0.5001665353775024,
"logps/chosen": -48.21821594238281,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.08464813232422,
"loss": 1.0898,
"margin_dpo/margin_mean": 2.344935417175293,
"margin_dpo/margin_std": 2.62362003326416,
"step": 45
},
{
"KL/chosen_KL_mean": 0.17397499084472656,
"KL/mean": -1.2946666479110718,
"KL/rejected_KL_mean": -2.7633094787597656,
"KL/std": 2.6588382720947266,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.14704476296901703,
"fcm_dpo/delta": -0.05587401241064072,
"fcm_dpo/margin": 2.9372873306274414,
"fcm_dpo/q_t": 0.3960358500480652,
"grad_norm": 88.10020446777344,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.517956018447876,
"logits/rejected": -0.46634984016418457,
"logps/chosen": -55.8254508972168,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.4158935546875,
"loss": 1.0542,
"margin_dpo/margin_mean": 2.9372870922088623,
"margin_dpo/margin_std": 3.269395351409912,
"step": 46
},
{
"KL/chosen_KL_mean": 0.4055042266845703,
"KL/mean": -1.0942176580429077,
"KL/rejected_KL_mean": -2.5939369201660156,
"KL/std": 2.6453235149383545,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.13928335905075073,
"fcm_dpo/delta": -0.032578371465206146,
"fcm_dpo/margin": 2.9994425773620605,
"fcm_dpo/q_t": 0.3979858160018921,
"grad_norm": 82.3074951171875,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5774295330047607,
"logits/rejected": -0.5251759886741638,
"logps/chosen": -57.52057647705078,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.27314758300781,
"loss": 1.0483,
"margin_dpo/margin_mean": 2.999443769454956,
"margin_dpo/margin_std": 2.69224214553833,
"step": 47
},
{
"KL/chosen_KL_mean": 0.05373954772949219,
"KL/mean": -1.51072359085083,
"KL/rejected_KL_mean": -3.075183868408203,
"KL/std": 2.700470447540283,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.12641848623752594,
"fcm_dpo/delta": 0.0021915659308433533,
"fcm_dpo/margin": 3.1289291381835938,
"fcm_dpo/q_t": 0.406181663274765,
"grad_norm": 87.43419647216797,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5810247659683228,
"logits/rejected": -0.5221822261810303,
"logps/chosen": -57.134334564208984,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.09178161621094,
"loss": 1.0689,
"margin_dpo/margin_mean": 3.1289291381835938,
"margin_dpo/margin_std": 2.9688820838928223,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3045005798339844,
"KL/mean": -1.5968005657196045,
"KL/rejected_KL_mean": -3.498096466064453,
"KL/std": 3.415726661682129,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.11936050653457642,
"fcm_dpo/delta": -0.0978364571928978,
"fcm_dpo/margin": 3.8025965690612793,
"fcm_dpo/q_t": 0.38723063468933105,
"grad_norm": 69.9267578125,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5347216725349426,
"logits/rejected": -0.4745851755142212,
"logps/chosen": -61.380767822265625,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.26557159423828,
"loss": 1.0376,
"margin_dpo/margin_mean": 3.8025965690612793,
"margin_dpo/margin_std": 3.9496779441833496,
"step": 49
},
{
"KL/chosen_KL_mean": -0.15890121459960938,
"KL/mean": -2.2356748580932617,
"KL/rejected_KL_mean": -4.31243896484375,
"KL/std": 3.5314018726348877,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.10124868154525757,
"fcm_dpo/delta": -0.03730197250843048,
"fcm_dpo/margin": 4.153538227081299,
"fcm_dpo/q_t": 0.3976140022277832,
"grad_norm": 63.207149505615234,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5480292439460754,
"logits/rejected": -0.512259304523468,
"logps/chosen": -58.88304138183594,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.67058563232422,
"loss": 1.0489,
"margin_dpo/margin_mean": 4.153538703918457,
"margin_dpo/margin_std": 4.076920986175537,
"step": 50
},
{
"KL/chosen_KL_mean": -0.2769756317138672,
"KL/mean": -2.2801334857940674,
"KL/rejected_KL_mean": -4.283294677734375,
"KL/std": 4.116324424743652,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.09770938754081726,
"fcm_dpo/delta": 0.00890747457742691,
"fcm_dpo/margin": 4.006317138671875,
"fcm_dpo/q_t": 0.4107528030872345,
"grad_norm": 52.39238357543945,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.539941132068634,
"logits/rejected": -0.5076016187667847,
"logps/chosen": -61.65064239501953,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.28529357910156,
"loss": 1.0914,
"margin_dpo/margin_mean": 4.006316661834717,
"margin_dpo/margin_std": 5.059848308563232,
"step": 51
},
{
"KL/chosen_KL_mean": 0.4057025909423828,
"KL/mean": -2.6870739459991455,
"KL/rejected_KL_mean": -5.779850006103516,
"KL/std": 4.8188886642456055,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.077643021941185,
"fcm_dpo/delta": -0.17084355652332306,
"fcm_dpo/margin": 6.185553073883057,
"fcm_dpo/q_t": 0.3687407672405243,
"grad_norm": 52.488014221191406,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5524188876152039,
"logits/rejected": -0.4968685805797577,
"logps/chosen": -51.93165588378906,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.75376892089844,
"loss": 0.9997,
"margin_dpo/margin_mean": 6.185553073883057,
"margin_dpo/margin_std": 5.159676551818848,
"step": 52
},
{
"KL/chosen_KL_mean": -0.19318580627441406,
"KL/mean": -3.297855854034424,
"KL/rejected_KL_mean": -6.402523040771484,
"KL/std": 5.35381555557251,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.07053211331367493,
"fcm_dpo/delta": -0.07264071702957153,
"fcm_dpo/margin": 6.209340572357178,
"fcm_dpo/q_t": 0.3935110867023468,
"grad_norm": 48.63420104980469,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6061801910400391,
"logits/rejected": -0.5843982696533203,
"logps/chosen": -53.507835388183594,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.18611907958984,
"loss": 1.0485,
"margin_dpo/margin_mean": 6.209340572357178,
"margin_dpo/margin_std": 6.154953479766846,
"step": 53
},
{
"KL/chosen_KL_mean": -0.3580665588378906,
"KL/mean": -3.091987133026123,
"KL/rejected_KL_mean": -5.825908660888672,
"KL/std": 5.11324405670166,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.06830751895904541,
"fcm_dpo/delta": 0.039879053831100464,
"fcm_dpo/margin": 5.467842102050781,
"fcm_dpo/q_t": 0.41404271125793457,
"grad_norm": 49.40464782714844,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5767498016357422,
"logits/rejected": -0.5225586295127869,
"logps/chosen": -51.046730041503906,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.54130554199219,
"loss": 1.0757,
"margin_dpo/margin_mean": 5.467841148376465,
"margin_dpo/margin_std": 5.10695743560791,
"step": 54
},
{
"KL/chosen_KL_mean": -0.9405612945556641,
"KL/mean": -4.179176330566406,
"KL/rejected_KL_mean": -7.417789459228516,
"KL/std": 6.182194232940674,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.06581026315689087,
"fcm_dpo/delta": -0.04618923366069794,
"fcm_dpo/margin": 6.477225303649902,
"fcm_dpo/q_t": 0.3977408707141876,
"grad_norm": 41.99592971801758,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6314051151275635,
"logits/rejected": -0.5677323341369629,
"logps/chosen": -63.55579376220703,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.41129302978516,
"loss": 1.0627,
"margin_dpo/margin_mean": 6.4772257804870605,
"margin_dpo/margin_std": 7.436443328857422,
"step": 55
},
{
"KL/chosen_KL_mean": -0.7300090789794922,
"KL/mean": -3.821871280670166,
"KL/rejected_KL_mean": -6.913738250732422,
"KL/std": 6.0643815994262695,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.06379474699497223,
"fcm_dpo/delta": 0.00866011157631874,
"fcm_dpo/margin": 6.183729648590088,
"fcm_dpo/q_t": 0.4088730216026306,
"grad_norm": 37.987911224365234,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.5912456512451172,
"logits/rejected": -0.5477725267410278,
"logps/chosen": -58.66273498535156,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -101.08818054199219,
"loss": 1.0816,
"margin_dpo/margin_mean": 6.183730602264404,
"margin_dpo/margin_std": 7.2952046394348145,
"step": 56
},
{
"KL/chosen_KL_mean": -0.7435302734375,
"KL/mean": -4.197587966918945,
"KL/rejected_KL_mean": -7.651645660400391,
"KL/std": 5.97504997253418,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.061574921011924744,
"fcm_dpo/delta": -0.04527352750301361,
"fcm_dpo/margin": 6.908114910125732,
"fcm_dpo/q_t": 0.3953145742416382,
"grad_norm": 41.76232147216797,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5989680290222168,
"logits/rejected": -0.571702241897583,
"logps/chosen": -71.23881530761719,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -103.21710205078125,
"loss": 1.0479,
"margin_dpo/margin_mean": 6.908115386962891,
"margin_dpo/margin_std": 6.625408172607422,
"step": 57
},
{
"KL/chosen_KL_mean": -1.0145072937011719,
"KL/mean": -4.882469177246094,
"KL/rejected_KL_mean": -8.75042724609375,
"KL/std": 6.831453323364258,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.0555315725505352,
"fcm_dpo/delta": -0.05244975909590721,
"fcm_dpo/margin": 7.735917568206787,
"fcm_dpo/q_t": 0.3955305814743042,
"grad_norm": 40.50571060180664,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5726555585861206,
"logits/rejected": -0.49422115087509155,
"logps/chosen": -63.14744567871094,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -93.36772155761719,
"loss": 1.0532,
"margin_dpo/margin_mean": 7.7359185218811035,
"margin_dpo/margin_std": 8.176453590393066,
"step": 58
},
{
"KL/chosen_KL_mean": -1.488332748413086,
"KL/mean": -5.7610859870910645,
"KL/rejected_KL_mean": -10.033843994140625,
"KL/std": 7.584994792938232,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.047585196793079376,
"fcm_dpo/delta": -0.022728845477104187,
"fcm_dpo/margin": 8.545510292053223,
"fcm_dpo/q_t": 0.40144288539886475,
"grad_norm": 37.327362060546875,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6131638288497925,
"logits/rejected": -0.572055459022522,
"logps/chosen": -53.420860290527344,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.9190444946289,
"loss": 1.0661,
"margin_dpo/margin_mean": 8.545509338378906,
"margin_dpo/margin_std": 8.80247688293457,
"step": 59
},
{
"KL/chosen_KL_mean": -2.7225379943847656,
"KL/mean": -6.128490447998047,
"KL/rejected_KL_mean": -9.534439086914062,
"KL/std": 6.866521835327148,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.053369902074337006,
"fcm_dpo/delta": 0.05756625533103943,
"fcm_dpo/margin": 6.811899185180664,
"fcm_dpo/q_t": 0.4179559648036957,
"grad_norm": 40.32707214355469,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6027133464813232,
"logits/rejected": -0.5420011878013611,
"logps/chosen": -63.66472625732422,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.92784118652344,
"loss": 1.0996,
"margin_dpo/margin_mean": 6.811899185180664,
"margin_dpo/margin_std": 7.766169548034668,
"step": 60
},
{
"KL/chosen_KL_mean": -1.580484390258789,
"KL/mean": -5.808887481689453,
"KL/rejected_KL_mean": -10.03729248046875,
"KL/std": 9.263717651367188,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.05006622523069382,
"fcm_dpo/delta": -0.04818345606327057,
"fcm_dpo/margin": 8.456808090209961,
"fcm_dpo/q_t": 0.40202924609184265,
"grad_norm": 35.04801940917969,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6034557819366455,
"logits/rejected": -0.5694935917854309,
"logps/chosen": -62.21400833129883,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -99.88978576660156,
"loss": 1.0795,
"margin_dpo/margin_mean": 8.456808090209961,
"margin_dpo/margin_std": 11.565231323242188,
"step": 61
},
{
"KL/chosen_KL_mean": -1.7315292358398438,
"KL/mean": -4.907068729400635,
"KL/rejected_KL_mean": -8.08260726928711,
"KL/std": 6.785726070404053,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.057009097188711166,
"fcm_dpo/delta": 0.05251770839095116,
"fcm_dpo/margin": 6.351075649261475,
"fcm_dpo/q_t": 0.4195292592048645,
"grad_norm": 36.52075958251953,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.6032343506813049,
"logits/rejected": -0.5685884952545166,
"logps/chosen": -57.882301330566406,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -83.6488037109375,
"loss": 1.1108,
"margin_dpo/margin_mean": 6.351076126098633,
"margin_dpo/margin_std": 8.331364631652832,
"step": 62
},
{
"KL/chosen_KL_mean": -2.8989200592041016,
"KL/mean": -7.218653678894043,
"KL/rejected_KL_mean": -11.538387298583984,
"KL/std": 8.756481170654297,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.04904904216527939,
"fcm_dpo/delta": -0.0558142215013504,
"fcm_dpo/margin": 8.639472961425781,
"fcm_dpo/q_t": 0.3941209316253662,
"grad_norm": 36.34617614746094,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.608661949634552,
"logits/rejected": -0.5645633935928345,
"logps/chosen": -76.04631042480469,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -109.14845275878906,
"loss": 1.0578,
"margin_dpo/margin_mean": 8.639472007751465,
"margin_dpo/margin_std": 9.21734619140625,
"step": 63
},
{
"KL/chosen_KL_mean": -1.1949291229248047,
"KL/mean": -6.278130054473877,
"KL/rejected_KL_mean": -11.361335754394531,
"KL/std": 9.37601089477539,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.04156487435102463,
"fcm_dpo/delta": -0.07142579555511475,
"fcm_dpo/margin": 10.166400909423828,
"fcm_dpo/q_t": 0.39335623383522034,
"grad_norm": 31.952661514282227,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.6194932460784912,
"logits/rejected": -0.5888575911521912,
"logps/chosen": -55.19352722167969,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -104.89154052734375,
"loss": 1.0654,
"margin_dpo/margin_mean": 10.166400909423828,
"margin_dpo/margin_std": 11.463658332824707,
"step": 64
},
{
"KL/chosen_KL_mean": -3.5524654388427734,
"KL/mean": -8.784185409545898,
"KL/rejected_KL_mean": -14.01590347290039,
"KL/std": 10.731781005859375,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.03978603705763817,
"fcm_dpo/delta": -0.03537318855524063,
"fcm_dpo/margin": 10.463440895080566,
"fcm_dpo/q_t": 0.39846062660217285,
"grad_norm": 32.09275436401367,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6814702153205872,
"logits/rejected": -0.6694004535675049,
"logps/chosen": -68.38846588134766,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -123.96236419677734,
"loss": 1.0664,
"margin_dpo/margin_mean": 10.46343994140625,
"margin_dpo/margin_std": 11.84306812286377,
"step": 65
},
{
"KL/chosen_KL_mean": -3.371671676635742,
"KL/mean": -8.132331848144531,
"KL/rejected_KL_mean": -12.892993927001953,
"KL/std": 9.663976669311523,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.04018988832831383,
"fcm_dpo/delta": 0.02319152280688286,
"fcm_dpo/margin": 9.521313667297363,
"fcm_dpo/q_t": 0.4129948616027832,
"grad_norm": 28.723054885864258,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6573896408081055,
"logits/rejected": -0.6258237957954407,
"logps/chosen": -54.8151969909668,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -88.529296875,
"loss": 1.0948,
"margin_dpo/margin_mean": 9.521312713623047,
"margin_dpo/margin_std": 11.660235404968262,
"step": 66
},
{
"KL/chosen_KL_mean": -3.1228866577148438,
"KL/mean": -8.022209167480469,
"KL/rejected_KL_mean": -12.92153549194336,
"KL/std": 10.03524398803711,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.042240217328071594,
"fcm_dpo/delta": -0.024514786899089813,
"fcm_dpo/margin": 9.798646926879883,
"fcm_dpo/q_t": 0.40358948707580566,
"grad_norm": 30.853675842285156,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.6088730096817017,
"logits/rejected": -0.5662700533866882,
"logps/chosen": -62.46369171142578,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -85.70881652832031,
"loss": 1.0751,
"margin_dpo/margin_mean": 9.798646926879883,
"margin_dpo/margin_std": 12.186027526855469,
"step": 67
},
{
"KL/chosen_KL_mean": -3.8463668823242188,
"KL/mean": -8.590458869934082,
"KL/rejected_KL_mean": -13.334552764892578,
"KL/std": 9.031107902526855,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.041834283620119095,
"fcm_dpo/delta": 0.003476101905107498,
"fcm_dpo/margin": 9.48818588256836,
"fcm_dpo/q_t": 0.4069967567920685,
"grad_norm": 29.675251007080078,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6760765314102173,
"logits/rejected": -0.6185558438301086,
"logps/chosen": -69.05220031738281,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -90.54180145263672,
"loss": 1.0714,
"margin_dpo/margin_mean": 9.48818588256836,
"margin_dpo/margin_std": 10.152603149414062,
"step": 68
},
{
"KL/chosen_KL_mean": -4.753917694091797,
"KL/mean": -10.989311218261719,
"KL/rejected_KL_mean": -17.224708557128906,
"KL/std": 11.462624549865723,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.034877799451351166,
"fcm_dpo/delta": -0.07517345249652863,
"fcm_dpo/margin": 12.470787048339844,
"fcm_dpo/q_t": 0.39176517724990845,
"grad_norm": 30.214656829833984,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.6015123724937439,
"logits/rejected": -0.5792900323867798,
"logps/chosen": -64.57315826416016,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -120.61357116699219,
"loss": 1.0442,
"margin_dpo/margin_mean": 12.470787048339844,
"margin_dpo/margin_std": 12.772706985473633,
"step": 69
},
{
"KL/chosen_KL_mean": -7.221284866333008,
"KL/mean": -13.940620422363281,
"KL/rejected_KL_mean": -20.659957885742188,
"KL/std": 14.507774353027344,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.0326962023973465,
"fcm_dpo/delta": -0.06987478584051132,
"fcm_dpo/margin": 13.438671112060547,
"fcm_dpo/q_t": 0.39314454793930054,
"grad_norm": 29.549423217773438,
"learning_rate": 5e-07,
"logits/chosen": -0.6086961627006531,
"logits/rejected": -0.5738969445228577,
"logps/chosen": -69.15192413330078,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -111.72074127197266,
"loss": 1.0564,
"margin_dpo/margin_mean": 13.438671112060547,
"margin_dpo/margin_std": 15.399250984191895,
"step": 70
},
{
"KL/chosen_KL_mean": -7.312736511230469,
"KL/mean": -15.442537307739258,
"KL/rejected_KL_mean": -23.572338104248047,
"KL/std": 15.629602432250977,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.026657406240701675,
"fcm_dpo/delta": -0.06790647655725479,
"fcm_dpo/margin": 16.259611129760742,
"fcm_dpo/q_t": 0.3945091962814331,
"grad_norm": 26.020174026489258,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6621614694595337,
"logits/rejected": -0.6253249645233154,
"logps/chosen": -69.06307220458984,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -120.9089584350586,
"loss": 1.0485,
"margin_dpo/margin_mean": 16.259613037109375,
"margin_dpo/margin_std": 17.08768653869629,
"step": 71
},
{
"KL/chosen_KL_mean": -8.572813034057617,
"KL/mean": -16.923320770263672,
"KL/rejected_KL_mean": -25.273826599121094,
"KL/std": 16.826129913330078,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.025071250274777412,
"fcm_dpo/delta": -0.03367580845952034,
"fcm_dpo/margin": 16.70101547241211,
"fcm_dpo/q_t": 0.4001430869102478,
"grad_norm": 24.218080520629883,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6549113988876343,
"logits/rejected": -0.617749810218811,
"logps/chosen": -74.626220703125,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -120.5608139038086,
"loss": 1.0698,
"margin_dpo/margin_mean": 16.70101547241211,
"margin_dpo/margin_std": 20.266984939575195,
"step": 72
},
{
"KL/chosen_KL_mean": -10.759519577026367,
"KL/mean": -19.278858184814453,
"KL/rejected_KL_mean": -27.798202514648438,
"KL/std": 21.811187744140625,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.02377324365079403,
"fcm_dpo/delta": -0.011964067816734314,
"fcm_dpo/margin": 17.038679122924805,
"fcm_dpo/q_t": 0.40983206033706665,
"grad_norm": 23.957334518432617,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.6897900104522705,
"logits/rejected": -0.6500684022903442,
"logps/chosen": -77.01579284667969,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -118.25433349609375,
"loss": 1.1199,
"margin_dpo/margin_mean": 17.038681030273438,
"margin_dpo/margin_std": 27.641098022460938,
"step": 73
},
{
"KL/chosen_KL_mean": -11.2384033203125,
"KL/mean": -20.760555267333984,
"KL/rejected_KL_mean": -30.28270721435547,
"KL/std": 24.14501190185547,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.022344253957271576,
"fcm_dpo/delta": -0.0470757819712162,
"fcm_dpo/margin": 19.044301986694336,
"fcm_dpo/q_t": 0.40436846017837524,
"grad_norm": 24.303987503051758,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.6677048206329346,
"logits/rejected": -0.6622331142425537,
"logps/chosen": -64.66328430175781,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -126.22964477539062,
"loss": 1.0957,
"margin_dpo/margin_mean": 19.044300079345703,
"margin_dpo/margin_std": 30.3118896484375,
"step": 74
},
{
"KL/chosen_KL_mean": -11.227849960327148,
"KL/mean": -24.636085510253906,
"KL/rejected_KL_mean": -38.04431915283203,
"KL/std": 24.73340606689453,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.018182795494794846,
"fcm_dpo/delta": -0.16822758316993713,
"fcm_dpo/margin": 26.81647491455078,
"fcm_dpo/q_t": 0.3744775354862213,
"grad_norm": 22.551315307617188,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.7105200290679932,
"logits/rejected": -0.70478755235672,
"logps/chosen": -63.08951187133789,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -149.29830932617188,
"loss": 1.0253,
"margin_dpo/margin_mean": 26.81647491455078,
"margin_dpo/margin_std": 28.454429626464844,
"step": 75
},
{
"KL/chosen_KL_mean": -13.570169448852539,
"KL/mean": -21.888824462890625,
"KL/rejected_KL_mean": -30.207481384277344,
"KL/std": 20.21454620361328,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.019159602001309395,
"fcm_dpo/delta": 0.1141112893819809,
"fcm_dpo/margin": 16.637313842773438,
"fcm_dpo/q_t": 0.43109238147735596,
"grad_norm": 21.629785537719727,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.7143449783325195,
"logits/rejected": -0.6758897304534912,
"logps/chosen": -66.83621215820312,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -108.42411041259766,
"loss": 1.1366,
"margin_dpo/margin_mean": 16.63731575012207,
"margin_dpo/margin_std": 22.22722816467285,
"step": 76
},
{
"KL/chosen_KL_mean": -12.503726959228516,
"KL/mean": -26.28228759765625,
"KL/rejected_KL_mean": -40.06085205078125,
"KL/std": 27.661182403564453,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.016155380755662918,
"fcm_dpo/delta": -0.10240481793880463,
"fcm_dpo/margin": 27.55712127685547,
"fcm_dpo/q_t": 0.38546115159988403,
"grad_norm": 21.97159767150879,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6718529462814331,
"logits/rejected": -0.635248601436615,
"logps/chosen": -70.60040283203125,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -133.83447265625,
"loss": 1.0496,
"margin_dpo/margin_mean": 27.55712127685547,
"margin_dpo/margin_std": 31.21552085876465,
"step": 77
},
{
"KL/chosen_KL_mean": -12.715435028076172,
"KL/mean": -22.89756202697754,
"KL/rejected_KL_mean": -33.07969284057617,
"KL/std": 22.80880355834961,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.0179641991853714,
"fcm_dpo/delta": 0.04957953840494156,
"fcm_dpo/margin": 20.364255905151367,
"fcm_dpo/q_t": 0.41824695467948914,
"grad_norm": 20.99672508239746,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6376138925552368,
"logits/rejected": -0.6148805618286133,
"logps/chosen": -68.3292236328125,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -118.01405334472656,
"loss": 1.1079,
"margin_dpo/margin_mean": 20.364255905151367,
"margin_dpo/margin_std": 26.547138214111328,
"step": 78
},
{
"KL/chosen_KL_mean": -13.37647819519043,
"KL/mean": -24.498859405517578,
"KL/rejected_KL_mean": -35.62124252319336,
"KL/std": 25.48963737487793,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.01794726401567459,
"fcm_dpo/delta": 0.0012769072782248259,
"fcm_dpo/margin": 22.24475860595703,
"fcm_dpo/q_t": 0.41002583503723145,
"grad_norm": 19.79173469543457,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6492550373077393,
"logits/rejected": -0.6207257509231567,
"logps/chosen": -68.82696533203125,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -123.26880645751953,
"loss": 1.0989,
"margin_dpo/margin_mean": 22.24475860595703,
"margin_dpo/margin_std": 31.258174896240234,
"step": 79
},
{
"KL/chosen_KL_mean": -17.12893295288086,
"KL/mean": -28.049373626708984,
"KL/rejected_KL_mean": -38.969810485839844,
"KL/std": 28.13406753540039,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.017532825469970703,
"fcm_dpo/delta": 0.020619504153728485,
"fcm_dpo/margin": 21.840877532958984,
"fcm_dpo/q_t": 0.41682252287864685,
"grad_norm": 21.230684280395508,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7177095413208008,
"logits/rejected": -0.6798745393753052,
"logps/chosen": -75.64822387695312,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -126.5173110961914,
"loss": 1.1204,
"margin_dpo/margin_mean": 21.840877532958984,
"margin_dpo/margin_std": 32.85547637939453,
"step": 80
},
{
"KL/chosen_KL_mean": -16.53759765625,
"KL/mean": -31.449586868286133,
"KL/rejected_KL_mean": -46.361572265625,
"KL/std": 33.21380615234375,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.015006015077233315,
"fcm_dpo/delta": -0.10901469737291336,
"fcm_dpo/margin": 29.823978424072266,
"fcm_dpo/q_t": 0.3896971344947815,
"grad_norm": 20.577526092529297,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7138460874557495,
"logits/rejected": -0.6892383694648743,
"logps/chosen": -82.98646545410156,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -176.02427673339844,
"loss": 1.0665,
"margin_dpo/margin_mean": 29.823976516723633,
"margin_dpo/margin_std": 39.704833984375,
"step": 81
},
{
"KL/chosen_KL_mean": -19.28495216369629,
"KL/mean": -31.137420654296875,
"KL/rejected_KL_mean": -42.98988723754883,
"KL/std": 30.5316104888916,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.016485687345266342,
"fcm_dpo/delta": -0.008920453488826752,
"fcm_dpo/margin": 23.70493507385254,
"fcm_dpo/q_t": 0.4126429855823517,
"grad_norm": 23.881196975708008,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.6434165239334106,
"logits/rejected": -0.6364510655403137,
"logps/chosen": -71.517333984375,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -133.73313903808594,
"loss": 1.1577,
"margin_dpo/margin_mean": 23.704936981201172,
"margin_dpo/margin_std": 42.49738311767578,
"step": 82
},
{
"KL/chosen_KL_mean": -19.908788681030273,
"KL/mean": -34.38926696777344,
"KL/rejected_KL_mean": -48.869747161865234,
"KL/std": 31.844987869262695,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.014069490134716034,
"fcm_dpo/delta": -0.0153765007853508,
"fcm_dpo/margin": 28.960956573486328,
"fcm_dpo/q_t": 0.4053932726383209,
"grad_norm": 19.797786712646484,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6841074228286743,
"logits/rejected": -0.657940149307251,
"logps/chosen": -75.73617553710938,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -152.5856475830078,
"loss": 1.0873,
"margin_dpo/margin_mean": 28.96095848083496,
"margin_dpo/margin_std": 38.147422790527344,
"step": 83
},
{
"KL/chosen_KL_mean": -19.783796310424805,
"KL/mean": -31.47045135498047,
"KL/rejected_KL_mean": -43.157108306884766,
"KL/std": 27.439373016357422,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.015063371509313583,
"fcm_dpo/delta": 0.07021048665046692,
"fcm_dpo/margin": 23.373310089111328,
"fcm_dpo/q_t": 0.4214816689491272,
"grad_norm": 20.031166076660156,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6388710737228394,
"logits/rejected": -0.5998183488845825,
"logps/chosen": -86.9599609375,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -130.45571899414062,
"loss": 1.1181,
"margin_dpo/margin_mean": 23.373310089111328,
"margin_dpo/margin_std": 27.697301864624023,
"step": 84
},
{
"KL/chosen_KL_mean": -19.78733253479004,
"KL/mean": -32.103843688964844,
"KL/rejected_KL_mean": -44.42035675048828,
"KL/std": 27.902463912963867,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.016904689371585846,
"fcm_dpo/delta": -0.03945329040288925,
"fcm_dpo/margin": 24.633026123046875,
"fcm_dpo/q_t": 0.4018494188785553,
"grad_norm": 21.946027755737305,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6458739638328552,
"logits/rejected": -0.6141970157623291,
"logps/chosen": -78.19395446777344,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -123.05915832519531,
"loss": 1.099,
"margin_dpo/margin_mean": 24.633026123046875,
"margin_dpo/margin_std": 33.796836853027344,
"step": 85
},
{
"KL/chosen_KL_mean": -26.077333450317383,
"KL/mean": -39.15126037597656,
"KL/rejected_KL_mean": -52.225189208984375,
"KL/std": 39.593040466308594,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.015605635941028595,
"fcm_dpo/delta": -0.01844988390803337,
"fcm_dpo/margin": 26.14786148071289,
"fcm_dpo/q_t": 0.4177693724632263,
"grad_norm": 25.58841896057129,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.6518966555595398,
"logits/rejected": -0.6431847810745239,
"logps/chosen": -82.21479797363281,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -140.34683227539062,
"loss": 1.1793,
"margin_dpo/margin_mean": 26.147863388061523,
"margin_dpo/margin_std": 55.436424255371094,
"step": 86
},
{
"KL/chosen_KL_mean": -25.545059204101562,
"KL/mean": -41.57789611816406,
"KL/rejected_KL_mean": -57.61074447631836,
"KL/std": 39.62342071533203,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.013056030496954918,
"fcm_dpo/delta": -0.044095173478126526,
"fcm_dpo/margin": 32.0656852722168,
"fcm_dpo/q_t": 0.40036922693252563,
"grad_norm": 19.81938362121582,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.6956198215484619,
"logits/rejected": -0.6780788898468018,
"logps/chosen": -81.18115234375,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -153.0783233642578,
"loss": 1.1109,
"margin_dpo/margin_mean": 32.0656852722168,
"margin_dpo/margin_std": 48.967384338378906,
"step": 87
},
{
"KL/chosen_KL_mean": -32.21333694458008,
"KL/mean": -44.31005096435547,
"KL/rejected_KL_mean": -56.40675735473633,
"KL/std": 42.025611877441406,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.014491596259176731,
"fcm_dpo/delta": 0.07550811022520065,
"fcm_dpo/margin": 24.193424224853516,
"fcm_dpo/q_t": 0.4285997748374939,
"grad_norm": 21.64423179626465,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6556172370910645,
"logits/rejected": -0.6542757749557495,
"logps/chosen": -105.88449096679688,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -163.11526489257812,
"loss": 1.1734,
"margin_dpo/margin_mean": 24.193424224853516,
"margin_dpo/margin_std": 45.42322540283203,
"step": 88
},
{
"KL/chosen_KL_mean": -20.79347801208496,
"KL/mean": -34.439876556396484,
"KL/rejected_KL_mean": -48.08627700805664,
"KL/std": 35.92295837402344,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.01547418162226677,
"fcm_dpo/delta": -0.04180596396327019,
"fcm_dpo/margin": 27.29279136657715,
"fcm_dpo/q_t": 0.4021642208099365,
"grad_norm": 21.672611236572266,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7387075424194336,
"logits/rejected": -0.7136551141738892,
"logps/chosen": -81.41839599609375,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -130.16983032226562,
"loss": 1.1057,
"margin_dpo/margin_mean": 27.29279327392578,
"margin_dpo/margin_std": 42.178958892822266,
"step": 89
},
{
"KL/chosen_KL_mean": -25.24016761779785,
"KL/mean": -44.94635772705078,
"KL/rejected_KL_mean": -64.65254211425781,
"KL/std": 48.8939208984375,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.012419058009982109,
"fcm_dpo/delta": -0.1753031611442566,
"fcm_dpo/margin": 39.412376403808594,
"fcm_dpo/q_t": 0.38558393716812134,
"grad_norm": 20.073680877685547,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6283840537071228,
"logits/rejected": -0.6491606831550598,
"logps/chosen": -78.52548217773438,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -176.197265625,
"loss": 1.1068,
"margin_dpo/margin_mean": 39.41238021850586,
"margin_dpo/margin_std": 61.327945709228516,
"step": 90
},
{
"KL/chosen_KL_mean": -27.13432502746582,
"KL/mean": -41.85686111450195,
"KL/rejected_KL_mean": -56.57939910888672,
"KL/std": 37.675636291503906,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.011744784191250801,
"fcm_dpo/delta": 0.07527971267700195,
"fcm_dpo/margin": 29.445079803466797,
"fcm_dpo/q_t": 0.42634034156799316,
"grad_norm": 17.86397361755371,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.6645762920379639,
"logits/rejected": -0.635959267616272,
"logps/chosen": -88.93728637695312,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -144.45335388183594,
"loss": 1.151,
"margin_dpo/margin_mean": 29.445079803466797,
"margin_dpo/margin_std": 50.49188995361328,
"step": 91
},
{
"KL/chosen_KL_mean": -23.93476676940918,
"KL/mean": -39.31444549560547,
"KL/rejected_KL_mean": -54.69412612915039,
"KL/std": 39.209224700927734,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.012650266289710999,
"fcm_dpo/delta": 0.008795984089374542,
"fcm_dpo/margin": 30.75935935974121,
"fcm_dpo/q_t": 0.41261714696884155,
"grad_norm": 18.466337203979492,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.6946055889129639,
"logits/rejected": -0.6752103567123413,
"logps/chosen": -75.57553100585938,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -132.5753173828125,
"loss": 1.1221,
"margin_dpo/margin_mean": 30.759361267089844,
"margin_dpo/margin_std": 47.41279602050781,
"step": 92
},
{
"KL/chosen_KL_mean": -27.03689193725586,
"KL/mean": -42.25169372558594,
"KL/rejected_KL_mean": -57.466495513916016,
"KL/std": 36.04038619995117,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.012704822234809399,
"fcm_dpo/delta": 0.01873387210071087,
"fcm_dpo/margin": 30.429601669311523,
"fcm_dpo/q_t": 0.4136204719543457,
"grad_norm": 19.59035873413086,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7035015821456909,
"logits/rejected": -0.6627704501152039,
"logps/chosen": -79.56613159179688,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -134.6272430419922,
"loss": 1.103,
"margin_dpo/margin_mean": 30.42959976196289,
"margin_dpo/margin_std": 42.63709259033203,
"step": 93
},
{
"KL/chosen_KL_mean": -28.926584243774414,
"KL/mean": -47.09135818481445,
"KL/rejected_KL_mean": -65.25613403320312,
"KL/std": 42.653038024902344,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.011295975185930729,
"fcm_dpo/delta": -0.028756409883499146,
"fcm_dpo/margin": 36.32954788208008,
"fcm_dpo/q_t": 0.40284472703933716,
"grad_norm": 17.930683135986328,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.626760721206665,
"logits/rejected": -0.618344783782959,
"logps/chosen": -90.14920043945312,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -164.85516357421875,
"loss": 1.0816,
"margin_dpo/margin_mean": 36.32954788208008,
"margin_dpo/margin_std": 45.59572219848633,
"step": 94
},
{
"KL/chosen_KL_mean": -28.432294845581055,
"KL/mean": -43.03887176513672,
"KL/rejected_KL_mean": -57.64545440673828,
"KL/std": 43.11521530151367,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.012412654235959053,
"fcm_dpo/delta": 0.05812495946884155,
"fcm_dpo/margin": 29.213159561157227,
"fcm_dpo/q_t": 0.42024633288383484,
"grad_norm": 19.021087646484375,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.694536030292511,
"logits/rejected": -0.673601508140564,
"logps/chosen": -80.95594024658203,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -133.5258026123047,
"loss": 1.133,
"margin_dpo/margin_mean": 29.213159561157227,
"margin_dpo/margin_std": 45.21534729003906,
"step": 95
},
{
"KL/chosen_KL_mean": -28.583940505981445,
"KL/mean": -48.43400573730469,
"KL/rejected_KL_mean": -68.28407287597656,
"KL/std": 46.205230712890625,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.010704778134822845,
"fcm_dpo/delta": -0.0798785462975502,
"fcm_dpo/margin": 39.70012664794922,
"fcm_dpo/q_t": 0.3951551914215088,
"grad_norm": 17.751604080200195,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.6716827154159546,
"logits/rejected": -0.6496946811676025,
"logps/chosen": -90.74091339111328,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -164.88009643554688,
"loss": 1.0813,
"margin_dpo/margin_mean": 39.70012664794922,
"margin_dpo/margin_std": 52.09516906738281,
"step": 96
},
{
"KL/chosen_KL_mean": -29.381603240966797,
"KL/mean": -44.338138580322266,
"KL/rejected_KL_mean": -59.294677734375,
"KL/std": 39.46461486816406,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.010876108892261982,
"fcm_dpo/delta": -0.03550150245428085,
"fcm_dpo/margin": 29.913070678710938,
"fcm_dpo/q_t": 0.4207790791988373,
"grad_norm": 18.363679885864258,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.5955668091773987,
"logits/rejected": -0.5594468116760254,
"logps/chosen": -84.02796936035156,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -136.25942993164062,
"loss": 1.1386,
"margin_dpo/margin_mean": 29.913070678710938,
"margin_dpo/margin_std": 39.46156311035156,
"step": 97
},
{
"KL/chosen_KL_mean": -37.65998458862305,
"KL/mean": -54.85460662841797,
"KL/rejected_KL_mean": -72.04922485351562,
"KL/std": 45.98478698730469,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.0109539944678545,
"fcm_dpo/delta": 0.03750079125165939,
"fcm_dpo/margin": 34.38923645019531,
"fcm_dpo/q_t": 0.4147164225578308,
"grad_norm": 20.02430534362793,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.652593731880188,
"logits/rejected": -0.6433310508728027,
"logps/chosen": -102.91860961914062,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -168.57672119140625,
"loss": 1.1337,
"margin_dpo/margin_mean": 34.38923645019531,
"margin_dpo/margin_std": 55.96516799926758,
"step": 98
},
{
"KL/chosen_KL_mean": -28.500194549560547,
"KL/mean": -50.64104461669922,
"KL/rejected_KL_mean": -72.78189849853516,
"KL/std": 48.286502838134766,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.00977346021682024,
"fcm_dpo/delta": -0.06876382231712341,
"fcm_dpo/margin": 44.28170394897461,
"fcm_dpo/q_t": 0.3981579542160034,
"grad_norm": 16.403064727783203,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6286486387252808,
"logits/rejected": -0.6337354183197021,
"logps/chosen": -74.13867950439453,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -159.21983337402344,
"loss": 1.0777,
"margin_dpo/margin_mean": 44.28170394897461,
"margin_dpo/margin_std": 60.85698699951172,
"step": 99
},
{
"KL/chosen_KL_mean": -32.810997009277344,
"KL/mean": -44.14629364013672,
"KL/rejected_KL_mean": -55.4815788269043,
"KL/std": 39.670440673828125,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.010382324457168579,
"fcm_dpo/delta": 0.05089900270104408,
"fcm_dpo/margin": 22.670581817626953,
"fcm_dpo/q_t": 0.4473540782928467,
"grad_norm": 18.2591495513916,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6274121999740601,
"logits/rejected": -0.5915518999099731,
"logps/chosen": -90.40498352050781,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -129.5417938232422,
"loss": 1.23,
"margin_dpo/margin_mean": 22.670583724975586,
"margin_dpo/margin_std": 50.92362976074219,
"step": 100
},
{
"KL/chosen_KL_mean": -40.26070022583008,
"KL/mean": -54.877384185791016,
"KL/rejected_KL_mean": -69.49407958984375,
"KL/std": 49.381065368652344,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.010775712318718433,
"fcm_dpo/delta": -0.0062071713618934155,
"fcm_dpo/margin": 29.23336410522461,
"fcm_dpo/q_t": 0.43012934923171997,
"grad_norm": 19.166645050048828,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6265345811843872,
"logits/rejected": -0.6024205684661865,
"logps/chosen": -101.90955352783203,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -152.6837615966797,
"loss": 1.1904,
"margin_dpo/margin_mean": 29.23336410522461,
"margin_dpo/margin_std": 58.819854736328125,
"step": 101
},
{
"KL/chosen_KL_mean": -42.29863739013672,
"KL/mean": -52.66799545288086,
"KL/rejected_KL_mean": -63.037353515625,
"KL/std": 45.68564224243164,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.011057091876864433,
"fcm_dpo/delta": 0.02545342966914177,
"fcm_dpo/margin": 20.73870849609375,
"fcm_dpo/q_t": 0.4484948515892029,
"grad_norm": 22.39993667602539,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.6203207969665527,
"logits/rejected": -0.580852210521698,
"logps/chosen": -106.37751770019531,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -131.22442626953125,
"loss": 1.2607,
"margin_dpo/margin_mean": 20.73870849609375,
"margin_dpo/margin_std": 55.18003463745117,
"step": 102
},
{
"KL/chosen_KL_mean": -35.84786605834961,
"KL/mean": -58.045082092285156,
"KL/rejected_KL_mean": -80.24229431152344,
"KL/std": 51.89337921142578,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.009846476837992668,
"fcm_dpo/delta": -0.07356812059879303,
"fcm_dpo/margin": 44.39442443847656,
"fcm_dpo/q_t": 0.3977162837982178,
"grad_norm": 18.495681762695312,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.6326062679290771,
"logits/rejected": -0.603610634803772,
"logps/chosen": -97.14714050292969,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -173.81500244140625,
"loss": 1.0847,
"margin_dpo/margin_mean": 44.39442443847656,
"margin_dpo/margin_std": 64.85426330566406,
"step": 103
},
{
"KL/chosen_KL_mean": -36.992637634277344,
"KL/mean": -58.22779846191406,
"KL/rejected_KL_mean": -79.46296691894531,
"KL/std": 55.633026123046875,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.00963391549885273,
"fcm_dpo/delta": -0.015489146113395691,
"fcm_dpo/margin": 42.47032928466797,
"fcm_dpo/q_t": 0.410830020904541,
"grad_norm": 17.655324935913086,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.6385079622268677,
"logits/rejected": -0.6165839433670044,
"logps/chosen": -91.36540985107422,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -169.02767944335938,
"loss": 1.1177,
"margin_dpo/margin_mean": 42.47032928466797,
"margin_dpo/margin_std": 69.24513244628906,
"step": 104
},
{
"KL/chosen_KL_mean": -35.22704315185547,
"KL/mean": -65.53314208984375,
"KL/rejected_KL_mean": -95.83924865722656,
"KL/std": 52.481048583984375,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.007748795207589865,
"fcm_dpo/delta": -0.13560038805007935,
"fcm_dpo/margin": 60.61220932006836,
"fcm_dpo/q_t": 0.37679198384284973,
"grad_norm": 16.846649169921875,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.5944653749465942,
"logits/rejected": -0.5763468742370605,
"logps/chosen": -89.86598205566406,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -193.81277465820312,
"loss": 1.0206,
"margin_dpo/margin_mean": 60.612213134765625,
"margin_dpo/margin_std": 59.409732818603516,
"step": 105
},
{
"KL/chosen_KL_mean": -40.11956024169922,
"KL/mean": -58.95640563964844,
"KL/rejected_KL_mean": -77.79325866699219,
"KL/std": 47.83055114746094,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.0085395947098732,
"fcm_dpo/delta": 0.11511291563510895,
"fcm_dpo/margin": 37.6736946105957,
"fcm_dpo/q_t": 0.43202945590019226,
"grad_norm": 16.822023391723633,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.5823087096214294,
"logits/rejected": -0.56536865234375,
"logps/chosen": -94.95245361328125,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -163.01788330078125,
"loss": 1.1419,
"margin_dpo/margin_mean": 37.67369079589844,
"margin_dpo/margin_std": 51.76994323730469,
"step": 106
},
{
"KL/chosen_KL_mean": -47.187808990478516,
"KL/mean": -68.33793640136719,
"KL/rejected_KL_mean": -89.4880599975586,
"KL/std": 58.32842254638672,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.0090809166431427,
"fcm_dpo/delta": 0.023154791444540024,
"fcm_dpo/margin": 42.300254821777344,
"fcm_dpo/q_t": 0.41329509019851685,
"grad_norm": 18.076017379760742,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.6128166913986206,
"logits/rejected": -0.5746489763259888,
"logps/chosen": -116.89562225341797,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -184.22756958007812,
"loss": 1.1189,
"margin_dpo/margin_mean": 42.300254821777344,
"margin_dpo/margin_std": 62.243568420410156,
"step": 107
},
{
"KL/chosen_KL_mean": -40.908172607421875,
"KL/mean": -67.95404052734375,
"KL/rejected_KL_mean": -94.99990844726562,
"KL/std": 65.11196899414062,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.008320159278810024,
"fcm_dpo/delta": -0.09254781901836395,
"fcm_dpo/margin": 54.09173583984375,
"fcm_dpo/q_t": 0.3973488211631775,
"grad_norm": 16.829160690307617,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.5930562019348145,
"logits/rejected": -0.5809206962585449,
"logps/chosen": -96.91806030273438,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -190.79592895507812,
"loss": 1.0985,
"margin_dpo/margin_mean": 54.09173583984375,
"margin_dpo/margin_std": 84.54571533203125,
"step": 108
},
{
"KL/chosen_KL_mean": -39.185829162597656,
"KL/mean": -66.94438171386719,
"KL/rejected_KL_mean": -94.70292663574219,
"KL/std": 61.541465759277344,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.007376614026725292,
"fcm_dpo/delta": -0.021881788969039917,
"fcm_dpo/margin": 55.51709747314453,
"fcm_dpo/q_t": 0.4035341143608093,
"grad_norm": 17.845430374145508,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.5359349250793457,
"logits/rejected": -0.5162516832351685,
"logps/chosen": -102.07131958007812,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -193.388671875,
"loss": 1.0947,
"margin_dpo/margin_mean": 55.51709747314453,
"margin_dpo/margin_std": 77.26396179199219,
"step": 109
},
{
"KL/chosen_KL_mean": -41.10560989379883,
"KL/mean": -66.01383972167969,
"KL/rejected_KL_mean": -90.92208099365234,
"KL/std": 61.80369567871094,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.006370611023157835,
"fcm_dpo/delta": -0.08743350207805634,
"fcm_dpo/margin": 49.81646728515625,
"fcm_dpo/q_t": 0.41799625754356384,
"grad_norm": 12.691235542297363,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.5346049666404724,
"logits/rejected": -0.5050790309906006,
"logps/chosen": -99.85929870605469,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -170.67208862304688,
"loss": 1.1549,
"margin_dpo/margin_mean": 49.81646728515625,
"margin_dpo/margin_std": 75.78208923339844,
"step": 110
},
{
"KL/chosen_KL_mean": -45.89485168457031,
"KL/mean": -75.09423828125,
"KL/rejected_KL_mean": -104.29363250732422,
"KL/std": 70.21835327148438,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.006470158696174622,
"fcm_dpo/delta": 0.03387834131717682,
"fcm_dpo/margin": 58.398780822753906,
"fcm_dpo/q_t": 0.41569167375564575,
"grad_norm": 14.799457550048828,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.5766105055809021,
"logits/rejected": -0.5566138029098511,
"logps/chosen": -114.51895904541016,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -202.72250366210938,
"loss": 1.1264,
"margin_dpo/margin_mean": 58.39877700805664,
"margin_dpo/margin_std": 89.97933959960938,
"step": 111
},
{
"KL/chosen_KL_mean": -42.42522048950195,
"KL/mean": -60.012123107910156,
"KL/rejected_KL_mean": -77.59901428222656,
"KL/std": 46.559471130371094,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.00751154450699687,
"fcm_dpo/delta": 0.04849451780319214,
"fcm_dpo/margin": 35.173797607421875,
"fcm_dpo/q_t": 0.4408394694328308,
"grad_norm": 15.346757888793945,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.53926682472229,
"logits/rejected": -0.5091655254364014,
"logps/chosen": -92.67486572265625,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -142.37344360351562,
"loss": 1.1874,
"margin_dpo/margin_mean": 35.173797607421875,
"margin_dpo/margin_std": 58.55726623535156,
"step": 112
},
{
"KL/chosen_KL_mean": -55.43156433105469,
"KL/mean": -73.22611999511719,
"KL/rejected_KL_mean": -91.02066802978516,
"KL/std": 52.921966552734375,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.009013749659061432,
"fcm_dpo/delta": 0.11262636631727219,
"fcm_dpo/margin": 35.58909606933594,
"fcm_dpo/q_t": 0.4331721365451813,
"grad_norm": 19.263153076171875,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.5204916596412659,
"logits/rejected": -0.483734130859375,
"logps/chosen": -122.14451599121094,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -168.9893798828125,
"loss": 1.1549,
"margin_dpo/margin_mean": 35.58909606933594,
"margin_dpo/margin_std": 58.14237976074219,
"step": 113
},
{
"KL/chosen_KL_mean": -49.490875244140625,
"KL/mean": -79.34724426269531,
"KL/rejected_KL_mean": -109.20361328125,
"KL/std": 74.02787780761719,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.007360486313700676,
"fcm_dpo/delta": -0.10784411430358887,
"fcm_dpo/margin": 59.712745666503906,
"fcm_dpo/q_t": 0.3967573046684265,
"grad_norm": 18.57329750061035,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.5290780067443848,
"logits/rejected": -0.5046583414077759,
"logps/chosen": -107.27595520019531,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -196.3132781982422,
"loss": 1.0889,
"margin_dpo/margin_mean": 59.712745666503906,
"margin_dpo/margin_std": 90.68304443359375,
"step": 114
},
{
"KL/chosen_KL_mean": -62.15449523925781,
"KL/mean": -82.36180114746094,
"KL/rejected_KL_mean": -102.5691146850586,
"KL/std": 71.60066223144531,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.008717365562915802,
"fcm_dpo/delta": 0.06397520750761032,
"fcm_dpo/margin": 40.414615631103516,
"fcm_dpo/q_t": 0.4292001724243164,
"grad_norm": 26.535457611083984,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.5466749668121338,
"logits/rejected": -0.5373319387435913,
"logps/chosen": -127.73713684082031,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -201.1346435546875,
"loss": 1.2197,
"margin_dpo/margin_mean": 40.41461944580078,
"margin_dpo/margin_std": 96.31269073486328,
"step": 115
},
{
"KL/chosen_KL_mean": -45.30133819580078,
"KL/mean": -68.48527526855469,
"KL/rejected_KL_mean": -91.66920471191406,
"KL/std": 58.61573028564453,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.008865797892212868,
"fcm_dpo/delta": -0.027695897966623306,
"fcm_dpo/margin": 46.367855072021484,
"fcm_dpo/q_t": 0.4044142961502075,
"grad_norm": 19.043041229248047,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.5231291055679321,
"logits/rejected": -0.5219400525093079,
"logps/chosen": -96.70165252685547,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -172.19105529785156,
"loss": 1.1011,
"margin_dpo/margin_mean": 46.367855072021484,
"margin_dpo/margin_std": 64.93318939208984,
"step": 116
},
{
"KL/chosen_KL_mean": -58.65241241455078,
"KL/mean": -76.01322937011719,
"KL/rejected_KL_mean": -93.37403869628906,
"KL/std": 61.22138977050781,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.009560597129166126,
"fcm_dpo/delta": 0.09886346757411957,
"fcm_dpo/margin": 34.72163009643555,
"fcm_dpo/q_t": 0.4316639304161072,
"grad_norm": 31.035114288330078,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.49749547243118286,
"logits/rejected": -0.46514734625816345,
"logps/chosen": -127.95082092285156,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -159.95803833007812,
"loss": 1.1874,
"margin_dpo/margin_mean": 34.72163391113281,
"margin_dpo/margin_std": 71.40594482421875,
"step": 117
},
{
"KL/chosen_KL_mean": -44.07014465332031,
"KL/mean": -66.22931671142578,
"KL/rejected_KL_mean": -88.38848114013672,
"KL/std": 55.144588470458984,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.009519684128463268,
"fcm_dpo/delta": -0.037742018699645996,
"fcm_dpo/margin": 44.318336486816406,
"fcm_dpo/q_t": 0.40347346663475037,
"grad_norm": 20.083301544189453,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.563313364982605,
"logits/rejected": -0.5480994582176208,
"logps/chosen": -99.71112060546875,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -164.0575408935547,
"loss": 1.0928,
"margin_dpo/margin_mean": 44.318336486816406,
"margin_dpo/margin_std": 63.96471405029297,
"step": 118
},
{
"KL/chosen_KL_mean": -60.53836441040039,
"KL/mean": -82.12613677978516,
"KL/rejected_KL_mean": -103.71391296386719,
"KL/std": 61.16850280761719,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.00799720548093319,
"fcm_dpo/delta": -0.07085751742124557,
"fcm_dpo/margin": 43.175559997558594,
"fcm_dpo/q_t": 0.41732871532440186,
"grad_norm": 19.45294952392578,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.553020715713501,
"logits/rejected": -0.545320987701416,
"logps/chosen": -134.04855346679688,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -206.6912078857422,
"loss": 1.1596,
"margin_dpo/margin_mean": 43.175559997558594,
"margin_dpo/margin_std": 76.69107055664062,
"step": 119
},
{
"KL/chosen_KL_mean": -60.02840805053711,
"KL/mean": -90.28742980957031,
"KL/rejected_KL_mean": -120.54644775390625,
"KL/std": 74.63396453857422,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.007171543315052986,
"fcm_dpo/delta": -0.06302839517593384,
"fcm_dpo/margin": 60.51802444458008,
"fcm_dpo/q_t": 0.3962337374687195,
"grad_norm": 22.441465377807617,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.5601464509963989,
"logits/rejected": -0.5344448089599609,
"logps/chosen": -136.80923461914062,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -228.5701904296875,
"loss": 1.0734,
"margin_dpo/margin_mean": 60.51802062988281,
"margin_dpo/margin_std": 79.95449829101562,
"step": 120
},
{
"KL/chosen_KL_mean": -56.952545166015625,
"KL/mean": -91.06138610839844,
"KL/rejected_KL_mean": -125.17024230957031,
"KL/std": 72.70287322998047,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.0062125981785357,
"fcm_dpo/delta": -0.05240712687373161,
"fcm_dpo/margin": 68.21770477294922,
"fcm_dpo/q_t": 0.39837193489074707,
"grad_norm": 17.03141212463379,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.5124868154525757,
"logits/rejected": -0.511413037776947,
"logps/chosen": -118.74243927001953,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -235.16481018066406,
"loss": 1.0811,
"margin_dpo/margin_mean": 68.21769714355469,
"margin_dpo/margin_std": 90.72044372558594,
"step": 121
},
{
"KL/chosen_KL_mean": -51.569435119628906,
"KL/mean": -93.51173400878906,
"KL/rejected_KL_mean": -135.4540252685547,
"KL/std": 81.56539916992188,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.005438469350337982,
"fcm_dpo/delta": -0.10527084767818451,
"fcm_dpo/margin": 83.88459777832031,
"fcm_dpo/q_t": 0.3847019076347351,
"grad_norm": 17.03737449645996,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.4612678587436676,
"logits/rejected": -0.4794490337371826,
"logps/chosen": -98.47164916992188,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -242.168212890625,
"loss": 1.0341,
"margin_dpo/margin_mean": 83.88459777832031,
"margin_dpo/margin_std": 86.80702209472656,
"step": 122
},
{
"KL/chosen_KL_mean": -59.32091522216797,
"KL/mean": -86.37959289550781,
"KL/rejected_KL_mean": -113.43827819824219,
"KL/std": 70.35043334960938,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.005368525628000498,
"fcm_dpo/delta": 0.021436911076307297,
"fcm_dpo/margin": 54.11735534667969,
"fcm_dpo/q_t": 0.4345107674598694,
"grad_norm": 15.537219047546387,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.4674427807331085,
"logits/rejected": -0.4554457664489746,
"logps/chosen": -120.65955352783203,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -201.2136688232422,
"loss": 1.1808,
"margin_dpo/margin_mean": 54.11735534667969,
"margin_dpo/margin_std": 95.79205322265625,
"step": 123
},
{
"KL/chosen_KL_mean": -70.40459442138672,
"KL/mean": -111.80335998535156,
"KL/rejected_KL_mean": -153.20213317871094,
"KL/std": 99.7587890625,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.00502355583012104,
"fcm_dpo/delta": -0.029820293188095093,
"fcm_dpo/margin": 82.79753112792969,
"fcm_dpo/q_t": 0.40601056814193726,
"grad_norm": 14.011857986450195,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.4481953978538513,
"logits/rejected": -0.4538896083831787,
"logps/chosen": -141.85293579101562,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -270.78271484375,
"loss": 1.1052,
"margin_dpo/margin_mean": 82.79753875732422,
"margin_dpo/margin_std": 125.91355895996094,
"step": 124
},
{
"KL/chosen_KL_mean": -57.097442626953125,
"KL/mean": -92.41481018066406,
"KL/rejected_KL_mean": -127.73218536376953,
"KL/std": 83.84275817871094,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.005386849865317345,
"fcm_dpo/delta": 0.029525037854909897,
"fcm_dpo/margin": 70.63475036621094,
"fcm_dpo/q_t": 0.4158494174480438,
"grad_norm": 14.315393447875977,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.4188230633735657,
"logits/rejected": -0.42580023407936096,
"logps/chosen": -107.23438262939453,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -211.72079467773438,
"loss": 1.1187,
"margin_dpo/margin_mean": 70.63475036621094,
"margin_dpo/margin_std": 106.31303405761719,
"step": 125
},
{
"KL/chosen_KL_mean": -62.320526123046875,
"KL/mean": -96.66703796386719,
"KL/rejected_KL_mean": -131.0135498046875,
"KL/std": 78.08503723144531,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.005546340718865395,
"fcm_dpo/delta": 0.030749067664146423,
"fcm_dpo/margin": 68.69302368164062,
"fcm_dpo/q_t": 0.4163479208946228,
"grad_norm": 15.782130241394043,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.42107057571411133,
"logits/rejected": -0.42162832617759705,
"logps/chosen": -117.98759460449219,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -229.143310546875,
"loss": 1.1145,
"margin_dpo/margin_mean": 68.69302368164062,
"margin_dpo/margin_std": 100.53376007080078,
"step": 126
},
{
"KL/chosen_KL_mean": -61.44911193847656,
"KL/mean": -84.82119750976562,
"KL/rejected_KL_mean": -108.19326782226562,
"KL/std": 68.35600280761719,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.006859184708446264,
"fcm_dpo/delta": 0.11228857934474945,
"fcm_dpo/margin": 46.74416732788086,
"fcm_dpo/q_t": 0.43170320987701416,
"grad_norm": 19.79861831665039,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.45557230710983276,
"logits/rejected": -0.44303327798843384,
"logps/chosen": -118.00379180908203,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -184.98904418945312,
"loss": 1.1732,
"margin_dpo/margin_mean": 46.744171142578125,
"margin_dpo/margin_std": 86.41580200195312,
"step": 127
},
{
"KL/chosen_KL_mean": -64.04533386230469,
"KL/mean": -89.68724060058594,
"KL/rejected_KL_mean": -115.32914733886719,
"KL/std": 80.13230895996094,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.007653497625142336,
"fcm_dpo/delta": 0.0008252784609794617,
"fcm_dpo/margin": 51.28382110595703,
"fcm_dpo/q_t": 0.4136595129966736,
"grad_norm": 32.553951263427734,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.439397931098938,
"logits/rejected": -0.42928874492645264,
"logps/chosen": -122.16629028320312,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -191.76809692382812,
"loss": 1.1786,
"margin_dpo/margin_mean": 51.283817291259766,
"margin_dpo/margin_std": 102.103515625,
"step": 128
},
{
"KL/chosen_KL_mean": -78.18240356445312,
"KL/mean": -104.9556884765625,
"KL/rejected_KL_mean": -131.72897338867188,
"KL/std": 91.75530242919922,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.0073263440281152725,
"fcm_dpo/delta": 0.01247208472341299,
"fcm_dpo/margin": 53.54655075073242,
"fcm_dpo/q_t": 0.4221315085887909,
"grad_norm": 23.202054977416992,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.4406889081001282,
"logits/rejected": -0.4349208474159241,
"logps/chosen": -145.09878540039062,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -228.37118530273438,
"loss": 1.1973,
"margin_dpo/margin_mean": 53.54655456542969,
"margin_dpo/margin_std": 122.08485412597656,
"step": 129
},
{
"KL/chosen_KL_mean": -56.172401428222656,
"KL/mean": -96.15986633300781,
"KL/rejected_KL_mean": -136.1473388671875,
"KL/std": 85.45478820800781,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.006229862105101347,
"fcm_dpo/delta": -0.19545677304267883,
"fcm_dpo/margin": 79.97492980957031,
"fcm_dpo/q_t": 0.37694329023361206,
"grad_norm": 17.230426788330078,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.42614853382110596,
"logits/rejected": -0.4212262034416199,
"logps/chosen": -100.83924865722656,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -218.92898559570312,
"loss": 1.0744,
"margin_dpo/margin_mean": 79.97492980957031,
"margin_dpo/margin_std": 107.18331146240234,
"step": 130
},
{
"KL/chosen_KL_mean": -52.116981506347656,
"KL/mean": -93.28705596923828,
"KL/rejected_KL_mean": -134.45712280273438,
"KL/std": 90.56842041015625,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.004888694733381271,
"fcm_dpo/delta": -0.004482886288315058,
"fcm_dpo/margin": 82.34013366699219,
"fcm_dpo/q_t": 0.40506911277770996,
"grad_norm": 26.67673683166504,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.42915642261505127,
"logits/rejected": -0.4353182315826416,
"logps/chosen": -97.04157257080078,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -222.90113830566406,
"loss": 1.0791,
"margin_dpo/margin_mean": 82.34013366699219,
"margin_dpo/margin_std": 98.78244018554688,
"step": 131
},
{
"KL/chosen_KL_mean": -67.49314880371094,
"KL/mean": -99.89106750488281,
"KL/rejected_KL_mean": -132.28900146484375,
"KL/std": 87.30780029296875,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.005424443632364273,
"fcm_dpo/delta": 0.07522791624069214,
"fcm_dpo/margin": 64.79584503173828,
"fcm_dpo/q_t": 0.42780178785324097,
"grad_norm": 16.56679916381836,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.46568989753723145,
"logits/rejected": -0.4727591872215271,
"logps/chosen": -126.49423217773438,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -220.18115234375,
"loss": 1.1552,
"margin_dpo/margin_mean": 64.79584503173828,
"margin_dpo/margin_std": 114.17736053466797,
"step": 132
},
{
"KL/chosen_KL_mean": -78.84221649169922,
"KL/mean": -106.69076538085938,
"KL/rejected_KL_mean": -134.53932189941406,
"KL/std": 77.18131256103516,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.006308811716735363,
"fcm_dpo/delta": 0.0753559023141861,
"fcm_dpo/margin": 55.69710159301758,
"fcm_dpo/q_t": 0.4273163676261902,
"grad_norm": 21.23187255859375,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.46139687299728394,
"logits/rejected": -0.4528288245201111,
"logps/chosen": -145.44671630859375,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -230.8728790283203,
"loss": 1.157,
"margin_dpo/margin_mean": 55.69709777832031,
"margin_dpo/margin_std": 99.82786560058594,
"step": 133
},
{
"KL/chosen_KL_mean": -59.535091400146484,
"KL/mean": -90.11434173583984,
"KL/rejected_KL_mean": -120.69359588623047,
"KL/std": 71.90062713623047,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.006723651196807623,
"fcm_dpo/delta": -0.022656653076410294,
"fcm_dpo/margin": 61.15849304199219,
"fcm_dpo/q_t": 0.40606844425201416,
"grad_norm": 17.838544845581055,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.4176589846611023,
"logits/rejected": -0.4087156057357788,
"logps/chosen": -111.6043472290039,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -208.34811401367188,
"loss": 1.0923,
"margin_dpo/margin_mean": 61.15850067138672,
"margin_dpo/margin_std": 85.12577056884766,
"step": 134
},
{
"KL/chosen_KL_mean": -64.89012145996094,
"KL/mean": -110.44607543945312,
"KL/rejected_KL_mean": -156.00204467773438,
"KL/std": 98.89341735839844,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.005103720352053642,
"fcm_dpo/delta": -0.13181009888648987,
"fcm_dpo/margin": 91.11190795898438,
"fcm_dpo/q_t": 0.3853245973587036,
"grad_norm": 24.731225967407227,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.41161245107650757,
"logits/rejected": -0.4464326500892639,
"logps/chosen": -115.24398803710938,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -271.9817810058594,
"loss": 1.061,
"margin_dpo/margin_mean": 91.11190795898438,
"margin_dpo/margin_std": 121.44206237792969,
"step": 135
},
{
"KL/chosen_KL_mean": -77.75067138671875,
"KL/mean": -107.7598876953125,
"KL/rejected_KL_mean": -137.76910400390625,
"KL/std": 92.06062316894531,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.005009594839066267,
"fcm_dpo/delta": 0.013183820061385632,
"fcm_dpo/margin": 60.01844024658203,
"fcm_dpo/q_t": 0.4368482232093811,
"grad_norm": 20.24871063232422,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.41852521896362305,
"logits/rejected": -0.41355031728744507,
"logps/chosen": -142.82318115234375,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -234.09033203125,
"loss": 1.1918,
"margin_dpo/margin_mean": 60.01844024658203,
"margin_dpo/margin_std": 120.7431640625,
"step": 136
},
{
"KL/chosen_KL_mean": -70.75765991210938,
"KL/mean": -118.09391784667969,
"KL/rejected_KL_mean": -165.43017578125,
"KL/std": 117.8000717163086,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.0045467158779501915,
"fcm_dpo/delta": -0.05582471936941147,
"fcm_dpo/margin": 94.67250061035156,
"fcm_dpo/q_t": 0.402421772480011,
"grad_norm": 18.368188858032227,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.40962690114974976,
"logits/rejected": -0.43391185998916626,
"logps/chosen": -119.51677703857422,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -279.2939453125,
"loss": 1.096,
"margin_dpo/margin_mean": 94.67250061035156,
"margin_dpo/margin_std": 143.3657989501953,
"step": 137
},
{
"KL/chosen_KL_mean": -75.74452209472656,
"KL/mean": -111.02808380126953,
"KL/rejected_KL_mean": -146.31163024902344,
"KL/std": 89.46757507324219,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.005052464548498392,
"fcm_dpo/delta": 0.06267927587032318,
"fcm_dpo/margin": 70.56710815429688,
"fcm_dpo/q_t": 0.42063024640083313,
"grad_norm": 20.598432540893555,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.40584802627563477,
"logits/rejected": -0.3970094919204712,
"logps/chosen": -136.26417541503906,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -239.50857543945312,
"loss": 1.1109,
"margin_dpo/margin_mean": 70.56710815429688,
"margin_dpo/margin_std": 90.8817367553711,
"step": 138
},
{
"KL/chosen_KL_mean": -64.63111114501953,
"KL/mean": -105.27629089355469,
"KL/rejected_KL_mean": -145.92144775390625,
"KL/std": 83.68637084960938,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.005053876433521509,
"fcm_dpo/delta": -0.019144825637340546,
"fcm_dpo/margin": 81.29034423828125,
"fcm_dpo/q_t": 0.4056248664855957,
"grad_norm": 15.781937599182129,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.38525593280792236,
"logits/rejected": -0.37835609912872314,
"logps/chosen": -111.52249145507812,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -225.64944458007812,
"loss": 1.0888,
"margin_dpo/margin_mean": 81.29034423828125,
"margin_dpo/margin_std": 111.37606811523438,
"step": 139
},
{
"KL/chosen_KL_mean": -77.94660949707031,
"KL/mean": -114.49009704589844,
"KL/rejected_KL_mean": -151.03359985351562,
"KL/std": 91.72400665283203,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.0051695131696760654,
"fcm_dpo/delta": 0.03525704890489578,
"fcm_dpo/margin": 73.08699035644531,
"fcm_dpo/q_t": 0.4163084924221039,
"grad_norm": 16.439830780029297,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.3429441452026367,
"logits/rejected": -0.3306434154510498,
"logps/chosen": -136.92132568359375,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -234.3177032470703,
"loss": 1.1224,
"margin_dpo/margin_mean": 73.08699035644531,
"margin_dpo/margin_std": 112.87854766845703,
"step": 140
},
{
"KL/chosen_KL_mean": -88.09237670898438,
"KL/mean": -122.84248352050781,
"KL/rejected_KL_mean": -157.59259033203125,
"KL/std": 105.68617248535156,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.005516390781849623,
"fcm_dpo/delta": 0.02600482851266861,
"fcm_dpo/margin": 69.5002212524414,
"fcm_dpo/q_t": 0.4152142405509949,
"grad_norm": 25.1342716217041,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.3684391975402832,
"logits/rejected": -0.35982775688171387,
"logps/chosen": -163.16802978515625,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -255.78485107421875,
"loss": 1.1451,
"margin_dpo/margin_mean": 69.50021362304688,
"margin_dpo/margin_std": 123.44959259033203,
"step": 141
},
{
"KL/chosen_KL_mean": -85.69429016113281,
"KL/mean": -125.2984848022461,
"KL/rejected_KL_mean": -164.9027099609375,
"KL/std": 109.44830322265625,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.0053951190784573555,
"fcm_dpo/delta": -0.05040793865919113,
"fcm_dpo/margin": 79.20841217041016,
"fcm_dpo/q_t": 0.4017929434776306,
"grad_norm": 22.805734634399414,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.3611358404159546,
"logits/rejected": -0.36291757225990295,
"logps/chosen": -143.7222137451172,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -259.48492431640625,
"loss": 1.1263,
"margin_dpo/margin_mean": 79.20841217041016,
"margin_dpo/margin_std": 128.5096893310547,
"step": 142
},
{
"KL/chosen_KL_mean": -88.5622329711914,
"KL/mean": -114.20449829101562,
"KL/rejected_KL_mean": -139.84677124023438,
"KL/std": 94.44708251953125,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.005363130010664463,
"fcm_dpo/delta": 0.035532694309949875,
"fcm_dpo/margin": 51.28453063964844,
"fcm_dpo/q_t": 0.4414982795715332,
"grad_norm": 20.507484436035156,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.3294772803783417,
"logits/rejected": -0.3267552852630615,
"logps/chosen": -146.15869140625,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -218.84634399414062,
"loss": 1.2134,
"margin_dpo/margin_mean": 51.28453063964844,
"margin_dpo/margin_std": 110.63523864746094,
"step": 143
},
{
"KL/chosen_KL_mean": -80.96118927001953,
"KL/mean": -111.06156921386719,
"KL/rejected_KL_mean": -141.16195678710938,
"KL/std": 81.7334213256836,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.005931549705564976,
"fcm_dpo/delta": 0.0660046860575676,
"fcm_dpo/margin": 60.200782775878906,
"fcm_dpo/q_t": 0.4232863783836365,
"grad_norm": 20.33156394958496,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.3503091037273407,
"logits/rejected": -0.3469555974006653,
"logps/chosen": -140.8675537109375,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -223.1622314453125,
"loss": 1.1353,
"margin_dpo/margin_mean": 60.200782775878906,
"margin_dpo/margin_std": 95.25154113769531,
"step": 144
},
{
"KL/chosen_KL_mean": -77.64210510253906,
"KL/mean": -109.43412780761719,
"KL/rejected_KL_mean": -141.22616577148438,
"KL/std": 79.6153564453125,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.006183978170156479,
"fcm_dpo/delta": 0.011190323159098625,
"fcm_dpo/margin": 63.584068298339844,
"fcm_dpo/q_t": 0.41196322441101074,
"grad_norm": 25.684057235717773,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.376176118850708,
"logits/rejected": -0.3664652109146118,
"logps/chosen": -134.24276733398438,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -219.0924835205078,
"loss": 1.1163,
"margin_dpo/margin_mean": 63.584068298339844,
"margin_dpo/margin_std": 97.96441650390625,
"step": 145
},
{
"KL/chosen_KL_mean": -102.16140747070312,
"KL/mean": -126.88875579833984,
"KL/rejected_KL_mean": -151.61610412597656,
"KL/std": 86.07960510253906,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.006922571919858456,
"fcm_dpo/delta": 0.08818544447422028,
"fcm_dpo/margin": 49.45469665527344,
"fcm_dpo/q_t": 0.42923611402511597,
"grad_norm": 36.29349136352539,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.372928261756897,
"logits/rejected": -0.35708069801330566,
"logps/chosen": -168.161865234375,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -233.3188934326172,
"loss": 1.2057,
"margin_dpo/margin_mean": 49.45469665527344,
"margin_dpo/margin_std": 103.38996887207031,
"step": 146
},
{
"KL/chosen_KL_mean": -74.98482513427734,
"KL/mean": -108.55328369140625,
"KL/rejected_KL_mean": -142.12173461914062,
"KL/std": 86.59007263183594,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.006506077013909817,
"fcm_dpo/delta": -0.07235047966241837,
"fcm_dpo/margin": 67.13691711425781,
"fcm_dpo/q_t": 0.3988075852394104,
"grad_norm": 22.362390518188477,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.37692469358444214,
"logits/rejected": -0.3635402321815491,
"logps/chosen": -128.39031982421875,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -213.51235961914062,
"loss": 1.1083,
"margin_dpo/margin_mean": 67.13691711425781,
"margin_dpo/margin_std": 108.45335388183594,
"step": 147
},
{
"KL/chosen_KL_mean": -73.58265686035156,
"KL/mean": -103.37672424316406,
"KL/rejected_KL_mean": -133.17080688476562,
"KL/std": 88.8348388671875,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.005902706179767847,
"fcm_dpo/delta": -0.09467141330242157,
"fcm_dpo/margin": 59.5881462097168,
"fcm_dpo/q_t": 0.4176294803619385,
"grad_norm": 16.640806198120117,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.4213970899581909,
"logits/rejected": -0.4161149859428406,
"logps/chosen": -138.5197296142578,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -236.2646484375,
"loss": 1.1526,
"margin_dpo/margin_mean": 59.5881462097168,
"margin_dpo/margin_std": 94.07696533203125,
"step": 148
},
{
"KL/chosen_KL_mean": -68.221435546875,
"KL/mean": -104.93063354492188,
"KL/rejected_KL_mean": -141.63983154296875,
"KL/std": 77.2359619140625,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.005145592149347067,
"fcm_dpo/delta": 0.023085758090019226,
"fcm_dpo/margin": 73.41839599609375,
"fcm_dpo/q_t": 0.412888765335083,
"grad_norm": 15.609312057495117,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.43985214829444885,
"logits/rejected": -0.4274490475654602,
"logps/chosen": -126.69520568847656,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -240.95458984375,
"loss": 1.1062,
"margin_dpo/margin_mean": 73.41839599609375,
"margin_dpo/margin_std": 91.98696899414062,
"step": 149
},
{
"KL/chosen_KL_mean": -59.703102111816406,
"KL/mean": -95.98683166503906,
"KL/rejected_KL_mean": -132.27056884765625,
"KL/std": 89.55494689941406,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.005316386930644512,
"fcm_dpo/delta": 0.011053726077079773,
"fcm_dpo/margin": 72.56747436523438,
"fcm_dpo/q_t": 0.4142092168331146,
"grad_norm": 16.736658096313477,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.41275283694267273,
"logits/rejected": -0.41830599308013916,
"logps/chosen": -105.4089126586914,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -215.61817932128906,
"loss": 1.1153,
"margin_dpo/margin_mean": 72.56747436523438,
"margin_dpo/margin_std": 106.04312133789062,
"step": 150
},
{
"KL/chosen_KL_mean": -77.2176513671875,
"KL/mean": -112.65522766113281,
"KL/rejected_KL_mean": -148.09280395507812,
"KL/std": 87.22770690917969,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.005660551600158215,
"fcm_dpo/delta": -0.002181790303438902,
"fcm_dpo/margin": 70.87516784667969,
"fcm_dpo/q_t": 0.40790289640426636,
"grad_norm": 21.483774185180664,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.4776611328125,
"logits/rejected": -0.4529585838317871,
"logps/chosen": -147.78848266601562,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -248.55662536621094,
"loss": 1.0927,
"margin_dpo/margin_mean": 70.87516784667969,
"margin_dpo/margin_std": 95.74323272705078,
"step": 151
},
{
"KL/chosen_KL_mean": -65.65343475341797,
"KL/mean": -108.78428649902344,
"KL/rejected_KL_mean": -151.91513061523438,
"KL/std": 86.67942810058594,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.005145312752574682,
"fcm_dpo/delta": -0.07856889814138412,
"fcm_dpo/margin": 86.26168823242188,
"fcm_dpo/q_t": 0.39314860105514526,
"grad_norm": 18.017948150634766,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.4266291558742523,
"logits/rejected": -0.42508143186569214,
"logps/chosen": -125.81782531738281,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -258.05560302734375,
"loss": 1.063,
"margin_dpo/margin_mean": 86.26168823242188,
"margin_dpo/margin_std": 107.51152038574219,
"step": 152
},
{
"KL/chosen_KL_mean": -65.02685546875,
"KL/mean": -102.04881286621094,
"KL/rejected_KL_mean": -139.07077026367188,
"KL/std": 95.22334289550781,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.005174952559173107,
"fcm_dpo/delta": 0.024773182347416878,
"fcm_dpo/margin": 74.04390716552734,
"fcm_dpo/q_t": 0.4167134463787079,
"grad_norm": 14.81849193572998,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.44491156935691833,
"logits/rejected": -0.44107282161712646,
"logps/chosen": -121.34213256835938,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -224.72659301757812,
"loss": 1.1225,
"margin_dpo/margin_mean": 74.04390716552734,
"margin_dpo/margin_std": 117.32939147949219,
"step": 153
},
{
"KL/chosen_KL_mean": -78.82939910888672,
"KL/mean": -113.46821594238281,
"KL/rejected_KL_mean": -148.10702514648438,
"KL/std": 93.49275207519531,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.005523172207176685,
"fcm_dpo/delta": 0.02156968042254448,
"fcm_dpo/margin": 69.27762603759766,
"fcm_dpo/q_t": 0.41571739315986633,
"grad_norm": 18.01680564880371,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.4389742910861969,
"logits/rejected": -0.44377291202545166,
"logps/chosen": -141.57196044921875,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -252.35122680664062,
"loss": 1.1476,
"margin_dpo/margin_mean": 69.27761840820312,
"margin_dpo/margin_std": 122.24600982666016,
"step": 154
},
{
"KL/chosen_KL_mean": -71.37049102783203,
"KL/mean": -105.14212036132812,
"KL/rejected_KL_mean": -138.9137420654297,
"KL/std": 85.17216491699219,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.0056947702541947365,
"fcm_dpo/delta": 0.021101046353578568,
"fcm_dpo/margin": 67.54327392578125,
"fcm_dpo/q_t": 0.41405290365219116,
"grad_norm": 18.612998962402344,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.45382189750671387,
"logits/rejected": -0.43611928820610046,
"logps/chosen": -132.023681640625,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -216.40594482421875,
"loss": 1.1265,
"margin_dpo/margin_mean": 67.54327392578125,
"margin_dpo/margin_std": 108.4144287109375,
"step": 155
},
{
"KL/chosen_KL_mean": -96.85684204101562,
"KL/mean": -116.4049301147461,
"KL/rejected_KL_mean": -135.95301818847656,
"KL/std": 89.89227294921875,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.006321663036942482,
"fcm_dpo/delta": 0.06302600353956223,
"fcm_dpo/margin": 39.09617614746094,
"fcm_dpo/q_t": 0.44758695363998413,
"grad_norm": 30.659738540649414,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.4037202298641205,
"logits/rejected": -0.3993948698043823,
"logps/chosen": -166.34872436523438,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -213.122314453125,
"loss": 1.2862,
"margin_dpo/margin_mean": 39.09617614746094,
"margin_dpo/margin_std": 119.15692138671875,
"step": 156
},
{
"KL/chosen_KL_mean": -85.81370544433594,
"KL/mean": -130.08242797851562,
"KL/rejected_KL_mean": -174.35113525390625,
"KL/std": 98.00643920898438,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.004979247227311134,
"fcm_dpo/delta": -0.10685013234615326,
"fcm_dpo/margin": 88.53744506835938,
"fcm_dpo/q_t": 0.38473406434059143,
"grad_norm": 23.637866973876953,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.37596505880355835,
"logits/rejected": -0.38393884897232056,
"logps/chosen": -147.18214416503906,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -281.99749755859375,
"loss": 1.073,
"margin_dpo/margin_mean": 88.53744506835938,
"margin_dpo/margin_std": 110.94694519042969,
"step": 157
},
{
"KL/chosen_KL_mean": -86.83956909179688,
"KL/mean": -135.0098876953125,
"KL/rejected_KL_mean": -183.18020629882812,
"KL/std": 121.85169982910156,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.004572378005832434,
"fcm_dpo/delta": -0.07351740449666977,
"fcm_dpo/margin": 96.34066009521484,
"fcm_dpo/q_t": 0.3994863033294678,
"grad_norm": 16.999359130859375,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.37230780720710754,
"logits/rejected": -0.391128271818161,
"logps/chosen": -144.45248413085938,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -296.8748779296875,
"loss": 1.0944,
"margin_dpo/margin_mean": 96.34065246582031,
"margin_dpo/margin_std": 147.7144775390625,
"step": 158
},
{
"KL/chosen_KL_mean": -100.27496337890625,
"KL/mean": -131.74691772460938,
"KL/rejected_KL_mean": -163.2188720703125,
"KL/std": 107.50895690917969,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.004534922074526548,
"fcm_dpo/delta": 0.02816414088010788,
"fcm_dpo/margin": 62.94389343261719,
"fcm_dpo/q_t": 0.435373455286026,
"grad_norm": 18.136377334594727,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.3773287534713745,
"logits/rejected": -0.34688425064086914,
"logps/chosen": -181.83531188964844,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -252.11758422851562,
"loss": 1.1876,
"margin_dpo/margin_mean": 62.94389343261719,
"margin_dpo/margin_std": 117.23402404785156,
"step": 159
},
{
"KL/chosen_KL_mean": -103.02062225341797,
"KL/mean": -146.32418823242188,
"KL/rejected_KL_mean": -189.62774658203125,
"KL/std": 114.74168395996094,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.0045206244103610516,
"fcm_dpo/delta": 0.009057410061359406,
"fcm_dpo/margin": 86.60713958740234,
"fcm_dpo/q_t": 0.4141198396682739,
"grad_norm": 18.559383392333984,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.3365239202976227,
"logits/rejected": -0.3285549283027649,
"logps/chosen": -168.75149536132812,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -286.8455810546875,
"loss": 1.1313,
"margin_dpo/margin_mean": 86.60713958740234,
"margin_dpo/margin_std": 143.93826293945312,
"step": 160
},
{
"KL/chosen_KL_mean": -87.73458862304688,
"KL/mean": -126.02169799804688,
"KL/rejected_KL_mean": -164.30880737304688,
"KL/std": 91.14512634277344,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.004974995274096727,
"fcm_dpo/delta": 0.030578553676605225,
"fcm_dpo/margin": 76.57421875,
"fcm_dpo/q_t": 0.41616570949554443,
"grad_norm": 17.85219383239746,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.34718695282936096,
"logits/rejected": -0.34764522314071655,
"logps/chosen": -140.17105102539062,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -247.73976135253906,
"loss": 1.1167,
"margin_dpo/margin_mean": 76.57421875,
"margin_dpo/margin_std": 114.59629821777344,
"step": 161
},
{
"KL/chosen_KL_mean": -87.06214904785156,
"KL/mean": -124.53192901611328,
"KL/rejected_KL_mean": -162.001708984375,
"KL/std": 101.81707763671875,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.004378783982247114,
"fcm_dpo/delta": -0.07103817164897919,
"fcm_dpo/margin": 74.93955993652344,
"fcm_dpo/q_t": 0.41890764236450195,
"grad_norm": 17.84354019165039,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.40842747688293457,
"logits/rejected": -0.3954179883003235,
"logps/chosen": -149.6727294921875,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -251.3922882080078,
"loss": 1.1551,
"margin_dpo/margin_mean": 74.93955993652344,
"margin_dpo/margin_std": 121.97283935546875,
"step": 162
},
{
"KL/chosen_KL_mean": -97.13140869140625,
"KL/mean": -133.43283081054688,
"KL/rejected_KL_mean": -169.73428344726562,
"KL/std": 102.79348754882812,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.004893934354186058,
"fcm_dpo/delta": 0.06739577651023865,
"fcm_dpo/margin": 72.60286712646484,
"fcm_dpo/q_t": 0.4250032901763916,
"grad_norm": 18.780895233154297,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.2808549404144287,
"logits/rejected": -0.2610132694244385,
"logps/chosen": -152.15304565429688,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -245.15249633789062,
"loss": 1.1469,
"margin_dpo/margin_mean": 72.60285949707031,
"margin_dpo/margin_std": 123.78816223144531,
"step": 163
},
{
"KL/chosen_KL_mean": -90.34315490722656,
"KL/mean": -135.48098754882812,
"KL/rejected_KL_mean": -180.6188201904297,
"KL/std": 100.77694702148438,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.004474318120628595,
"fcm_dpo/delta": -0.025637470185756683,
"fcm_dpo/margin": 90.27565002441406,
"fcm_dpo/q_t": 0.40300068259239197,
"grad_norm": 18.992538452148438,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.33539408445358276,
"logits/rejected": -0.32457149028778076,
"logps/chosen": -145.98382568359375,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -260.283447265625,
"loss": 1.0914,
"margin_dpo/margin_mean": 90.27565002441406,
"margin_dpo/margin_std": 116.16989135742188,
"step": 164
},
{
"KL/chosen_KL_mean": -97.24948120117188,
"KL/mean": -127.26203155517578,
"KL/rejected_KL_mean": -157.2745819091797,
"KL/std": 88.08245849609375,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.005466942209750414,
"fcm_dpo/delta": 0.10794621706008911,
"fcm_dpo/margin": 60.025089263916016,
"fcm_dpo/q_t": 0.43372809886932373,
"grad_norm": 26.800586700439453,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.3530671000480652,
"logits/rejected": -0.3305925726890564,
"logps/chosen": -158.5601806640625,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -230.9451904296875,
"loss": 1.1889,
"margin_dpo/margin_mean": 60.025089263916016,
"margin_dpo/margin_std": 118.67210388183594,
"step": 165
},
{
"KL/chosen_KL_mean": -87.03776550292969,
"KL/mean": -144.3040771484375,
"KL/rejected_KL_mean": -201.57040405273438,
"KL/std": 130.58055114746094,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.004105499014258385,
"fcm_dpo/delta": -0.187312051653862,
"fcm_dpo/margin": 114.53263854980469,
"fcm_dpo/q_t": 0.3861845135688782,
"grad_norm": 15.165700912475586,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.2735704779624939,
"logits/rejected": -0.2837375998497009,
"logps/chosen": -138.0213623046875,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -299.66552734375,
"loss": 1.0692,
"margin_dpo/margin_mean": 114.53263854980469,
"margin_dpo/margin_std": 165.4405059814453,
"step": 166
},
{
"KL/chosen_KL_mean": -88.03388977050781,
"KL/mean": -147.37371826171875,
"KL/rejected_KL_mean": -206.71356201171875,
"KL/std": 116.15589904785156,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.0036684987135231495,
"fcm_dpo/delta": -0.06418722122907639,
"fcm_dpo/margin": 118.67967987060547,
"fcm_dpo/q_t": 0.3941141664981842,
"grad_norm": 17.225709915161133,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.2889915704727173,
"logits/rejected": -0.29361122846603394,
"logps/chosen": -138.45797729492188,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -302.7439880371094,
"loss": 1.0561,
"margin_dpo/margin_mean": 118.6796875,
"margin_dpo/margin_std": 136.93655395507812,
"step": 167
},
{
"KL/chosen_KL_mean": -97.47393798828125,
"KL/mean": -140.19537353515625,
"KL/rejected_KL_mean": -182.9167938232422,
"KL/std": 109.24185180664062,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.004024019930511713,
"fcm_dpo/delta": 0.08574581891298294,
"fcm_dpo/margin": 85.44287109375,
"fcm_dpo/q_t": 0.4270554780960083,
"grad_norm": 18.420507431030273,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.29539740085601807,
"logits/rejected": -0.2888868451118469,
"logps/chosen": -146.936767578125,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -258.225341796875,
"loss": 1.1323,
"margin_dpo/margin_mean": 85.44287109375,
"margin_dpo/margin_std": 124.32752990722656,
"step": 168
},
{
"KL/chosen_KL_mean": -101.9376449584961,
"KL/mean": -139.52291870117188,
"KL/rejected_KL_mean": -177.1082000732422,
"KL/std": 107.82769012451172,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.0039043950382620096,
"fcm_dpo/delta": -0.04128513112664223,
"fcm_dpo/margin": 75.17054748535156,
"fcm_dpo/q_t": 0.4314765930175781,
"grad_norm": 17.273651123046875,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.3079976439476013,
"logits/rejected": -0.2939789295196533,
"logps/chosen": -161.7410888671875,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -260.4539489746094,
"loss": 1.1953,
"margin_dpo/margin_mean": 75.17054748535156,
"margin_dpo/margin_std": 148.5015869140625,
"step": 169
},
{
"KL/chosen_KL_mean": -92.11761474609375,
"KL/mean": -135.65200805664062,
"KL/rejected_KL_mean": -179.18641662597656,
"KL/std": 106.36629486083984,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.004083374515175819,
"fcm_dpo/delta": 0.06564062833786011,
"fcm_dpo/margin": 87.06880187988281,
"fcm_dpo/q_t": 0.422102689743042,
"grad_norm": 16.470184326171875,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.2703856825828552,
"logits/rejected": -0.26214104890823364,
"logps/chosen": -141.58938598632812,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -255.103759765625,
"loss": 1.1365,
"margin_dpo/margin_mean": 87.06879425048828,
"margin_dpo/margin_std": 128.26217651367188,
"step": 170
},
{
"KL/chosen_KL_mean": -134.2930908203125,
"KL/mean": -169.67434692382812,
"KL/rejected_KL_mean": -205.0555877685547,
"KL/std": 124.128662109375,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.00497001688927412,
"fcm_dpo/delta": 0.07393293082714081,
"fcm_dpo/margin": 70.76249694824219,
"fcm_dpo/q_t": 0.4315279722213745,
"grad_norm": 27.584213256835938,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.29333311319351196,
"logits/rejected": -0.2810017764568329,
"logps/chosen": -218.7924041748047,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -314.43768310546875,
"loss": 1.2014,
"margin_dpo/margin_mean": 70.76250457763672,
"margin_dpo/margin_std": 159.743896484375,
"step": 171
},
{
"KL/chosen_KL_mean": -115.03340148925781,
"KL/mean": -152.57827758789062,
"KL/rejected_KL_mean": -190.12313842773438,
"KL/std": 118.62234497070312,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.0052231112495064735,
"fcm_dpo/delta": 0.01281255204230547,
"fcm_dpo/margin": 75.0897445678711,
"fcm_dpo/q_t": 0.4145969748497009,
"grad_norm": 22.732559204101562,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.3138948976993561,
"logits/rejected": -0.29758453369140625,
"logps/chosen": -183.68731689453125,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -275.559814453125,
"loss": 1.1405,
"margin_dpo/margin_mean": 75.08975219726562,
"margin_dpo/margin_std": 133.15069580078125,
"step": 172
},
{
"KL/chosen_KL_mean": -104.79826354980469,
"KL/mean": -142.23779296875,
"KL/rejected_KL_mean": -179.67730712890625,
"KL/std": 111.67139434814453,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.00543089909479022,
"fcm_dpo/delta": -0.016082536429166794,
"fcm_dpo/margin": 74.8790512084961,
"fcm_dpo/q_t": 0.40886810421943665,
"grad_norm": 25.01607322692871,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.377377450466156,
"logits/rejected": -0.3528056740760803,
"logps/chosen": -167.84915161132812,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -258.3612365722656,
"loss": 1.1119,
"margin_dpo/margin_mean": 74.8790512084961,
"margin_dpo/margin_std": 115.37376403808594,
"step": 173
},
{
"KL/chosen_KL_mean": -96.8199462890625,
"KL/mean": -141.00047302246094,
"KL/rejected_KL_mean": -185.18099975585938,
"KL/std": 111.30195617675781,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.0049737137742340565,
"fcm_dpo/delta": -0.0776623785495758,
"fcm_dpo/margin": 88.36105346679688,
"fcm_dpo/q_t": 0.3986249268054962,
"grad_norm": 24.379945755004883,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.30099165439605713,
"logits/rejected": -0.3199496269226074,
"logps/chosen": -150.18292236328125,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -287.09222412109375,
"loss": 1.1075,
"margin_dpo/margin_mean": 88.36104583740234,
"margin_dpo/margin_std": 135.80799865722656,
"step": 174
},
{
"KL/chosen_KL_mean": -88.39326477050781,
"KL/mean": -147.26588439941406,
"KL/rejected_KL_mean": -206.13851928710938,
"KL/std": 116.80426788330078,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.0037730231415480375,
"fcm_dpo/delta": -0.0859874039888382,
"fcm_dpo/margin": 117.74525451660156,
"fcm_dpo/q_t": 0.3884362578392029,
"grad_norm": 26.734390258789062,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.3031741976737976,
"logits/rejected": -0.28989243507385254,
"logps/chosen": -133.81103515625,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -295.6443176269531,
"loss": 1.0409,
"margin_dpo/margin_mean": 117.74525451660156,
"margin_dpo/margin_std": 123.38618469238281,
"step": 175
},
{
"KL/chosen_KL_mean": -90.31321716308594,
"KL/mean": -141.83433532714844,
"KL/rejected_KL_mean": -193.35546875,
"KL/std": 116.11126708984375,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.0038136134389787912,
"fcm_dpo/delta": 0.010705736465752125,
"fcm_dpo/margin": 103.042236328125,
"fcm_dpo/q_t": 0.4118039608001709,
"grad_norm": 16.391658782958984,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.309362530708313,
"logits/rejected": -0.325575590133667,
"logps/chosen": -140.76605224609375,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -288.9144287109375,
"loss": 1.0923,
"margin_dpo/margin_mean": 103.042236328125,
"margin_dpo/margin_std": 134.9136199951172,
"step": 176
},
{
"KL/chosen_KL_mean": -105.74380493164062,
"KL/mean": -156.05882263183594,
"KL/rejected_KL_mean": -206.3738250732422,
"KL/std": 128.35159301757812,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.003937173634767532,
"fcm_dpo/delta": 0.002443891018629074,
"fcm_dpo/margin": 100.63003540039062,
"fcm_dpo/q_t": 0.41030067205429077,
"grad_norm": 20.429180145263672,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.3264981508255005,
"logits/rejected": -0.31806203722953796,
"logps/chosen": -166.96026611328125,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -302.2676086425781,
"loss": 1.0932,
"margin_dpo/margin_mean": 100.63003540039062,
"margin_dpo/margin_std": 134.44273376464844,
"step": 177
},
{
"KL/chosen_KL_mean": -116.06280517578125,
"KL/mean": -179.25152587890625,
"KL/rejected_KL_mean": -242.44021606445312,
"KL/std": 147.4817657470703,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.0033705285750329494,
"fcm_dpo/delta": -0.053363341838121414,
"fcm_dpo/margin": 126.37743377685547,
"fcm_dpo/q_t": 0.39890462160110474,
"grad_norm": 19.848587036132812,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.22271186113357544,
"logits/rejected": -0.21497488021850586,
"logps/chosen": -174.3275909423828,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -347.8055419921875,
"loss": 1.0721,
"margin_dpo/margin_mean": 126.37743377685547,
"margin_dpo/margin_std": 161.1014862060547,
"step": 178
},
{
"KL/chosen_KL_mean": -124.49735260009766,
"KL/mean": -168.00872802734375,
"KL/rejected_KL_mean": -211.52008056640625,
"KL/std": 130.20875549316406,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.003844078164547682,
"fcm_dpo/delta": 0.0992623046040535,
"fcm_dpo/margin": 87.0227279663086,
"fcm_dpo/q_t": 0.43123751878738403,
"grad_norm": 34.565582275390625,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.27690017223358154,
"logits/rejected": -0.2880879044532776,
"logps/chosen": -185.55567932128906,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -302.04791259765625,
"loss": 1.1778,
"margin_dpo/margin_mean": 87.0227279663086,
"margin_dpo/margin_std": 163.81394958496094,
"step": 179
},
{
"KL/chosen_KL_mean": -107.85954284667969,
"KL/mean": -166.43894958496094,
"KL/rejected_KL_mean": -225.0183563232422,
"KL/std": 115.33984375,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.0037114559672772884,
"fcm_dpo/delta": -0.06508003920316696,
"fcm_dpo/margin": 117.1588134765625,
"fcm_dpo/q_t": 0.39406827092170715,
"grad_norm": 16.350337982177734,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.23366385698318481,
"logits/rejected": -0.22989457845687866,
"logps/chosen": -162.2022705078125,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -323.2301940917969,
"loss": 1.0637,
"margin_dpo/margin_mean": 117.1588134765625,
"margin_dpo/margin_std": 143.82162475585938,
"step": 180
},
{
"KL/chosen_KL_mean": -94.5392074584961,
"KL/mean": -129.8724365234375,
"KL/rejected_KL_mean": -165.2056884765625,
"KL/std": 107.95272827148438,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.004456082358956337,
"fcm_dpo/delta": 0.11765934526920319,
"fcm_dpo/margin": 70.6664810180664,
"fcm_dpo/q_t": 0.4351644515991211,
"grad_norm": 19.006668090820312,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.2771533727645874,
"logits/rejected": -0.2554428279399872,
"logps/chosen": -149.5396728515625,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -226.86184692382812,
"loss": 1.1826,
"margin_dpo/margin_mean": 70.6664810180664,
"margin_dpo/margin_std": 136.99830627441406,
"step": 181
},
{
"KL/chosen_KL_mean": -89.14537811279297,
"KL/mean": -153.301513671875,
"KL/rejected_KL_mean": -217.45767211914062,
"KL/std": 129.44326782226562,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.0035601062700152397,
"fcm_dpo/delta": -0.1331714242696762,
"fcm_dpo/margin": 128.31228637695312,
"fcm_dpo/q_t": 0.38162392377853394,
"grad_norm": 15.877484321594238,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.24869897961616516,
"logits/rejected": -0.259911447763443,
"logps/chosen": -130.25323486328125,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -306.979248046875,
"loss": 1.054,
"margin_dpo/margin_mean": 128.31228637695312,
"margin_dpo/margin_std": 157.36795043945312,
"step": 182
},
{
"KL/chosen_KL_mean": -132.1239776611328,
"KL/mean": -164.78997802734375,
"KL/rejected_KL_mean": -197.4559783935547,
"KL/std": 107.94178771972656,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.003544195555150509,
"fcm_dpo/delta": 0.000493988860398531,
"fcm_dpo/margin": 65.33200073242188,
"fcm_dpo/q_t": 0.4455084502696991,
"grad_norm": 17.170263290405273,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.2581177055835724,
"logits/rejected": -0.2515699863433838,
"logps/chosen": -189.64854431152344,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -273.43170166015625,
"loss": 1.2172,
"margin_dpo/margin_mean": 65.33200073242188,
"margin_dpo/margin_std": 118.64833068847656,
"step": 183
},
{
"KL/chosen_KL_mean": -111.37728881835938,
"KL/mean": -146.77032470703125,
"KL/rejected_KL_mean": -182.16336059570312,
"KL/std": 101.87884521484375,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.003663355251774192,
"fcm_dpo/delta": 0.03253895044326782,
"fcm_dpo/margin": 70.78607177734375,
"fcm_dpo/q_t": 0.4417157769203186,
"grad_norm": 16.201236724853516,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.28559648990631104,
"logits/rejected": -0.27648642659187317,
"logps/chosen": -169.9222412109375,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -258.79742431640625,
"loss": 1.1955,
"margin_dpo/margin_mean": 70.78607177734375,
"margin_dpo/margin_std": 128.29718017578125,
"step": 184
},
{
"KL/chosen_KL_mean": -122.75660705566406,
"KL/mean": -149.69287109375,
"KL/rejected_KL_mean": -176.62911987304688,
"KL/std": 117.6004638671875,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.003782514948397875,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 53.87251663208008,
"fcm_dpo/q_t": 0.4551408886909485,
"grad_norm": 18.46157455444336,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.2715925872325897,
"logits/rejected": -0.25268396735191345,
"logps/chosen": -184.78245544433594,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -250.3917236328125,
"loss": 1.2595,
"margin_dpo/margin_mean": 53.872520446777344,
"margin_dpo/margin_std": 141.81597900390625,
"step": 185
},
{
"KL/chosen_KL_mean": -115.03028869628906,
"KL/mean": -167.48983764648438,
"KL/rejected_KL_mean": -219.94937133789062,
"KL/std": 118.58721160888672,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.0037781421560794115,
"fcm_dpo/delta": 0.005793063435703516,
"fcm_dpo/margin": 104.91907501220703,
"fcm_dpo/q_t": 0.40997469425201416,
"grad_norm": 19.549184799194336,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.3371797502040863,
"logits/rejected": -0.3270975947380066,
"logps/chosen": -184.38375854492188,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -308.0218200683594,
"loss": 1.0931,
"margin_dpo/margin_mean": 104.9190673828125,
"margin_dpo/margin_std": 139.93943786621094,
"step": 186
},
{
"KL/chosen_KL_mean": -102.89697265625,
"KL/mean": -148.2735137939453,
"KL/rejected_KL_mean": -193.65003967285156,
"KL/std": 112.91288757324219,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.00417533703148365,
"fcm_dpo/delta": 0.025836054235696793,
"fcm_dpo/margin": 90.75306701660156,
"fcm_dpo/q_t": 0.4136863350868225,
"grad_norm": 21.79146385192871,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.2833201587200165,
"logits/rejected": -0.27770209312438965,
"logps/chosen": -155.6534423828125,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -275.619140625,
"loss": 1.1064,
"margin_dpo/margin_mean": 90.75306701660156,
"margin_dpo/margin_std": 121.31842041015625,
"step": 187
},
{
"KL/chosen_KL_mean": -96.85295104980469,
"KL/mean": -152.17840576171875,
"KL/rejected_KL_mean": -207.50384521484375,
"KL/std": 124.83462524414062,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.003823988139629364,
"fcm_dpo/delta": -0.04001890495419502,
"fcm_dpo/margin": 110.6509017944336,
"fcm_dpo/q_t": 0.4005782902240753,
"grad_norm": 25.183116912841797,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.2504292130470276,
"logits/rejected": -0.2585201859474182,
"logps/chosen": -146.2684326171875,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -297.04425048828125,
"loss": 1.0712,
"margin_dpo/margin_mean": 110.6509017944336,
"margin_dpo/margin_std": 138.2140655517578,
"step": 188
},
{
"KL/chosen_KL_mean": -115.58564758300781,
"KL/mean": -160.01602172851562,
"KL/rejected_KL_mean": -204.44639587402344,
"KL/std": 127.6397933959961,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.004090734291821718,
"fcm_dpo/delta": 0.055509161204099655,
"fcm_dpo/margin": 88.86076354980469,
"fcm_dpo/q_t": 0.42382127046585083,
"grad_norm": 27.4331111907959,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.271611750125885,
"logits/rejected": -0.2570100426673889,
"logps/chosen": -167.98460388183594,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -276.6137390136719,
"loss": 1.1472,
"margin_dpo/margin_mean": 88.86076354980469,
"margin_dpo/margin_std": 158.424560546875,
"step": 189
},
{
"KL/chosen_KL_mean": -124.001708984375,
"KL/mean": -178.27090454101562,
"KL/rejected_KL_mean": -232.54010009765625,
"KL/std": 137.54371643066406,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.004083447623997927,
"fcm_dpo/delta": -0.09416892379522324,
"fcm_dpo/margin": 108.53840637207031,
"fcm_dpo/q_t": 0.3950834274291992,
"grad_norm": 19.966215133666992,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.3038903474807739,
"logits/rejected": -0.29708611965179443,
"logps/chosen": -188.68478393554688,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -335.09063720703125,
"loss": 1.1112,
"margin_dpo/margin_mean": 108.53840637207031,
"margin_dpo/margin_std": 159.21173095703125,
"step": 190
},
{
"KL/chosen_KL_mean": -116.2418212890625,
"KL/mean": -192.55517578125,
"KL/rejected_KL_mean": -268.8685302734375,
"KL/std": 155.87002563476562,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.0028747329488396645,
"fcm_dpo/delta": -0.08564233034849167,
"fcm_dpo/margin": 152.626708984375,
"fcm_dpo/q_t": 0.3907443881034851,
"grad_norm": 17.792726516723633,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.2973872721195221,
"logits/rejected": -0.2730400264263153,
"logps/chosen": -184.90069580078125,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -379.0082092285156,
"loss": 1.0538,
"margin_dpo/margin_mean": 152.626708984375,
"margin_dpo/margin_std": 175.80136108398438,
"step": 191
},
{
"KL/chosen_KL_mean": -149.68504333496094,
"KL/mean": -200.70745849609375,
"KL/rejected_KL_mean": -251.7298583984375,
"KL/std": 139.31866455078125,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.003366319462656975,
"fcm_dpo/delta": 0.079339399933815,
"fcm_dpo/margin": 102.04479217529297,
"fcm_dpo/q_t": 0.42711859941482544,
"grad_norm": 20.006580352783203,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.2600586414337158,
"logits/rejected": -0.26135045289993286,
"logps/chosen": -219.41195678710938,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -355.05120849609375,
"loss": 1.1503,
"margin_dpo/margin_mean": 102.0447998046875,
"margin_dpo/margin_std": 174.04171752929688,
"step": 192
},
{
"KL/chosen_KL_mean": -149.03636169433594,
"KL/mean": -177.56857299804688,
"KL/rejected_KL_mean": -206.10079956054688,
"KL/std": 129.29698181152344,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.003643119242042303,
"fcm_dpo/delta": 0.06691880524158478,
"fcm_dpo/margin": 57.06443786621094,
"fcm_dpo/q_t": 0.45318782329559326,
"grad_norm": 28.55337142944336,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.27016884088516235,
"logits/rejected": -0.2688116431236267,
"logps/chosen": -209.22686767578125,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -282.50836181640625,
"loss": 1.2736,
"margin_dpo/margin_mean": 57.06443786621094,
"margin_dpo/margin_std": 165.7989501953125,
"step": 193
},
{
"KL/chosen_KL_mean": -90.87747955322266,
"KL/mean": -138.6411590576172,
"KL/rejected_KL_mean": -186.40484619140625,
"KL/std": 105.31715393066406,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.0039482079446315765,
"fcm_dpo/delta": 0.034811198711395264,
"fcm_dpo/margin": 95.52735900878906,
"fcm_dpo/q_t": 0.4142157733440399,
"grad_norm": 17.623666763305664,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.1908482015132904,
"logits/rejected": -0.17822426557540894,
"logps/chosen": -128.7178497314453,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -247.08963012695312,
"loss": 1.0982,
"margin_dpo/margin_mean": 95.52735900878906,
"margin_dpo/margin_std": 116.02978515625,
"step": 194
},
{
"KL/chosen_KL_mean": -144.2040557861328,
"KL/mean": -196.9357147216797,
"KL/rejected_KL_mean": -249.66738891601562,
"KL/std": 133.17889404296875,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.0039472421631217,
"fcm_dpo/delta": -0.028900716453790665,
"fcm_dpo/margin": 105.46333312988281,
"fcm_dpo/q_t": 0.40308016538619995,
"grad_norm": 22.52235984802246,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.23734885454177856,
"logits/rejected": -0.25579947233200073,
"logps/chosen": -199.0956268310547,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -346.4383544921875,
"loss": 1.0821,
"margin_dpo/margin_mean": 105.46334075927734,
"margin_dpo/margin_std": 139.16427612304688,
"step": 195
},
{
"KL/chosen_KL_mean": -111.86429595947266,
"KL/mean": -169.64901733398438,
"KL/rejected_KL_mean": -227.4337158203125,
"KL/std": 129.91050720214844,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.003521410282701254,
"fcm_dpo/delta": -0.028155531734228134,
"fcm_dpo/margin": 115.56942749023438,
"fcm_dpo/q_t": 0.40525445342063904,
"grad_norm": 16.956315994262695,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.24712811410427094,
"logits/rejected": -0.2348405420780182,
"logps/chosen": -165.1095428466797,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -303.4866638183594,
"loss": 1.0964,
"margin_dpo/margin_mean": 115.56942749023438,
"margin_dpo/margin_std": 156.6468505859375,
"step": 196
},
{
"KL/chosen_KL_mean": -114.23474884033203,
"KL/mean": -157.83114624023438,
"KL/rejected_KL_mean": -201.42755126953125,
"KL/std": 111.18183135986328,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.004171357490122318,
"fcm_dpo/delta": 0.04900194704532623,
"fcm_dpo/margin": 87.19279479980469,
"fcm_dpo/q_t": 0.4201440215110779,
"grad_norm": 31.859424591064453,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.24368175864219666,
"logits/rejected": -0.24027302861213684,
"logps/chosen": -174.65509033203125,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -278.636474609375,
"loss": 1.1232,
"margin_dpo/margin_mean": 87.19279479980469,
"margin_dpo/margin_std": 128.5411376953125,
"step": 197
},
{
"KL/chosen_KL_mean": -124.44345092773438,
"KL/mean": -183.01571655273438,
"KL/rejected_KL_mean": -241.5879669189453,
"KL/std": 141.48675537109375,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.003705207258462906,
"fcm_dpo/delta": -0.06200866401195526,
"fcm_dpo/margin": 117.14451599121094,
"fcm_dpo/q_t": 0.39838171005249023,
"grad_norm": 22.00115966796875,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.29106560349464417,
"logits/rejected": -0.2917174696922302,
"logps/chosen": -179.47964477539062,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -338.83123779296875,
"loss": 1.0749,
"margin_dpo/margin_mean": 117.14451599121094,
"margin_dpo/margin_std": 159.01614379882812,
"step": 198
},
{
"KL/chosen_KL_mean": -120.00575256347656,
"KL/mean": -176.61056518554688,
"KL/rejected_KL_mean": -233.21536254882812,
"KL/std": 125.2437515258789,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.00345216179266572,
"fcm_dpo/delta": 0.006332229822874069,
"fcm_dpo/margin": 113.2095947265625,
"fcm_dpo/q_t": 0.4100239872932434,
"grad_norm": 18.751543045043945,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.2945285141468048,
"logits/rejected": -0.2768348157405853,
"logps/chosen": -176.8345947265625,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -317.86358642578125,
"loss": 1.106,
"margin_dpo/margin_mean": 113.2095947265625,
"margin_dpo/margin_std": 155.6266632080078,
"step": 199
},
{
"KL/chosen_KL_mean": -117.3885498046875,
"KL/mean": -176.23748779296875,
"KL/rejected_KL_mean": -235.08642578125,
"KL/std": 138.85617065429688,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.0034625413827598095,
"fcm_dpo/delta": -0.016614485532045364,
"fcm_dpo/margin": 117.6978988647461,
"fcm_dpo/q_t": 0.4070635437965393,
"grad_norm": 20.583110809326172,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.2415538728237152,
"logits/rejected": -0.22881929576396942,
"logps/chosen": -170.4556121826172,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -315.69488525390625,
"loss": 1.1079,
"margin_dpo/margin_mean": 117.6978988647461,
"margin_dpo/margin_std": 178.77822875976562,
"step": 200
},
{
"KL/chosen_KL_mean": -128.0514373779297,
"KL/mean": -184.71365356445312,
"KL/rejected_KL_mean": -241.3758544921875,
"KL/std": 148.18923950195312,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.00359878852032125,
"fcm_dpo/delta": -0.015944721177220345,
"fcm_dpo/margin": 113.32442474365234,
"fcm_dpo/q_t": 0.40622514486312866,
"grad_norm": 24.77713966369629,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.3444761037826538,
"logits/rejected": -0.3457464575767517,
"logps/chosen": -203.4536590576172,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -356.18408203125,
"loss": 1.1082,
"margin_dpo/margin_mean": 113.32442474365234,
"margin_dpo/margin_std": 168.1522979736328,
"step": 201
},
{
"KL/chosen_KL_mean": -129.18331909179688,
"KL/mean": -168.78958129882812,
"KL/rejected_KL_mean": -208.39584350585938,
"KL/std": 127.8756103515625,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.004097277298569679,
"fcm_dpo/delta": 0.10765650868415833,
"fcm_dpo/margin": 79.21253204345703,
"fcm_dpo/q_t": 0.4345342516899109,
"grad_norm": 22.265583038330078,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.27213054895401,
"logits/rejected": -0.28951022028923035,
"logps/chosen": -179.28463745117188,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -295.380859375,
"loss": 1.1947,
"margin_dpo/margin_mean": 79.21253967285156,
"margin_dpo/margin_std": 168.17898559570312,
"step": 202
},
{
"KL/chosen_KL_mean": -128.6010284423828,
"KL/mean": -178.1983642578125,
"KL/rejected_KL_mean": -227.79568481445312,
"KL/std": 133.08248901367188,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.004127629101276398,
"fcm_dpo/delta": -0.01639389619231224,
"fcm_dpo/margin": 99.19467163085938,
"fcm_dpo/q_t": 0.40751299262046814,
"grad_norm": 20.381248474121094,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.303178995847702,
"logits/rejected": -0.29665249586105347,
"logps/chosen": -189.21072387695312,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -313.691650390625,
"loss": 1.1046,
"margin_dpo/margin_mean": 99.19467163085938,
"margin_dpo/margin_std": 150.01708984375,
"step": 203
},
{
"KL/chosen_KL_mean": -141.3732452392578,
"KL/mean": -185.82644653320312,
"KL/rejected_KL_mean": -230.27967834472656,
"KL/std": 142.70265197753906,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.0043134503066539764,
"fcm_dpo/delta": 0.026382368057966232,
"fcm_dpo/margin": 88.90643310546875,
"fcm_dpo/q_t": 0.4196096360683441,
"grad_norm": 31.27920150756836,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.38292059302330017,
"logits/rejected": -0.34853625297546387,
"logps/chosen": -221.5382080078125,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -317.9755859375,
"loss": 1.1628,
"margin_dpo/margin_mean": 88.90642547607422,
"margin_dpo/margin_std": 170.709228515625,
"step": 204
},
{
"KL/chosen_KL_mean": -131.44627380371094,
"KL/mean": -190.60824584960938,
"KL/rejected_KL_mean": -249.77024841308594,
"KL/std": 140.37823486328125,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.0035104253329336643,
"fcm_dpo/delta": -0.07186352461576462,
"fcm_dpo/margin": 118.3239517211914,
"fcm_dpo/q_t": 0.3940643072128296,
"grad_norm": 19.9980411529541,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.290398508310318,
"logits/rejected": -0.2605670094490051,
"logps/chosen": -190.8310089111328,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -334.8952941894531,
"loss": 1.0949,
"margin_dpo/margin_mean": 118.3239517211914,
"margin_dpo/margin_std": 153.96115112304688,
"step": 205
},
{
"KL/chosen_KL_mean": -112.48219299316406,
"KL/mean": -175.43797302246094,
"KL/rejected_KL_mean": -238.3937530517578,
"KL/std": 123.48064422607422,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.0033509135246276855,
"fcm_dpo/delta": -0.047678425908088684,
"fcm_dpo/margin": 125.91154479980469,
"fcm_dpo/q_t": 0.39560678601264954,
"grad_norm": 30.06896209716797,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.26723650097846985,
"logits/rejected": -0.2706051170825958,
"logps/chosen": -159.44668579101562,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -337.34722900390625,
"loss": 1.0582,
"margin_dpo/margin_mean": 125.91154479980469,
"margin_dpo/margin_std": 135.0882568359375,
"step": 206
},
{
"KL/chosen_KL_mean": -106.53071594238281,
"KL/mean": -176.65072631835938,
"KL/rejected_KL_mean": -246.77072143554688,
"KL/std": 144.35787963867188,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.0031110041309148073,
"fcm_dpo/delta": -0.06504091620445251,
"fcm_dpo/margin": 140.24000549316406,
"fcm_dpo/q_t": 0.3961068391799927,
"grad_norm": 30.773921966552734,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.35365355014801025,
"logits/rejected": -0.3321405053138733,
"logps/chosen": -162.58697509765625,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -331.218505859375,
"loss": 1.0617,
"margin_dpo/margin_mean": 140.24000549316406,
"margin_dpo/margin_std": 172.24667358398438,
"step": 207
},
{
"KL/chosen_KL_mean": -163.8634490966797,
"KL/mean": -219.51443481445312,
"KL/rejected_KL_mean": -275.1654052734375,
"KL/std": 141.97909545898438,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.0032390430569648743,
"fcm_dpo/delta": 0.06209877133369446,
"fcm_dpo/margin": 111.30195617675781,
"fcm_dpo/q_t": 0.42106446623802185,
"grad_norm": 23.454050064086914,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.34735146164894104,
"logits/rejected": -0.33190101385116577,
"logps/chosen": -230.93106079101562,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -369.4523010253906,
"loss": 1.1334,
"margin_dpo/margin_mean": 111.30195617675781,
"margin_dpo/margin_std": 172.6982879638672,
"step": 208
},
{
"KL/chosen_KL_mean": -133.59063720703125,
"KL/mean": -184.37820434570312,
"KL/rejected_KL_mean": -235.165771484375,
"KL/std": 125.87327575683594,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.0036668735556304455,
"fcm_dpo/delta": 0.042608536779880524,
"fcm_dpo/margin": 101.57514190673828,
"fcm_dpo/q_t": 0.41783520579338074,
"grad_norm": 21.281137466430664,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.257140576839447,
"logits/rejected": -0.24332758784294128,
"logps/chosen": -189.77232360839844,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -316.1072998046875,
"loss": 1.127,
"margin_dpo/margin_mean": 101.57514953613281,
"margin_dpo/margin_std": 156.05691528320312,
"step": 209
},
{
"KL/chosen_KL_mean": -117.53192138671875,
"KL/mean": -174.57296752929688,
"KL/rejected_KL_mean": -231.614013671875,
"KL/std": 124.3192367553711,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.0036693743895739317,
"fcm_dpo/delta": -0.03410163149237633,
"fcm_dpo/margin": 114.08209228515625,
"fcm_dpo/q_t": 0.4016813635826111,
"grad_norm": 30.14597511291504,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.3081471621990204,
"logits/rejected": -0.30119192600250244,
"logps/chosen": -163.90374755859375,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -308.2956237792969,
"loss": 1.0831,
"margin_dpo/margin_mean": 114.08209228515625,
"margin_dpo/margin_std": 149.70401000976562,
"step": 210
},
{
"KL/chosen_KL_mean": -176.67413330078125,
"KL/mean": -227.01296997070312,
"KL/rejected_KL_mean": -277.351806640625,
"KL/std": 148.22705078125,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.0037167323753237724,
"fcm_dpo/delta": 0.03998677805066109,
"fcm_dpo/margin": 100.67768859863281,
"fcm_dpo/q_t": 0.42228320240974426,
"grad_norm": 28.76727294921875,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.31403297185897827,
"logits/rejected": -0.27679428458213806,
"logps/chosen": -255.60647583007812,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -364.17279052734375,
"loss": 1.1458,
"margin_dpo/margin_mean": 100.67768859863281,
"margin_dpo/margin_std": 173.43338012695312,
"step": 211
},
{
"KL/chosen_KL_mean": -147.537109375,
"KL/mean": -220.67324829101562,
"KL/rejected_KL_mean": -293.80938720703125,
"KL/std": 159.14773559570312,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.0029512690380215645,
"fcm_dpo/delta": -0.09743355959653854,
"fcm_dpo/margin": 146.27227783203125,
"fcm_dpo/q_t": 0.3903607130050659,
"grad_norm": 18.54225730895996,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.26962530612945557,
"logits/rejected": -0.26270633935928345,
"logps/chosen": -205.73411560058594,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -396.86724853515625,
"loss": 1.0686,
"margin_dpo/margin_mean": 146.27227783203125,
"margin_dpo/margin_std": 174.09445190429688,
"step": 212
},
{
"KL/chosen_KL_mean": -143.3365478515625,
"KL/mean": -215.00723266601562,
"KL/rejected_KL_mean": -286.67791748046875,
"KL/std": 147.20899963378906,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.002868292387574911,
"fcm_dpo/delta": -0.02426821179687977,
"fcm_dpo/margin": 143.34136962890625,
"fcm_dpo/q_t": 0.4022514820098877,
"grad_norm": 18.237730026245117,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.2986302077770233,
"logits/rejected": -0.27630025148391724,
"logps/chosen": -210.84925842285156,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -380.5926513671875,
"loss": 1.0765,
"margin_dpo/margin_mean": 143.3413848876953,
"margin_dpo/margin_std": 173.92288208007812,
"step": 213
},
{
"KL/chosen_KL_mean": -125.64663696289062,
"KL/mean": -189.3560028076172,
"KL/rejected_KL_mean": -253.06533813476562,
"KL/std": 145.16001892089844,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.0030471929349005222,
"fcm_dpo/delta": 0.018730763345956802,
"fcm_dpo/margin": 127.41871643066406,
"fcm_dpo/q_t": 0.4126247763633728,
"grad_norm": 18.90703010559082,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.21006491780281067,
"logits/rejected": -0.2164653241634369,
"logps/chosen": -167.25152587890625,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -330.582763671875,
"loss": 1.0976,
"margin_dpo/margin_mean": 127.41871643066406,
"margin_dpo/margin_std": 170.29165649414062,
"step": 214
},
{
"KL/chosen_KL_mean": -150.6676025390625,
"KL/mean": -216.49374389648438,
"KL/rejected_KL_mean": -282.31988525390625,
"KL/std": 148.04367065429688,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.0030098496936261654,
"fcm_dpo/delta": 0.005100354552268982,
"fcm_dpo/margin": 131.6522979736328,
"fcm_dpo/q_t": 0.4081616997718811,
"grad_norm": 22.060007095336914,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.24693366885185242,
"logits/rejected": -0.23978078365325928,
"logps/chosen": -203.94686889648438,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -372.2845458984375,
"loss": 1.0802,
"margin_dpo/margin_mean": 131.6522979736328,
"margin_dpo/margin_std": 154.73556518554688,
"step": 215
},
{
"KL/chosen_KL_mean": -155.56214904785156,
"KL/mean": -220.8539581298828,
"KL/rejected_KL_mean": -286.145751953125,
"KL/std": 156.0887451171875,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.0031287854071706533,
"fcm_dpo/delta": -0.01758376508951187,
"fcm_dpo/margin": 130.5836181640625,
"fcm_dpo/q_t": 0.40592771768569946,
"grad_norm": 22.239622116088867,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.2485334426164627,
"logits/rejected": -0.23272496461868286,
"logps/chosen": -204.449951171875,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -363.3446960449219,
"loss": 1.1032,
"margin_dpo/margin_mean": 130.5836181640625,
"margin_dpo/margin_std": 189.82901000976562,
"step": 216
},
{
"KL/chosen_KL_mean": -158.0336456298828,
"KL/mean": -235.93881225585938,
"KL/rejected_KL_mean": -313.843994140625,
"KL/std": 162.12493896484375,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.0027228975668549538,
"fcm_dpo/delta": -0.045385316014289856,
"fcm_dpo/margin": 155.81036376953125,
"fcm_dpo/q_t": 0.39862823486328125,
"grad_norm": 20.71597671508789,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.25405189394950867,
"logits/rejected": -0.2514030337333679,
"logps/chosen": -207.8789520263672,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -413.92230224609375,
"loss": 1.0651,
"margin_dpo/margin_mean": 155.81036376953125,
"margin_dpo/margin_std": 185.34475708007812,
"step": 217
},
{
"KL/chosen_KL_mean": -167.77676391601562,
"KL/mean": -227.49313354492188,
"KL/rejected_KL_mean": -287.2095031738281,
"KL/std": 159.71775817871094,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.0029854662716388702,
"fcm_dpo/delta": 0.0675802081823349,
"fcm_dpo/margin": 119.43275451660156,
"fcm_dpo/q_t": 0.42418262362480164,
"grad_norm": 21.031944274902344,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.2911514639854431,
"logits/rejected": -0.28862863779067993,
"logps/chosen": -226.3534393310547,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -375.055908203125,
"loss": 1.1395,
"margin_dpo/margin_mean": 119.43275451660156,
"margin_dpo/margin_std": 189.45452880859375,
"step": 218
},
{
"KL/chosen_KL_mean": -179.43991088867188,
"KL/mean": -230.53575134277344,
"KL/rejected_KL_mean": -281.631591796875,
"KL/std": 166.12942504882812,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.003073344938457012,
"fcm_dpo/delta": -0.012668075039982796,
"fcm_dpo/margin": 102.19166564941406,
"fcm_dpo/q_t": 0.42826542258262634,
"grad_norm": 32.95735549926758,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.2939431965351105,
"logits/rejected": -0.29159048199653625,
"logps/chosen": -240.52377319335938,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -367.4620361328125,
"loss": 1.1924,
"margin_dpo/margin_mean": 102.19166564941406,
"margin_dpo/margin_std": 205.6397247314453,
"step": 219
},
{
"KL/chosen_KL_mean": -201.14923095703125,
"KL/mean": -243.14886474609375,
"KL/rejected_KL_mean": -285.1484680175781,
"KL/std": 140.5084228515625,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.003665705444291234,
"fcm_dpo/delta": 0.13320602476596832,
"fcm_dpo/margin": 83.9992446899414,
"fcm_dpo/q_t": 0.4367624521255493,
"grad_norm": 45.616695404052734,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.3121609687805176,
"logits/rejected": -0.2908840775489807,
"logps/chosen": -271.1805419921875,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -372.833984375,
"loss": 1.176,
"margin_dpo/margin_mean": 83.9992446899414,
"margin_dpo/margin_std": 149.43328857421875,
"step": 220
},
{
"KL/chosen_KL_mean": -161.646728515625,
"KL/mean": -256.31317138671875,
"KL/rejected_KL_mean": -350.9796142578125,
"KL/std": 178.03521728515625,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.0027057200204581022,
"fcm_dpo/delta": -0.25114211440086365,
"fcm_dpo/margin": 189.3328857421875,
"fcm_dpo/q_t": 0.35794326663017273,
"grad_norm": 40.98723602294922,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.3036612868309021,
"logits/rejected": -0.3129799962043762,
"logps/chosen": -213.80145263671875,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -457.44732666015625,
"loss": 1.0037,
"margin_dpo/margin_mean": 189.3328857421875,
"margin_dpo/margin_std": 184.73748779296875,
"step": 221
},
{
"KL/chosen_KL_mean": -166.11956787109375,
"KL/mean": -236.33322143554688,
"KL/rejected_KL_mean": -306.546875,
"KL/std": 162.46240234375,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.00264683086425066,
"fcm_dpo/delta": 0.03757670894265175,
"fcm_dpo/margin": 140.42733764648438,
"fcm_dpo/q_t": 0.4158090353012085,
"grad_norm": 24.47178840637207,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.3462597131729126,
"logits/rejected": -0.3537750244140625,
"logps/chosen": -227.09066772460938,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -406.54803466796875,
"loss": 1.1057,
"margin_dpo/margin_mean": 140.42733764648438,
"margin_dpo/margin_std": 183.39102172851562,
"step": 222
},
{
"KL/chosen_KL_mean": -183.58261108398438,
"KL/mean": -243.48226928710938,
"KL/rejected_KL_mean": -303.38189697265625,
"KL/std": 159.4056396484375,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.0028988937847316265,
"fcm_dpo/delta": 0.0806143581867218,
"fcm_dpo/margin": 119.79930114746094,
"fcm_dpo/q_t": 0.4264669120311737,
"grad_norm": 21.68163299560547,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.2645799219608307,
"logits/rejected": -0.2657017111778259,
"logps/chosen": -236.22317504882812,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -386.2069396972656,
"loss": 1.1509,
"margin_dpo/margin_mean": 119.79930114746094,
"margin_dpo/margin_std": 204.67282104492188,
"step": 223
},
{
"KL/chosen_KL_mean": -170.26473999023438,
"KL/mean": -239.272216796875,
"KL/rejected_KL_mean": -308.27972412109375,
"KL/std": 185.91464233398438,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.002813777420669794,
"fcm_dpo/delta": 0.0016366764903068542,
"fcm_dpo/margin": 138.0149383544922,
"fcm_dpo/q_t": 0.41247671842575073,
"grad_norm": 31.99540901184082,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.27732497453689575,
"logits/rejected": -0.26288285851478577,
"logps/chosen": -218.86013793945312,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -385.39617919921875,
"loss": 1.1282,
"margin_dpo/margin_mean": 138.0149383544922,
"margin_dpo/margin_std": 214.30078125,
"step": 224
},
{
"KL/chosen_KL_mean": -199.21405029296875,
"KL/mean": -281.2215576171875,
"KL/rejected_KL_mean": -363.2290954589844,
"KL/std": 178.91641235351562,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.0026704357005655766,
"fcm_dpo/delta": -0.07286377251148224,
"fcm_dpo/margin": 164.01502990722656,
"fcm_dpo/q_t": 0.3952982723712921,
"grad_norm": 19.550594329833984,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.24977388978004456,
"logits/rejected": -0.24200385808944702,
"logps/chosen": -257.21453857421875,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -463.13201904296875,
"loss": 1.0792,
"margin_dpo/margin_mean": 164.01502990722656,
"margin_dpo/margin_std": 227.45672607421875,
"step": 225
},
{
"KL/chosen_KL_mean": -171.4937286376953,
"KL/mean": -237.76144409179688,
"KL/rejected_KL_mean": -304.0291748046875,
"KL/std": 178.0089111328125,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.0027313155587762594,
"fcm_dpo/delta": 0.056763019412755966,
"fcm_dpo/margin": 132.5354461669922,
"fcm_dpo/q_t": 0.42256051301956177,
"grad_norm": 24.24056053161621,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.32699283957481384,
"logits/rejected": -0.29330500960350037,
"logps/chosen": -230.39251708984375,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -382.7169189453125,
"loss": 1.139,
"margin_dpo/margin_mean": 132.53546142578125,
"margin_dpo/margin_std": 209.61676025390625,
"step": 226
},
{
"KL/chosen_KL_mean": -197.4924774169922,
"KL/mean": -280.9031677246094,
"KL/rejected_KL_mean": -364.3138427734375,
"KL/std": 201.27569580078125,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.002555706538259983,
"fcm_dpo/delta": -0.05992633104324341,
"fcm_dpo/margin": 166.82138061523438,
"fcm_dpo/q_t": 0.3967394530773163,
"grad_norm": 19.361919403076172,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.2924739718437195,
"logits/rejected": -0.2799687385559082,
"logps/chosen": -256.56463623046875,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -463.7261962890625,
"loss": 1.072,
"margin_dpo/margin_mean": 166.82138061523438,
"margin_dpo/margin_std": 210.5462646484375,
"step": 227
},
{
"KL/chosen_KL_mean": -204.37904357910156,
"KL/mean": -260.72320556640625,
"KL/rejected_KL_mean": -317.0673828125,
"KL/std": 166.0177001953125,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.0030394818168133497,
"fcm_dpo/delta": 0.08101288229227066,
"fcm_dpo/margin": 112.68833923339844,
"fcm_dpo/q_t": 0.427123486995697,
"grad_norm": 26.379526138305664,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.28412914276123047,
"logits/rejected": -0.26933881640434265,
"logps/chosen": -270.27032470703125,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -408.1161193847656,
"loss": 1.1678,
"margin_dpo/margin_mean": 112.68833923339844,
"margin_dpo/margin_std": 209.85467529296875,
"step": 228
},
{
"KL/chosen_KL_mean": -214.5729522705078,
"KL/mean": -271.60009765625,
"KL/rejected_KL_mean": -328.62725830078125,
"KL/std": 190.02706909179688,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.0032839591149240732,
"fcm_dpo/delta": 0.04005454480648041,
"fcm_dpo/margin": 114.0543212890625,
"fcm_dpo/q_t": 0.4181785583496094,
"grad_norm": 38.73786544799805,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.3614084720611572,
"logits/rejected": -0.3334274888038635,
"logps/chosen": -285.2793273925781,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -413.1546630859375,
"loss": 1.1432,
"margin_dpo/margin_mean": 114.0543212890625,
"margin_dpo/margin_std": 198.13516235351562,
"step": 229
},
{
"KL/chosen_KL_mean": -155.9505157470703,
"KL/mean": -249.08929443359375,
"KL/rejected_KL_mean": -342.22808837890625,
"KL/std": 169.89035034179688,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.0025410507805645466,
"fcm_dpo/delta": -0.15831325948238373,
"fcm_dpo/margin": 186.27757263183594,
"fcm_dpo/q_t": 0.3732534348964691,
"grad_norm": 28.544626235961914,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.25178319215774536,
"logits/rejected": -0.25868701934814453,
"logps/chosen": -195.23251342773438,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -427.8499755859375,
"loss": 1.0197,
"margin_dpo/margin_mean": 186.27755737304688,
"margin_dpo/margin_std": 184.69554138183594,
"step": 230
},
{
"KL/chosen_KL_mean": -179.1609344482422,
"KL/mean": -240.4979705810547,
"KL/rejected_KL_mean": -301.8349914550781,
"KL/std": 151.0618896484375,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.0027899318374693394,
"fcm_dpo/delta": 0.0851934552192688,
"fcm_dpo/margin": 122.67406463623047,
"fcm_dpo/q_t": 0.427597314119339,
"grad_norm": 24.108116149902344,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.3525630831718445,
"logits/rejected": -0.32520967721939087,
"logps/chosen": -242.4373779296875,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -375.9588928222656,
"loss": 1.1322,
"margin_dpo/margin_mean": 122.67406463623047,
"margin_dpo/margin_std": 179.02493286132812,
"step": 231
},
{
"KL/chosen_KL_mean": -216.9395294189453,
"KL/mean": -265.2451171875,
"KL/rejected_KL_mean": -313.55072021484375,
"KL/std": 175.61700439453125,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.0033727658446878195,
"fcm_dpo/delta": 0.10968147218227386,
"fcm_dpo/margin": 96.61114501953125,
"fcm_dpo/q_t": 0.435283899307251,
"grad_norm": 27.411245346069336,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.3760000467300415,
"logits/rejected": -0.35748612880706787,
"logps/chosen": -287.68829345703125,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -397.52777099609375,
"loss": 1.1718,
"margin_dpo/margin_mean": 96.61115264892578,
"margin_dpo/margin_std": 181.7001495361328,
"step": 232
},
{
"KL/chosen_KL_mean": -181.86721801757812,
"KL/mean": -262.99560546875,
"KL/rejected_KL_mean": -344.12396240234375,
"KL/std": 183.73455810546875,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.0029199281707406044,
"fcm_dpo/delta": -0.14302438497543335,
"fcm_dpo/margin": 162.25674438476562,
"fcm_dpo/q_t": 0.38662809133529663,
"grad_norm": 21.30043601989746,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.3732132017612457,
"logits/rejected": -0.37482750415802,
"logps/chosen": -236.75015258789062,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -451.6040344238281,
"loss": 1.0754,
"margin_dpo/margin_mean": 162.25674438476562,
"margin_dpo/margin_std": 236.0518035888672,
"step": 233
},
{
"KL/chosen_KL_mean": -163.74453735351562,
"KL/mean": -245.33642578125,
"KL/rejected_KL_mean": -326.9283447265625,
"KL/std": 157.0659942626953,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.0024456209503114223,
"fcm_dpo/delta": -0.015596166253089905,
"fcm_dpo/margin": 163.18380737304688,
"fcm_dpo/q_t": 0.403796911239624,
"grad_norm": 22.082944869995117,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.36095279455184937,
"logits/rejected": -0.3873627483844757,
"logps/chosen": -207.83905029296875,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -426.93499755859375,
"loss": 1.0777,
"margin_dpo/margin_mean": 163.18380737304688,
"margin_dpo/margin_std": 176.453857421875,
"step": 234
},
{
"KL/chosen_KL_mean": -201.49075317382812,
"KL/mean": -251.1799774169922,
"KL/rejected_KL_mean": -300.86920166015625,
"KL/std": 150.57394409179688,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.002739574294537306,
"fcm_dpo/delta": 0.046936918050050735,
"fcm_dpo/margin": 99.37845611572266,
"fcm_dpo/q_t": 0.43904921412467957,
"grad_norm": 21.031633377075195,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.38550442457199097,
"logits/rejected": -0.3654111921787262,
"logps/chosen": -263.7286682128906,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -391.2642822265625,
"loss": 1.1915,
"margin_dpo/margin_mean": 99.37844848632812,
"margin_dpo/margin_std": 185.09054565429688,
"step": 235
},
{
"KL/chosen_KL_mean": -145.79144287109375,
"KL/mean": -228.65101623535156,
"KL/rejected_KL_mean": -311.5105895996094,
"KL/std": 160.1380157470703,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.0026162322610616684,
"fcm_dpo/delta": -0.06015552580356598,
"fcm_dpo/margin": 165.71914672851562,
"fcm_dpo/q_t": 0.3923841118812561,
"grad_norm": 37.16407775878906,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.3568821847438812,
"logits/rejected": -0.3578595221042633,
"logps/chosen": -195.1328125,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -415.022216796875,
"loss": 1.0388,
"margin_dpo/margin_mean": 165.71914672851562,
"margin_dpo/margin_std": 155.69921875,
"step": 236
},
{
"KL/chosen_KL_mean": -203.95053100585938,
"KL/mean": -268.9967041015625,
"KL/rejected_KL_mean": -334.04290771484375,
"KL/std": 166.29592895507812,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.0028002336621284485,
"fcm_dpo/delta": 0.05379777029156685,
"fcm_dpo/margin": 130.09239196777344,
"fcm_dpo/q_t": 0.4198486804962158,
"grad_norm": 27.22985076904297,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.32136499881744385,
"logits/rejected": -0.3153060972690582,
"logps/chosen": -258.11865234375,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -428.82330322265625,
"loss": 1.1202,
"margin_dpo/margin_mean": 130.09239196777344,
"margin_dpo/margin_std": 190.1111602783203,
"step": 237
},
{
"KL/chosen_KL_mean": -188.8069610595703,
"KL/mean": -254.78656005859375,
"KL/rejected_KL_mean": -320.7662048339844,
"KL/std": 168.50390625,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.0028396020643413067,
"fcm_dpo/delta": 0.035806380212306976,
"fcm_dpo/margin": 131.95924377441406,
"fcm_dpo/q_t": 0.418282687664032,
"grad_norm": 23.320514678955078,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.31921494007110596,
"logits/rejected": -0.33235523104667664,
"logps/chosen": -242.78009033203125,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -410.18414306640625,
"loss": 1.116,
"margin_dpo/margin_mean": 131.95924377441406,
"margin_dpo/margin_std": 188.6099395751953,
"step": 238
},
{
"KL/chosen_KL_mean": -209.04046630859375,
"KL/mean": -283.45135498046875,
"KL/rejected_KL_mean": -357.8622741699219,
"KL/std": 165.2271728515625,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.0027087791822850704,
"fcm_dpo/delta": -0.025651507079601288,
"fcm_dpo/margin": 148.8218231201172,
"fcm_dpo/q_t": 0.4025055170059204,
"grad_norm": 26.025190353393555,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3110630214214325,
"logits/rejected": -0.3010329604148865,
"logps/chosen": -267.1383056640625,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -451.4552307128906,
"loss": 1.0886,
"margin_dpo/margin_mean": 148.8218231201172,
"margin_dpo/margin_std": 182.97344970703125,
"step": 239
},
{
"KL/chosen_KL_mean": -216.27822875976562,
"KL/mean": -275.22637939453125,
"KL/rejected_KL_mean": -334.17449951171875,
"KL/std": 174.17791748046875,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.003052940359339118,
"fcm_dpo/delta": 0.06172945722937584,
"fcm_dpo/margin": 117.89629364013672,
"fcm_dpo/q_t": 0.42239585518836975,
"grad_norm": 30.3389949798584,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.34589529037475586,
"logits/rejected": -0.32597455382347107,
"logps/chosen": -276.8927001953125,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -408.29302978515625,
"loss": 1.1595,
"margin_dpo/margin_mean": 117.89629364013672,
"margin_dpo/margin_std": 205.7227020263672,
"step": 240
},
{
"KL/chosen_KL_mean": -189.63717651367188,
"KL/mean": -283.8388366699219,
"KL/rejected_KL_mean": -378.04052734375,
"KL/std": 193.572021484375,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.002562090754508972,
"fcm_dpo/delta": -0.16558168828487396,
"fcm_dpo/margin": 188.4033203125,
"fcm_dpo/q_t": 0.3756968379020691,
"grad_norm": 19.404237747192383,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.3840920329093933,
"logits/rejected": -0.36620739102363586,
"logps/chosen": -255.72824096679688,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -466.1014099121094,
"loss": 1.033,
"margin_dpo/margin_mean": 188.4033203125,
"margin_dpo/margin_std": 218.57650756835938,
"step": 241
},
{
"KL/chosen_KL_mean": -209.334716796875,
"KL/mean": -275.63055419921875,
"KL/rejected_KL_mean": -341.9263610839844,
"KL/std": 168.1392364501953,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.0026945569552481174,
"fcm_dpo/delta": 0.052527863532304764,
"fcm_dpo/margin": 132.59165954589844,
"fcm_dpo/q_t": 0.4214203357696533,
"grad_norm": 27.632558822631836,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.3013432025909424,
"logits/rejected": -0.27941906452178955,
"logps/chosen": -277.1986389160156,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -425.2867126464844,
"loss": 1.1318,
"margin_dpo/margin_mean": 132.59165954589844,
"margin_dpo/margin_std": 201.74139404296875,
"step": 242
},
{
"KL/chosen_KL_mean": -208.54391479492188,
"KL/mean": -289.72686767578125,
"KL/rejected_KL_mean": -370.9097900390625,
"KL/std": 181.60488891601562,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.0025174664333462715,
"fcm_dpo/delta": -0.015545245260000229,
"fcm_dpo/margin": 162.36587524414062,
"fcm_dpo/q_t": 0.4042058289051056,
"grad_norm": 18.131187438964844,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.2907285690307617,
"logits/rejected": -0.26282238960266113,
"logps/chosen": -271.628173828125,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -447.24542236328125,
"loss": 1.0785,
"margin_dpo/margin_mean": 162.36585998535156,
"margin_dpo/margin_std": 201.73919677734375,
"step": 243
},
{
"KL/chosen_KL_mean": -176.4449462890625,
"KL/mean": -262.936767578125,
"KL/rejected_KL_mean": -349.4285888671875,
"KL/std": 177.73191833496094,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.002386314095929265,
"fcm_dpo/delta": -0.024043917655944824,
"fcm_dpo/margin": 172.983642578125,
"fcm_dpo/q_t": 0.4017675518989563,
"grad_norm": 22.16912841796875,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.37594205141067505,
"logits/rejected": -0.3709765076637268,
"logps/chosen": -237.58563232421875,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -444.3205261230469,
"loss": 1.061,
"margin_dpo/margin_mean": 172.983642578125,
"margin_dpo/margin_std": 182.35865783691406,
"step": 244
},
{
"KL/chosen_KL_mean": -210.67608642578125,
"KL/mean": -275.7692565917969,
"KL/rejected_KL_mean": -340.8624267578125,
"KL/std": 175.44842529296875,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.002276795916259289,
"fcm_dpo/delta": -0.028318271040916443,
"fcm_dpo/margin": 130.18634033203125,
"fcm_dpo/q_t": 0.42754846811294556,
"grad_norm": 21.60332489013672,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.38238322734832764,
"logits/rejected": -0.35831815004348755,
"logps/chosen": -277.9383544921875,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -428.50250244140625,
"loss": 1.1617,
"margin_dpo/margin_mean": 130.1863250732422,
"margin_dpo/margin_std": 192.90383911132812,
"step": 245
},
{
"KL/chosen_KL_mean": -203.3482666015625,
"KL/mean": -271.3511657714844,
"KL/rejected_KL_mean": -339.3540344238281,
"KL/std": 193.9945068359375,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.002540751826018095,
"fcm_dpo/delta": 0.08369514346122742,
"fcm_dpo/margin": 136.00575256347656,
"fcm_dpo/q_t": 0.42736515402793884,
"grad_norm": 29.36928939819336,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.43162864446640015,
"logits/rejected": -0.39635908603668213,
"logps/chosen": -270.04522705078125,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -423.70037841796875,
"loss": 1.139,
"margin_dpo/margin_mean": 136.00576782226562,
"margin_dpo/margin_std": 211.0897216796875,
"step": 246
},
{
"KL/chosen_KL_mean": -238.23548889160156,
"KL/mean": -336.6441345214844,
"KL/rejected_KL_mean": -435.05279541015625,
"KL/std": 201.9400634765625,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.002273306716233492,
"fcm_dpo/delta": -0.09434907883405685,
"fcm_dpo/margin": 196.8173065185547,
"fcm_dpo/q_t": 0.3884933590888977,
"grad_norm": 24.273239135742188,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.35952645540237427,
"logits/rejected": -0.35816654562950134,
"logps/chosen": -294.8408203125,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -541.3460693359375,
"loss": 1.0487,
"margin_dpo/margin_mean": 196.81732177734375,
"margin_dpo/margin_std": 221.4100799560547,
"step": 247
},
{
"KL/chosen_KL_mean": -203.11044311523438,
"KL/mean": -287.45843505859375,
"KL/rejected_KL_mean": -371.806396484375,
"KL/std": 163.28530883789062,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.002244535367935896,
"fcm_dpo/delta": 0.031689293682575226,
"fcm_dpo/margin": 168.69598388671875,
"fcm_dpo/q_t": 0.4118279814720154,
"grad_norm": 20.322803497314453,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.37604576349258423,
"logits/rejected": -0.39418306946754456,
"logps/chosen": -247.15365600585938,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -463.66326904296875,
"loss": 1.0792,
"margin_dpo/margin_mean": 168.69598388671875,
"margin_dpo/margin_std": 161.9207763671875,
"step": 248
},
{
"KL/chosen_KL_mean": -261.7310791015625,
"KL/mean": -307.686279296875,
"KL/rejected_KL_mean": -353.6415100097656,
"KL/std": 179.0096435546875,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.002784975804388523,
"fcm_dpo/delta": 0.0796995759010315,
"fcm_dpo/margin": 91.9104232788086,
"fcm_dpo/q_t": 0.4464285671710968,
"grad_norm": 43.7970085144043,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.40871942043304443,
"logits/rejected": -0.38268932700157166,
"logps/chosen": -324.1734313964844,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -434.10955810546875,
"loss": 1.2311,
"margin_dpo/margin_mean": 91.9104232788086,
"margin_dpo/margin_std": 214.4239959716797,
"step": 249
},
{
"KL/chosen_KL_mean": -232.26882934570312,
"KL/mean": -313.94488525390625,
"KL/rejected_KL_mean": -395.6209716796875,
"KL/std": 176.32708740234375,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.002672810573130846,
"fcm_dpo/delta": -0.06773751974105835,
"fcm_dpo/margin": 163.3521270751953,
"fcm_dpo/q_t": 0.3925377130508423,
"grad_norm": 46.0150032043457,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.4073798656463623,
"logits/rejected": -0.3506365418434143,
"logps/chosen": -297.905517578125,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -469.4928283691406,
"loss": 1.059,
"margin_dpo/margin_mean": 163.3521270751953,
"margin_dpo/margin_std": 180.181884765625,
"step": 250
},
{
"KL/chosen_KL_mean": -246.01177978515625,
"KL/mean": -302.828857421875,
"KL/rejected_KL_mean": -359.64593505859375,
"KL/std": 187.79159545898438,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.0028411387465894222,
"fcm_dpo/delta": 0.11464589834213257,
"fcm_dpo/margin": 113.63416290283203,
"fcm_dpo/q_t": 0.43381422758102417,
"grad_norm": 41.136512756347656,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.37190985679626465,
"logits/rejected": -0.3395519554615021,
"logps/chosen": -303.1944885253906,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -437.30938720703125,
"loss": 1.1687,
"margin_dpo/margin_mean": 113.63416290283203,
"margin_dpo/margin_std": 201.274169921875,
"step": 251
},
{
"KL/chosen_KL_mean": -241.49786376953125,
"KL/mean": -317.1058654785156,
"KL/rejected_KL_mean": -392.7138671875,
"KL/std": 162.52325439453125,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.0027777920477092266,
"fcm_dpo/delta": -0.0407252199947834,
"fcm_dpo/margin": 151.21600341796875,
"fcm_dpo/q_t": 0.3957163393497467,
"grad_norm": 25.602100372314453,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.42717283964157104,
"logits/rejected": -0.38557177782058716,
"logps/chosen": -313.1835021972656,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -477.47186279296875,
"loss": 1.0596,
"margin_dpo/margin_mean": 151.21600341796875,
"margin_dpo/margin_std": 162.9154815673828,
"step": 252
},
{
"KL/chosen_KL_mean": -202.2740478515625,
"KL/mean": -278.83538818359375,
"KL/rejected_KL_mean": -355.396728515625,
"KL/std": 169.63818359375,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.0027958948630839586,
"fcm_dpo/delta": -0.05647484213113785,
"fcm_dpo/margin": 153.12266540527344,
"fcm_dpo/q_t": 0.3956605792045593,
"grad_norm": 26.383848190307617,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.42702776193618774,
"logits/rejected": -0.4189242720603943,
"logps/chosen": -271.4079895019531,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -454.0992736816406,
"loss": 1.0808,
"margin_dpo/margin_mean": 153.1226806640625,
"margin_dpo/margin_std": 188.74114990234375,
"step": 253
},
{
"KL/chosen_KL_mean": -190.6053924560547,
"KL/mean": -255.74807739257812,
"KL/rejected_KL_mean": -320.8907470703125,
"KL/std": 174.94302368164062,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.002762971445918083,
"fcm_dpo/delta": 0.06207642704248428,
"fcm_dpo/margin": 130.28536987304688,
"fcm_dpo/q_t": 0.4237990379333496,
"grad_norm": 26.932371139526367,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.40052559971809387,
"logits/rejected": -0.38795092701911926,
"logps/chosen": -244.76039123535156,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -411.1983947753906,
"loss": 1.1403,
"margin_dpo/margin_mean": 130.28536987304688,
"margin_dpo/margin_std": 217.71649169921875,
"step": 254
},
{
"KL/chosen_KL_mean": -196.3863983154297,
"KL/mean": -262.91351318359375,
"KL/rejected_KL_mean": -329.44061279296875,
"KL/std": 153.0009002685547,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.002854404039680958,
"fcm_dpo/delta": 0.02931365743279457,
"fcm_dpo/margin": 133.05422973632812,
"fcm_dpo/q_t": 0.41438183188438416,
"grad_norm": 25.421916961669922,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.35080671310424805,
"logits/rejected": -0.3392289876937866,
"logps/chosen": -253.528076171875,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -419.64923095703125,
"loss": 1.1183,
"margin_dpo/margin_mean": 133.05422973632812,
"margin_dpo/margin_std": 194.57473754882812,
"step": 255
},
{
"KL/chosen_KL_mean": -169.05984497070312,
"KL/mean": -243.46347045898438,
"KL/rejected_KL_mean": -317.8670959472656,
"KL/std": 165.87603759765625,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.002850246848538518,
"fcm_dpo/delta": -0.04200413078069687,
"fcm_dpo/margin": 148.8072509765625,
"fcm_dpo/q_t": 0.39909690618515015,
"grad_norm": 28.732080459594727,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.2951539158821106,
"logits/rejected": -0.3040773868560791,
"logps/chosen": -224.22332763671875,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -410.42999267578125,
"loss": 1.075,
"margin_dpo/margin_mean": 148.8072509765625,
"margin_dpo/margin_std": 190.7834014892578,
"step": 256
},
{
"KL/chosen_KL_mean": -165.35116577148438,
"KL/mean": -242.6610107421875,
"KL/rejected_KL_mean": -319.9708557128906,
"KL/std": 172.49417114257812,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.002569081261754036,
"fcm_dpo/delta": -0.008627064526081085,
"fcm_dpo/margin": 154.6196746826172,
"fcm_dpo/q_t": 0.40691322088241577,
"grad_norm": 21.039451599121094,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.33848869800567627,
"logits/rejected": -0.3249668478965759,
"logps/chosen": -214.77487182617188,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -399.5087890625,
"loss": 1.0937,
"margin_dpo/margin_mean": 154.6196746826172,
"margin_dpo/margin_std": 196.26806640625,
"step": 257
},
{
"KL/chosen_KL_mean": -220.045654296875,
"KL/mean": -296.9225158691406,
"KL/rejected_KL_mean": -373.79937744140625,
"KL/std": 184.12759399414062,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.0026925005950033665,
"fcm_dpo/delta": -0.024290982633829117,
"fcm_dpo/margin": 153.7537078857422,
"fcm_dpo/q_t": 0.4024388790130615,
"grad_norm": 26.345495223999023,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.3109281659126282,
"logits/rejected": -0.3115309476852417,
"logps/chosen": -279.4297790527344,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -469.78948974609375,
"loss": 1.0982,
"margin_dpo/margin_mean": 153.7537078857422,
"margin_dpo/margin_std": 222.58351135253906,
"step": 258
},
{
"KL/chosen_KL_mean": -209.99713134765625,
"KL/mean": -273.5628356933594,
"KL/rejected_KL_mean": -337.1285400390625,
"KL/std": 172.61416625976562,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.0028006834909319878,
"fcm_dpo/delta": 0.06836118549108505,
"fcm_dpo/margin": 127.13138580322266,
"fcm_dpo/q_t": 0.42346620559692383,
"grad_norm": 21.578977584838867,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.3506283462047577,
"logits/rejected": -0.3485710322856903,
"logps/chosen": -262.82550048828125,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -426.3201904296875,
"loss": 1.137,
"margin_dpo/margin_mean": 127.13138580322266,
"margin_dpo/margin_std": 197.51239013671875,
"step": 259
},
{
"KL/chosen_KL_mean": -210.90850830078125,
"KL/mean": -298.40771484375,
"KL/rejected_KL_mean": -385.90692138671875,
"KL/std": 181.40826416015625,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.002633176278322935,
"fcm_dpo/delta": -0.11144484579563141,
"fcm_dpo/margin": 174.99844360351562,
"fcm_dpo/q_t": 0.3861405551433563,
"grad_norm": 28.88026237487793,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.31534501910209656,
"logits/rejected": -0.33186644315719604,
"logps/chosen": -258.326171875,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -480.9967041015625,
"loss": 1.0402,
"margin_dpo/margin_mean": 174.99844360351562,
"margin_dpo/margin_std": 197.2706298828125,
"step": 260
},
{
"KL/chosen_KL_mean": -220.949462890625,
"KL/mean": -301.2371826171875,
"KL/rejected_KL_mean": -381.52496337890625,
"KL/std": 196.11915588378906,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.0024081161245703697,
"fcm_dpo/delta": 0.020973602309823036,
"fcm_dpo/margin": 160.57550048828125,
"fcm_dpo/q_t": 0.41286349296569824,
"grad_norm": 18.689228057861328,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.2601136565208435,
"logits/rejected": -0.2584311366081238,
"logps/chosen": -273.9808349609375,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -470.0399169921875,
"loss": 1.1028,
"margin_dpo/margin_mean": 160.57550048828125,
"margin_dpo/margin_std": 221.1414337158203,
"step": 261
},
{
"KL/chosen_KL_mean": -260.35552978515625,
"KL/mean": -325.7707214355469,
"KL/rejected_KL_mean": -391.1859130859375,
"KL/std": 179.41098022460938,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.002791064791381359,
"fcm_dpo/delta": 0.04618440195918083,
"fcm_dpo/margin": 130.8303680419922,
"fcm_dpo/q_t": 0.4176996350288391,
"grad_norm": 33.41985321044922,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.29340440034866333,
"logits/rejected": -0.26309913396835327,
"logps/chosen": -319.9756774902344,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -477.60443115234375,
"loss": 1.1146,
"margin_dpo/margin_mean": 130.83035278320312,
"margin_dpo/margin_std": 180.4952392578125,
"step": 262
},
{
"KL/chosen_KL_mean": -233.8137969970703,
"KL/mean": -327.53448486328125,
"KL/rejected_KL_mean": -421.2552185058594,
"KL/std": 211.76744079589844,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.0023032082244753838,
"fcm_dpo/delta": -0.07411213964223862,
"fcm_dpo/margin": 187.44140625,
"fcm_dpo/q_t": 0.39327767491340637,
"grad_norm": 18.634008407592773,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.3248862028121948,
"logits/rejected": -0.30047309398651123,
"logps/chosen": -293.2347412109375,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -518.1124267578125,
"loss": 1.0674,
"margin_dpo/margin_mean": 187.44142150878906,
"margin_dpo/margin_std": 231.066162109375,
"step": 263
},
{
"KL/chosen_KL_mean": -241.53146362304688,
"KL/mean": -323.41162109375,
"KL/rejected_KL_mean": -405.291748046875,
"KL/std": 194.24461364746094,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.002298943232744932,
"fcm_dpo/delta": 0.028040196746587753,
"fcm_dpo/margin": 163.76028442382812,
"fcm_dpo/q_t": 0.4143482446670532,
"grad_norm": 20.764347076416016,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.3569568693637848,
"logits/rejected": -0.34849122166633606,
"logps/chosen": -304.2535400390625,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -499.14794921875,
"loss": 1.1174,
"margin_dpo/margin_mean": 163.76028442382812,
"margin_dpo/margin_std": 224.1405487060547,
"step": 264
},
{
"KL/chosen_KL_mean": -264.88433837890625,
"KL/mean": -340.1336669921875,
"KL/rejected_KL_mean": -415.3829650878906,
"KL/std": 215.2719268798828,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.0025752345100045204,
"fcm_dpo/delta": 0.020102664828300476,
"fcm_dpo/margin": 150.49859619140625,
"fcm_dpo/q_t": 0.4153306484222412,
"grad_norm": 23.874309539794922,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.31058114767074585,
"logits/rejected": -0.2928779423236847,
"logps/chosen": -326.8558349609375,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -503.403564453125,
"loss": 1.1349,
"margin_dpo/margin_mean": 150.49859619140625,
"margin_dpo/margin_std": 257.16748046875,
"step": 265
},
{
"KL/chosen_KL_mean": -264.1705322265625,
"KL/mean": -323.71405029296875,
"KL/rejected_KL_mean": -383.25750732421875,
"KL/std": 179.75607299804688,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.0029307270888239145,
"fcm_dpo/delta": 0.07655003666877747,
"fcm_dpo/margin": 119.08696746826172,
"fcm_dpo/q_t": 0.42644163966178894,
"grad_norm": 46.22469711303711,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3636462092399597,
"logits/rejected": -0.3230510354042053,
"logps/chosen": -331.27020263671875,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -451.228759765625,
"loss": 1.1556,
"margin_dpo/margin_mean": 119.08696746826172,
"margin_dpo/margin_std": 211.2481689453125,
"step": 266
},
{
"KL/chosen_KL_mean": -236.5076904296875,
"KL/mean": -311.24676513671875,
"KL/rejected_KL_mean": -385.98590087890625,
"KL/std": 193.96978759765625,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.00275492575019598,
"fcm_dpo/delta": -0.029174409806728363,
"fcm_dpo/margin": 149.47821044921875,
"fcm_dpo/q_t": 0.4047339856624603,
"grad_norm": 41.7883415222168,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.3469699025154114,
"logits/rejected": -0.3102799355983734,
"logps/chosen": -305.47845458984375,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -476.1543273925781,
"loss": 1.1054,
"margin_dpo/margin_mean": 149.47821044921875,
"margin_dpo/margin_std": 224.31707763671875,
"step": 267
},
{
"KL/chosen_KL_mean": -240.66119384765625,
"KL/mean": -317.0205078125,
"KL/rejected_KL_mean": -393.37982177734375,
"KL/std": 185.6231689453125,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.002668085042387247,
"fcm_dpo/delta": -0.01804983615875244,
"fcm_dpo/margin": 152.71859741210938,
"fcm_dpo/q_t": 0.40581750869750977,
"grad_norm": 28.408159255981445,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.35881197452545166,
"logits/rejected": -0.36506927013397217,
"logps/chosen": -296.5615234375,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -495.0274353027344,
"loss": 1.1018,
"margin_dpo/margin_mean": 152.71859741210938,
"margin_dpo/margin_std": 222.39694213867188,
"step": 268
},
{
"KL/chosen_KL_mean": -257.70648193359375,
"KL/mean": -346.0221862792969,
"KL/rejected_KL_mean": -434.337890625,
"KL/std": 184.19381713867188,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.00247764540836215,
"fcm_dpo/delta": -0.06811105459928513,
"fcm_dpo/margin": 176.63140869140625,
"fcm_dpo/q_t": 0.3938090205192566,
"grad_norm": 22.41498374938965,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.38506025075912476,
"logits/rejected": -0.3642171025276184,
"logps/chosen": -327.74603271484375,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -541.687255859375,
"loss": 1.0757,
"margin_dpo/margin_mean": 176.6313934326172,
"margin_dpo/margin_std": 235.69393920898438,
"step": 269
},
{
"KL/chosen_KL_mean": -222.79367065429688,
"KL/mean": -289.4638671875,
"KL/rejected_KL_mean": -356.134033203125,
"KL/std": 165.93325805664062,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.0027032734360545874,
"fcm_dpo/delta": 0.05463853478431702,
"fcm_dpo/margin": 133.3403778076172,
"fcm_dpo/q_t": 0.4207903742790222,
"grad_norm": 25.0926570892334,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.43856489658355713,
"logits/rejected": -0.4321529269218445,
"logps/chosen": -292.3271484375,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -466.06268310546875,
"loss": 1.1388,
"margin_dpo/margin_mean": 133.3403778076172,
"margin_dpo/margin_std": 216.5740509033203,
"step": 270
},
{
"KL/chosen_KL_mean": -203.2021942138672,
"KL/mean": -287.8795166015625,
"KL/rejected_KL_mean": -372.5568542480469,
"KL/std": 168.32081604003906,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.0024533187970519066,
"fcm_dpo/delta": -0.03273054212331772,
"fcm_dpo/margin": 169.3546905517578,
"fcm_dpo/q_t": 0.4006693959236145,
"grad_norm": 23.280786514282227,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.3510928452014923,
"logits/rejected": -0.33294203877449036,
"logps/chosen": -259.9667663574219,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -465.0706787109375,
"loss": 1.0663,
"margin_dpo/margin_mean": 169.3546905517578,
"margin_dpo/margin_std": 192.665283203125,
"step": 271
},
{
"KL/chosen_KL_mean": -191.9091339111328,
"KL/mean": -292.7960205078125,
"KL/rejected_KL_mean": -393.68292236328125,
"KL/std": 186.9519805908203,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.002250020857900381,
"fcm_dpo/delta": -0.09791300445795059,
"fcm_dpo/margin": 201.77377319335938,
"fcm_dpo/q_t": 0.38448095321655273,
"grad_norm": 28.866960525512695,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.29558974504470825,
"logits/rejected": -0.3049769401550293,
"logps/chosen": -241.4062957763672,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -499.2257080078125,
"loss": 1.0321,
"margin_dpo/margin_mean": 201.7737579345703,
"margin_dpo/margin_std": 194.00624084472656,
"step": 272
},
{
"KL/chosen_KL_mean": -226.32107543945312,
"KL/mean": -325.10498046875,
"KL/rejected_KL_mean": -423.8888244628906,
"KL/std": 189.16091918945312,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.0020193979144096375,
"fcm_dpo/delta": 0.0005125030875205994,
"fcm_dpo/margin": 197.56777954101562,
"fcm_dpo/q_t": 0.4059392511844635,
"grad_norm": 17.918615341186523,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.31307101249694824,
"logits/rejected": -0.29330283403396606,
"logps/chosen": -289.2964782714844,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -516.3873901367188,
"loss": 1.0715,
"margin_dpo/margin_mean": 197.56777954101562,
"margin_dpo/margin_std": 214.41448974609375,
"step": 273
},
{
"KL/chosen_KL_mean": -267.08245849609375,
"KL/mean": -342.52392578125,
"KL/rejected_KL_mean": -417.9654541015625,
"KL/std": 174.84378051757812,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.002351115457713604,
"fcm_dpo/delta": 0.06252269446849823,
"fcm_dpo/margin": 150.8830108642578,
"fcm_dpo/q_t": 0.420854389667511,
"grad_norm": 22.64203453063965,
"learning_rate": 3.75e-07,
"logits/chosen": -0.23737865686416626,
"logits/rejected": -0.22025717794895172,
"logps/chosen": -322.7501525878906,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -495.29852294921875,
"loss": 1.1162,
"margin_dpo/margin_mean": 150.88299560546875,
"margin_dpo/margin_std": 200.55410766601562,
"step": 274
},
{
"KL/chosen_KL_mean": -211.66796875,
"KL/mean": -292.65130615234375,
"KL/rejected_KL_mean": -373.6346435546875,
"KL/std": 179.12078857421875,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.002439269330352545,
"fcm_dpo/delta": 0.004620321094989777,
"fcm_dpo/margin": 161.9666748046875,
"fcm_dpo/q_t": 0.40825700759887695,
"grad_norm": 23.12116241455078,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.3093240261077881,
"logits/rejected": -0.316268652677536,
"logps/chosen": -260.2626953125,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -466.9383544921875,
"loss": 1.0894,
"margin_dpo/margin_mean": 161.9666748046875,
"margin_dpo/margin_std": 203.50302124023438,
"step": 275
},
{
"KL/chosen_KL_mean": -241.7791290283203,
"KL/mean": -322.0965576171875,
"KL/rejected_KL_mean": -402.4139404296875,
"KL/std": 190.34759521484375,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.0024003759026527405,
"fcm_dpo/delta": 0.02319333702325821,
"fcm_dpo/margin": 160.63482666015625,
"fcm_dpo/q_t": 0.4141028821468353,
"grad_norm": 29.356189727783203,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.23649513721466064,
"logits/rejected": -0.20997360348701477,
"logps/chosen": -298.3565368652344,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -472.77960205078125,
"loss": 1.1107,
"margin_dpo/margin_mean": 160.63482666015625,
"margin_dpo/margin_std": 233.87583923339844,
"step": 276
},
{
"KL/chosen_KL_mean": -263.56640625,
"KL/mean": -349.49713134765625,
"KL/rejected_KL_mean": -435.42779541015625,
"KL/std": 192.81695556640625,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.002432500245049596,
"fcm_dpo/delta": -0.032088592648506165,
"fcm_dpo/margin": 171.86138916015625,
"fcm_dpo/q_t": 0.4016081690788269,
"grad_norm": 26.00898551940918,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.26711416244506836,
"logits/rejected": -0.24979907274246216,
"logps/chosen": -319.8379821777344,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -528.30908203125,
"loss": 1.0858,
"margin_dpo/margin_mean": 171.86138916015625,
"margin_dpo/margin_std": 228.77777099609375,
"step": 277
},
{
"KL/chosen_KL_mean": -235.44161987304688,
"KL/mean": -336.4327392578125,
"KL/rejected_KL_mean": -437.42388916015625,
"KL/std": 201.9891357421875,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.002094062976539135,
"fcm_dpo/delta": -0.045741938054561615,
"fcm_dpo/margin": 201.98226928710938,
"fcm_dpo/q_t": 0.3976041078567505,
"grad_norm": 22.05190658569336,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.24980834126472473,
"logits/rejected": -0.25416260957717896,
"logps/chosen": -288.383544921875,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -528.677490234375,
"loss": 1.0605,
"margin_dpo/margin_mean": 201.9822998046875,
"margin_dpo/margin_std": 226.97244262695312,
"step": 278
},
{
"KL/chosen_KL_mean": -271.568115234375,
"KL/mean": -371.5149841308594,
"KL/rejected_KL_mean": -471.46185302734375,
"KL/std": 216.6588897705078,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.0019927441608160734,
"fcm_dpo/delta": -0.005218621343374252,
"fcm_dpo/margin": 199.89370727539062,
"fcm_dpo/q_t": 0.40749794244766235,
"grad_norm": 26.85765838623047,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.19662827253341675,
"logits/rejected": -0.19574469327926636,
"logps/chosen": -320.20947265625,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -559.3132934570312,
"loss": 1.0963,
"margin_dpo/margin_mean": 199.89370727539062,
"margin_dpo/margin_std": 265.7958984375,
"step": 279
},
{
"KL/chosen_KL_mean": -264.79876708984375,
"KL/mean": -365.7987060546875,
"KL/rejected_KL_mean": -466.79864501953125,
"KL/std": 195.9139862060547,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.0020362103823572397,
"fcm_dpo/delta": -0.01925138756632805,
"fcm_dpo/margin": 201.9998779296875,
"fcm_dpo/q_t": 0.4014127254486084,
"grad_norm": 20.498315811157227,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.24768587946891785,
"logits/rejected": -0.24966652691364288,
"logps/chosen": -323.59588623046875,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -565.41748046875,
"loss": 1.0628,
"margin_dpo/margin_mean": 201.99989318847656,
"margin_dpo/margin_std": 216.71530151367188,
"step": 280
},
{
"KL/chosen_KL_mean": -247.58489990234375,
"KL/mean": -335.40484619140625,
"KL/rejected_KL_mean": -423.2247619628906,
"KL/std": 196.88302612304688,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.0021747334394603968,
"fcm_dpo/delta": 0.022978195920586586,
"fcm_dpo/margin": 175.6398162841797,
"fcm_dpo/q_t": 0.4103030562400818,
"grad_norm": 21.528810501098633,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.3240095376968384,
"logits/rejected": -0.3004445433616638,
"logps/chosen": -303.07342529296875,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -504.1073303222656,
"loss": 1.0948,
"margin_dpo/margin_mean": 175.6398162841797,
"margin_dpo/margin_std": 213.6923828125,
"step": 281
},
{
"KL/chosen_KL_mean": -268.12432861328125,
"KL/mean": -337.4071044921875,
"KL/rejected_KL_mean": -406.68988037109375,
"KL/std": 196.039794921875,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.0021595642901957035,
"fcm_dpo/delta": 0.02335960976779461,
"fcm_dpo/margin": 138.56552124023438,
"fcm_dpo/q_t": 0.4331875145435333,
"grad_norm": 21.117961883544922,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.3865717053413391,
"logits/rejected": -0.3700195550918579,
"logps/chosen": -341.1944885253906,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -502.0408935546875,
"loss": 1.1627,
"margin_dpo/margin_mean": 138.56552124023438,
"margin_dpo/margin_std": 225.9492645263672,
"step": 282
},
{
"KL/chosen_KL_mean": -283.1618347167969,
"KL/mean": -388.3852233886719,
"KL/rejected_KL_mean": -493.608642578125,
"KL/std": 241.67697143554688,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0020841769874095917,
"fcm_dpo/delta": -0.06936129927635193,
"fcm_dpo/margin": 210.44679260253906,
"fcm_dpo/q_t": 0.3913659453392029,
"grad_norm": 29.183977127075195,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.33291661739349365,
"logits/rejected": -0.3372425436973572,
"logps/chosen": -345.0602722167969,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -590.59521484375,
"loss": 1.057,
"margin_dpo/margin_mean": 210.44679260253906,
"margin_dpo/margin_std": 231.2135009765625,
"step": 283
},
{
"KL/chosen_KL_mean": -270.32830810546875,
"KL/mean": -369.215087890625,
"KL/rejected_KL_mean": -468.1018371582031,
"KL/std": 219.54708862304688,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.0019610195886343718,
"fcm_dpo/delta": 0.01984664611518383,
"fcm_dpo/margin": 197.77349853515625,
"fcm_dpo/q_t": 0.4114644527435303,
"grad_norm": 25.379192352294922,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.37090370059013367,
"logits/rejected": -0.3533616065979004,
"logps/chosen": -328.76385498046875,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -561.5711059570312,
"loss": 1.0843,
"margin_dpo/margin_mean": 197.77349853515625,
"margin_dpo/margin_std": 232.99339294433594,
"step": 284
},
{
"KL/chosen_KL_mean": -313.6859130859375,
"KL/mean": -400.13983154296875,
"KL/rejected_KL_mean": -486.59375,
"KL/std": 192.8905029296875,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.002196455840021372,
"fcm_dpo/delta": 0.021015789359807968,
"fcm_dpo/margin": 172.90782165527344,
"fcm_dpo/q_t": 0.41028088331222534,
"grad_norm": 23.027700424194336,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.36354196071624756,
"logits/rejected": -0.3486790955066681,
"logps/chosen": -379.9181213378906,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -585.7205810546875,
"loss": 1.1035,
"margin_dpo/margin_mean": 172.9078369140625,
"margin_dpo/margin_std": 218.47084045410156,
"step": 285
},
{
"KL/chosen_KL_mean": -327.0200500488281,
"KL/mean": -422.5919189453125,
"KL/rejected_KL_mean": -518.163818359375,
"KL/std": 230.7489776611328,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.002110408851876855,
"fcm_dpo/delta": -0.006586743518710136,
"fcm_dpo/margin": 191.14376831054688,
"fcm_dpo/q_t": 0.4062705934047699,
"grad_norm": 23.25826644897461,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.35514867305755615,
"logits/rejected": -0.3393225073814392,
"logps/chosen": -399.9710693359375,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -606.7522583007812,
"loss": 1.0915,
"margin_dpo/margin_mean": 191.14376831054688,
"margin_dpo/margin_std": 254.77488708496094,
"step": 286
},
{
"KL/chosen_KL_mean": -300.4835510253906,
"KL/mean": -379.34942626953125,
"KL/rejected_KL_mean": -458.21527099609375,
"KL/std": 201.38296508789062,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.0022800066508352757,
"fcm_dpo/delta": 0.06143353134393692,
"fcm_dpo/margin": 157.731689453125,
"fcm_dpo/q_t": 0.42126035690307617,
"grad_norm": 27.160978317260742,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.30109989643096924,
"logits/rejected": -0.27528104186058044,
"logps/chosen": -362.02471923828125,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -535.9113159179688,
"loss": 1.1154,
"margin_dpo/margin_mean": 157.731689453125,
"margin_dpo/margin_std": 216.85557556152344,
"step": 287
},
{
"KL/chosen_KL_mean": -276.4866943359375,
"KL/mean": -374.0993347167969,
"KL/rejected_KL_mean": -471.71197509765625,
"KL/std": 193.3370361328125,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.002058174926787615,
"fcm_dpo/delta": -0.026749789714813232,
"fcm_dpo/margin": 195.2252655029297,
"fcm_dpo/q_t": 0.40044307708740234,
"grad_norm": 21.408998489379883,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.34782230854034424,
"logits/rejected": -0.3332597613334656,
"logps/chosen": -333.1479187011719,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -559.0476684570312,
"loss": 1.0755,
"margin_dpo/margin_mean": 195.2252655029297,
"margin_dpo/margin_std": 204.141357421875,
"step": 288
},
{
"KL/chosen_KL_mean": -259.8858337402344,
"KL/mean": -367.64276123046875,
"KL/rejected_KL_mean": -475.3996887207031,
"KL/std": 213.38723754882812,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.0020438190549612045,
"fcm_dpo/delta": -0.07187651097774506,
"fcm_dpo/margin": 215.51380920410156,
"fcm_dpo/q_t": 0.3916912078857422,
"grad_norm": 24.203519821166992,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.3048250675201416,
"logits/rejected": -0.3042747378349304,
"logps/chosen": -305.1162414550781,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -563.0423583984375,
"loss": 1.0505,
"margin_dpo/margin_mean": 215.51380920410156,
"margin_dpo/margin_std": 233.81417846679688,
"step": 289
},
{
"KL/chosen_KL_mean": -269.5323791503906,
"KL/mean": -379.81329345703125,
"KL/rejected_KL_mean": -490.09423828125,
"KL/std": 221.88650512695312,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.0018460073042660952,
"fcm_dpo/delta": -0.013119934126734734,
"fcm_dpo/margin": 220.56182861328125,
"fcm_dpo/q_t": 0.4045451879501343,
"grad_norm": 20.667232513427734,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.2957327961921692,
"logits/rejected": -0.31704530119895935,
"logps/chosen": -325.0038757324219,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -606.8027954101562,
"loss": 1.0762,
"margin_dpo/margin_mean": 220.56182861328125,
"margin_dpo/margin_std": 266.3393859863281,
"step": 290
},
{
"KL/chosen_KL_mean": -240.43844604492188,
"KL/mean": -341.17254638671875,
"KL/rejected_KL_mean": -441.9066467285156,
"KL/std": 196.33567810058594,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.001962371403351426,
"fcm_dpo/delta": 0.0012248605489730835,
"fcm_dpo/margin": 201.46820068359375,
"fcm_dpo/q_t": 0.40641385316848755,
"grad_norm": 25.23223114013672,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.3080219626426697,
"logits/rejected": -0.29609572887420654,
"logps/chosen": -303.69879150390625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -531.2037353515625,
"loss": 1.0851,
"margin_dpo/margin_mean": 201.46820068359375,
"margin_dpo/margin_std": 237.0089874267578,
"step": 291
},
{
"KL/chosen_KL_mean": -267.9251708984375,
"KL/mean": -372.0924987792969,
"KL/rejected_KL_mean": -476.25982666015625,
"KL/std": 229.24436950683594,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.0019257422536611557,
"fcm_dpo/delta": -0.005529198795557022,
"fcm_dpo/margin": 208.33460998535156,
"fcm_dpo/q_t": 0.40545937418937683,
"grad_norm": 21.263906478881836,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.3315708637237549,
"logits/rejected": -0.3213496208190918,
"logps/chosen": -321.8437194824219,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -566.22119140625,
"loss": 1.0907,
"margin_dpo/margin_mean": 208.33460998535156,
"margin_dpo/margin_std": 271.5143737792969,
"step": 292
},
{
"KL/chosen_KL_mean": -278.5457763671875,
"KL/mean": -362.8836975097656,
"KL/rejected_KL_mean": -447.22161865234375,
"KL/std": 229.81143188476562,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.002093075541779399,
"fcm_dpo/delta": 0.06820766627788544,
"fcm_dpo/margin": 168.67581176757812,
"fcm_dpo/q_t": 0.42393064498901367,
"grad_norm": 32.55642318725586,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.3256397843360901,
"logits/rejected": -0.3210792541503906,
"logps/chosen": -338.92181396484375,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -525.0740356445312,
"loss": 1.1334,
"margin_dpo/margin_mean": 168.67581176757812,
"margin_dpo/margin_std": 262.6167907714844,
"step": 293
},
{
"KL/chosen_KL_mean": -259.8635559082031,
"KL/mean": -352.86517333984375,
"KL/rejected_KL_mean": -445.8667907714844,
"KL/std": 204.66827392578125,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.002107615116983652,
"fcm_dpo/delta": 0.012143617495894432,
"fcm_dpo/margin": 186.0032196044922,
"fcm_dpo/q_t": 0.4118291139602661,
"grad_norm": 22.22089195251465,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.3044741451740265,
"logits/rejected": -0.311285138130188,
"logps/chosen": -307.95111083984375,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -527.7637939453125,
"loss": 1.1023,
"margin_dpo/margin_mean": 186.0032196044922,
"margin_dpo/margin_std": 260.2086181640625,
"step": 294
},
{
"KL/chosen_KL_mean": -320.80810546875,
"KL/mean": -423.1761474609375,
"KL/rejected_KL_mean": -525.5442504882812,
"KL/std": 238.36337280273438,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.002073909156024456,
"fcm_dpo/delta": -0.04264184087514877,
"fcm_dpo/margin": 204.73614501953125,
"fcm_dpo/q_t": 0.4011147618293762,
"grad_norm": 27.336824417114258,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.3063458800315857,
"logits/rejected": -0.31039565801620483,
"logps/chosen": -370.7327880859375,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -613.0005493164062,
"loss": 1.0927,
"margin_dpo/margin_mean": 204.73614501953125,
"margin_dpo/margin_std": 289.61798095703125,
"step": 295
},
{
"KL/chosen_KL_mean": -394.3350830078125,
"KL/mean": -464.76324462890625,
"KL/rejected_KL_mean": -535.19140625,
"KL/std": 229.95870971679688,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0020087361335754395,
"fcm_dpo/delta": 0.00424933061003685,
"fcm_dpo/margin": 140.85630798339844,
"fcm_dpo/q_t": 0.4354059398174286,
"grad_norm": 33.60810852050781,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.3529552221298218,
"logits/rejected": -0.3394961953163147,
"logps/chosen": -459.8262939453125,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -628.2804565429688,
"loss": 1.2109,
"margin_dpo/margin_mean": 140.85629272460938,
"margin_dpo/margin_std": 299.87567138671875,
"step": 296
},
{
"KL/chosen_KL_mean": -344.30731201171875,
"KL/mean": -423.664794921875,
"KL/rejected_KL_mean": -503.0222473144531,
"KL/std": 215.01690673828125,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.002223505638539791,
"fcm_dpo/delta": 0.07282247394323349,
"fcm_dpo/margin": 158.7149200439453,
"fcm_dpo/q_t": 0.4236125946044922,
"grad_norm": 42.23075485229492,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.3576158583164215,
"logits/rejected": -0.36280083656311035,
"logps/chosen": -400.7842712402344,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -598.1607666015625,
"loss": 1.1244,
"margin_dpo/margin_mean": 158.71493530273438,
"margin_dpo/margin_std": 226.28025817871094,
"step": 297
},
{
"KL/chosen_KL_mean": -358.29290771484375,
"KL/mean": -466.8505859375,
"KL/rejected_KL_mean": -575.408203125,
"KL/std": 281.4233093261719,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.0020931584294885397,
"fcm_dpo/delta": -0.0989355742931366,
"fcm_dpo/margin": 217.11532592773438,
"fcm_dpo/q_t": 0.39393287897109985,
"grad_norm": 29.66607093811035,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.3804464340209961,
"logits/rejected": -0.39784708619117737,
"logps/chosen": -425.61810302734375,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -692.0704345703125,
"loss": 1.0936,
"margin_dpo/margin_mean": 217.1153106689453,
"margin_dpo/margin_std": 334.7890930175781,
"step": 298
},
{
"KL/chosen_KL_mean": -290.83984375,
"KL/mean": -384.0028076171875,
"KL/rejected_KL_mean": -477.165771484375,
"KL/std": 211.8370361328125,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.0020763694774359465,
"fcm_dpo/delta": 0.006563708186149597,
"fcm_dpo/margin": 186.32595825195312,
"fcm_dpo/q_t": 0.41148841381073,
"grad_norm": 29.92995262145996,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.35604220628738403,
"logits/rejected": -0.3601064085960388,
"logps/chosen": -339.80194091796875,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -561.4940185546875,
"loss": 1.1168,
"margin_dpo/margin_mean": 186.32595825195312,
"margin_dpo/margin_std": 269.77178955078125,
"step": 299
},
{
"KL/chosen_KL_mean": -361.88922119140625,
"KL/mean": -479.45281982421875,
"KL/rejected_KL_mean": -597.016357421875,
"KL/std": 256.1278381347656,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.0018352132756263018,
"fcm_dpo/delta": -0.055538419634103775,
"fcm_dpo/margin": 235.1271514892578,
"fcm_dpo/q_t": 0.3963469862937927,
"grad_norm": 22.620500564575195,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.35153138637542725,
"logits/rejected": -0.3557186424732208,
"logps/chosen": -420.96295166015625,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -692.9827880859375,
"loss": 1.0815,
"margin_dpo/margin_mean": 235.12716674804688,
"margin_dpo/margin_std": 311.29791259765625,
"step": 300
},
{
"KL/chosen_KL_mean": -296.9462890625,
"KL/mean": -410.1385498046875,
"KL/rejected_KL_mean": -523.3307495117188,
"KL/std": 233.12229919433594,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.0017267554067075253,
"fcm_dpo/delta": 0.014057676307857037,
"fcm_dpo/margin": 226.38442993164062,
"fcm_dpo/q_t": 0.4108361601829529,
"grad_norm": 18.437192916870117,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.3524608612060547,
"logits/rejected": -0.3491095006465912,
"logps/chosen": -354.1956787109375,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -615.684326171875,
"loss": 1.0883,
"margin_dpo/margin_mean": 226.38442993164062,
"margin_dpo/margin_std": 280.14007568359375,
"step": 301
},
{
"KL/chosen_KL_mean": -269.1903076171875,
"KL/mean": -361.92718505859375,
"KL/rejected_KL_mean": -454.6640625,
"KL/std": 212.72869873046875,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.001909639802761376,
"fcm_dpo/delta": 0.07072426378726959,
"fcm_dpo/margin": 185.4737548828125,
"fcm_dpo/q_t": 0.42330536246299744,
"grad_norm": 20.071674346923828,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.34278440475463867,
"logits/rejected": -0.34728020429611206,
"logps/chosen": -320.3883056640625,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -551.8904418945312,
"loss": 1.1251,
"margin_dpo/margin_mean": 185.4737548828125,
"margin_dpo/margin_std": 256.4941101074219,
"step": 302
},
{
"KL/chosen_KL_mean": -279.39471435546875,
"KL/mean": -365.64605712890625,
"KL/rejected_KL_mean": -451.89739990234375,
"KL/std": 218.69338989257812,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.0021402090787887573,
"fcm_dpo/delta": 0.04843229800462723,
"fcm_dpo/margin": 172.50267028808594,
"fcm_dpo/q_t": 0.41948962211608887,
"grad_norm": 23.985990524291992,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.38367849588394165,
"logits/rejected": -0.3712496757507324,
"logps/chosen": -346.1086730957031,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -538.8428344726562,
"loss": 1.1289,
"margin_dpo/margin_mean": 172.50267028808594,
"margin_dpo/margin_std": 265.6961669921875,
"step": 303
},
{
"KL/chosen_KL_mean": -255.20101928710938,
"KL/mean": -347.8723449707031,
"KL/rejected_KL_mean": -440.54364013671875,
"KL/std": 201.79244995117188,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.0022568381391465664,
"fcm_dpo/delta": -0.03559745475649834,
"fcm_dpo/margin": 185.34263610839844,
"fcm_dpo/q_t": 0.3972512185573578,
"grad_norm": 40.94980239868164,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.40435200929641724,
"logits/rejected": -0.39599835872650146,
"logps/chosen": -327.1517333984375,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -531.0156860351562,
"loss": 1.0579,
"margin_dpo/margin_mean": 185.34262084960938,
"margin_dpo/margin_std": 180.45718383789062,
"step": 304
},
{
"KL/chosen_KL_mean": -266.6454772949219,
"KL/mean": -351.7230224609375,
"KL/rejected_KL_mean": -436.8005676269531,
"KL/std": 221.42990112304688,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.0021582874469459057,
"fcm_dpo/delta": 0.04888404905796051,
"fcm_dpo/margin": 170.1551055908203,
"fcm_dpo/q_t": 0.41974300146102905,
"grad_norm": 22.135257720947266,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.4137483537197113,
"logits/rejected": -0.39837193489074707,
"logps/chosen": -333.4407043457031,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -529.55517578125,
"loss": 1.1212,
"margin_dpo/margin_mean": 170.1551055908203,
"margin_dpo/margin_std": 243.96319580078125,
"step": 305
},
{
"KL/chosen_KL_mean": -261.8374938964844,
"KL/mean": -343.0050354003906,
"KL/rejected_KL_mean": -424.1725769042969,
"KL/std": 194.5423583984375,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.002403826918452978,
"fcm_dpo/delta": 0.015124435536563396,
"fcm_dpo/margin": 162.33505249023438,
"fcm_dpo/q_t": 0.4113726019859314,
"grad_norm": 23.578460693359375,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.3788735866546631,
"logits/rejected": -0.35274261236190796,
"logps/chosen": -331.5213928222656,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -509.3317565917969,
"loss": 1.0925,
"margin_dpo/margin_mean": 162.33505249023438,
"margin_dpo/margin_std": 210.62339782714844,
"step": 306
},
{
"KL/chosen_KL_mean": -237.50689697265625,
"KL/mean": -322.65093994140625,
"KL/rejected_KL_mean": -407.7949523925781,
"KL/std": 183.3559112548828,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0023515899665653706,
"fcm_dpo/delta": -0.0012738704681396484,
"fcm_dpo/margin": 170.28805541992188,
"fcm_dpo/q_t": 0.40794315934181213,
"grad_norm": 24.69138526916504,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.37078455090522766,
"logits/rejected": -0.35139191150665283,
"logps/chosen": -307.67230224609375,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -494.76727294921875,
"loss": 1.0939,
"margin_dpo/margin_mean": 170.28805541992188,
"margin_dpo/margin_std": 231.32247924804688,
"step": 307
},
{
"KL/chosen_KL_mean": -242.2062225341797,
"KL/mean": -339.5050964355469,
"KL/rejected_KL_mean": -436.803955078125,
"KL/std": 205.6837158203125,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.002198460977524519,
"fcm_dpo/delta": -0.049677349627017975,
"fcm_dpo/margin": 194.59776306152344,
"fcm_dpo/q_t": 0.3950149416923523,
"grad_norm": 29.672807693481445,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.3967479467391968,
"logits/rejected": -0.3871362507343292,
"logps/chosen": -297.45123291015625,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -516.1762084960938,
"loss": 1.05,
"margin_dpo/margin_mean": 194.59774780273438,
"margin_dpo/margin_std": 198.84432983398438,
"step": 308
},
{
"KL/chosen_KL_mean": -249.229248046875,
"KL/mean": -345.9730224609375,
"KL/rejected_KL_mean": -442.71673583984375,
"KL/std": 216.7500457763672,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.00216893432661891,
"fcm_dpo/delta": -0.040406569838523865,
"fcm_dpo/margin": 193.48745727539062,
"fcm_dpo/q_t": 0.3975731432437897,
"grad_norm": 26.32208251953125,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.32539457082748413,
"logits/rejected": -0.3188524842262268,
"logps/chosen": -298.1883544921875,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -525.0574951171875,
"loss": 1.0796,
"margin_dpo/margin_mean": 193.4874725341797,
"margin_dpo/margin_std": 231.7923583984375,
"step": 309
},
{
"KL/chosen_KL_mean": -287.5223388671875,
"KL/mean": -375.1282958984375,
"KL/rejected_KL_mean": -462.7342224121094,
"KL/std": 191.27525329589844,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.002167648635804653,
"fcm_dpo/delta": 0.02439631149172783,
"fcm_dpo/margin": 175.21188354492188,
"fcm_dpo/q_t": 0.41132253408432007,
"grad_norm": 37.333168029785156,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": -0.37113621830940247,
"logits/rejected": -0.35927319526672363,
"logps/chosen": -350.2641296386719,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -542.66455078125,
"loss": 1.0851,
"margin_dpo/margin_mean": 175.21188354492188,
"margin_dpo/margin_std": 188.38232421875,
"step": 310
},
{
"KL/chosen_KL_mean": -305.24285888671875,
"KL/mean": -408.00628662109375,
"KL/rejected_KL_mean": -510.7697448730469,
"KL/std": 250.8982696533203,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.002006505150347948,
"fcm_dpo/delta": -0.021621335297822952,
"fcm_dpo/margin": 205.52688598632812,
"fcm_dpo/q_t": 0.40415090322494507,
"grad_norm": 23.370407104492188,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.3467109501361847,
"logits/rejected": -0.32372719049453735,
"logps/chosen": -358.2708435058594,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -588.2079467773438,
"loss": 1.0815,
"margin_dpo/margin_mean": 205.52688598632812,
"margin_dpo/margin_std": 265.2549743652344,
"step": 311
},
{
"KL/chosen_KL_mean": -289.9778747558594,
"KL/mean": -387.76654052734375,
"KL/rejected_KL_mean": -485.55523681640625,
"KL/std": 229.4159393310547,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.0020645607728511095,
"fcm_dpo/delta": -0.011410795152187347,
"fcm_dpo/margin": 195.57733154296875,
"fcm_dpo/q_t": 0.4080260396003723,
"grad_norm": 22.71463394165039,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": -0.2678723633289337,
"logits/rejected": -0.2562296390533447,
"logps/chosen": -339.3700866699219,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -561.3480224609375,
"loss": 1.0977,
"margin_dpo/margin_mean": 195.57733154296875,
"margin_dpo/margin_std": 275.55340576171875,
"step": 312
},
{
"KL/chosen_KL_mean": -276.6165771484375,
"KL/mean": -375.31329345703125,
"KL/rejected_KL_mean": -474.01007080078125,
"KL/std": 237.2071533203125,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.0020586608443409204,
"fcm_dpo/delta": -0.02704712748527527,
"fcm_dpo/margin": 197.3934326171875,
"fcm_dpo/q_t": 0.40283501148223877,
"grad_norm": 26.37751579284668,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": -0.3565303385257721,
"logits/rejected": -0.3560790419578552,
"logps/chosen": -326.76934814453125,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -560.416259765625,
"loss": 1.1012,
"margin_dpo/margin_mean": 197.3934326171875,
"margin_dpo/margin_std": 258.5960693359375,
"step": 313
},
{
"KL/chosen_KL_mean": -297.798828125,
"KL/mean": -391.349853515625,
"KL/rejected_KL_mean": -484.90087890625,
"KL/std": 228.58383178710938,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.0019475116860121489,
"fcm_dpo/delta": 0.055439822375774384,
"fcm_dpo/margin": 187.10211181640625,
"fcm_dpo/q_t": 0.4213699698448181,
"grad_norm": 22.879201889038086,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.39260512590408325,
"logits/rejected": -0.39466455578804016,
"logps/chosen": -355.036376953125,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -582.4974365234375,
"loss": 1.1347,
"margin_dpo/margin_mean": 187.10211181640625,
"margin_dpo/margin_std": 293.64080810546875,
"step": 314
},
{
"KL/chosen_KL_mean": -271.39031982421875,
"KL/mean": -353.8436279296875,
"KL/rejected_KL_mean": -436.296875,
"KL/std": 202.65625,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.0022210520692169666,
"fcm_dpo/delta": 0.05339854210615158,
"fcm_dpo/margin": 164.90655517578125,
"fcm_dpo/q_t": 0.41980287432670593,
"grad_norm": 24.21739959716797,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": -0.3449453115463257,
"logits/rejected": -0.3495122194290161,
"logps/chosen": -320.45989990234375,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -521.977783203125,
"loss": 1.1145,
"margin_dpo/margin_mean": 164.90655517578125,
"margin_dpo/margin_std": 229.23095703125,
"step": 315
},
{
"KL/chosen_KL_mean": -270.35205078125,
"KL/mean": -393.6520690917969,
"KL/rejected_KL_mean": -516.9520263671875,
"KL/std": 250.11061096191406,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.0017885593697428703,
"fcm_dpo/delta": -0.11213979125022888,
"fcm_dpo/margin": 246.60000610351562,
"fcm_dpo/q_t": 0.3872528076171875,
"grad_norm": 22.613483428955078,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.40570446848869324,
"logits/rejected": -0.40869593620300293,
"logps/chosen": -324.6128234863281,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -618.2335205078125,
"loss": 1.0531,
"margin_dpo/margin_mean": 246.60000610351562,
"margin_dpo/margin_std": 279.0342102050781,
"step": 316
},
{
"KL/chosen_KL_mean": -269.0587158203125,
"KL/mean": -400.5162658691406,
"KL/rejected_KL_mean": -531.9737548828125,
"KL/std": 222.888427734375,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.0017005549743771553,
"fcm_dpo/delta": -0.08523563295602798,
"fcm_dpo/margin": 262.91510009765625,
"fcm_dpo/q_t": 0.3873101472854614,
"grad_norm": 26.969892501831055,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.3938244581222534,
"logits/rejected": -0.3835000693798065,
"logps/chosen": -325.1529235839844,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -632.6728515625,
"loss": 1.0363,
"margin_dpo/margin_mean": 262.91510009765625,
"margin_dpo/margin_std": 250.01504516601562,
"step": 317
},
{
"KL/chosen_KL_mean": -297.5737609863281,
"KL/mean": -394.74908447265625,
"KL/rejected_KL_mean": -491.9244384765625,
"KL/std": 229.82669067382812,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.0017898095538839698,
"fcm_dpo/delta": 0.07254049926996231,
"fcm_dpo/margin": 194.35072326660156,
"fcm_dpo/q_t": 0.42293781042099,
"grad_norm": 21.67091178894043,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.45766058564186096,
"logits/rejected": -0.4315134584903717,
"logps/chosen": -362.2194519042969,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -574.688720703125,
"loss": 1.1147,
"margin_dpo/margin_mean": 194.35073852539062,
"margin_dpo/margin_std": 247.84156799316406,
"step": 318
},
{
"KL/chosen_KL_mean": -268.0478820800781,
"KL/mean": -386.1382751464844,
"KL/rejected_KL_mean": -504.2286682128906,
"KL/std": 255.11056518554688,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.0017438679933547974,
"fcm_dpo/delta": -0.02019650489091873,
"fcm_dpo/margin": 236.18075561523438,
"fcm_dpo/q_t": 0.40276533365249634,
"grad_norm": 24.052961349487305,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.40444713830947876,
"logits/rejected": -0.4185525178909302,
"logps/chosen": -317.431640625,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -618.1351318359375,
"loss": 1.0723,
"margin_dpo/margin_mean": 236.18075561523438,
"margin_dpo/margin_std": 282.4808349609375,
"step": 319
},
{
"KL/chosen_KL_mean": -281.9687194824219,
"KL/mean": -408.4984130859375,
"KL/rejected_KL_mean": -535.0281372070312,
"KL/std": 261.2070007324219,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.0016149728326126933,
"fcm_dpo/delta": -0.019406653940677643,
"fcm_dpo/margin": 253.0594482421875,
"fcm_dpo/q_t": 0.40273725986480713,
"grad_norm": 27.48386001586914,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.39280545711517334,
"logits/rejected": -0.397488534450531,
"logps/chosen": -341.4736022949219,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -632.6953125,
"loss": 1.0693,
"margin_dpo/margin_mean": 253.05943298339844,
"margin_dpo/margin_std": 285.04510498046875,
"step": 320
},
{
"KL/chosen_KL_mean": -347.18927001953125,
"KL/mean": -454.4679260253906,
"KL/rejected_KL_mean": -561.74658203125,
"KL/std": 268.3890686035156,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.0017120807897299528,
"fcm_dpo/delta": 0.0495997853577137,
"fcm_dpo/margin": 214.55735778808594,
"fcm_dpo/q_t": 0.41874903440475464,
"grad_norm": 21.842844009399414,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": -0.3386973738670349,
"logits/rejected": -0.3240874409675598,
"logps/chosen": -408.7379150390625,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -653.3876953125,
"loss": 1.1147,
"margin_dpo/margin_mean": 214.55735778808594,
"margin_dpo/margin_std": 290.58978271484375,
"step": 321
},
{
"KL/chosen_KL_mean": -306.6402893066406,
"KL/mean": -424.40594482421875,
"KL/rejected_KL_mean": -542.171630859375,
"KL/std": 241.84188842773438,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.0017245409544557333,
"fcm_dpo/delta": -0.015119031071662903,
"fcm_dpo/margin": 235.53131103515625,
"fcm_dpo/q_t": 0.4033818244934082,
"grad_norm": 24.67373275756836,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": -0.3635835349559784,
"logits/rejected": -0.36071500182151794,
"logps/chosen": -363.9303283691406,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -637.9215698242188,
"loss": 1.0722,
"margin_dpo/margin_mean": 235.5313262939453,
"margin_dpo/margin_std": 266.08941650390625,
"step": 322
},
{
"KL/chosen_KL_mean": -327.5718994140625,
"KL/mean": -426.56634521484375,
"KL/rejected_KL_mean": -525.560791015625,
"KL/std": 240.03286743164062,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.0018759286031126976,
"fcm_dpo/delta": 0.04566890001296997,
"fcm_dpo/margin": 197.98890686035156,
"fcm_dpo/q_t": 0.41689807176589966,
"grad_norm": 30.36820411682129,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": -0.391653448343277,
"logits/rejected": -0.3899046778678894,
"logps/chosen": -378.8058776855469,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -600.6227416992188,
"loss": 1.1077,
"margin_dpo/margin_mean": 197.98890686035156,
"margin_dpo/margin_std": 260.3305969238281,
"step": 323
},
{
"KL/chosen_KL_mean": -376.3759765625,
"KL/mean": -470.93865966796875,
"KL/rejected_KL_mean": -565.5013427734375,
"KL/std": 256.7306823730469,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.002103945007547736,
"fcm_dpo/delta": -0.02135651558637619,
"fcm_dpo/margin": 189.12530517578125,
"fcm_dpo/q_t": 0.41041427850723267,
"grad_norm": 50.850955963134766,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": -0.3911237120628357,
"logits/rejected": -0.37464720010757446,
"logps/chosen": -441.51116943359375,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -651.9788818359375,
"loss": 1.1359,
"margin_dpo/margin_mean": 189.12530517578125,
"margin_dpo/margin_std": 302.71319580078125,
"step": 324
},
{
"KL/chosen_KL_mean": -296.9546203613281,
"KL/mean": -379.6038513183594,
"KL/rejected_KL_mean": -462.25311279296875,
"KL/std": 226.74127197265625,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.002113199792802334,
"fcm_dpo/delta": 0.0753774344921112,
"fcm_dpo/margin": 165.2984619140625,
"fcm_dpo/q_t": 0.4240267872810364,
"grad_norm": 27.387161254882812,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.4187164306640625,
"logits/rejected": -0.3953896760940552,
"logps/chosen": -353.17022705078125,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -532.3390502929688,
"loss": 1.1291,
"margin_dpo/margin_mean": 165.2984619140625,
"margin_dpo/margin_std": 237.98086547851562,
"step": 325
},
{
"KL/chosen_KL_mean": -324.411865234375,
"KL/mean": -408.02496337890625,
"KL/rejected_KL_mean": -491.6380920410156,
"KL/std": 210.09243774414062,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.0022600889205932617,
"fcm_dpo/delta": 0.03556614741683006,
"fcm_dpo/margin": 167.2262420654297,
"fcm_dpo/q_t": 0.414311021566391,
"grad_norm": 37.292301177978516,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.44619232416152954,
"logits/rejected": -0.42077717185020447,
"logps/chosen": -397.1368408203125,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -571.48486328125,
"loss": 1.0978,
"margin_dpo/margin_mean": 167.2262420654297,
"margin_dpo/margin_std": 205.65249633789062,
"step": 326
},
{
"KL/chosen_KL_mean": -280.4002685546875,
"KL/mean": -404.8081970214844,
"KL/rejected_KL_mean": -529.2161254882812,
"KL/std": 223.6777801513672,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.0018240961944684386,
"fcm_dpo/delta": -0.12385020405054092,
"fcm_dpo/margin": 248.81582641601562,
"fcm_dpo/q_t": 0.3799220621585846,
"grad_norm": 32.990352630615234,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.41425788402557373,
"logits/rejected": -0.4029478132724762,
"logps/chosen": -349.53472900390625,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -641.14990234375,
"loss": 1.0343,
"margin_dpo/margin_mean": 248.81582641601562,
"margin_dpo/margin_std": 254.188720703125,
"step": 327
},
{
"KL/chosen_KL_mean": -289.2528381347656,
"KL/mean": -397.5467224121094,
"KL/rejected_KL_mean": -505.84063720703125,
"KL/std": 236.40066528320312,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.0018365057185292244,
"fcm_dpo/delta": 0.003600106341764331,
"fcm_dpo/margin": 216.58779907226562,
"fcm_dpo/q_t": 0.4082072973251343,
"grad_norm": 25.665807723999023,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.35888296365737915,
"logits/rejected": -0.36398810148239136,
"logps/chosen": -348.9400329589844,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -596.6956176757812,
"loss": 1.0842,
"margin_dpo/margin_mean": 216.58779907226562,
"margin_dpo/margin_std": 268.95428466796875,
"step": 328
},
{
"KL/chosen_KL_mean": -328.11090087890625,
"KL/mean": -442.0875244140625,
"KL/rejected_KL_mean": -556.0640869140625,
"KL/std": 269.01434326171875,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.0017261970788240433,
"fcm_dpo/delta": 0.0008079372346401215,
"fcm_dpo/margin": 227.95321655273438,
"fcm_dpo/q_t": 0.40766558051109314,
"grad_norm": 24.41579818725586,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": -0.3722694516181946,
"logits/rejected": -0.3635803461074829,
"logps/chosen": -393.35711669921875,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -656.7618408203125,
"loss": 1.1045,
"margin_dpo/margin_mean": 227.95321655273438,
"margin_dpo/margin_std": 301.1634521484375,
"step": 329
},
{
"KL/chosen_KL_mean": -275.129150390625,
"KL/mean": -394.5272216796875,
"KL/rejected_KL_mean": -513.92529296875,
"KL/std": 246.2072296142578,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.0017482815310359001,
"fcm_dpo/delta": -0.03028486669063568,
"fcm_dpo/margin": 238.79615783691406,
"fcm_dpo/q_t": 0.400304913520813,
"grad_norm": 20.50057029724121,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": -0.3955565094947815,
"logits/rejected": -0.39654165506362915,
"logps/chosen": -322.12750244140625,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -600.8021240234375,
"loss": 1.0642,
"margin_dpo/margin_mean": 238.79615783691406,
"margin_dpo/margin_std": 271.3484191894531,
"step": 330
},
{
"KL/chosen_KL_mean": -305.6733093261719,
"KL/mean": -425.43841552734375,
"KL/rejected_KL_mean": -545.2034912109375,
"KL/std": 206.7650146484375,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.0017030672170221806,
"fcm_dpo/delta": -0.013409378007054329,
"fcm_dpo/margin": 239.53012084960938,
"fcm_dpo/q_t": 0.40146273374557495,
"grad_norm": 22.680070877075195,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": -0.37283700704574585,
"logits/rejected": -0.3688333034515381,
"logps/chosen": -356.19757080078125,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -634.2188720703125,
"loss": 1.0483,
"margin_dpo/margin_mean": 239.5301513671875,
"margin_dpo/margin_std": 205.19898986816406,
"step": 331
},
{
"KL/chosen_KL_mean": -309.19793701171875,
"KL/mean": -390.8100891113281,
"KL/rejected_KL_mean": -472.4222412109375,
"KL/std": 211.47320556640625,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.0020041917450726032,
"fcm_dpo/delta": 0.10211435705423355,
"fcm_dpo/margin": 163.22427368164062,
"fcm_dpo/q_t": 0.43064454197883606,
"grad_norm": 23.14905548095703,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": -0.360334187746048,
"logits/rejected": -0.33985477685928345,
"logps/chosen": -358.37823486328125,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -548.9073486328125,
"loss": 1.139,
"margin_dpo/margin_mean": 163.22427368164062,
"margin_dpo/margin_std": 238.90945434570312,
"step": 332
},
{
"KL/chosen_KL_mean": -336.2814636230469,
"KL/mean": -449.5226135253906,
"KL/rejected_KL_mean": -562.7637939453125,
"KL/std": 258.14825439453125,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.001787829678505659,
"fcm_dpo/delta": -0.0357210636138916,
"fcm_dpo/margin": 226.48231506347656,
"fcm_dpo/q_t": 0.4021400809288025,
"grad_norm": 37.613563537597656,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": -0.34511464834213257,
"logits/rejected": -0.3372688889503479,
"logps/chosen": -400.0372314453125,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -657.807861328125,
"loss": 1.1071,
"margin_dpo/margin_mean": 226.48233032226562,
"margin_dpo/margin_std": 314.77203369140625,
"step": 333
},
{
"KL/chosen_KL_mean": -324.66705322265625,
"KL/mean": -422.1019287109375,
"KL/rejected_KL_mean": -519.5368041992188,
"KL/std": 276.22686767578125,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.0019669593311846256,
"fcm_dpo/delta": 0.0270000621676445,
"fcm_dpo/margin": 194.86973571777344,
"fcm_dpo/q_t": 0.41571369767189026,
"grad_norm": 21.602262496948242,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.38413316011428833,
"logits/rejected": -0.3805462718009949,
"logps/chosen": -391.6468200683594,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -614.853759765625,
"loss": 1.1271,
"margin_dpo/margin_mean": 194.86972045898438,
"margin_dpo/margin_std": 309.7537536621094,
"step": 334
},
{
"KL/chosen_KL_mean": -354.06585693359375,
"KL/mean": -434.64276123046875,
"KL/rejected_KL_mean": -515.2196655273438,
"KL/std": 258.8491516113281,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.002282127272337675,
"fcm_dpo/delta": 0.03639799728989601,
"fcm_dpo/margin": 161.15380859375,
"fcm_dpo/q_t": 0.41994214057922363,
"grad_norm": 31.488386154174805,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": -0.41236335039138794,
"logits/rejected": -0.39202946424484253,
"logps/chosen": -416.60833740234375,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -602.83740234375,
"loss": 1.1504,
"margin_dpo/margin_mean": 161.15379333496094,
"margin_dpo/margin_std": 279.69989013671875,
"step": 335
},
{
"KL/chosen_KL_mean": -347.204345703125,
"KL/mean": -475.4479064941406,
"KL/rejected_KL_mean": -603.69140625,
"KL/std": 295.57818603515625,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.0018394170328974724,
"fcm_dpo/delta": -0.1347188502550125,
"fcm_dpo/margin": 256.4870910644531,
"fcm_dpo/q_t": 0.3836958408355713,
"grad_norm": 27.45176887512207,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": -0.38979193568229675,
"logits/rejected": -0.3902202248573303,
"logps/chosen": -401.7355041503906,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -708.095703125,
"loss": 1.0657,
"margin_dpo/margin_mean": 256.4870910644531,
"margin_dpo/margin_std": 350.75457763671875,
"step": 336
},
{
"KL/chosen_KL_mean": -316.01263427734375,
"KL/mean": -441.3327941894531,
"KL/rejected_KL_mean": -566.6529541015625,
"KL/std": 243.086181640625,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.001608079532161355,
"fcm_dpo/delta": -0.006862752139568329,
"fcm_dpo/margin": 250.64028930664062,
"fcm_dpo/q_t": 0.4053046405315399,
"grad_norm": 20.08615493774414,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": -0.41743162274360657,
"logits/rejected": -0.40591827034950256,
"logps/chosen": -381.1413269042969,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -668.3799438476562,
"loss": 1.0743,
"margin_dpo/margin_mean": 250.6403045654297,
"margin_dpo/margin_std": 288.3606872558594,
"step": 337
},
{
"KL/chosen_KL_mean": -287.96209716796875,
"KL/mean": -410.0386962890625,
"KL/rejected_KL_mean": -532.1152954101562,
"KL/std": 230.76153564453125,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.0016326969489455223,
"fcm_dpo/delta": 0.002140873111784458,
"fcm_dpo/margin": 244.1531524658203,
"fcm_dpo/q_t": 0.4057929515838623,
"grad_norm": 22.323871612548828,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": -0.42750677466392517,
"logits/rejected": -0.4222105145454407,
"logps/chosen": -346.3847961425781,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -621.183837890625,
"loss": 1.0645,
"margin_dpo/margin_mean": 244.15316772460938,
"margin_dpo/margin_std": 244.08584594726562,
"step": 338
},
{
"KL/chosen_KL_mean": -303.424560546875,
"KL/mean": -419.2534484863281,
"KL/rejected_KL_mean": -535.0823974609375,
"KL/std": 239.62149047851562,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0017491846811026335,
"fcm_dpo/delta": -0.029546871781349182,
"fcm_dpo/margin": 231.65774536132812,
"fcm_dpo/q_t": 0.40035557746887207,
"grad_norm": 24.648235321044922,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": -0.43223291635513306,
"logits/rejected": -0.42046141624450684,
"logps/chosen": -363.4198913574219,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -638.9932861328125,
"loss": 1.0808,
"margin_dpo/margin_mean": 231.65774536132812,
"margin_dpo/margin_std": 242.7178497314453,
"step": 339
},
{
"KL/chosen_KL_mean": -324.7750244140625,
"KL/mean": -426.1435546875,
"KL/rejected_KL_mean": -527.5120239257812,
"KL/std": 234.62620544433594,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0017105863662436604,
"fcm_dpo/delta": 0.08201248943805695,
"fcm_dpo/margin": 202.73699951171875,
"fcm_dpo/q_t": 0.425212025642395,
"grad_norm": 34.08943176269531,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": -0.3556247353553772,
"logits/rejected": -0.33346790075302124,
"logps/chosen": -377.60528564453125,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -600.6192626953125,
"loss": 1.1282,
"margin_dpo/margin_mean": 202.73699951171875,
"margin_dpo/margin_std": 283.2412109375,
"step": 340
},
{
"KL/chosen_KL_mean": -331.2342224121094,
"KL/mean": -450.0869140625,
"KL/rejected_KL_mean": -568.9395751953125,
"KL/std": 259.69732666015625,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.0017094245413318276,
"fcm_dpo/delta": -0.016936201602220535,
"fcm_dpo/margin": 237.70541381835938,
"fcm_dpo/q_t": 0.4038848578929901,
"grad_norm": 25.37321662902832,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.37913063168525696,
"logits/rejected": -0.37905052304267883,
"logps/chosen": -379.134033203125,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -670.74951171875,
"loss": 1.0753,
"margin_dpo/margin_mean": 237.70538330078125,
"margin_dpo/margin_std": 275.2982177734375,
"step": 341
},
{
"KL/chosen_KL_mean": -320.9766540527344,
"KL/mean": -426.2503967285156,
"KL/rejected_KL_mean": -531.524169921875,
"KL/std": 245.80764770507812,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.0018736084457486868,
"fcm_dpo/delta": 0.0011573433876037598,
"fcm_dpo/margin": 210.54747009277344,
"fcm_dpo/q_t": 0.4087076187133789,
"grad_norm": 27.59446144104004,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.45016488432884216,
"logits/rejected": -0.42709416151046753,
"logps/chosen": -392.9732971191406,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -624.11376953125,
"loss": 1.1059,
"margin_dpo/margin_mean": 210.5474853515625,
"margin_dpo/margin_std": 294.087158203125,
"step": 342
},
{
"KL/chosen_KL_mean": -307.4436950683594,
"KL/mean": -433.9495849609375,
"KL/rejected_KL_mean": -560.4554443359375,
"KL/std": 248.22842407226562,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.001662734430283308,
"fcm_dpo/delta": -0.036752138286828995,
"fcm_dpo/margin": 253.01181030273438,
"fcm_dpo/q_t": 0.3963671922683716,
"grad_norm": 21.48690414428711,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.4163956344127655,
"logits/rejected": -0.4220476746559143,
"logps/chosen": -361.84930419921875,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -671.4969482421875,
"loss": 1.042,
"margin_dpo/margin_mean": 253.01181030273438,
"margin_dpo/margin_std": 219.65103149414062,
"step": 343
},
{
"KL/chosen_KL_mean": -313.14569091796875,
"KL/mean": -424.7760925292969,
"KL/rejected_KL_mean": -536.406494140625,
"KL/std": 274.6375427246094,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.0015421407297253609,
"fcm_dpo/delta": -0.06822478026151657,
"fcm_dpo/margin": 223.26084899902344,
"fcm_dpo/q_t": 0.41599759459495544,
"grad_norm": 17.046581268310547,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.3974248170852661,
"logits/rejected": -0.39894139766693115,
"logps/chosen": -367.1103515625,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -627.0298461914062,
"loss": 1.1263,
"margin_dpo/margin_mean": 223.26084899902344,
"margin_dpo/margin_std": 279.2683410644531,
"step": 344
},
{
"KL/chosen_KL_mean": -370.6162414550781,
"KL/mean": -496.4261474609375,
"KL/rejected_KL_mean": -622.236083984375,
"KL/std": 260.423095703125,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.001530057517811656,
"fcm_dpo/delta": 0.020667294040322304,
"fcm_dpo/margin": 251.619873046875,
"fcm_dpo/q_t": 0.4105876684188843,
"grad_norm": 20.95164680480957,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.42145538330078125,
"logits/rejected": -0.41829943656921387,
"logps/chosen": -432.30194091796875,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -721.7265014648438,
"loss": 1.084,
"margin_dpo/margin_mean": 251.619873046875,
"margin_dpo/margin_std": 285.166259765625,
"step": 345
},
{
"KL/chosen_KL_mean": -365.87664794921875,
"KL/mean": -480.89178466796875,
"KL/rejected_KL_mean": -595.906982421875,
"KL/std": 254.52523803710938,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.001609054277651012,
"fcm_dpo/delta": 0.04702121764421463,
"fcm_dpo/margin": 230.03033447265625,
"fcm_dpo/q_t": 0.415949285030365,
"grad_norm": 20.622108459472656,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": -0.3826184570789337,
"logits/rejected": -0.37241339683532715,
"logps/chosen": -421.1329040527344,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -673.322265625,
"loss": 1.0974,
"margin_dpo/margin_mean": 230.03033447265625,
"margin_dpo/margin_std": 276.1398620605469,
"step": 346
},
{
"KL/chosen_KL_mean": -361.59759521484375,
"KL/mean": -479.4732666015625,
"KL/rejected_KL_mean": -597.348876953125,
"KL/std": 272.4322509765625,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.0016747593181207776,
"fcm_dpo/delta": 0.00839798804372549,
"fcm_dpo/margin": 235.7513427734375,
"fcm_dpo/q_t": 0.4093061089515686,
"grad_norm": 21.934925079345703,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": -0.4126300811767578,
"logits/rejected": -0.40708625316619873,
"logps/chosen": -419.163818359375,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -689.7039794921875,
"loss": 1.0905,
"margin_dpo/margin_mean": 235.7513427734375,
"margin_dpo/margin_std": 303.5050964355469,
"step": 347
},
{
"KL/chosen_KL_mean": -326.0738220214844,
"KL/mean": -424.66363525390625,
"KL/rejected_KL_mean": -523.25341796875,
"KL/std": 231.39590454101562,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.001778720528818667,
"fcm_dpo/delta": 0.07379812002182007,
"fcm_dpo/margin": 197.17959594726562,
"fcm_dpo/q_t": 0.42386046051979065,
"grad_norm": 24.249563217163086,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": -0.44509440660476685,
"logits/rejected": -0.44437336921691895,
"logps/chosen": -382.39154052734375,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -612.391845703125,
"loss": 1.1286,
"margin_dpo/margin_mean": 197.1796112060547,
"margin_dpo/margin_std": 269.8843994140625,
"step": 348
},
{
"KL/chosen_KL_mean": -308.29888916015625,
"KL/mean": -417.86456298828125,
"KL/rejected_KL_mean": -527.4302978515625,
"KL/std": 253.47750854492188,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.0019209685269743204,
"fcm_dpo/delta": -0.03923729062080383,
"fcm_dpo/margin": 219.1314239501953,
"fcm_dpo/q_t": 0.40259507298469543,
"grad_norm": 23.687149047851562,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": -0.45386502146720886,
"logits/rejected": -0.45663371682167053,
"logps/chosen": -366.32440185546875,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -624.9354248046875,
"loss": 1.0951,
"margin_dpo/margin_mean": 219.13143920898438,
"margin_dpo/margin_std": 312.9615478515625,
"step": 349
},
{
"KL/chosen_KL_mean": -332.3165283203125,
"KL/mean": -443.63629150390625,
"KL/rejected_KL_mean": -554.9560546875,
"KL/std": 248.83816528320312,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.0017454567132517695,
"fcm_dpo/delta": 0.013433754444122314,
"fcm_dpo/margin": 222.63954162597656,
"fcm_dpo/q_t": 0.4100583791732788,
"grad_norm": 26.530868530273438,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": -0.4750177562236786,
"logits/rejected": -0.4820059835910797,
"logps/chosen": -396.64703369140625,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -644.8277587890625,
"loss": 1.1064,
"margin_dpo/margin_mean": 222.63955688476562,
"margin_dpo/margin_std": 307.46893310546875,
"step": 350
},
{
"KL/chosen_KL_mean": -296.4862060546875,
"KL/mean": -429.1023254394531,
"KL/rejected_KL_mean": -561.718505859375,
"KL/std": 283.80633544921875,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.0016240214463323355,
"fcm_dpo/delta": -0.061422545462846756,
"fcm_dpo/margin": 265.2322998046875,
"fcm_dpo/q_t": 0.3965432643890381,
"grad_norm": 31.153547286987305,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": -0.4499565362930298,
"logits/rejected": -0.4501733183860779,
"logps/chosen": -357.1583251953125,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -663.283935546875,
"loss": 1.0671,
"margin_dpo/margin_mean": 265.2322998046875,
"margin_dpo/margin_std": 327.75909423828125,
"step": 351
},
{
"KL/chosen_KL_mean": -349.35406494140625,
"KL/mean": -436.3336181640625,
"KL/rejected_KL_mean": -523.313232421875,
"KL/std": 263.14996337890625,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.0018782506231218576,
"fcm_dpo/delta": 0.10981118679046631,
"fcm_dpo/margin": 173.9591064453125,
"fcm_dpo/q_t": 0.4318041503429413,
"grad_norm": 46.439903259277344,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": -0.45800480246543884,
"logits/rejected": -0.4245471954345703,
"logps/chosen": -420.2975158691406,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -599.9552001953125,
"loss": 1.155,
"margin_dpo/margin_mean": 173.9591064453125,
"margin_dpo/margin_std": 280.7679748535156,
"step": 352
},
{
"KL/chosen_KL_mean": -317.61041259765625,
"KL/mean": -430.0752258300781,
"KL/rejected_KL_mean": -542.5400390625,
"KL/std": 259.1496276855469,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.0018013006774708629,
"fcm_dpo/delta": -0.02872556447982788,
"fcm_dpo/margin": 224.9296112060547,
"fcm_dpo/q_t": 0.40363016724586487,
"grad_norm": 23.421403884887695,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": -0.4782019555568695,
"logits/rejected": -0.4654806852340698,
"logps/chosen": -376.0057373046875,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -622.8755493164062,
"loss": 1.0947,
"margin_dpo/margin_mean": 224.92959594726562,
"margin_dpo/margin_std": 295.62744140625,
"step": 353
},
{
"KL/chosen_KL_mean": -272.10107421875,
"KL/mean": -383.7615966796875,
"KL/rejected_KL_mean": -495.422119140625,
"KL/std": 226.24905395507812,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.001837678486481309,
"fcm_dpo/delta": -0.01862015575170517,
"fcm_dpo/margin": 223.32101440429688,
"fcm_dpo/q_t": 0.4018193483352661,
"grad_norm": 24.753582000732422,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": -0.45796817541122437,
"logits/rejected": -0.4484265446662903,
"logps/chosen": -331.9040832519531,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -584.1796264648438,
"loss": 1.0623,
"margin_dpo/margin_mean": 223.32101440429688,
"margin_dpo/margin_std": 235.79531860351562,
"step": 354
},
{
"KL/chosen_KL_mean": -267.42645263671875,
"KL/mean": -380.90118408203125,
"KL/rejected_KL_mean": -494.37591552734375,
"KL/std": 236.37852478027344,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.0017689072992652655,
"fcm_dpo/delta": -0.005881348624825478,
"fcm_dpo/margin": 226.94949340820312,
"fcm_dpo/q_t": 0.40436455607414246,
"grad_norm": 38.343528747558594,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": -0.48708575963974,
"logits/rejected": -0.4752500653266907,
"logps/chosen": -321.554931640625,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -576.781982421875,
"loss": 1.0671,
"margin_dpo/margin_mean": 226.9495086669922,
"margin_dpo/margin_std": 235.01458740234375,
"step": 355
},
{
"KL/chosen_KL_mean": -329.96331787109375,
"KL/mean": -393.64111328125,
"KL/rejected_KL_mean": -457.3189697265625,
"KL/std": 250.74232482910156,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.00188022933434695,
"fcm_dpo/delta": 0.029842089861631393,
"fcm_dpo/margin": 127.35562896728516,
"fcm_dpo/q_t": 0.4473668932914734,
"grad_norm": 28.56047248840332,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": -0.47035759687423706,
"logits/rejected": -0.44759708642959595,
"logps/chosen": -394.6371154785156,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -533.21826171875,
"loss": 1.2407,
"margin_dpo/margin_mean": 127.35563659667969,
"margin_dpo/margin_std": 307.847412109375,
"step": 356
},
{
"KL/chosen_KL_mean": -290.78863525390625,
"KL/mean": -393.8023681640625,
"KL/rejected_KL_mean": -496.8161315917969,
"KL/std": 240.8219757080078,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.0018875239184126258,
"fcm_dpo/delta": 0.013352084904909134,
"fcm_dpo/margin": 206.02752685546875,
"fcm_dpo/q_t": 0.4096463918685913,
"grad_norm": 27.79951286315918,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": -0.45739173889160156,
"logits/rejected": -0.44537484645843506,
"logps/chosen": -343.514404296875,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -583.6572875976562,
"loss": 1.0937,
"margin_dpo/margin_mean": 206.02752685546875,
"margin_dpo/margin_std": 253.822509765625,
"step": 357
},
{
"KL/chosen_KL_mean": -269.3954162597656,
"KL/mean": -368.242919921875,
"KL/rejected_KL_mean": -467.09039306640625,
"KL/std": 235.274658203125,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.0020065298303961754,
"fcm_dpo/delta": 0.005489196628332138,
"fcm_dpo/margin": 197.69497680664062,
"fcm_dpo/q_t": 0.410032719373703,
"grad_norm": 28.569393157958984,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.48102307319641113,
"logits/rejected": -0.463295042514801,
"logps/chosen": -332.6008605957031,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -555.4637451171875,
"loss": 1.0932,
"margin_dpo/margin_mean": 197.69496154785156,
"margin_dpo/margin_std": 264.3070983886719,
"step": 358
},
{
"KL/chosen_KL_mean": -304.4889221191406,
"KL/mean": -419.318359375,
"KL/rejected_KL_mean": -534.1477661132812,
"KL/std": 241.59100341796875,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.0018390321638435125,
"fcm_dpo/delta": -0.04261445254087448,
"fcm_dpo/margin": 229.6588134765625,
"fcm_dpo/q_t": 0.3999358117580414,
"grad_norm": 27.864505767822266,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": -0.4233545958995819,
"logits/rejected": -0.40909135341644287,
"logps/chosen": -360.859130859375,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -616.321533203125,
"loss": 1.0733,
"margin_dpo/margin_mean": 229.6588134765625,
"margin_dpo/margin_std": 284.9479064941406,
"step": 359
},
{
"KL/chosen_KL_mean": -301.6378479003906,
"KL/mean": -391.5552978515625,
"KL/rejected_KL_mean": -481.4727478027344,
"KL/std": 206.68411254882812,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.0020297402516007423,
"fcm_dpo/delta": 0.05278204008936882,
"fcm_dpo/margin": 179.8349151611328,
"fcm_dpo/q_t": 0.4177628755569458,
"grad_norm": 40.045162200927734,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": -0.43576139211654663,
"logits/rejected": -0.422180712223053,
"logps/chosen": -353.0982360839844,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -551.3116455078125,
"loss": 1.0958,
"margin_dpo/margin_mean": 179.83489990234375,
"margin_dpo/margin_std": 205.51583862304688,
"step": 360
},
{
"KL/chosen_KL_mean": -332.83001708984375,
"KL/mean": -420.3536376953125,
"KL/rejected_KL_mean": -507.8772277832031,
"KL/std": 242.87753295898438,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.0021930173970758915,
"fcm_dpo/delta": 0.021988222375512123,
"fcm_dpo/margin": 175.04721069335938,
"fcm_dpo/q_t": 0.4129222631454468,
"grad_norm": 37.02199172973633,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": -0.4316215217113495,
"logits/rejected": -0.427333801984787,
"logps/chosen": -386.69952392578125,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -598.646484375,
"loss": 1.115,
"margin_dpo/margin_mean": 175.04721069335938,
"margin_dpo/margin_std": 257.5013427734375,
"step": 361
},
{
"KL/chosen_KL_mean": -281.8548583984375,
"KL/mean": -425.60888671875,
"KL/rejected_KL_mean": -569.3629150390625,
"KL/std": 254.64712524414062,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0015978740993887186,
"fcm_dpo/delta": -0.14796875417232513,
"fcm_dpo/margin": 287.50799560546875,
"fcm_dpo/q_t": 0.37716490030288696,
"grad_norm": 23.246320724487305,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": -0.4437373876571655,
"logits/rejected": -0.45072150230407715,
"logps/chosen": -340.49395751953125,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -674.94482421875,
"loss": 1.0236,
"margin_dpo/margin_mean": 287.50799560546875,
"margin_dpo/margin_std": 274.0611572265625,
"step": 362
},
{
"KL/chosen_KL_mean": -271.1270751953125,
"KL/mean": -413.6645202636719,
"KL/rejected_KL_mean": -556.2019653320312,
"KL/std": 249.9840087890625,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.0015145809156820178,
"fcm_dpo/delta": -0.056021787226200104,
"fcm_dpo/margin": 285.07489013671875,
"fcm_dpo/q_t": 0.39367860555648804,
"grad_norm": 20.8101749420166,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": -0.43102937936782837,
"logits/rejected": -0.4243617057800293,
"logps/chosen": -315.6854553222656,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -630.8969116210938,
"loss": 1.0437,
"margin_dpo/margin_mean": 285.07489013671875,
"margin_dpo/margin_std": 276.6128234863281,
"step": 363
},
{
"KL/chosen_KL_mean": -300.5224914550781,
"KL/mean": -406.80938720703125,
"KL/rejected_KL_mean": -513.0963134765625,
"KL/std": 243.1720733642578,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.0016487597022205591,
"fcm_dpo/delta": 0.06513446569442749,
"fcm_dpo/margin": 212.57382202148438,
"fcm_dpo/q_t": 0.42112141847610474,
"grad_norm": 21.724754333496094,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.44869840145111084,
"logits/rejected": -0.4586051106452942,
"logps/chosen": -349.4171142578125,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -604.4920654296875,
"loss": 1.1204,
"margin_dpo/margin_mean": 212.57383728027344,
"margin_dpo/margin_std": 284.5193176269531,
"step": 364
},
{
"KL/chosen_KL_mean": -304.2714538574219,
"KL/mean": -416.9530029296875,
"KL/rejected_KL_mean": -529.634521484375,
"KL/std": 258.83819580078125,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.0016951878787949681,
"fcm_dpo/delta": 0.02898254618048668,
"fcm_dpo/margin": 225.36306762695312,
"fcm_dpo/q_t": 0.41397157311439514,
"grad_norm": 20.870635986328125,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": -0.43810129165649414,
"logits/rejected": -0.44645947217941284,
"logps/chosen": -355.7641906738281,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -622.336181640625,
"loss": 1.0909,
"margin_dpo/margin_mean": 225.36306762695312,
"margin_dpo/margin_std": 272.38427734375,
"step": 365
},
{
"KL/chosen_KL_mean": -287.7502746582031,
"KL/mean": -404.9134826660156,
"KL/rejected_KL_mean": -522.0767211914062,
"KL/std": 256.9874267578125,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.0016629542224109173,
"fcm_dpo/delta": 0.010643713176250458,
"fcm_dpo/margin": 234.326416015625,
"fcm_dpo/q_t": 0.4102753698825836,
"grad_norm": 19.262928009033203,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": -0.4263162612915039,
"logits/rejected": -0.44171929359436035,
"logps/chosen": -332.4708557128906,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -605.3870849609375,
"loss": 1.0992,
"margin_dpo/margin_mean": 234.326416015625,
"margin_dpo/margin_std": 304.7642517089844,
"step": 366
},
{
"KL/chosen_KL_mean": -303.0289611816406,
"KL/mean": -399.514892578125,
"KL/rejected_KL_mean": -496.0008544921875,
"KL/std": 225.67916870117188,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.0019213332561776042,
"fcm_dpo/delta": 0.04300880804657936,
"fcm_dpo/margin": 192.971923828125,
"fcm_dpo/q_t": 0.4157758355140686,
"grad_norm": 21.952829360961914,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": -0.48216262459754944,
"logits/rejected": -0.4676123857498169,
"logps/chosen": -361.43438720703125,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -572.752197265625,
"loss": 1.1045,
"margin_dpo/margin_mean": 192.971923828125,
"margin_dpo/margin_std": 250.1029052734375,
"step": 367
},
{
"KL/chosen_KL_mean": -282.49639892578125,
"KL/mean": -436.16912841796875,
"KL/rejected_KL_mean": -589.8418579101562,
"KL/std": 253.77154541015625,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.0015261652879416943,
"fcm_dpo/delta": -0.14009416103363037,
"fcm_dpo/margin": 307.345458984375,
"fcm_dpo/q_t": 0.37267598509788513,
"grad_norm": 34.17594528198242,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": -0.4614467918872833,
"logits/rejected": -0.4720000922679901,
"logps/chosen": -326.94891357421875,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -688.3970947265625,
"loss": 0.9994,
"margin_dpo/margin_mean": 307.345458984375,
"margin_dpo/margin_std": 224.4804229736328,
"step": 368
},
{
"KL/chosen_KL_mean": -367.19921875,
"KL/mean": -449.5543212890625,
"KL/rejected_KL_mean": -531.909423828125,
"KL/std": 254.9052276611328,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0015358589589595795,
"fcm_dpo/delta": 0.05729483440518379,
"fcm_dpo/margin": 164.710205078125,
"fcm_dpo/q_t": 0.4428786039352417,
"grad_norm": 24.15337371826172,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": -0.4904573857784271,
"logits/rejected": -0.4741830825805664,
"logps/chosen": -438.58074951171875,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -623.2052001953125,
"loss": 1.1956,
"margin_dpo/margin_mean": 164.710205078125,
"margin_dpo/margin_std": 289.94000244140625,
"step": 369
},
{
"KL/chosen_KL_mean": -390.330810546875,
"KL/mean": -462.7243347167969,
"KL/rejected_KL_mean": -535.1178588867188,
"KL/std": 265.82781982421875,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.0017272934783250093,
"fcm_dpo/delta": 0.06001188978552818,
"fcm_dpo/margin": 144.7870635986328,
"fcm_dpo/q_t": 0.44495099782943726,
"grad_norm": 28.623899459838867,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": -0.4887716770172119,
"logits/rejected": -0.4825999140739441,
"logps/chosen": -461.93829345703125,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -632.377685546875,
"loss": 1.2167,
"margin_dpo/margin_mean": 144.7870635986328,
"margin_dpo/margin_std": 307.05999755859375,
"step": 370
},
{
"KL/chosen_KL_mean": -373.00421142578125,
"KL/mean": -484.24078369140625,
"KL/rejected_KL_mean": -595.477294921875,
"KL/std": 274.64886474609375,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.0018450857605785131,
"fcm_dpo/delta": -0.020257379859685898,
"fcm_dpo/margin": 222.4730987548828,
"fcm_dpo/q_t": 0.4056010842323303,
"grad_norm": 29.032577514648438,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": -0.5206031799316406,
"logits/rejected": -0.5084042549133301,
"logps/chosen": -442.418701171875,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -694.6495361328125,
"loss": 1.1008,
"margin_dpo/margin_mean": 222.4730987548828,
"margin_dpo/margin_std": 321.52008056640625,
"step": 371
},
{
"KL/chosen_KL_mean": -360.9215393066406,
"KL/mean": -495.5597229003906,
"KL/rejected_KL_mean": -630.1979370117188,
"KL/std": 315.78912353515625,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.001595328445546329,
"fcm_dpo/delta": -0.05453532934188843,
"fcm_dpo/margin": 269.2763671875,
"fcm_dpo/q_t": 0.39929690957069397,
"grad_norm": 26.51140594482422,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": -0.4976493716239929,
"logits/rejected": -0.4843401610851288,
"logps/chosen": -422.7395324707031,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -708.7374267578125,
"loss": 1.0767,
"margin_dpo/margin_mean": 269.2763671875,
"margin_dpo/margin_std": 363.7861328125,
"step": 372
},
{
"KL/chosen_KL_mean": -385.86328125,
"KL/mean": -511.8228759765625,
"KL/rejected_KL_mean": -637.7823486328125,
"KL/std": 299.7609558105469,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.001618098933249712,
"fcm_dpo/delta": -0.01854773610830307,
"fcm_dpo/margin": 251.91912841796875,
"fcm_dpo/q_t": 0.40265533328056335,
"grad_norm": 23.972576141357422,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": -0.46941909193992615,
"logits/rejected": -0.4492412209510803,
"logps/chosen": -450.0804138183594,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -723.7420043945312,
"loss": 1.0837,
"margin_dpo/margin_mean": 251.91912841796875,
"margin_dpo/margin_std": 299.7061767578125,
"step": 373
},
{
"KL/chosen_KL_mean": -349.892333984375,
"KL/mean": -465.20263671875,
"KL/rejected_KL_mean": -580.512939453125,
"KL/std": 325.4590148925781,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.0015723302494734526,
"fcm_dpo/delta": 0.05567358061671257,
"fcm_dpo/margin": 230.62063598632812,
"fcm_dpo/q_t": 0.42263877391815186,
"grad_norm": 22.6434383392334,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": -0.5414470434188843,
"logits/rejected": -0.524927020072937,
"logps/chosen": -420.54254150390625,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -674.1531372070312,
"loss": 1.1406,
"margin_dpo/margin_mean": 230.62063598632812,
"margin_dpo/margin_std": 373.28961181640625,
"step": 374
},
{
"KL/chosen_KL_mean": -364.77288818359375,
"KL/mean": -473.304931640625,
"KL/rejected_KL_mean": -581.8369750976562,
"KL/std": 262.2808837890625,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.0017019468359649181,
"fcm_dpo/delta": 0.04178474843502045,
"fcm_dpo/margin": 217.06411743164062,
"fcm_dpo/q_t": 0.41703087091445923,
"grad_norm": 31.42391586303711,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": -0.4924180507659912,
"logits/rejected": -0.4902787208557129,
"logps/chosen": -424.85308837890625,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -670.7752685546875,
"loss": 1.1287,
"margin_dpo/margin_mean": 217.06411743164062,
"margin_dpo/margin_std": 315.9866943359375,
"step": 375
},
{
"KL/chosen_KL_mean": -347.1155090332031,
"KL/mean": -484.00567626953125,
"KL/rejected_KL_mean": -620.8958740234375,
"KL/std": 285.42181396484375,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0016071631107479334,
"fcm_dpo/delta": -0.07716603577136993,
"fcm_dpo/margin": 273.7803955078125,
"fcm_dpo/q_t": 0.39170658588409424,
"grad_norm": 24.55660629272461,
"learning_rate": 2.5e-07,
"logits/chosen": -0.49682873487472534,
"logits/rejected": -0.48542171716690063,
"logps/chosen": -409.77581787109375,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -726.4224853515625,
"loss": 1.0607,
"margin_dpo/margin_mean": 273.7803955078125,
"margin_dpo/margin_std": 332.501708984375,
"step": 376
},
{
"KL/chosen_KL_mean": -349.8211975097656,
"KL/mean": -483.8100280761719,
"KL/rejected_KL_mean": -617.7988891601562,
"KL/std": 303.7757263183594,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.0015564009081572294,
"fcm_dpo/delta": -0.030124623328447342,
"fcm_dpo/margin": 267.9776916503906,
"fcm_dpo/q_t": 0.401019811630249,
"grad_norm": 21.409746170043945,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": -0.4336928129196167,
"logits/rejected": -0.432683527469635,
"logps/chosen": -404.2999267578125,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -716.502197265625,
"loss": 1.072,
"margin_dpo/margin_mean": 267.9776611328125,
"margin_dpo/margin_std": 321.41864013671875,
"step": 377
},
{
"KL/chosen_KL_mean": -324.5780029296875,
"KL/mean": -463.517333984375,
"KL/rejected_KL_mean": -602.4566650390625,
"KL/std": 269.98370361328125,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0014617801643908024,
"fcm_dpo/delta": -0.010532225482165813,
"fcm_dpo/margin": 277.878662109375,
"fcm_dpo/q_t": 0.40280961990356445,
"grad_norm": 24.20557403564453,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": -0.43811696767807007,
"logits/rejected": -0.4507744312286377,
"logps/chosen": -369.5985107421875,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -690.503662109375,
"loss": 1.0649,
"margin_dpo/margin_mean": 277.878662109375,
"margin_dpo/margin_std": 294.4283447265625,
"step": 378
},
{
"KL/chosen_KL_mean": -359.47418212890625,
"KL/mean": -495.9452209472656,
"KL/rejected_KL_mean": -632.416259765625,
"KL/std": 277.9902038574219,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.0014259706949815154,
"fcm_dpo/delta": 0.0134861059486866,
"fcm_dpo/margin": 272.94207763671875,
"fcm_dpo/q_t": 0.41009724140167236,
"grad_norm": 26.755817413330078,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": -0.38554373383522034,
"logits/rejected": -0.4059056043624878,
"logps/chosen": -412.65625,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -746.7164306640625,
"loss": 1.0908,
"margin_dpo/margin_mean": 272.9421081542969,
"margin_dpo/margin_std": 334.01593017578125,
"step": 379
},
{
"KL/chosen_KL_mean": -353.51654052734375,
"KL/mean": -504.0564270019531,
"KL/rejected_KL_mean": -654.5963134765625,
"KL/std": 314.81463623046875,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.0014171417569741607,
"fcm_dpo/delta": -0.046321943402290344,
"fcm_dpo/margin": 301.0798034667969,
"fcm_dpo/q_t": 0.39565783739089966,
"grad_norm": 22.568410873413086,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": -0.4838710427284241,
"logits/rejected": -0.5105802416801453,
"logps/chosen": -404.86956787109375,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -758.7880249023438,
"loss": 1.0598,
"margin_dpo/margin_mean": 301.079833984375,
"margin_dpo/margin_std": 341.14447021484375,
"step": 380
},
{
"KL/chosen_KL_mean": -365.193359375,
"KL/mean": -466.3520812988281,
"KL/rejected_KL_mean": -567.5108642578125,
"KL/std": 262.38165283203125,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.001597442664206028,
"fcm_dpo/delta": 0.11401475220918655,
"fcm_dpo/margin": 202.31747436523438,
"fcm_dpo/q_t": 0.43302297592163086,
"grad_norm": 25.53712272644043,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": -0.4542039632797241,
"logits/rejected": -0.4321713149547577,
"logps/chosen": -422.99639892578125,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -646.730224609375,
"loss": 1.1718,
"margin_dpo/margin_mean": 202.31747436523438,
"margin_dpo/margin_std": 359.52716064453125,
"step": 381
},
{
"KL/chosen_KL_mean": -346.081298828125,
"KL/mean": -476.01800537109375,
"KL/rejected_KL_mean": -605.9547119140625,
"KL/std": 252.94512939453125,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.0016131680458784103,
"fcm_dpo/delta": -0.03418818861246109,
"fcm_dpo/margin": 259.8733825683594,
"fcm_dpo/q_t": 0.3991495966911316,
"grad_norm": 20.419843673706055,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": -0.5054559707641602,
"logits/rejected": -0.5107045769691467,
"logps/chosen": -412.10162353515625,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -716.6648559570312,
"loss": 1.0605,
"margin_dpo/margin_mean": 259.8733825683594,
"margin_dpo/margin_std": 285.6113586425781,
"step": 382
},
{
"KL/chosen_KL_mean": -361.154541015625,
"KL/mean": -490.5584716796875,
"KL/rejected_KL_mean": -619.96240234375,
"KL/std": 283.15301513671875,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.0015578651800751686,
"fcm_dpo/delta": -0.00645710714161396,
"fcm_dpo/margin": 258.8077392578125,
"fcm_dpo/q_t": 0.4064059555530548,
"grad_norm": 27.733203887939453,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": -0.4914437532424927,
"logits/rejected": -0.5008552670478821,
"logps/chosen": -411.5460510253906,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -713.67822265625,
"loss": 1.1016,
"margin_dpo/margin_mean": 258.8077392578125,
"margin_dpo/margin_std": 370.54718017578125,
"step": 383
},
{
"KL/chosen_KL_mean": -375.8470458984375,
"KL/mean": -478.16192626953125,
"KL/rejected_KL_mean": -580.4768676757812,
"KL/std": 256.4949645996094,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.0017840572400018573,
"fcm_dpo/delta": 0.04306403547525406,
"fcm_dpo/margin": 204.6298065185547,
"fcm_dpo/q_t": 0.41565489768981934,
"grad_norm": 26.314607620239258,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": -0.501458466053009,
"logits/rejected": -0.4933622181415558,
"logps/chosen": -427.89312744140625,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -666.23779296875,
"loss": 1.1048,
"margin_dpo/margin_mean": 204.62982177734375,
"margin_dpo/margin_std": 249.5997314453125,
"step": 384
},
{
"KL/chosen_KL_mean": -345.30267333984375,
"KL/mean": -467.48583984375,
"KL/rejected_KL_mean": -589.6690063476562,
"KL/std": 224.0086669921875,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.0016768588684499264,
"fcm_dpo/delta": -0.016689352691173553,
"fcm_dpo/margin": 244.36630249023438,
"fcm_dpo/q_t": 0.40174418687820435,
"grad_norm": 26.779409408569336,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": -0.5289189219474792,
"logits/rejected": -0.500619649887085,
"logps/chosen": -410.8548583984375,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -667.4969482421875,
"loss": 1.0615,
"margin_dpo/margin_mean": 244.36631774902344,
"margin_dpo/margin_std": 253.60830688476562,
"step": 385
},
{
"KL/chosen_KL_mean": -361.995361328125,
"KL/mean": -493.2794494628906,
"KL/rejected_KL_mean": -624.5635375976562,
"KL/std": 274.184326171875,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.0015988796949386597,
"fcm_dpo/delta": -0.034051790833473206,
"fcm_dpo/margin": 262.5681457519531,
"fcm_dpo/q_t": 0.400490939617157,
"grad_norm": 25.8364200592041,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": -0.5255472660064697,
"logits/rejected": -0.5233632922172546,
"logps/chosen": -420.21722412109375,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -716.8909912109375,
"loss": 1.0737,
"margin_dpo/margin_mean": 262.56817626953125,
"margin_dpo/margin_std": 327.55181884765625,
"step": 386
},
{
"KL/chosen_KL_mean": -404.6907958984375,
"KL/mean": -515.4647827148438,
"KL/rejected_KL_mean": -626.23876953125,
"KL/std": 265.8984375,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.0016458687605336308,
"fcm_dpo/delta": 0.05550061911344528,
"fcm_dpo/margin": 221.54798889160156,
"fcm_dpo/q_t": 0.4187946617603302,
"grad_norm": 29.908674240112305,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": -0.5720622539520264,
"logits/rejected": -0.5503807067871094,
"logps/chosen": -471.1102600097656,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -718.407958984375,
"loss": 1.1074,
"margin_dpo/margin_mean": 221.54798889160156,
"margin_dpo/margin_std": 279.3563232421875,
"step": 387
},
{
"KL/chosen_KL_mean": -380.31915283203125,
"KL/mean": -527.940185546875,
"KL/rejected_KL_mean": -675.5611572265625,
"KL/std": 304.70587158203125,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.001483785454183817,
"fcm_dpo/delta": -0.07916902750730515,
"fcm_dpo/margin": 295.241943359375,
"fcm_dpo/q_t": 0.39270198345184326,
"grad_norm": 27.048213958740234,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": -0.4747141897678375,
"logits/rejected": -0.4871397018432617,
"logps/chosen": -430.4486083984375,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -779.9942016601562,
"loss": 1.0573,
"margin_dpo/margin_mean": 295.2419738769531,
"margin_dpo/margin_std": 350.07147216796875,
"step": 388
},
{
"KL/chosen_KL_mean": -417.1252136230469,
"KL/mean": -543.790771484375,
"KL/rejected_KL_mean": -670.4564208984375,
"KL/std": 307.47637939453125,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.001513138646259904,
"fcm_dpo/delta": 0.026595965027809143,
"fcm_dpo/margin": 253.3311767578125,
"fcm_dpo/q_t": 0.41341066360473633,
"grad_norm": 33.48180389404297,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": -0.5469924211502075,
"logits/rejected": -0.5381365418434143,
"logps/chosen": -475.03179931640625,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -748.3709716796875,
"loss": 1.1045,
"margin_dpo/margin_mean": 253.3311767578125,
"margin_dpo/margin_std": 345.8287353515625,
"step": 389
},
{
"KL/chosen_KL_mean": -417.957763671875,
"KL/mean": -541.5369262695312,
"KL/rejected_KL_mean": -665.1160888671875,
"KL/std": 302.0540771484375,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.0015666976105421782,
"fcm_dpo/delta": 0.019385023042559624,
"fcm_dpo/margin": 247.1583251953125,
"fcm_dpo/q_t": 0.4156228303909302,
"grad_norm": 23.625947952270508,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": -0.46019166707992554,
"logits/rejected": -0.45252659916877747,
"logps/chosen": -467.1836853027344,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -750.644287109375,
"loss": 1.1168,
"margin_dpo/margin_mean": 247.1583251953125,
"margin_dpo/margin_std": 383.4603271484375,
"step": 390
},
{
"KL/chosen_KL_mean": -418.42572021484375,
"KL/mean": -490.8452453613281,
"KL/rejected_KL_mean": -563.2647705078125,
"KL/std": 288.76385498046875,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.0020397088956087828,
"fcm_dpo/delta": 0.14491389691829681,
"fcm_dpo/margin": 144.83901977539062,
"fcm_dpo/q_t": 0.4414842128753662,
"grad_norm": 52.0184440612793,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": -0.472456693649292,
"logits/rejected": -0.4619515538215637,
"logps/chosen": -482.75537109375,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -650.0029296875,
"loss": 1.2088,
"margin_dpo/margin_mean": 144.83901977539062,
"margin_dpo/margin_std": 298.0270690917969,
"step": 391
},
{
"KL/chosen_KL_mean": -343.19525146484375,
"KL/mean": -493.396484375,
"KL/rejected_KL_mean": -643.5977783203125,
"KL/std": 300.6968994140625,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.0016628424637019634,
"fcm_dpo/delta": -0.20023079216480255,
"fcm_dpo/margin": 300.4024658203125,
"fcm_dpo/q_t": 0.36630189418792725,
"grad_norm": 26.650352478027344,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": -0.4742690920829773,
"logits/rejected": -0.46840304136276245,
"logps/chosen": -396.69921875,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -745.943603515625,
"loss": 1.0092,
"margin_dpo/margin_mean": 300.4024658203125,
"margin_dpo/margin_std": 297.166748046875,
"step": 392
},
{
"KL/chosen_KL_mean": -344.81060791015625,
"KL/mean": -455.6504821777344,
"KL/rejected_KL_mean": -566.4903564453125,
"KL/std": 291.33221435546875,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.0016052678693085909,
"fcm_dpo/delta": 0.06851550936698914,
"fcm_dpo/margin": 221.67974853515625,
"fcm_dpo/q_t": 0.4255250096321106,
"grad_norm": 21.591970443725586,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": -0.4540834426879883,
"logits/rejected": -0.451620489358902,
"logps/chosen": -391.2845458984375,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -638.459228515625,
"loss": 1.1418,
"margin_dpo/margin_mean": 221.67974853515625,
"margin_dpo/margin_std": 372.8785400390625,
"step": 393
},
{
"KL/chosen_KL_mean": -376.7562561035156,
"KL/mean": -496.7586669921875,
"KL/rejected_KL_mean": -616.7610473632812,
"KL/std": 303.6330261230469,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.001655565807595849,
"fcm_dpo/delta": 0.003836357966065407,
"fcm_dpo/margin": 240.00477600097656,
"fcm_dpo/q_t": 0.4095136225223541,
"grad_norm": 21.343175888061523,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": -0.46841922402381897,
"logits/rejected": -0.4693748354911804,
"logps/chosen": -429.6678161621094,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -707.5836791992188,
"loss": 1.0918,
"margin_dpo/margin_mean": 240.00477600097656,
"margin_dpo/margin_std": 317.42437744140625,
"step": 394
},
{
"KL/chosen_KL_mean": -362.38995361328125,
"KL/mean": -493.2774658203125,
"KL/rejected_KL_mean": -624.1649780273438,
"KL/std": 293.3901672363281,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.0015726612182334065,
"fcm_dpo/delta": -0.024214228615164757,
"fcm_dpo/margin": 261.7750244140625,
"fcm_dpo/q_t": 0.4036139249801636,
"grad_norm": 21.289175033569336,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": -0.5199534893035889,
"logits/rejected": -0.5158529281616211,
"logps/chosen": -424.9360656738281,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -707.9476318359375,
"loss": 1.0849,
"margin_dpo/margin_mean": 261.7750244140625,
"margin_dpo/margin_std": 343.44354248046875,
"step": 395
},
{
"KL/chosen_KL_mean": -373.7277526855469,
"KL/mean": -489.446533203125,
"KL/rejected_KL_mean": -605.1654052734375,
"KL/std": 288.4603576660156,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.0016512691508978605,
"fcm_dpo/delta": 0.028826599940657616,
"fcm_dpo/margin": 231.4375457763672,
"fcm_dpo/q_t": 0.41250765323638916,
"grad_norm": 22.668109893798828,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.4854172468185425,
"logits/rejected": -0.4645005464553833,
"logps/chosen": -442.72369384765625,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -693.81201171875,
"loss": 1.0875,
"margin_dpo/margin_mean": 231.43756103515625,
"margin_dpo/margin_std": 270.1766662597656,
"step": 396
},
{
"KL/chosen_KL_mean": -345.4547119140625,
"KL/mean": -491.2574462890625,
"KL/rejected_KL_mean": -637.0601806640625,
"KL/std": 283.0622253417969,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.0014739397447556257,
"fcm_dpo/delta": -0.06422150880098343,
"fcm_dpo/margin": 291.6054382324219,
"fcm_dpo/q_t": 0.39274823665618896,
"grad_norm": 29.06831169128418,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": -0.46980020403862,
"logits/rejected": -0.47478750348091125,
"logps/chosen": -406.73187255859375,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -740.17626953125,
"loss": 1.0495,
"margin_dpo/margin_mean": 291.6054382324219,
"margin_dpo/margin_std": 302.38970947265625,
"step": 397
},
{
"KL/chosen_KL_mean": -366.6300354003906,
"KL/mean": -499.76983642578125,
"KL/rejected_KL_mean": -632.90966796875,
"KL/std": 293.9355773925781,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.0014921356923878193,
"fcm_dpo/delta": 0.004290161654353142,
"fcm_dpo/margin": 266.2795715332031,
"fcm_dpo/q_t": 0.40941908955574036,
"grad_norm": 20.657440185546875,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.5278111696243286,
"logits/rejected": -0.5256218314170837,
"logps/chosen": -434.7815856933594,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -741.4332275390625,
"loss": 1.0907,
"margin_dpo/margin_mean": 266.2795715332031,
"margin_dpo/margin_std": 350.179443359375,
"step": 398
},
{
"KL/chosen_KL_mean": -315.19482421875,
"KL/mean": -432.66326904296875,
"KL/rejected_KL_mean": -550.1317138671875,
"KL/std": 257.30950927734375,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.0015734569169580936,
"fcm_dpo/delta": 0.047637663781642914,
"fcm_dpo/margin": 234.9368133544922,
"fcm_dpo/q_t": 0.41743379831314087,
"grad_norm": 21.86760711669922,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": -0.4569128751754761,
"logits/rejected": -0.41413915157318115,
"logps/chosen": -376.0846252441406,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -628.0972900390625,
"loss": 1.1148,
"margin_dpo/margin_mean": 234.93682861328125,
"margin_dpo/margin_std": 323.26910400390625,
"step": 399
},
{
"KL/chosen_KL_mean": -298.4742431640625,
"KL/mean": -467.8236389160156,
"KL/rejected_KL_mean": -637.1730346679688,
"KL/std": 272.68536376953125,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.00134001555852592,
"fcm_dpo/delta": -0.11473602056503296,
"fcm_dpo/margin": 338.6988220214844,
"fcm_dpo/q_t": 0.37980222702026367,
"grad_norm": 19.321056365966797,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": -0.45275765657424927,
"logits/rejected": -0.4457949995994568,
"logps/chosen": -362.1178283691406,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -742.4257202148438,
"loss": 1.0206,
"margin_dpo/margin_mean": 338.6988220214844,
"margin_dpo/margin_std": 297.51031494140625,
"step": 400
},
{
"KL/chosen_KL_mean": -347.76898193359375,
"KL/mean": -442.2925109863281,
"KL/rejected_KL_mean": -536.8160400390625,
"KL/std": 280.5494384765625,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.00160554563626647,
"fcm_dpo/delta": 0.13878443837165833,
"fcm_dpo/margin": 189.04698181152344,
"fcm_dpo/q_t": 0.43937820196151733,
"grad_norm": 21.756427764892578,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": -0.4988451600074768,
"logits/rejected": -0.492392897605896,
"logps/chosen": -404.9320373535156,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -620.6085205078125,
"loss": 1.1925,
"margin_dpo/margin_mean": 189.0469970703125,
"margin_dpo/margin_std": 374.1167297363281,
"step": 401
},
{
"KL/chosen_KL_mean": -258.14959716796875,
"KL/mean": -438.02703857421875,
"KL/rejected_KL_mean": -617.9044799804688,
"KL/std": 310.0885009765625,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.0013267784379422665,
"fcm_dpo/delta": -0.1641661822795868,
"fcm_dpo/margin": 359.75494384765625,
"fcm_dpo/q_t": 0.37116238474845886,
"grad_norm": 20.190622329711914,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": -0.4920314848423004,
"logits/rejected": -0.480332612991333,
"logps/chosen": -308.88995361328125,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -698.9505615234375,
"loss": 1.0061,
"margin_dpo/margin_mean": 359.75494384765625,
"margin_dpo/margin_std": 319.1515808105469,
"step": 402
},
{
"KL/chosen_KL_mean": -292.50390625,
"KL/mean": -428.873291015625,
"KL/rejected_KL_mean": -565.24267578125,
"KL/std": 291.69244384765625,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.0013493448495864868,
"fcm_dpo/delta": 0.050415799021720886,
"fcm_dpo/margin": 272.73876953125,
"fcm_dpo/q_t": 0.4170788526535034,
"grad_norm": 22.705007553100586,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": -0.4615529775619507,
"logits/rejected": -0.4557945728302002,
"logps/chosen": -339.6512145996094,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -642.50927734375,
"loss": 1.0941,
"margin_dpo/margin_mean": 272.73876953125,
"margin_dpo/margin_std": 306.66265869140625,
"step": 403
},
{
"KL/chosen_KL_mean": -336.2072448730469,
"KL/mean": -466.8104248046875,
"KL/rejected_KL_mean": -597.41357421875,
"KL/std": 278.45245361328125,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.001444081193767488,
"fcm_dpo/delta": 0.03668762370944023,
"fcm_dpo/margin": 261.20635986328125,
"fcm_dpo/q_t": 0.41537854075431824,
"grad_norm": 29.176677703857422,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": -0.45944684743881226,
"logits/rejected": -0.4487238824367523,
"logps/chosen": -384.08251953125,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -674.568603515625,
"loss": 1.1104,
"margin_dpo/margin_mean": 261.20635986328125,
"margin_dpo/margin_std": 361.2408447265625,
"step": 404
},
{
"KL/chosen_KL_mean": -388.7740478515625,
"KL/mean": -496.8353271484375,
"KL/rejected_KL_mean": -604.8966064453125,
"KL/std": 312.28668212890625,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.0014538425020873547,
"fcm_dpo/delta": -0.026516491547226906,
"fcm_dpo/margin": 216.12255859375,
"fcm_dpo/q_t": 0.42757102847099304,
"grad_norm": 29.22574234008789,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": -0.5201287865638733,
"logits/rejected": -0.5093380212783813,
"logps/chosen": -453.9369812011719,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -692.0834350585938,
"loss": 1.1806,
"margin_dpo/margin_mean": 216.12255859375,
"margin_dpo/margin_std": 396.87908935546875,
"step": 405
},
{
"KL/chosen_KL_mean": -355.637451171875,
"KL/mean": -501.5604248046875,
"KL/rejected_KL_mean": -647.4833984375,
"KL/std": 317.0197448730469,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.0014266908401623368,
"fcm_dpo/delta": -0.033352650701999664,
"fcm_dpo/margin": 291.8459167480469,
"fcm_dpo/q_t": 0.4017193913459778,
"grad_norm": 21.75078773498535,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": -0.5137743949890137,
"logits/rejected": -0.5227109789848328,
"logps/chosen": -405.3782958984375,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -739.56201171875,
"loss": 1.0792,
"margin_dpo/margin_mean": 291.8459167480469,
"margin_dpo/margin_std": 360.86712646484375,
"step": 406
},
{
"KL/chosen_KL_mean": -375.3631896972656,
"KL/mean": -470.50189208984375,
"KL/rejected_KL_mean": -565.6405639648438,
"KL/std": 258.78857421875,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.0013535934267565608,
"fcm_dpo/delta": 0.02188667096197605,
"fcm_dpo/margin": 190.27740478515625,
"fcm_dpo/q_t": 0.43996331095695496,
"grad_norm": 23.94710350036621,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": -0.5585223436355591,
"logits/rejected": -0.5322338938713074,
"logps/chosen": -431.69390869140625,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -643.1527099609375,
"loss": 1.1953,
"margin_dpo/margin_mean": 190.2774200439453,
"margin_dpo/margin_std": 333.7882385253906,
"step": 407
},
{
"KL/chosen_KL_mean": -389.05035400390625,
"KL/mean": -490.5750427246094,
"KL/rejected_KL_mean": -592.0997314453125,
"KL/std": 244.0631866455078,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.0016291262581944466,
"fcm_dpo/delta": 0.09969305992126465,
"fcm_dpo/margin": 203.04937744140625,
"fcm_dpo/q_t": 0.4286070466041565,
"grad_norm": 28.127901077270508,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": -0.5659944415092468,
"logits/rejected": -0.5408339500427246,
"logps/chosen": -458.83966064453125,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -682.1966552734375,
"loss": 1.1308,
"margin_dpo/margin_mean": 203.04937744140625,
"margin_dpo/margin_std": 274.0846862792969,
"step": 408
},
{
"KL/chosen_KL_mean": -381.6026306152344,
"KL/mean": -485.71112060546875,
"KL/rejected_KL_mean": -589.819580078125,
"KL/std": 286.2164306640625,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0018248842097818851,
"fcm_dpo/delta": 0.02699420042335987,
"fcm_dpo/margin": 208.21694946289062,
"fcm_dpo/q_t": 0.4149779677391052,
"grad_norm": 42.70933532714844,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": -0.4948524832725525,
"logits/rejected": -0.47649848461151123,
"logps/chosen": -448.9200744628906,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -674.723876953125,
"loss": 1.1244,
"margin_dpo/margin_mean": 208.21694946289062,
"margin_dpo/margin_std": 319.69189453125,
"step": 409
},
{
"KL/chosen_KL_mean": -351.9015808105469,
"KL/mean": -478.4359130859375,
"KL/rejected_KL_mean": -604.9702758789062,
"KL/std": 285.3361511230469,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.0016538414638489485,
"fcm_dpo/delta": -0.03480095416307449,
"fcm_dpo/margin": 253.06863403320312,
"fcm_dpo/q_t": 0.3992757201194763,
"grad_norm": 35.037715911865234,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": -0.5824193954467773,
"logits/rejected": -0.5856485366821289,
"logps/chosen": -403.366943359375,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -688.169189453125,
"loss": 1.079,
"margin_dpo/margin_mean": 253.06863403320312,
"margin_dpo/margin_std": 324.7110595703125,
"step": 410
},
{
"KL/chosen_KL_mean": -374.0163269042969,
"KL/mean": -491.8541259765625,
"KL/rejected_KL_mean": -609.69189453125,
"KL/std": 309.1568603515625,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.0014088283060118556,
"fcm_dpo/delta": -0.08271211385726929,
"fcm_dpo/margin": 235.67556762695312,
"fcm_dpo/q_t": 0.4166325628757477,
"grad_norm": 32.5038948059082,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": -0.5615779757499695,
"logits/rejected": -0.566271185874939,
"logps/chosen": -426.3236083984375,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -690.3868408203125,
"loss": 1.1475,
"margin_dpo/margin_mean": 235.67556762695312,
"margin_dpo/margin_std": 364.743408203125,
"step": 411
},
{
"KL/chosen_KL_mean": -376.010009765625,
"KL/mean": -504.2699279785156,
"KL/rejected_KL_mean": -632.5298461914062,
"KL/std": 297.99005126953125,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0014877468347549438,
"fcm_dpo/delta": 0.028690597042441368,
"fcm_dpo/margin": 256.51983642578125,
"fcm_dpo/q_t": 0.4138166904449463,
"grad_norm": 27.262081146240234,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": -0.6011987924575806,
"logits/rejected": -0.6305391788482666,
"logps/chosen": -429.1541442871094,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -732.5906372070312,
"loss": 1.1073,
"margin_dpo/margin_mean": 256.51983642578125,
"margin_dpo/margin_std": 358.5880126953125,
"step": 412
},
{
"KL/chosen_KL_mean": -413.91949462890625,
"KL/mean": -544.1796264648438,
"KL/rejected_KL_mean": -674.4398193359375,
"KL/std": 308.14947509765625,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.0015282519161701202,
"fcm_dpo/delta": 0.0018488653004169464,
"fcm_dpo/margin": 260.52032470703125,
"fcm_dpo/q_t": 0.4066236615180969,
"grad_norm": 31.31689453125,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": -0.5308142304420471,
"logits/rejected": -0.5381065607070923,
"logps/chosen": -475.5014343261719,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -773.9132080078125,
"loss": 1.0953,
"margin_dpo/margin_mean": 260.52032470703125,
"margin_dpo/margin_std": 349.33416748046875,
"step": 413
},
{
"KL/chosen_KL_mean": -376.8102722167969,
"KL/mean": -513.2676391601562,
"KL/rejected_KL_mean": -649.7249755859375,
"KL/std": 290.8931884765625,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.0015321837272495031,
"fcm_dpo/delta": -0.043847713619470596,
"fcm_dpo/margin": 272.91473388671875,
"fcm_dpo/q_t": 0.3983882963657379,
"grad_norm": 32.68032455444336,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": -0.5271749496459961,
"logits/rejected": -0.5175067186355591,
"logps/chosen": -423.4417724609375,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -737.3715209960938,
"loss": 1.0877,
"margin_dpo/margin_mean": 272.91473388671875,
"margin_dpo/margin_std": 341.33642578125,
"step": 414
},
{
"KL/chosen_KL_mean": -399.9847412109375,
"KL/mean": -505.3280029296875,
"KL/rejected_KL_mean": -610.6712646484375,
"KL/std": 276.31341552734375,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.0015684034442529082,
"fcm_dpo/delta": 0.10484690964221954,
"fcm_dpo/margin": 210.68653869628906,
"fcm_dpo/q_t": 0.42983919382095337,
"grad_norm": 25.915630340576172,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": -0.5725701451301575,
"logits/rejected": -0.567946195602417,
"logps/chosen": -478.60302734375,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -711.1488037109375,
"loss": 1.144,
"margin_dpo/margin_mean": 210.68653869628906,
"margin_dpo/margin_std": 311.3158264160156,
"step": 415
},
{
"KL/chosen_KL_mean": -365.5072021484375,
"KL/mean": -527.9451904296875,
"KL/rejected_KL_mean": -690.3831787109375,
"KL/std": 313.2249755859375,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0014323981013149023,
"fcm_dpo/delta": -0.1226118803024292,
"fcm_dpo/margin": 324.8759460449219,
"fcm_dpo/q_t": 0.37992632389068604,
"grad_norm": 27.974157333374023,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": -0.5598719120025635,
"logits/rejected": -0.564556896686554,
"logps/chosen": -423.7863464355469,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -780.951904296875,
"loss": 1.024,
"margin_dpo/margin_mean": 324.8759460449219,
"margin_dpo/margin_std": 312.55938720703125,
"step": 416
},
{
"KL/chosen_KL_mean": -353.57537841796875,
"KL/mean": -489.65478515625,
"KL/rejected_KL_mean": -625.7341918945312,
"KL/std": 285.030029296875,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.0013993415050208569,
"fcm_dpo/delta": 0.030016159638762474,
"fcm_dpo/margin": 272.1588134765625,
"fcm_dpo/q_t": 0.4119567573070526,
"grad_norm": 29.711999893188477,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": -0.5493444204330444,
"logits/rejected": -0.5241958498954773,
"logps/chosen": -403.7740783691406,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -693.8860473632812,
"loss": 1.0801,
"margin_dpo/margin_mean": 272.1588439941406,
"margin_dpo/margin_std": 288.48846435546875,
"step": 417
},
{
"KL/chosen_KL_mean": -389.98394775390625,
"KL/mean": -531.188232421875,
"KL/rejected_KL_mean": -672.392578125,
"KL/std": 321.02679443359375,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.0014645163901150227,
"fcm_dpo/delta": -0.04145345836877823,
"fcm_dpo/margin": 282.40869140625,
"fcm_dpo/q_t": 0.4018082022666931,
"grad_norm": 22.506240844726562,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": -0.6127409338951111,
"logits/rejected": -0.5916212201118469,
"logps/chosen": -471.96240234375,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -777.0841064453125,
"loss": 1.1022,
"margin_dpo/margin_mean": 282.40869140625,
"margin_dpo/margin_std": 376.09161376953125,
"step": 418
},
{
"KL/chosen_KL_mean": -354.58636474609375,
"KL/mean": -506.98297119140625,
"KL/rejected_KL_mean": -659.379638671875,
"KL/std": 276.9751281738281,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.0013055275194346905,
"fcm_dpo/delta": 0.0034003122709691525,
"fcm_dpo/margin": 304.79327392578125,
"fcm_dpo/q_t": 0.4058445692062378,
"grad_norm": 25.18451499938965,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": -0.5361112356185913,
"logits/rejected": -0.5325924158096313,
"logps/chosen": -407.5350341796875,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -750.9627685546875,
"loss": 1.0692,
"margin_dpo/margin_mean": 304.79327392578125,
"margin_dpo/margin_std": 320.16162109375,
"step": 419
},
{
"KL/chosen_KL_mean": -448.81329345703125,
"KL/mean": -544.2679443359375,
"KL/rejected_KL_mean": -639.7225341796875,
"KL/std": 302.1771240234375,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.001609490136615932,
"fcm_dpo/delta": 0.12760579586029053,
"fcm_dpo/margin": 190.9093475341797,
"fcm_dpo/q_t": 0.43786126375198364,
"grad_norm": 57.40446090698242,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": -0.5600706338882446,
"logits/rejected": -0.5160965919494629,
"logps/chosen": -526.5831909179688,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -709.04248046875,
"loss": 1.2016,
"margin_dpo/margin_mean": 190.9093475341797,
"margin_dpo/margin_std": 403.58026123046875,
"step": 420
},
{
"KL/chosen_KL_mean": -383.8134765625,
"KL/mean": -487.9678955078125,
"KL/rejected_KL_mean": -592.122314453125,
"KL/std": 293.6025390625,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.0018344377167522907,
"fcm_dpo/delta": 0.018156960606575012,
"fcm_dpo/margin": 208.3088836669922,
"fcm_dpo/q_t": 0.4109729826450348,
"grad_norm": 32.440040588378906,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": -0.6202581524848938,
"logits/rejected": -0.6075379848480225,
"logps/chosen": -437.579345703125,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -681.40380859375,
"loss": 1.1322,
"margin_dpo/margin_mean": 208.3088836669922,
"margin_dpo/margin_std": 319.2204895019531,
"step": 421
},
{
"KL/chosen_KL_mean": -423.02252197265625,
"KL/mean": -545.48046875,
"KL/rejected_KL_mean": -667.9382934570312,
"KL/std": 296.6886901855469,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.0016887241508811712,
"fcm_dpo/delta": -0.023193474858999252,
"fcm_dpo/margin": 244.91578674316406,
"fcm_dpo/q_t": 0.40264299511909485,
"grad_norm": 44.55949783325195,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": -0.5801171064376831,
"logits/rejected": -0.5803545713424683,
"logps/chosen": -491.65631103515625,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -755.8018188476562,
"loss": 1.0928,
"margin_dpo/margin_mean": 244.91578674316406,
"margin_dpo/margin_std": 343.1561584472656,
"step": 422
},
{
"KL/chosen_KL_mean": -408.7502136230469,
"KL/mean": -549.06884765625,
"KL/rejected_KL_mean": -689.3875122070312,
"KL/std": 292.96270751953125,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.0015233319718390703,
"fcm_dpo/delta": -0.05018993839621544,
"fcm_dpo/margin": 280.6373291015625,
"fcm_dpo/q_t": 0.3961368799209595,
"grad_norm": 26.388099670410156,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": -0.5856159329414368,
"logits/rejected": -0.5923899412155151,
"logps/chosen": -463.7405090332031,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -775.694091796875,
"loss": 1.0592,
"margin_dpo/margin_mean": 280.6373596191406,
"margin_dpo/margin_std": 320.4229431152344,
"step": 423
},
{
"KL/chosen_KL_mean": -368.3785400390625,
"KL/mean": -496.5576477050781,
"KL/rejected_KL_mean": -624.7366943359375,
"KL/std": 285.81842041015625,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.0015009802300482988,
"fcm_dpo/delta": 0.022251751273870468,
"fcm_dpo/margin": 256.358154296875,
"fcm_dpo/q_t": 0.4135265052318573,
"grad_norm": 33.62800598144531,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": -0.6009132266044617,
"logits/rejected": -0.5754563808441162,
"logps/chosen": -424.39044189453125,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -691.2156982421875,
"loss": 1.106,
"margin_dpo/margin_mean": 256.358154296875,
"margin_dpo/margin_std": 352.6119384765625,
"step": 424
},
{
"KL/chosen_KL_mean": -417.15570068359375,
"KL/mean": -515.7978515625,
"KL/rejected_KL_mean": -614.43994140625,
"KL/std": 278.95025634765625,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.0017970511689782143,
"fcm_dpo/delta": 0.062237467616796494,
"fcm_dpo/margin": 197.28427124023438,
"fcm_dpo/q_t": 0.4205625057220459,
"grad_norm": 29.63746452331543,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": -0.5688070058822632,
"logits/rejected": -0.5690561532974243,
"logps/chosen": -464.024658203125,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -710.3654174804688,
"loss": 1.1556,
"margin_dpo/margin_mean": 197.28427124023438,
"margin_dpo/margin_std": 347.9555358886719,
"step": 425
},
{
"KL/chosen_KL_mean": -396.1029052734375,
"KL/mean": -517.9900512695312,
"KL/rejected_KL_mean": -639.877197265625,
"KL/std": 274.98834228515625,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0017517898231744766,
"fcm_dpo/delta": -0.0511149987578392,
"fcm_dpo/margin": 243.7742919921875,
"fcm_dpo/q_t": 0.3971126079559326,
"grad_norm": 32.52616882324219,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": -0.5753895044326782,
"logits/rejected": -0.5384079813957214,
"logps/chosen": -472.6864318847656,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -721.143798828125,
"loss": 1.0773,
"margin_dpo/margin_mean": 243.7742919921875,
"margin_dpo/margin_std": 307.3427734375,
"step": 426
},
{
"KL/chosen_KL_mean": -360.6373596191406,
"KL/mean": -457.64605712890625,
"KL/rejected_KL_mean": -554.6547241210938,
"KL/std": 240.3179473876953,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.001793712843209505,
"fcm_dpo/delta": 0.08003441989421844,
"fcm_dpo/margin": 194.01736450195312,
"fcm_dpo/q_t": 0.4255453646183014,
"grad_norm": 28.326711654663086,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": -0.6111181974411011,
"logits/rejected": -0.5922002792358398,
"logps/chosen": -425.4912414550781,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -633.220703125,
"loss": 1.1332,
"margin_dpo/margin_mean": 194.01736450195312,
"margin_dpo/margin_std": 291.666015625,
"step": 427
},
{
"KL/chosen_KL_mean": -443.464111328125,
"KL/mean": -577.4393310546875,
"KL/rejected_KL_mean": -711.41455078125,
"KL/std": 321.87640380859375,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.001614520326256752,
"fcm_dpo/delta": -0.07145649194717407,
"fcm_dpo/margin": 267.9504089355469,
"fcm_dpo/q_t": 0.394869327545166,
"grad_norm": 29.2352294921875,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": -0.5986416339874268,
"logits/rejected": -0.6018632650375366,
"logps/chosen": -506.10076904296875,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -814.6963500976562,
"loss": 1.0951,
"margin_dpo/margin_mean": 267.9504089355469,
"margin_dpo/margin_std": 396.5560302734375,
"step": 428
},
{
"KL/chosen_KL_mean": -453.14031982421875,
"KL/mean": -558.6182861328125,
"KL/rejected_KL_mean": -664.09619140625,
"KL/std": 281.262939453125,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.0015741197858005762,
"fcm_dpo/delta": -0.040317293256521225,
"fcm_dpo/margin": 210.95590209960938,
"fcm_dpo/q_t": 0.4206579625606537,
"grad_norm": 29.22441864013672,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": -0.6086622476577759,
"logits/rejected": -0.581855297088623,
"logps/chosen": -534.3743286132812,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -755.89111328125,
"loss": 1.1602,
"margin_dpo/margin_mean": 210.95590209960938,
"margin_dpo/margin_std": 341.42901611328125,
"step": 429
},
{
"KL/chosen_KL_mean": -381.8338623046875,
"KL/mean": -485.0985107421875,
"KL/rejected_KL_mean": -588.3631591796875,
"KL/std": 265.72589111328125,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.001708697760477662,
"fcm_dpo/delta": 0.06908033788204193,
"fcm_dpo/margin": 206.52932739257812,
"fcm_dpo/q_t": 0.4227146804332733,
"grad_norm": 40.607749938964844,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": -0.5892158150672913,
"logits/rejected": -0.5879628658294678,
"logps/chosen": -442.754150390625,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -692.7860107421875,
"loss": 1.1248,
"margin_dpo/margin_mean": 206.52932739257812,
"margin_dpo/margin_std": 297.7901611328125,
"step": 430
},
{
"KL/chosen_KL_mean": -354.79736328125,
"KL/mean": -504.74810791015625,
"KL/rejected_KL_mean": -654.6988525390625,
"KL/std": 294.78875732421875,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.0014870925806462765,
"fcm_dpo/delta": -0.0890204980969429,
"fcm_dpo/margin": 299.9014892578125,
"fcm_dpo/q_t": 0.3885713517665863,
"grad_norm": 27.04310417175293,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": -0.6147454977035522,
"logits/rejected": -0.6116945743560791,
"logps/chosen": -412.1461181640625,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -747.5390625,
"loss": 1.0492,
"margin_dpo/margin_mean": 299.9014892578125,
"margin_dpo/margin_std": 333.67730712890625,
"step": 431
},
{
"KL/chosen_KL_mean": -351.54315185546875,
"KL/mean": -501.9591979980469,
"KL/rejected_KL_mean": -652.3753051757812,
"KL/std": 292.12811279296875,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.0014298271853476763,
"fcm_dpo/delta": -0.05924157053232193,
"fcm_dpo/margin": 300.8321533203125,
"fcm_dpo/q_t": 0.39376306533813477,
"grad_norm": 31.670360565185547,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": -0.6010767221450806,
"logits/rejected": -0.5897285342216492,
"logps/chosen": -422.617919921875,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -750.954833984375,
"loss": 1.0523,
"margin_dpo/margin_mean": 300.8321533203125,
"margin_dpo/margin_std": 296.0356140136719,
"step": 432
},
{
"KL/chosen_KL_mean": -445.81146240234375,
"KL/mean": -543.8163452148438,
"KL/rejected_KL_mean": -641.8211669921875,
"KL/std": 302.93316650390625,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.0015859071863815188,
"fcm_dpo/delta": 0.12226778268814087,
"fcm_dpo/margin": 196.00973510742188,
"fcm_dpo/q_t": 0.43242147564888,
"grad_norm": 38.35377502441406,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": -0.6371247172355652,
"logits/rejected": -0.6237683892250061,
"logps/chosen": -504.08465576171875,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -737.7720947265625,
"loss": 1.1783,
"margin_dpo/margin_mean": 196.00973510742188,
"margin_dpo/margin_std": 365.50811767578125,
"step": 433
},
{
"KL/chosen_KL_mean": -385.5121765136719,
"KL/mean": -504.4186096191406,
"KL/rejected_KL_mean": -623.3250122070312,
"KL/std": 285.87884521484375,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.0016222589183598757,
"fcm_dpo/delta": 0.019110068678855896,
"fcm_dpo/margin": 237.81283569335938,
"fcm_dpo/q_t": 0.4163554310798645,
"grad_norm": 31.304502487182617,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": -0.6609657406806946,
"logits/rejected": -0.6543369889259338,
"logps/chosen": -447.48590087890625,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -701.8236083984375,
"loss": 1.113,
"margin_dpo/margin_mean": 237.8128204345703,
"margin_dpo/margin_std": 357.0226745605469,
"step": 434
},
{
"KL/chosen_KL_mean": -366.02606201171875,
"KL/mean": -493.39617919921875,
"KL/rejected_KL_mean": -620.766357421875,
"KL/std": 283.81414794921875,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.001610426465049386,
"fcm_dpo/delta": -0.020799141377210617,
"fcm_dpo/margin": 254.740234375,
"fcm_dpo/q_t": 0.4033718705177307,
"grad_norm": 30.6375789642334,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": -0.6943074464797974,
"logits/rejected": -0.6805083751678467,
"logps/chosen": -417.5281066894531,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -708.333251953125,
"loss": 1.0828,
"margin_dpo/margin_mean": 254.74024963378906,
"margin_dpo/margin_std": 321.0318603515625,
"step": 435
},
{
"KL/chosen_KL_mean": -373.72186279296875,
"KL/mean": -489.437744140625,
"KL/rejected_KL_mean": -605.153564453125,
"KL/std": 257.1832580566406,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.0016507278196513653,
"fcm_dpo/delta": 0.02712010033428669,
"fcm_dpo/margin": 231.43165588378906,
"fcm_dpo/q_t": 0.41315099596977234,
"grad_norm": 30.587175369262695,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": -0.6558730602264404,
"logits/rejected": -0.6176996827125549,
"logps/chosen": -445.1256103515625,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -687.88134765625,
"loss": 1.103,
"margin_dpo/margin_mean": 231.43167114257812,
"margin_dpo/margin_std": 304.65887451171875,
"step": 436
},
{
"KL/chosen_KL_mean": -369.8938293457031,
"KL/mean": -485.63037109375,
"KL/rejected_KL_mean": -601.366943359375,
"KL/std": 242.96083068847656,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.0017139371484518051,
"fcm_dpo/delta": 0.004689780995249748,
"fcm_dpo/margin": 231.47308349609375,
"fcm_dpo/q_t": 0.40768975019454956,
"grad_norm": 25.386877059936523,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": -0.6826403141021729,
"logits/rejected": -0.6562011241912842,
"logps/chosen": -434.6380920410156,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -683.4104614257812,
"loss": 1.0842,
"margin_dpo/margin_mean": 231.47308349609375,
"margin_dpo/margin_std": 282.5240478515625,
"step": 437
},
{
"KL/chosen_KL_mean": -385.8797302246094,
"KL/mean": -514.9544677734375,
"KL/rejected_KL_mean": -644.0291748046875,
"KL/std": 284.6157531738281,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.0015868828631937504,
"fcm_dpo/delta": -0.027686219662427902,
"fcm_dpo/margin": 258.14947509765625,
"fcm_dpo/q_t": 0.4011117219924927,
"grad_norm": 36.56245803833008,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": -0.6592687368392944,
"logits/rejected": -0.6328068971633911,
"logps/chosen": -444.8984069824219,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -727.1060180664062,
"loss": 1.0781,
"margin_dpo/margin_mean": 258.14947509765625,
"margin_dpo/margin_std": 307.8912658691406,
"step": 438
},
{
"KL/chosen_KL_mean": -397.440185546875,
"KL/mean": -502.10455322265625,
"KL/rejected_KL_mean": -606.7689208984375,
"KL/std": 281.3536682128906,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.0014228572836145759,
"fcm_dpo/delta": -0.07547041773796082,
"fcm_dpo/margin": 209.32879638671875,
"fcm_dpo/q_t": 0.4233711063861847,
"grad_norm": 26.591917037963867,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": -0.6513150930404663,
"logits/rejected": -0.6519348621368408,
"logps/chosen": -451.2242431640625,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -690.75439453125,
"loss": 1.1686,
"margin_dpo/margin_mean": 209.32876586914062,
"margin_dpo/margin_std": 326.3582763671875,
"step": 439
},
{
"KL/chosen_KL_mean": -424.1907043457031,
"KL/mean": -549.408203125,
"KL/rejected_KL_mean": -674.625732421875,
"KL/std": 355.38433837890625,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.0015238459454849362,
"fcm_dpo/delta": 0.02612234093248844,
"fcm_dpo/margin": 250.43495178222656,
"fcm_dpo/q_t": 0.41634005308151245,
"grad_norm": 36.65753936767578,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": -0.6994473934173584,
"logits/rejected": -0.6900968551635742,
"logps/chosen": -502.7574462890625,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -771.1234741210938,
"loss": 1.1072,
"margin_dpo/margin_mean": 250.43496704101562,
"margin_dpo/margin_std": 356.3500671386719,
"step": 440
},
{
"KL/chosen_KL_mean": -473.6324462890625,
"KL/mean": -599.330322265625,
"KL/rejected_KL_mean": -725.0281982421875,
"KL/std": 360.2265625,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0015725798439234495,
"fcm_dpo/delta": 0.00260746106505394,
"fcm_dpo/margin": 251.39569091796875,
"fcm_dpo/q_t": 0.41514575481414795,
"grad_norm": 36.14642333984375,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": -0.6896128058433533,
"logits/rejected": -0.6732909679412842,
"logps/chosen": -534.4569091796875,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -821.4990234375,
"loss": 1.1421,
"margin_dpo/margin_mean": 251.39569091796875,
"margin_dpo/margin_std": 446.50982666015625,
"step": 441
},
{
"KL/chosen_KL_mean": -402.5406494140625,
"KL/mean": -554.17041015625,
"KL/rejected_KL_mean": -705.8001098632812,
"KL/std": 349.5821533203125,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.0014577994588762522,
"fcm_dpo/delta": -0.08352486789226532,
"fcm_dpo/margin": 303.25946044921875,
"fcm_dpo/q_t": 0.3944917619228363,
"grad_norm": 34.06388473510742,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": -0.6561492085456848,
"logits/rejected": -0.6455131769180298,
"logps/chosen": -449.5518798828125,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -782.33935546875,
"loss": 1.0824,
"margin_dpo/margin_mean": 303.2594299316406,
"margin_dpo/margin_std": 401.3167419433594,
"step": 442
},
{
"KL/chosen_KL_mean": -470.502197265625,
"KL/mean": -563.5130615234375,
"KL/rejected_KL_mean": -656.52392578125,
"KL/std": 353.9302673339844,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.0013765229377895594,
"fcm_dpo/delta": 0.0382419228553772,
"fcm_dpo/margin": 186.021728515625,
"fcm_dpo/q_t": 0.4441724121570587,
"grad_norm": 40.15092086791992,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": -0.6763448119163513,
"logits/rejected": -0.6484371423721313,
"logps/chosen": -541.7752075195312,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -743.203857421875,
"loss": 1.2325,
"margin_dpo/margin_mean": 186.021728515625,
"margin_dpo/margin_std": 431.4246826171875,
"step": 443
},
{
"KL/chosen_KL_mean": -461.93658447265625,
"KL/mean": -613.6505126953125,
"KL/rejected_KL_mean": -765.364501953125,
"KL/std": 370.37298583984375,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.0013226409209892154,
"fcm_dpo/delta": -0.005307670682668686,
"fcm_dpo/margin": 303.42791748046875,
"fcm_dpo/q_t": 0.4082014858722687,
"grad_norm": 33.01715087890625,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": -0.6438364386558533,
"logits/rejected": -0.6463167667388916,
"logps/chosen": -519.1502685546875,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -862.619384765625,
"loss": 1.0982,
"margin_dpo/margin_mean": 303.42791748046875,
"margin_dpo/margin_std": 423.27032470703125,
"step": 444
},
{
"KL/chosen_KL_mean": -402.2928771972656,
"KL/mean": -540.4663696289062,
"KL/rejected_KL_mean": -678.639892578125,
"KL/std": 285.4811706542969,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.0014118524268269539,
"fcm_dpo/delta": 0.015325665473937988,
"fcm_dpo/margin": 276.34698486328125,
"fcm_dpo/q_t": 0.4115728735923767,
"grad_norm": 34.8456916809082,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": -0.6687377691268921,
"logits/rejected": -0.6554244756698608,
"logps/chosen": -469.5926818847656,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -771.322509765625,
"loss": 1.0992,
"margin_dpo/margin_mean": 276.34698486328125,
"margin_dpo/margin_std": 367.69573974609375,
"step": 445
},
{
"KL/chosen_KL_mean": -360.5770568847656,
"KL/mean": -518.53955078125,
"KL/rejected_KL_mean": -676.5020751953125,
"KL/std": 312.20379638671875,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.00137224025093019,
"fcm_dpo/delta": -0.06505385041236877,
"fcm_dpo/margin": 315.92498779296875,
"fcm_dpo/q_t": 0.39190033078193665,
"grad_norm": 26.528141021728516,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": -0.6326192617416382,
"logits/rejected": -0.6437954902648926,
"logps/chosen": -419.675537109375,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -777.7662353515625,
"loss": 1.0557,
"margin_dpo/margin_mean": 315.92498779296875,
"margin_dpo/margin_std": 325.07269287109375,
"step": 446
},
{
"KL/chosen_KL_mean": -383.163330078125,
"KL/mean": -541.930908203125,
"KL/rejected_KL_mean": -700.698486328125,
"KL/std": 360.7058410644531,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.0012335461797192693,
"fcm_dpo/delta": 0.013036707416176796,
"fcm_dpo/margin": 317.53509521484375,
"fcm_dpo/q_t": 0.41153937578201294,
"grad_norm": 25.36223602294922,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": -0.6875368356704712,
"logits/rejected": -0.7032911777496338,
"logps/chosen": -439.2386779785156,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -799.3931884765625,
"loss": 1.0899,
"margin_dpo/margin_mean": 317.5351257324219,
"margin_dpo/margin_std": 410.7313232421875,
"step": 447
},
{
"KL/chosen_KL_mean": -426.30487060546875,
"KL/mean": -575.4131469726562,
"KL/rejected_KL_mean": -724.521484375,
"KL/std": 317.2506103515625,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.0013242221903055906,
"fcm_dpo/delta": 0.004273921251296997,
"fcm_dpo/margin": 298.21661376953125,
"fcm_dpo/q_t": 0.4076346755027771,
"grad_norm": 29.84330940246582,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": -0.5890108942985535,
"logits/rejected": -0.5925085544586182,
"logps/chosen": -486.3087158203125,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -826.7861328125,
"loss": 1.0881,
"margin_dpo/margin_mean": 298.21661376953125,
"margin_dpo/margin_std": 366.0950927734375,
"step": 448
},
{
"KL/chosen_KL_mean": -432.274658203125,
"KL/mean": -588.0142822265625,
"KL/rejected_KL_mean": -743.7538452148438,
"KL/std": 391.2838134765625,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.0012739100493490696,
"fcm_dpo/delta": 0.005134426523000002,
"fcm_dpo/margin": 311.4792175292969,
"fcm_dpo/q_t": 0.4119264483451843,
"grad_norm": 24.36716079711914,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": -0.6395070552825928,
"logits/rejected": -0.6301548480987549,
"logps/chosen": -499.80126953125,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -832.3507690429688,
"loss": 1.1068,
"margin_dpo/margin_mean": 311.4792175292969,
"margin_dpo/margin_std": 466.9681396484375,
"step": 449
},
{
"KL/chosen_KL_mean": -350.74542236328125,
"KL/mean": -518.7066040039062,
"KL/rejected_KL_mean": -686.6678466796875,
"KL/std": 333.5655822753906,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.0012484046164900064,
"fcm_dpo/delta": -0.03335545212030411,
"fcm_dpo/margin": 335.9223937988281,
"fcm_dpo/q_t": 0.3976227641105652,
"grad_norm": 36.70199203491211,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": -0.6501277089118958,
"logits/rejected": -0.6594793796539307,
"logps/chosen": -407.853515625,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -789.4227294921875,
"loss": 1.052,
"margin_dpo/margin_mean": 335.92236328125,
"margin_dpo/margin_std": 331.643310546875,
"step": 450
},
{
"KL/chosen_KL_mean": -478.4472351074219,
"KL/mean": -596.5457763671875,
"KL/rejected_KL_mean": -714.6444091796875,
"KL/std": 388.7025146484375,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.0014245854690670967,
"fcm_dpo/delta": 0.08672139793634415,
"fcm_dpo/margin": 236.19717407226562,
"fcm_dpo/q_t": 0.42556342482566833,
"grad_norm": 40.344722747802734,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": -0.6830481290817261,
"logits/rejected": -0.6519405245780945,
"logps/chosen": -536.9160766601562,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -787.5737915039062,
"loss": 1.1783,
"margin_dpo/margin_mean": 236.19717407226562,
"margin_dpo/margin_std": 461.38665771484375,
"step": 451
},
{
"KL/chosen_KL_mean": -339.47760009765625,
"KL/mean": -487.0719299316406,
"KL/rejected_KL_mean": -634.666259765625,
"KL/std": 301.2289733886719,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.0013748779892921448,
"fcm_dpo/delta": -0.012026989832520485,
"fcm_dpo/margin": 295.1885986328125,
"fcm_dpo/q_t": 0.40248599648475647,
"grad_norm": 21.888635635375977,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": -0.6212879419326782,
"logits/rejected": -0.638474702835083,
"logps/chosen": -389.8681640625,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -732.4376831054688,
"loss": 1.0635,
"margin_dpo/margin_mean": 295.1885986328125,
"margin_dpo/margin_std": 304.22344970703125,
"step": 452
},
{
"KL/chosen_KL_mean": -394.113525390625,
"KL/mean": -535.056884765625,
"KL/rejected_KL_mean": -676.0001831054688,
"KL/std": 311.166259765625,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0013874982250854373,
"fcm_dpo/delta": 0.012876830995082855,
"fcm_dpo/margin": 281.88671875,
"fcm_dpo/q_t": 0.4120361804962158,
"grad_norm": 28.710693359375,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": -0.6545387506484985,
"logits/rejected": -0.635535478591919,
"logps/chosen": -451.828369140625,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -758.2076416015625,
"loss": 1.0973,
"margin_dpo/margin_mean": 281.8866882324219,
"margin_dpo/margin_std": 380.42938232421875,
"step": 453
},
{
"KL/chosen_KL_mean": -493.164306640625,
"KL/mean": -647.89892578125,
"KL/rejected_KL_mean": -802.6336669921875,
"KL/std": 370.6231689453125,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.0013456812594085932,
"fcm_dpo/delta": -0.030497819185256958,
"fcm_dpo/margin": 309.46929931640625,
"fcm_dpo/q_t": 0.4019313454627991,
"grad_norm": 25.663516998291016,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": -0.6898310780525208,
"logits/rejected": -0.676418662071228,
"logps/chosen": -554.1099853515625,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -887.5844116210938,
"loss": 1.0874,
"margin_dpo/margin_mean": 309.46929931640625,
"margin_dpo/margin_std": 427.5249938964844,
"step": 454
},
{
"KL/chosen_KL_mean": -418.80328369140625,
"KL/mean": -605.3193359375,
"KL/rejected_KL_mean": -791.8353881835938,
"KL/std": 368.81707763671875,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.0012299984227865934,
"fcm_dpo/delta": -0.10926243662834167,
"fcm_dpo/margin": 373.0321044921875,
"fcm_dpo/q_t": 0.38569873571395874,
"grad_norm": 30.85955810546875,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": -0.6721550226211548,
"logits/rejected": -0.6930463314056396,
"logps/chosen": -463.69000244140625,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -907.1368408203125,
"loss": 1.0505,
"margin_dpo/margin_mean": 373.0321044921875,
"margin_dpo/margin_std": 423.5682678222656,
"step": 455
},
{
"KL/chosen_KL_mean": -449.27508544921875,
"KL/mean": -635.3360595703125,
"KL/rejected_KL_mean": -821.3970947265625,
"KL/std": 394.14239501953125,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0010888128308579326,
"fcm_dpo/delta": -0.00909638125449419,
"fcm_dpo/margin": 372.12200927734375,
"fcm_dpo/q_t": 0.40475431084632874,
"grad_norm": 28.842674255371094,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": -0.6700366139411926,
"logits/rejected": -0.6916704773902893,
"logps/chosen": -506.3118591308594,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -926.6149291992188,
"loss": 1.063,
"margin_dpo/margin_mean": 372.12200927734375,
"margin_dpo/margin_std": 389.7081298828125,
"step": 456
},
{
"KL/chosen_KL_mean": -442.96075439453125,
"KL/mean": -624.3992919921875,
"KL/rejected_KL_mean": -805.837646484375,
"KL/std": 381.4212646484375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.0010792326647788286,
"fcm_dpo/delta": 0.01370345987379551,
"fcm_dpo/margin": 362.876953125,
"fcm_dpo/q_t": 0.40837815403938293,
"grad_norm": 28.204694747924805,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": -0.6881248950958252,
"logits/rejected": -0.68438720703125,
"logps/chosen": -497.20330810546875,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -890.947265625,
"loss": 1.0737,
"margin_dpo/margin_mean": 362.876953125,
"margin_dpo/margin_std": 379.9857177734375,
"step": 457
},
{
"KL/chosen_KL_mean": -499.1065368652344,
"KL/mean": -653.4949951171875,
"KL/rejected_KL_mean": -807.8834228515625,
"KL/std": 364.198974609375,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.0011557680554687977,
"fcm_dpo/delta": 0.06521759927272797,
"fcm_dpo/margin": 308.7769775390625,
"fcm_dpo/q_t": 0.4231434762477875,
"grad_norm": 32.81315231323242,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": -0.7310912609100342,
"logits/rejected": -0.7260030508041382,
"logps/chosen": -554.515380859375,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -905.5667114257812,
"loss": 1.123,
"margin_dpo/margin_mean": 308.7769775390625,
"margin_dpo/margin_std": 427.2161560058594,
"step": 458
},
{
"KL/chosen_KL_mean": -529.4247436523438,
"KL/mean": -706.0894165039062,
"KL/rejected_KL_mean": -882.7540283203125,
"KL/std": 432.647216796875,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0011564318556338549,
"fcm_dpo/delta": -0.021825630217790604,
"fcm_dpo/margin": 353.329345703125,
"fcm_dpo/q_t": 0.40472641587257385,
"grad_norm": 32.47217559814453,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": -0.7430839538574219,
"logits/rejected": -0.7598717212677002,
"logps/chosen": -575.9822387695312,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -968.922607421875,
"loss": 1.0937,
"margin_dpo/margin_mean": 353.329345703125,
"margin_dpo/margin_std": 485.6087341308594,
"step": 459
},
{
"KL/chosen_KL_mean": -571.9495239257812,
"KL/mean": -779.2578125,
"KL/rejected_KL_mean": -986.5660400390625,
"KL/std": 431.83953857421875,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0010669762268662453,
"fcm_dpo/delta": -0.07710142433643341,
"fcm_dpo/margin": 414.6165466308594,
"fcm_dpo/q_t": 0.3918076157569885,
"grad_norm": 30.98316764831543,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": -0.830254316329956,
"logits/rejected": -0.8624805212020874,
"logps/chosen": -623.58447265625,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -1090.6854248046875,
"loss": 1.0514,
"margin_dpo/margin_mean": 414.6165466308594,
"margin_dpo/margin_std": 476.18707275390625,
"step": 460
},
{
"KL/chosen_KL_mean": -605.7537841796875,
"KL/mean": -766.1744384765625,
"KL/rejected_KL_mean": -926.59521484375,
"KL/std": 430.42913818359375,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0011395696783438325,
"fcm_dpo/delta": 0.04857172816991806,
"fcm_dpo/margin": 320.8414611816406,
"fcm_dpo/q_t": 0.41997230052948,
"grad_norm": 24.931995391845703,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": -0.8122773170471191,
"logits/rejected": -0.8271867036819458,
"logps/chosen": -660.9356689453125,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -1013.0721435546875,
"loss": 1.1224,
"margin_dpo/margin_mean": 320.8414306640625,
"margin_dpo/margin_std": 479.56878662109375,
"step": 461
},
{
"KL/chosen_KL_mean": -618.9212036132812,
"KL/mean": -744.9371337890625,
"KL/rejected_KL_mean": -870.9530639648438,
"KL/std": 416.94464111328125,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.001313498243689537,
"fcm_dpo/delta": 0.10240471363067627,
"fcm_dpo/margin": 252.0318603515625,
"fcm_dpo/q_t": 0.43281617760658264,
"grad_norm": 40.46854019165039,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": -0.8632235527038574,
"logits/rejected": -0.8471617698669434,
"logps/chosen": -688.8492431640625,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -949.794189453125,
"loss": 1.1744,
"margin_dpo/margin_mean": 252.03182983398438,
"margin_dpo/margin_std": 479.5788269042969,
"step": 462
},
{
"KL/chosen_KL_mean": -603.7901611328125,
"KL/mean": -779.239990234375,
"KL/rejected_KL_mean": -954.6898803710938,
"KL/std": 431.516357421875,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.0012737201759591699,
"fcm_dpo/delta": -0.08446250110864639,
"fcm_dpo/margin": 350.899658203125,
"fcm_dpo/q_t": 0.39394649863243103,
"grad_norm": 37.86167526245117,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": -0.8599103689193726,
"logits/rejected": -0.8692607879638672,
"logps/chosen": -659.0645751953125,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -1043.71484375,
"loss": 1.0823,
"margin_dpo/margin_mean": 350.899658203125,
"margin_dpo/margin_std": 486.9836120605469,
"step": 463
},
{
"KL/chosen_KL_mean": -577.685302734375,
"KL/mean": -836.02734375,
"KL/rejected_KL_mean": -1094.369384765625,
"KL/std": 506.09954833984375,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0008683481719344854,
"fcm_dpo/delta": -0.13791523873806,
"fcm_dpo/margin": 516.6839599609375,
"fcm_dpo/q_t": 0.37923869490623474,
"grad_norm": 35.81996536254883,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": -0.8401867151260376,
"logits/rejected": -0.9024742841720581,
"logps/chosen": -628.59765625,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -1196.858642578125,
"loss": 1.0364,
"margin_dpo/margin_mean": 516.6839599609375,
"margin_dpo/margin_std": 519.8881225585938,
"step": 464
},
{
"KL/chosen_KL_mean": -585.9309692382812,
"KL/mean": -794.7791748046875,
"KL/rejected_KL_mean": -1003.6273193359375,
"KL/std": 513.0881958007812,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0009388852631673217,
"fcm_dpo/delta": 0.007542621344327927,
"fcm_dpo/margin": 417.69635009765625,
"fcm_dpo/q_t": 0.4073883295059204,
"grad_norm": 24.084501266479492,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": -0.7998400926589966,
"logits/rejected": -0.8214449882507324,
"logps/chosen": -646.0478515625,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -1117.5733642578125,
"loss": 1.0968,
"margin_dpo/margin_mean": 417.69635009765625,
"margin_dpo/margin_std": 542.79931640625,
"step": 465
},
{
"KL/chosen_KL_mean": -639.7386474609375,
"KL/mean": -817.8701171875,
"KL/rejected_KL_mean": -996.0015258789062,
"KL/std": 474.357666015625,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.0009801898850128055,
"fcm_dpo/delta": 0.0780898779630661,
"fcm_dpo/margin": 356.26287841796875,
"fcm_dpo/q_t": 0.42381781339645386,
"grad_norm": 24.35711669921875,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": -0.8397573232650757,
"logits/rejected": -0.840815007686615,
"logps/chosen": -692.6595458984375,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -1086.3170166015625,
"loss": 1.1348,
"margin_dpo/margin_mean": 356.26287841796875,
"margin_dpo/margin_std": 522.5521240234375,
"step": 466
},
{
"KL/chosen_KL_mean": -801.0821533203125,
"KL/mean": -986.1063232421875,
"KL/rejected_KL_mean": -1171.1304931640625,
"KL/std": 607.9132080078125,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0009247926063835621,
"fcm_dpo/delta": -0.06586673855781555,
"fcm_dpo/margin": 370.04833984375,
"fcm_dpo/q_t": 0.41659867763519287,
"grad_norm": 38.04802703857422,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": -0.9427855610847473,
"logits/rejected": -0.934941291809082,
"logps/chosen": -879.7979736328125,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -1273.99072265625,
"loss": 1.179,
"margin_dpo/margin_mean": 370.04833984375,
"margin_dpo/margin_std": 719.1068115234375,
"step": 467
},
{
"KL/chosen_KL_mean": -630.8428955078125,
"KL/mean": -867.4342041015625,
"KL/rejected_KL_mean": -1104.025634765625,
"KL/std": 526.3124389648438,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.0008766874088905752,
"fcm_dpo/delta": -0.026581648737192154,
"fcm_dpo/margin": 473.18267822265625,
"fcm_dpo/q_t": 0.4034311771392822,
"grad_norm": 31.51830291748047,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": -0.8827992677688599,
"logits/rejected": -0.8895440101623535,
"logps/chosen": -700.7782592773438,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -1205.054443359375,
"loss": 1.0778,
"margin_dpo/margin_mean": 473.18267822265625,
"margin_dpo/margin_std": 609.1591796875,
"step": 468
},
{
"KL/chosen_KL_mean": -650.935302734375,
"KL/mean": -885.42578125,
"KL/rejected_KL_mean": -1119.916259765625,
"KL/std": 477.8541259765625,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0008414213662035763,
"fcm_dpo/delta": 0.0032104477286338806,
"fcm_dpo/margin": 468.98089599609375,
"fcm_dpo/q_t": 0.4071598947048187,
"grad_norm": 36.97335433959961,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": -0.899463415145874,
"logits/rejected": -0.9126312732696533,
"logps/chosen": -719.0599975585938,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -1224.70263671875,
"loss": 1.0813,
"margin_dpo/margin_mean": 468.98089599609375,
"margin_dpo/margin_std": 533.3753662109375,
"step": 469
},
{
"KL/chosen_KL_mean": -553.0526123046875,
"KL/mean": -755.4017333984375,
"KL/rejected_KL_mean": -957.7508544921875,
"KL/std": 496.32080078125,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0009463757742196321,
"fcm_dpo/delta": 0.022916335612535477,
"fcm_dpo/margin": 404.69830322265625,
"fcm_dpo/q_t": 0.41216540336608887,
"grad_norm": 23.679851531982422,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": -0.8496056199073792,
"logits/rejected": -0.8591802716255188,
"logps/chosen": -596.844482421875,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -1040.4537353515625,
"loss": 1.1079,
"margin_dpo/margin_mean": 404.69830322265625,
"margin_dpo/margin_std": 562.834228515625,
"step": 470
},
{
"KL/chosen_KL_mean": -711.413330078125,
"KL/mean": -888.9149169921875,
"KL/rejected_KL_mean": -1066.4163818359375,
"KL/std": 547.1355590820312,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0008889293530955911,
"fcm_dpo/delta": -0.01932104118168354,
"fcm_dpo/margin": 355.0030517578125,
"fcm_dpo/q_t": 0.4268658757209778,
"grad_norm": 23.575862884521484,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": -0.9526507258415222,
"logits/rejected": -0.9646092057228088,
"logps/chosen": -774.7528686523438,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -1150.02685546875,
"loss": 1.1604,
"margin_dpo/margin_mean": 355.0030517578125,
"margin_dpo/margin_std": 585.3763427734375,
"step": 471
},
{
"KL/chosen_KL_mean": -698.0546875,
"KL/mean": -902.4146728515625,
"KL/rejected_KL_mean": -1106.774658203125,
"KL/std": 597.7765502929688,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0009217117330990732,
"fcm_dpo/delta": 0.0372597835958004,
"fcm_dpo/margin": 408.719970703125,
"fcm_dpo/q_t": 0.41907206177711487,
"grad_norm": 28.87525749206543,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": -0.9155426025390625,
"logits/rejected": -0.8952895402908325,
"logps/chosen": -781.7208251953125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -1223.98388671875,
"loss": 1.1323,
"margin_dpo/margin_mean": 408.719970703125,
"margin_dpo/margin_std": 667.8204345703125,
"step": 472
},
{
"KL/chosen_KL_mean": -824.3423461914062,
"KL/mean": -920.7506103515625,
"KL/rejected_KL_mean": -1017.1588134765625,
"KL/std": 608.287353515625,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0009577026357874274,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 192.81646728515625,
"fcm_dpo/q_t": 0.4590563476085663,
"grad_norm": 100.81672668457031,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": -0.9225292801856995,
"logits/rejected": -0.8966630697250366,
"logps/chosen": -887.83935546875,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -1098.305419921875,
"loss": 1.3432,
"margin_dpo/margin_mean": 192.81646728515625,
"margin_dpo/margin_std": 805.4339599609375,
"step": 473
},
{
"KL/chosen_KL_mean": -674.1361083984375,
"KL/mean": -858.5537109375,
"KL/rejected_KL_mean": -1042.971435546875,
"KL/std": 571.548583984375,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.0008901376277208328,
"fcm_dpo/delta": -0.0760503038764,
"fcm_dpo/margin": 368.83526611328125,
"fcm_dpo/q_t": 0.42114460468292236,
"grad_norm": 29.184354782104492,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": -0.8849027156829834,
"logits/rejected": -0.8879544734954834,
"logps/chosen": -726.748046875,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -1133.0517578125,
"loss": 1.1794,
"margin_dpo/margin_mean": 368.83526611328125,
"margin_dpo/margin_std": 658.4710693359375,
"step": 474
},
{
"KL/chosen_KL_mean": -489.204345703125,
"KL/mean": -716.9005737304688,
"KL/rejected_KL_mean": -944.5968627929688,
"KL/std": 447.86822509765625,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.0008446001447737217,
"fcm_dpo/delta": 0.025007635354995728,
"fcm_dpo/margin": 455.3924560546875,
"fcm_dpo/q_t": 0.4114686846733093,
"grad_norm": 35.35837173461914,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": -0.9520416855812073,
"logits/rejected": -0.979211688041687,
"logps/chosen": -531.6995849609375,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -1034.6597900390625,
"loss": 1.0813,
"margin_dpo/margin_mean": 455.3924560546875,
"margin_dpo/margin_std": 503.96844482421875,
"step": 475
},
{
"KL/chosen_KL_mean": -619.6912231445312,
"KL/mean": -803.4817504882812,
"KL/rejected_KL_mean": -987.272216796875,
"KL/std": 514.7440185546875,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.0009646163089200854,
"fcm_dpo/delta": 0.06908048689365387,
"fcm_dpo/margin": 367.5810546875,
"fcm_dpo/q_t": 0.42334458231925964,
"grad_norm": 42.48811721801758,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": -0.8881396055221558,
"logits/rejected": -0.904177188873291,
"logps/chosen": -662.640625,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -1060.982421875,
"loss": 1.1232,
"margin_dpo/margin_mean": 367.5810546875,
"margin_dpo/margin_std": 530.9556884765625,
"step": 476
},
{
"KL/chosen_KL_mean": -674.9053955078125,
"KL/mean": -861.569580078125,
"KL/rejected_KL_mean": -1048.2337646484375,
"KL/std": 548.8595581054688,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0010453263530507684,
"fcm_dpo/delta": 0.012534024193882942,
"fcm_dpo/margin": 373.328369140625,
"fcm_dpo/q_t": 0.4125494360923767,
"grad_norm": 29.967121124267578,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": -0.974365234375,
"logits/rejected": -0.9425362348556519,
"logps/chosen": -745.677978515625,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -1124.37109375,
"loss": 1.145,
"margin_dpo/margin_mean": 373.3283386230469,
"margin_dpo/margin_std": 659.33203125,
"step": 477
},
{
"KL/chosen_KL_mean": -521.8460693359375,
"KL/mean": -730.65234375,
"KL/rejected_KL_mean": -939.4588012695312,
"KL/std": 461.107421875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.001001200987957418,
"fcm_dpo/delta": -0.0321248322725296,
"fcm_dpo/margin": 417.6127014160156,
"fcm_dpo/q_t": 0.4019705057144165,
"grad_norm": 34.509376525878906,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -0.8223329782485962,
"logits/rejected": -0.8430248498916626,
"logps/chosen": -563.2865600585938,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -1024.82080078125,
"loss": 1.081,
"margin_dpo/margin_mean": 417.61273193359375,
"margin_dpo/margin_std": 542.504150390625,
"step": 478
},
{
"KL/chosen_KL_mean": -683.9410400390625,
"KL/mean": -882.384033203125,
"KL/rejected_KL_mean": -1080.8271484375,
"KL/std": 582.9780883789062,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.0010136861819773912,
"fcm_dpo/delta": -0.020786762237548828,
"fcm_dpo/margin": 396.88604736328125,
"fcm_dpo/q_t": 0.4105514883995056,
"grad_norm": 26.009479522705078,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": -0.8741980791091919,
"logits/rejected": -0.9063311815261841,
"logps/chosen": -737.8489990234375,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -1175.943359375,
"loss": 1.14,
"margin_dpo/margin_mean": 396.88604736328125,
"margin_dpo/margin_std": 691.9199829101562,
"step": 479
},
{
"KL/chosen_KL_mean": -823.0316772460938,
"KL/mean": -973.1109619140625,
"KL/rejected_KL_mean": -1123.190185546875,
"KL/std": 528.6361083984375,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0009059295989573002,
"fcm_dpo/delta": -0.010956360027194023,
"fcm_dpo/margin": 300.15850830078125,
"fcm_dpo/q_t": 0.43630528450012207,
"grad_norm": 46.897727966308594,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": -0.9384490251541138,
"logits/rejected": -0.9253517389297485,
"logps/chosen": -881.71435546875,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -1206.1226806640625,
"loss": 1.2068,
"margin_dpo/margin_mean": 300.15850830078125,
"margin_dpo/margin_std": 578.85498046875,
"step": 480
},
{
"KL/chosen_KL_mean": -666.0030517578125,
"KL/mean": -902.3043212890625,
"KL/rejected_KL_mean": -1138.605712890625,
"KL/std": 549.5169677734375,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.0008731147972866893,
"fcm_dpo/delta": -0.021470826119184494,
"fcm_dpo/margin": 472.6026611328125,
"fcm_dpo/q_t": 0.4040641784667969,
"grad_norm": 27.580970764160156,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": -0.9624149799346924,
"logits/rejected": -0.9881491661071777,
"logps/chosen": -720.96728515625,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -1231.026123046875,
"loss": 1.0827,
"margin_dpo/margin_mean": 472.6026611328125,
"margin_dpo/margin_std": 616.1715087890625,
"step": 481
},
{
"KL/chosen_KL_mean": -744.04248046875,
"KL/mean": -855.4127197265625,
"KL/rejected_KL_mean": -966.782958984375,
"KL/std": 563.21630859375,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0009502337779849768,
"fcm_dpo/delta": 0.09700674563646317,
"fcm_dpo/margin": 222.74044799804688,
"fcm_dpo/q_t": 0.4541202783584595,
"grad_norm": 35.13661575317383,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": -0.8878397941589355,
"logits/rejected": -0.8769594430923462,
"logps/chosen": -811.595947265625,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -1054.37255859375,
"loss": 1.2788,
"margin_dpo/margin_mean": 222.740478515625,
"margin_dpo/margin_std": 668.005859375,
"step": 482
},
{
"KL/chosen_KL_mean": -656.9723510742188,
"KL/mean": -896.6019897460938,
"KL/rejected_KL_mean": -1136.2315673828125,
"KL/std": 552.5379028320312,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.0009095891728065908,
"fcm_dpo/delta": -0.07079771906137466,
"fcm_dpo/margin": 479.25921630859375,
"fcm_dpo/q_t": 0.3960753083229065,
"grad_norm": 31.120384216308594,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": -0.8999602198600769,
"logits/rejected": -0.9098832607269287,
"logps/chosen": -720.3021850585938,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -1232.0185546875,
"loss": 1.0695,
"margin_dpo/margin_mean": 479.2591857910156,
"margin_dpo/margin_std": 622.5853271484375,
"step": 483
},
{
"KL/chosen_KL_mean": -550.8277587890625,
"KL/mean": -768.7025146484375,
"KL/rejected_KL_mean": -986.5772705078125,
"KL/std": 529.3361206054688,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0009340384276583791,
"fcm_dpo/delta": -0.01913183555006981,
"fcm_dpo/margin": 435.74957275390625,
"fcm_dpo/q_t": 0.40439409017562866,
"grad_norm": 38.8234748840332,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": -0.9227362871170044,
"logits/rejected": -0.9442682266235352,
"logps/chosen": -609.9659423828125,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -1070.94873046875,
"loss": 1.0868,
"margin_dpo/margin_mean": 435.74951171875,
"margin_dpo/margin_std": 546.264892578125,
"step": 484
},
{
"KL/chosen_KL_mean": -558.8453369140625,
"KL/mean": -764.8858642578125,
"KL/rejected_KL_mean": -970.9264526367188,
"KL/std": 525.61865234375,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.0009324835264123976,
"fcm_dpo/delta": 0.020539814606308937,
"fcm_dpo/margin": 412.0811767578125,
"fcm_dpo/q_t": 0.41461336612701416,
"grad_norm": 38.73042678833008,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": -0.8426438570022583,
"logits/rejected": -0.8627835512161255,
"logps/chosen": -617.6948852539062,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -1074.29052734375,
"loss": 1.1164,
"margin_dpo/margin_mean": 412.0811462402344,
"margin_dpo/margin_std": 618.0966186523438,
"step": 485
},
{
"KL/chosen_KL_mean": -644.1904296875,
"KL/mean": -887.1826782226562,
"KL/rejected_KL_mean": -1130.1748046875,
"KL/std": 590.3195190429688,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.0008638256113044918,
"fcm_dpo/delta": -0.034225355833768845,
"fcm_dpo/margin": 485.98443603515625,
"fcm_dpo/q_t": 0.4027414321899414,
"grad_norm": 39.733394622802734,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": -0.9089898467063904,
"logits/rejected": -0.9190336465835571,
"logps/chosen": -699.4500732421875,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -1222.314208984375,
"loss": 1.0991,
"margin_dpo/margin_mean": 485.9844665527344,
"margin_dpo/margin_std": 727.2791748046875,
"step": 486
},
{
"KL/chosen_KL_mean": -683.022216796875,
"KL/mean": -864.6463012695312,
"KL/rejected_KL_mean": -1046.2703857421875,
"KL/std": 544.7926025390625,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.0009704558178782463,
"fcm_dpo/delta": 0.06513335555791855,
"fcm_dpo/margin": 363.2481689453125,
"fcm_dpo/q_t": 0.4213024377822876,
"grad_norm": 40.45846939086914,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": -0.9383945465087891,
"logits/rejected": -0.9428784847259521,
"logps/chosen": -736.0855712890625,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -1138.689208984375,
"loss": 1.1385,
"margin_dpo/margin_mean": 363.2481689453125,
"margin_dpo/margin_std": 577.7797241210938,
"step": 487
},
{
"KL/chosen_KL_mean": -545.5859985351562,
"KL/mean": -734.630126953125,
"KL/rejected_KL_mean": -923.67431640625,
"KL/std": 473.2789306640625,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.0010280082933604717,
"fcm_dpo/delta": 0.012521795928478241,
"fcm_dpo/margin": 378.0882873535156,
"fcm_dpo/q_t": 0.41051536798477173,
"grad_norm": 22.53078269958496,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": -0.8381102681159973,
"logits/rejected": -0.8391183614730835,
"logps/chosen": -597.814208984375,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -1007.680908203125,
"loss": 1.0927,
"margin_dpo/margin_mean": 378.0882873535156,
"margin_dpo/margin_std": 473.35400390625,
"step": 488
},
{
"KL/chosen_KL_mean": -499.30194091796875,
"KL/mean": -672.4368286132812,
"KL/rejected_KL_mean": -845.5716552734375,
"KL/std": 457.94464111328125,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.00108251697383821,
"fcm_dpo/delta": 0.03405838832259178,
"fcm_dpo/margin": 346.2696838378906,
"fcm_dpo/q_t": 0.41736501455307007,
"grad_norm": 24.369661331176758,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": -0.8864847421646118,
"logits/rejected": -0.8804982900619507,
"logps/chosen": -555.2916259765625,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -924.9697265625,
"loss": 1.1145,
"margin_dpo/margin_mean": 346.2696838378906,
"margin_dpo/margin_std": 504.04510498046875,
"step": 489
},
{
"KL/chosen_KL_mean": -606.05126953125,
"KL/mean": -801.578125,
"KL/rejected_KL_mean": -997.1050415039062,
"KL/std": 577.8994140625,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.0010517244227230549,
"fcm_dpo/delta": -0.019802920520305634,
"fcm_dpo/margin": 391.0537414550781,
"fcm_dpo/q_t": 0.4080085754394531,
"grad_norm": 39.334407806396484,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": -0.8914596438407898,
"logits/rejected": -0.9301384687423706,
"logps/chosen": -658.4176025390625,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -1107.51416015625,
"loss": 1.1285,
"margin_dpo/margin_mean": 391.05377197265625,
"margin_dpo/margin_std": 660.88671875,
"step": 490
},
{
"KL/chosen_KL_mean": -556.3938598632812,
"KL/mean": -685.6237182617188,
"KL/rejected_KL_mean": -814.8536376953125,
"KL/std": 486.525634765625,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.0012323971604928374,
"fcm_dpo/delta": 0.1136833056807518,
"fcm_dpo/margin": 258.459716796875,
"fcm_dpo/q_t": 0.4345433712005615,
"grad_norm": 34.78307342529297,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": -0.8827264904975891,
"logits/rejected": -0.8719267845153809,
"logps/chosen": -616.5101318359375,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -888.1264038085938,
"loss": 1.175,
"margin_dpo/margin_mean": 258.4597473144531,
"margin_dpo/margin_std": 484.7692565917969,
"step": 491
},
{
"KL/chosen_KL_mean": -558.9266357421875,
"KL/mean": -683.21923828125,
"KL/rejected_KL_mean": -807.511962890625,
"KL/std": 487.9819030761719,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0014084703288972378,
"fcm_dpo/delta": 0.07727247476577759,
"fcm_dpo/margin": 248.58529663085938,
"fcm_dpo/q_t": 0.43000033497810364,
"grad_norm": 36.50166320800781,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": -0.8998019695281982,
"logits/rejected": -0.8745296001434326,
"logps/chosen": -612.9207763671875,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -880.1715087890625,
"loss": 1.209,
"margin_dpo/margin_mean": 248.58529663085938,
"margin_dpo/margin_std": 559.0795288085938,
"step": 492
},
{
"KL/chosen_KL_mean": -606.264892578125,
"KL/mean": -745.5481567382812,
"KL/rejected_KL_mean": -884.8314208984375,
"KL/std": 519.5693359375,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0013839509338140488,
"fcm_dpo/delta": 0.0048267170786857605,
"fcm_dpo/margin": 278.56646728515625,
"fcm_dpo/q_t": 0.412563681602478,
"grad_norm": 41.9803466796875,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": -0.9656150341033936,
"logits/rejected": -0.9288890361785889,
"logps/chosen": -681.7621459960938,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -972.1544189453125,
"loss": 1.1628,
"margin_dpo/margin_mean": 278.56646728515625,
"margin_dpo/margin_std": 508.8519287109375,
"step": 493
},
{
"KL/chosen_KL_mean": -456.39111328125,
"KL/mean": -672.3492431640625,
"KL/rejected_KL_mean": -888.307373046875,
"KL/std": 460.58941650390625,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.001108669675886631,
"fcm_dpo/delta": -0.17697980999946594,
"fcm_dpo/margin": 431.91619873046875,
"fcm_dpo/q_t": 0.37332338094711304,
"grad_norm": 37.33299255371094,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": -0.8224232196807861,
"logits/rejected": -0.8543698191642761,
"logps/chosen": -497.7503662109375,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -974.398681640625,
"loss": 1.0276,
"margin_dpo/margin_mean": 431.91619873046875,
"margin_dpo/margin_std": 471.99713134765625,
"step": 494
},
{
"KL/chosen_KL_mean": -516.5604248046875,
"KL/mean": -703.7841796875,
"KL/rejected_KL_mean": -891.0079956054688,
"KL/std": 482.6785888671875,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.0010678664548322558,
"fcm_dpo/delta": -4.4368207454681396e-05,
"fcm_dpo/margin": 374.44757080078125,
"fcm_dpo/q_t": 0.4114726483821869,
"grad_norm": 35.54365539550781,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": -0.9496263265609741,
"logits/rejected": -0.9522305727005005,
"logps/chosen": -580.095458984375,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -982.432373046875,
"loss": 1.1106,
"margin_dpo/margin_mean": 374.44757080078125,
"margin_dpo/margin_std": 582.944580078125,
"step": 495
},
{
"KL/chosen_KL_mean": -618.7561645507812,
"KL/mean": -733.871337890625,
"KL/rejected_KL_mean": -848.986572265625,
"KL/std": 380.02642822265625,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.0013148835860192776,
"fcm_dpo/delta": 0.13359305262565613,
"fcm_dpo/margin": 230.23040771484375,
"fcm_dpo/q_t": 0.43739110231399536,
"grad_norm": 59.39929962158203,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": -0.9213845729827881,
"logits/rejected": -0.8853435516357422,
"logps/chosen": -691.3480834960938,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -933.31591796875,
"loss": 1.1831,
"margin_dpo/margin_mean": 230.23040771484375,
"margin_dpo/margin_std": 436.0897216796875,
"step": 496
},
{
"KL/chosen_KL_mean": -592.6030883789062,
"KL/mean": -699.4375,
"KL/rejected_KL_mean": -806.27197265625,
"KL/std": 449.3829345703125,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.0013049728004261851,
"fcm_dpo/delta": -0.028186559677124023,
"fcm_dpo/margin": 213.6688232421875,
"fcm_dpo/q_t": 0.4349159598350525,
"grad_norm": 47.662078857421875,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": -0.8769658803939819,
"logits/rejected": -0.8549270629882812,
"logps/chosen": -651.1970825195312,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -882.560302734375,
"loss": 1.2114,
"margin_dpo/margin_mean": 213.6688232421875,
"margin_dpo/margin_std": 445.9394836425781,
"step": 497
},
{
"KL/chosen_KL_mean": -555.0482788085938,
"KL/mean": -724.8245849609375,
"KL/rejected_KL_mean": -894.6010131835938,
"KL/std": 492.70159912109375,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.0012368912575766444,
"fcm_dpo/delta": -0.03477949649095535,
"fcm_dpo/margin": 339.552734375,
"fcm_dpo/q_t": 0.4025610685348511,
"grad_norm": 32.19281768798828,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": -0.93885338306427,
"logits/rejected": -0.9247368574142456,
"logps/chosen": -626.25390625,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -978.55908203125,
"loss": 1.1152,
"margin_dpo/margin_mean": 339.552734375,
"margin_dpo/margin_std": 550.4183349609375,
"step": 498
},
{
"KL/chosen_KL_mean": -468.7248840332031,
"KL/mean": -666.8063354492188,
"KL/rejected_KL_mean": -864.8878173828125,
"KL/std": 467.00946044921875,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.0010947990231215954,
"fcm_dpo/delta": -0.062151163816452026,
"fcm_dpo/margin": 396.1629638671875,
"fcm_dpo/q_t": 0.39568614959716797,
"grad_norm": 36.04325485229492,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": -0.8788598775863647,
"logits/rejected": -0.9086207151412964,
"logps/chosen": -519.9801025390625,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -965.966552734375,
"loss": 1.0746,
"margin_dpo/margin_mean": 396.1629638671875,
"margin_dpo/margin_std": 527.5292358398438,
"step": 499
},
{
"KL/chosen_KL_mean": -586.3583374023438,
"KL/mean": -761.30419921875,
"KL/rejected_KL_mean": -936.25,
"KL/std": 430.88531494140625,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.0010993874166160822,
"fcm_dpo/delta": 0.024825483560562134,
"fcm_dpo/margin": 349.8916931152344,
"fcm_dpo/q_t": 0.4142201840877533,
"grad_norm": 32.015804290771484,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": -0.9118782877922058,
"logits/rejected": -0.9368890523910522,
"logps/chosen": -643.3857421875,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -1030.1842041015625,
"loss": 1.1177,
"margin_dpo/margin_mean": 349.8916931152344,
"margin_dpo/margin_std": 529.33837890625,
"step": 500
},
{
"KL/chosen_KL_mean": -496.50091552734375,
"KL/mean": -679.2710571289062,
"KL/rejected_KL_mean": -862.041259765625,
"KL/std": 453.33331298828125,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.001129016513004899,
"fcm_dpo/delta": -0.023394692689180374,
"fcm_dpo/margin": 365.5402526855469,
"fcm_dpo/q_t": 0.4061323404312134,
"grad_norm": 33.857051849365234,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": -0.8639429211616516,
"logits/rejected": -0.8466420769691467,
"logps/chosen": -550.8604736328125,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -942.1979370117188,
"loss": 1.1061,
"margin_dpo/margin_mean": 365.540283203125,
"margin_dpo/margin_std": 553.092041015625,
"step": 501
},
{
"KL/chosen_KL_mean": -463.086181640625,
"KL/mean": -645.1473388671875,
"KL/rejected_KL_mean": -827.20849609375,
"KL/std": 410.1119689941406,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.0010994459735229611,
"fcm_dpo/delta": -0.0017848927527666092,
"fcm_dpo/margin": 364.122314453125,
"fcm_dpo/q_t": 0.40732836723327637,
"grad_norm": 22.971820831298828,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": -0.8409848809242249,
"logits/rejected": -0.8154308795928955,
"logps/chosen": -530.6866455078125,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -910.1572265625,
"loss": 1.0803,
"margin_dpo/margin_mean": 364.1223449707031,
"margin_dpo/margin_std": 437.77301025390625,
"step": 502
},
{
"KL/chosen_KL_mean": -455.322998046875,
"KL/mean": -625.4855346679688,
"KL/rejected_KL_mean": -795.6480712890625,
"KL/std": 388.8305358886719,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.00111018109600991,
"fcm_dpo/delta": 0.03548618406057358,
"fcm_dpo/margin": 340.3250427246094,
"fcm_dpo/q_t": 0.4157389998435974,
"grad_norm": 23.465837478637695,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": -0.8313039541244507,
"logits/rejected": -0.8144758939743042,
"logps/chosen": -510.40142822265625,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -878.1535034179688,
"loss": 1.0957,
"margin_dpo/margin_mean": 340.3250427246094,
"margin_dpo/margin_std": 424.44854736328125,
"step": 503
},
{
"KL/chosen_KL_mean": -486.9976806640625,
"KL/mean": -691.9973754882812,
"KL/rejected_KL_mean": -896.9971313476562,
"KL/std": 449.76153564453125,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.001057411776855588,
"fcm_dpo/delta": -0.059654317796230316,
"fcm_dpo/margin": 409.99945068359375,
"fcm_dpo/q_t": 0.39566490054130554,
"grad_norm": 23.978740692138672,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": -0.8678739070892334,
"logits/rejected": -0.8862636089324951,
"logps/chosen": -546.96337890625,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -1000.75927734375,
"loss": 1.0674,
"margin_dpo/margin_mean": 409.9994812011719,
"margin_dpo/margin_std": 505.0823974609375,
"step": 504
},
{
"KL/chosen_KL_mean": -583.2789306640625,
"KL/mean": -693.8592529296875,
"KL/rejected_KL_mean": -804.4395751953125,
"KL/std": 454.05352783203125,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.0013018188765272498,
"fcm_dpo/delta": 0.1528121829032898,
"fcm_dpo/margin": 221.16064453125,
"fcm_dpo/q_t": 0.44206857681274414,
"grad_norm": 44.39717102050781,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": -0.9133349657058716,
"logits/rejected": -0.8864049911499023,
"logps/chosen": -659.433837890625,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -893.02490234375,
"loss": 1.1987,
"margin_dpo/margin_mean": 221.16065979003906,
"margin_dpo/margin_std": 430.33538818359375,
"step": 505
},
{
"KL/chosen_KL_mean": -461.5542907714844,
"KL/mean": -646.5904541015625,
"KL/rejected_KL_mean": -831.626708984375,
"KL/std": 457.093505859375,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0011723710922524333,
"fcm_dpo/delta": -0.07656194269657135,
"fcm_dpo/margin": 370.07244873046875,
"fcm_dpo/q_t": 0.3941624164581299,
"grad_norm": 31.93734359741211,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": -0.8899002075195312,
"logits/rejected": -0.8990935683250427,
"logps/chosen": -510.5147705078125,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -910.041748046875,
"loss": 1.083,
"margin_dpo/margin_mean": 370.07244873046875,
"margin_dpo/margin_std": 513.3609619140625,
"step": 506
},
{
"KL/chosen_KL_mean": -530.3336791992188,
"KL/mean": -686.3297729492188,
"KL/rejected_KL_mean": -842.3258666992188,
"KL/std": 511.7806701660156,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.0012264869874343276,
"fcm_dpo/delta": 0.028089439496397972,
"fcm_dpo/margin": 311.9921569824219,
"fcm_dpo/q_t": 0.41973698139190674,
"grad_norm": 32.90348434448242,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": -0.8918719291687012,
"logits/rejected": -0.889752984046936,
"logps/chosen": -597.1351928710938,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -937.69873046875,
"loss": 1.1451,
"margin_dpo/margin_mean": 311.9921569824219,
"margin_dpo/margin_std": 565.5931396484375,
"step": 507
},
{
"KL/chosen_KL_mean": -583.1611328125,
"KL/mean": -721.2760009765625,
"KL/rejected_KL_mean": -859.390869140625,
"KL/std": 509.71832275390625,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0013669917825609446,
"fcm_dpo/delta": 0.031157607212662697,
"fcm_dpo/margin": 276.22967529296875,
"fcm_dpo/q_t": 0.4213915169239044,
"grad_norm": 53.349178314208984,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": -0.8866854906082153,
"logits/rejected": -0.883845329284668,
"logps/chosen": -654.464599609375,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -955.0184326171875,
"loss": 1.1942,
"margin_dpo/margin_mean": 276.2297058105469,
"margin_dpo/margin_std": 601.1476440429688,
"step": 508
},
{
"KL/chosen_KL_mean": -447.341552734375,
"KL/mean": -596.3013916015625,
"KL/rejected_KL_mean": -745.26123046875,
"KL/std": 366.6440734863281,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0013803851325064898,
"fcm_dpo/delta": -0.028025750070810318,
"fcm_dpo/margin": 297.9196472167969,
"fcm_dpo/q_t": 0.4056088328361511,
"grad_norm": 36.49995803833008,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": -0.8121344447135925,
"logits/rejected": -0.8039395809173584,
"logps/chosen": -511.1605224609375,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -828.5177001953125,
"loss": 1.1197,
"margin_dpo/margin_mean": 297.9196472167969,
"margin_dpo/margin_std": 458.0248107910156,
"step": 509
},
{
"KL/chosen_KL_mean": -541.6021728515625,
"KL/mean": -733.678466796875,
"KL/rejected_KL_mean": -925.7547607421875,
"KL/std": 435.32470703125,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.0011522183194756508,
"fcm_dpo/delta": -0.07610686868429184,
"fcm_dpo/margin": 384.152587890625,
"fcm_dpo/q_t": 0.3902145326137543,
"grad_norm": 38.18740463256836,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": -0.7922756671905518,
"logits/rejected": -0.8119780421257019,
"logps/chosen": -593.4805908203125,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -1028.5198974609375,
"loss": 1.0452,
"margin_dpo/margin_mean": 384.152587890625,
"margin_dpo/margin_std": 409.4080810546875,
"step": 510
},
{
"KL/chosen_KL_mean": -491.181884765625,
"KL/mean": -649.146484375,
"KL/rejected_KL_mean": -807.1109619140625,
"KL/std": 461.1341247558594,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.0011797586921602488,
"fcm_dpo/delta": 0.0413035973906517,
"fcm_dpo/margin": 315.92913818359375,
"fcm_dpo/q_t": 0.41845548152923584,
"grad_norm": 29.213090896606445,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": -0.8354673385620117,
"logits/rejected": -0.8356435298919678,
"logps/chosen": -551.4199829101562,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -899.9677734375,
"loss": 1.1461,
"margin_dpo/margin_mean": 315.92913818359375,
"margin_dpo/margin_std": 560.2075805664062,
"step": 511
},
{
"KL/chosen_KL_mean": -411.56103515625,
"KL/mean": -608.6541137695312,
"KL/rejected_KL_mean": -805.7471923828125,
"KL/std": 410.18994140625,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.0010748871136456728,
"fcm_dpo/delta": -0.04578953981399536,
"fcm_dpo/margin": 394.1861572265625,
"fcm_dpo/q_t": 0.3963681757450104,
"grad_norm": 52.93358612060547,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": -0.8261533975601196,
"logits/rejected": -0.8413060307502747,
"logps/chosen": -466.466552734375,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -887.623046875,
"loss": 1.0543,
"margin_dpo/margin_mean": 394.1861572265625,
"margin_dpo/margin_std": 417.51800537109375,
"step": 512
},
{
"KL/chosen_KL_mean": -532.857421875,
"KL/mean": -685.5048828125,
"KL/rejected_KL_mean": -838.1524658203125,
"KL/std": 409.397705078125,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0009999822359532118,
"fcm_dpo/delta": -0.038525860756635666,
"fcm_dpo/margin": 305.2950134277344,
"fcm_dpo/q_t": 0.42581143975257874,
"grad_norm": 34.72964859008789,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": -0.8888071775436401,
"logits/rejected": -0.8702949285507202,
"logps/chosen": -597.7738647460938,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -914.2149047851562,
"loss": 1.1629,
"margin_dpo/margin_mean": 305.2950134277344,
"margin_dpo/margin_std": 478.2789306640625,
"step": 513
},
{
"KL/chosen_KL_mean": -564.9537353515625,
"KL/mean": -707.5296630859375,
"KL/rejected_KL_mean": -850.1055908203125,
"KL/std": 448.4078063964844,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0011754620354622602,
"fcm_dpo/delta": 0.09163933992385864,
"fcm_dpo/margin": 285.15191650390625,
"fcm_dpo/q_t": 0.4304603934288025,
"grad_norm": 32.26676559448242,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": -0.8373386859893799,
"logits/rejected": -0.8227113485336304,
"logps/chosen": -639.1832885742188,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -929.0511474609375,
"loss": 1.1577,
"margin_dpo/margin_mean": 285.15191650390625,
"margin_dpo/margin_std": 502.1484375,
"step": 514
},
{
"KL/chosen_KL_mean": -494.32861328125,
"KL/mean": -618.3717041015625,
"KL/rejected_KL_mean": -742.414794921875,
"KL/std": 406.7989501953125,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.0013804570771753788,
"fcm_dpo/delta": 0.08468376100063324,
"fcm_dpo/margin": 248.086181640625,
"fcm_dpo/q_t": 0.4263339042663574,
"grad_norm": 40.3935546875,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": -0.8123036623001099,
"logits/rejected": -0.8226903676986694,
"logps/chosen": -544.730224609375,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -829.5125122070312,
"loss": 1.1724,
"margin_dpo/margin_mean": 248.086181640625,
"margin_dpo/margin_std": 483.9718322753906,
"step": 515
},
{
"KL/chosen_KL_mean": -526.0533447265625,
"KL/mean": -701.6831665039062,
"KL/rejected_KL_mean": -877.3131103515625,
"KL/std": 438.8985900878906,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.0012882768642157316,
"fcm_dpo/delta": -0.0951680988073349,
"fcm_dpo/margin": 351.25970458984375,
"fcm_dpo/q_t": 0.3881225287914276,
"grad_norm": 36.28194046020508,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": -0.9183084964752197,
"logits/rejected": -0.9395666122436523,
"logps/chosen": -590.702880859375,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -989.0354614257812,
"loss": 1.0661,
"margin_dpo/margin_mean": 351.25970458984375,
"margin_dpo/margin_std": 446.1391906738281,
"step": 516
},
{
"KL/chosen_KL_mean": -552.42333984375,
"KL/mean": -758.86474609375,
"KL/rejected_KL_mean": -965.3060302734375,
"KL/std": 489.52093505859375,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0010481756180524826,
"fcm_dpo/delta": -0.0622154101729393,
"fcm_dpo/margin": 412.88262939453125,
"fcm_dpo/q_t": 0.39706072211265564,
"grad_norm": 25.22186279296875,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": -0.8430509567260742,
"logits/rejected": -0.837881326675415,
"logps/chosen": -613.3369140625,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -1054.38916015625,
"loss": 1.0697,
"margin_dpo/margin_mean": 412.88262939453125,
"margin_dpo/margin_std": 533.8587646484375,
"step": 517
},
{
"KL/chosen_KL_mean": -519.568603515625,
"KL/mean": -694.2239990234375,
"KL/rejected_KL_mean": -868.8794555664062,
"KL/std": 415.3473815917969,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.0010671718046069145,
"fcm_dpo/delta": 0.0419648140668869,
"fcm_dpo/margin": 349.3109130859375,
"fcm_dpo/q_t": 0.4160323143005371,
"grad_norm": 33.90839385986328,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": -0.875517725944519,
"logits/rejected": -0.8914189338684082,
"logps/chosen": -577.0244140625,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -954.192138671875,
"loss": 1.112,
"margin_dpo/margin_mean": 349.3109130859375,
"margin_dpo/margin_std": 470.837158203125,
"step": 518
},
{
"KL/chosen_KL_mean": -517.7393798828125,
"KL/mean": -643.4759521484375,
"KL/rejected_KL_mean": -769.21240234375,
"KL/std": 388.196044921875,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.0011561332503333688,
"fcm_dpo/delta": 0.02302495203912258,
"fcm_dpo/margin": 251.4730224609375,
"fcm_dpo/q_t": 0.4338497221469879,
"grad_norm": 42.36438751220703,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": -0.889582097530365,
"logits/rejected": -0.8709315061569214,
"logps/chosen": -591.802734375,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -873.6566162109375,
"loss": 1.1946,
"margin_dpo/margin_mean": 251.4730224609375,
"margin_dpo/margin_std": 490.158447265625,
"step": 519
},
{
"KL/chosen_KL_mean": -572.123291015625,
"KL/mean": -733.942138671875,
"KL/rejected_KL_mean": -895.7611083984375,
"KL/std": 462.9925537109375,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.001188506605103612,
"fcm_dpo/delta": 0.023475412279367447,
"fcm_dpo/margin": 323.6377868652344,
"fcm_dpo/q_t": 0.41749107837677,
"grad_norm": 31.222883224487305,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": -0.8482314348220825,
"logits/rejected": -0.8507933616638184,
"logps/chosen": -642.423095703125,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -995.742431640625,
"loss": 1.1305,
"margin_dpo/margin_mean": 323.6378173828125,
"margin_dpo/margin_std": 529.4666748046875,
"step": 520
},
{
"KL/chosen_KL_mean": -498.2612609863281,
"KL/mean": -677.0108032226562,
"KL/rejected_KL_mean": -855.76025390625,
"KL/std": 452.33221435546875,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.001218126737512648,
"fcm_dpo/delta": -0.07213665544986725,
"fcm_dpo/margin": 357.4990234375,
"fcm_dpo/q_t": 0.39487892389297485,
"grad_norm": 34.73341369628906,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": -0.8869306445121765,
"logits/rejected": -0.8860381841659546,
"logps/chosen": -556.4041748046875,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -939.0408935546875,
"loss": 1.0806,
"margin_dpo/margin_mean": 357.4990234375,
"margin_dpo/margin_std": 450.37408447265625,
"step": 521
},
{
"KL/chosen_KL_mean": -573.8514404296875,
"KL/mean": -739.8829345703125,
"KL/rejected_KL_mean": -905.91455078125,
"KL/std": 490.18310546875,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.0011361883953213692,
"fcm_dpo/delta": 0.036084454506635666,
"fcm_dpo/margin": 332.06304931640625,
"fcm_dpo/q_t": 0.41713184118270874,
"grad_norm": 28.38233757019043,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": -0.9024146199226379,
"logits/rejected": -0.9035881757736206,
"logps/chosen": -620.3991088867188,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -971.928466796875,
"loss": 1.128,
"margin_dpo/margin_mean": 332.0630798339844,
"margin_dpo/margin_std": 528.5509033203125,
"step": 522
},
{
"KL/chosen_KL_mean": -621.0657348632812,
"KL/mean": -792.4852905273438,
"KL/rejected_KL_mean": -963.9049072265625,
"KL/std": 544.6927490234375,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.0011524436995387077,
"fcm_dpo/delta": 0.007822713814675808,
"fcm_dpo/margin": 342.8391418457031,
"fcm_dpo/q_t": 0.4123551845550537,
"grad_norm": 35.35528564453125,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": -0.9692133665084839,
"logits/rejected": -0.9667251110076904,
"logps/chosen": -682.8353271484375,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -1047.666259765625,
"loss": 1.1316,
"margin_dpo/margin_mean": 342.8391418457031,
"margin_dpo/margin_std": 576.16796875,
"step": 523
},
{
"KL/chosen_KL_mean": -588.4208984375,
"KL/mean": -767.6956787109375,
"KL/rejected_KL_mean": -946.9705810546875,
"KL/std": 500.2874755859375,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0011021224781870842,
"fcm_dpo/delta": -0.0017808079719543457,
"fcm_dpo/margin": 358.5496826171875,
"fcm_dpo/q_t": 0.40673327445983887,
"grad_norm": 34.97015380859375,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": -0.9925479888916016,
"logits/rejected": -0.9842585325241089,
"logps/chosen": -666.492919921875,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -1028.2725830078125,
"loss": 1.1098,
"margin_dpo/margin_mean": 358.5496520996094,
"margin_dpo/margin_std": 513.2574462890625,
"step": 524
},
{
"KL/chosen_KL_mean": -575.912109375,
"KL/mean": -803.885986328125,
"KL/rejected_KL_mean": -1031.85986328125,
"KL/std": 526.4970092773438,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.0009674688335508108,
"fcm_dpo/delta": -0.09075939655303955,
"fcm_dpo/margin": 455.9477233886719,
"fcm_dpo/q_t": 0.39121919870376587,
"grad_norm": 35.0758056640625,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": -0.89338219165802,
"logits/rejected": -0.9303810596466064,
"logps/chosen": -626.739990234375,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -1131.912841796875,
"loss": 1.06,
"margin_dpo/margin_mean": 455.94775390625,
"margin_dpo/margin_std": 559.2699584960938,
"step": 525
},
{
"KL/chosen_KL_mean": -609.7196044921875,
"KL/mean": -831.7645874023438,
"KL/rejected_KL_mean": -1053.8095703125,
"KL/std": 516.262939453125,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.0009489007061347365,
"fcm_dpo/delta": -0.03767908737063408,
"fcm_dpo/margin": 444.0899658203125,
"fcm_dpo/q_t": 0.3990199565887451,
"grad_norm": 31.720500946044922,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": -0.921222984790802,
"logits/rejected": -0.9137049317359924,
"logps/chosen": -672.8868408203125,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -1140.118896484375,
"loss": 1.0826,
"margin_dpo/margin_mean": 444.0899658203125,
"margin_dpo/margin_std": 581.716064453125,
"step": 526
},
{
"KL/chosen_KL_mean": -647.0347290039062,
"KL/mean": -790.537841796875,
"KL/rejected_KL_mean": -934.0408935546875,
"KL/std": 527.8538208007812,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0010856003500521183,
"fcm_dpo/delta": 0.1282484382390976,
"fcm_dpo/margin": 287.0062255859375,
"fcm_dpo/q_t": 0.4351726174354553,
"grad_norm": 38.458126068115234,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": -1.1125977039337158,
"logits/rejected": -1.0749619007110596,
"logps/chosen": -718.1634521484375,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -1012.3834838867188,
"loss": 1.1598,
"margin_dpo/margin_mean": 287.0062255859375,
"margin_dpo/margin_std": 455.161376953125,
"step": 527
},
{
"KL/chosen_KL_mean": -650.195068359375,
"KL/mean": -832.568359375,
"KL/rejected_KL_mean": -1014.9417724609375,
"KL/std": 509.81414794921875,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.0011271900730207562,
"fcm_dpo/delta": -0.019259048625826836,
"fcm_dpo/margin": 364.7466735839844,
"fcm_dpo/q_t": 0.4092450439929962,
"grad_norm": 36.67366409301758,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": -0.981177568435669,
"logits/rejected": -0.9641728401184082,
"logps/chosen": -718.2845458984375,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -1108.851806640625,
"loss": 1.1235,
"margin_dpo/margin_mean": 364.7467041015625,
"margin_dpo/margin_std": 606.7515258789062,
"step": 528
},
{
"KL/chosen_KL_mean": -796.822509765625,
"KL/mean": -921.7469482421875,
"KL/rejected_KL_mean": -1046.67138671875,
"KL/std": 494.61181640625,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.0011494512436911464,
"fcm_dpo/delta": 0.01170584000647068,
"fcm_dpo/margin": 249.8488006591797,
"fcm_dpo/q_t": 0.4349808990955353,
"grad_norm": 53.19778823852539,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": -1.048140525817871,
"logits/rejected": -1.0602138042449951,
"logps/chosen": -852.3974609375,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -1135.88037109375,
"loss": 1.2429,
"margin_dpo/margin_mean": 249.84881591796875,
"margin_dpo/margin_std": 625.3258056640625,
"step": 529
},
{
"KL/chosen_KL_mean": -660.2740478515625,
"KL/mean": -862.5133666992188,
"KL/rejected_KL_mean": -1064.7528076171875,
"KL/std": 578.7081298828125,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.0010660856496542692,
"fcm_dpo/delta": -0.05491678789258003,
"fcm_dpo/margin": 404.4787902832031,
"fcm_dpo/q_t": 0.4025976061820984,
"grad_norm": 38.45912551879883,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": -0.9426896572113037,
"logits/rejected": -0.9390045404434204,
"logps/chosen": -707.8754272460938,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -1152.037353515625,
"loss": 1.136,
"margin_dpo/margin_mean": 404.4787902832031,
"margin_dpo/margin_std": 729.6232299804688,
"step": 530
},
{
"KL/chosen_KL_mean": -751.2568359375,
"KL/mean": -909.684814453125,
"KL/rejected_KL_mean": -1068.11279296875,
"KL/std": 594.5966186523438,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.001148285111412406,
"fcm_dpo/delta": 0.05123718082904816,
"fcm_dpo/margin": 316.85595703125,
"fcm_dpo/q_t": 0.42305827140808105,
"grad_norm": 43.48565673828125,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": -1.0459530353546143,
"logits/rejected": -1.0223256349563599,
"logps/chosen": -806.5028686523438,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -1138.71875,
"loss": 1.1531,
"margin_dpo/margin_mean": 316.85595703125,
"margin_dpo/margin_std": 567.612060546875,
"step": 531
},
{
"KL/chosen_KL_mean": -734.84375,
"KL/mean": -924.059326171875,
"KL/rejected_KL_mean": -1113.27490234375,
"KL/std": 581.4156494140625,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0010310335783287883,
"fcm_dpo/delta": -0.008013017475605011,
"fcm_dpo/margin": 378.4312438964844,
"fcm_dpo/q_t": 0.41106435656547546,
"grad_norm": 68.37248229980469,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": -0.9860376119613647,
"logits/rejected": -0.961615264415741,
"logps/chosen": -805.1297607421875,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -1199.8662109375,
"loss": 1.1619,
"margin_dpo/margin_mean": 378.43121337890625,
"margin_dpo/margin_std": 688.8643798828125,
"step": 532
},
{
"KL/chosen_KL_mean": -660.6729125976562,
"KL/mean": -794.4534301757812,
"KL/rejected_KL_mean": -928.23388671875,
"KL/std": 490.07489013671875,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.0011233469704166055,
"fcm_dpo/delta": 0.00041316391434520483,
"fcm_dpo/margin": 267.5610046386719,
"fcm_dpo/q_t": 0.4326951205730438,
"grad_norm": 41.96712875366211,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": -1.028921127319336,
"logits/rejected": -1.0061641931533813,
"logps/chosen": -725.5280151367188,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -1004.822021484375,
"loss": 1.2077,
"margin_dpo/margin_mean": 267.5610046386719,
"margin_dpo/margin_std": 582.5079345703125,
"step": 533
},
{
"KL/chosen_KL_mean": -676.6849365234375,
"KL/mean": -862.72900390625,
"KL/rejected_KL_mean": -1048.7730712890625,
"KL/std": 513.5587158203125,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.0010465008672326803,
"fcm_dpo/delta": 0.0019133538007736206,
"fcm_dpo/margin": 372.08819580078125,
"fcm_dpo/q_t": 0.4103556275367737,
"grad_norm": 26.502838134765625,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": -0.9815844297409058,
"logits/rejected": -0.9804989099502563,
"logps/chosen": -736.8043212890625,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -1127.3165283203125,
"loss": 1.1194,
"margin_dpo/margin_mean": 372.08819580078125,
"margin_dpo/margin_std": 547.061279296875,
"step": 534
},
{
"KL/chosen_KL_mean": -587.0618286132812,
"KL/mean": -801.0487060546875,
"KL/rejected_KL_mean": -1015.0355224609375,
"KL/std": 525.9488525390625,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.001057377434335649,
"fcm_dpo/delta": -0.10026499629020691,
"fcm_dpo/margin": 427.9737854003906,
"fcm_dpo/q_t": 0.390084445476532,
"grad_norm": 25.19883155822754,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": -0.9359257817268372,
"logits/rejected": -0.9411388039588928,
"logps/chosen": -641.39208984375,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -1111.34326171875,
"loss": 1.0808,
"margin_dpo/margin_mean": 427.9737854003906,
"margin_dpo/margin_std": 552.81982421875,
"step": 535
},
{
"KL/chosen_KL_mean": -524.899658203125,
"KL/mean": -780.73291015625,
"KL/rejected_KL_mean": -1036.5660400390625,
"KL/std": 589.3612060546875,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0008447887958027422,
"fcm_dpo/delta": -0.05769486725330353,
"fcm_dpo/margin": 511.6664733886719,
"fcm_dpo/q_t": 0.3999031186103821,
"grad_norm": 24.389259338378906,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": -0.9003403186798096,
"logits/rejected": -0.9327056407928467,
"logps/chosen": -571.980224609375,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -1125.6639404296875,
"loss": 1.0767,
"margin_dpo/margin_mean": 511.66644287109375,
"margin_dpo/margin_std": 714.7777709960938,
"step": 536
},
{
"KL/chosen_KL_mean": -562.1661376953125,
"KL/mean": -730.231201171875,
"KL/rejected_KL_mean": -898.2962646484375,
"KL/std": 485.2312316894531,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.0008401878294534981,
"fcm_dpo/delta": 0.021016618236899376,
"fcm_dpo/margin": 336.1300354003906,
"fcm_dpo/q_t": 0.4347127377986908,
"grad_norm": 33.1090202331543,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": -0.9427402019500732,
"logits/rejected": -0.9209805727005005,
"logps/chosen": -619.9136962890625,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -968.734619140625,
"loss": 1.1829,
"margin_dpo/margin_mean": 336.1300354003906,
"margin_dpo/margin_std": 590.2587890625,
"step": 537
},
{
"KL/chosen_KL_mean": -686.517822265625,
"KL/mean": -878.772705078125,
"KL/rejected_KL_mean": -1071.027587890625,
"KL/std": 506.6947937011719,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.0009338191011920571,
"fcm_dpo/delta": 0.064121313393116,
"fcm_dpo/margin": 384.5097351074219,
"fcm_dpo/q_t": 0.42180943489074707,
"grad_norm": 26.116352081298828,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": -0.9635825753211975,
"logits/rejected": -0.9489821195602417,
"logps/chosen": -752.9337768554688,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -1155.255615234375,
"loss": 1.1175,
"margin_dpo/margin_mean": 384.5097351074219,
"margin_dpo/margin_std": 529.1771240234375,
"step": 538
},
{
"KL/chosen_KL_mean": -581.152099609375,
"KL/mean": -763.506103515625,
"KL/rejected_KL_mean": -945.8601684570312,
"KL/std": 429.1192321777344,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.0010440616169944406,
"fcm_dpo/delta": 0.02792687714099884,
"fcm_dpo/margin": 364.7080383300781,
"fcm_dpo/q_t": 0.4141731858253479,
"grad_norm": 33.319786071777344,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": -0.9643697738647461,
"logits/rejected": -0.9533451795578003,
"logps/chosen": -639.6449584960938,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -1037.714111328125,
"loss": 1.101,
"margin_dpo/margin_mean": 364.70806884765625,
"margin_dpo/margin_std": 481.9061279296875,
"step": 539
},
{
"KL/chosen_KL_mean": -593.5488891601562,
"KL/mean": -833.126708984375,
"KL/rejected_KL_mean": -1072.70458984375,
"KL/std": 541.5235595703125,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.0009419465786777437,
"fcm_dpo/delta": -0.09322336316108704,
"fcm_dpo/margin": 479.15557861328125,
"fcm_dpo/q_t": 0.3929232060909271,
"grad_norm": 30.28093147277832,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": -0.9041993618011475,
"logits/rejected": -0.9246673583984375,
"logps/chosen": -657.0314331054688,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -1189.134521484375,
"loss": 1.0719,
"margin_dpo/margin_mean": 479.15557861328125,
"margin_dpo/margin_std": 642.5653686523438,
"step": 540
},
{
"KL/chosen_KL_mean": -716.245849609375,
"KL/mean": -844.7294921875,
"KL/rejected_KL_mean": -973.2132568359375,
"KL/std": 497.0907287597656,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0008998019620776176,
"fcm_dpo/delta": 0.02505185268819332,
"fcm_dpo/margin": 256.9674072265625,
"fcm_dpo/q_t": 0.44805556535720825,
"grad_norm": 39.39170455932617,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": -1.0489933490753174,
"logits/rejected": -1.043041706085205,
"logps/chosen": -794.5261840820312,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -1076.48681640625,
"loss": 1.241,
"margin_dpo/margin_mean": 256.9674377441406,
"margin_dpo/margin_std": 595.6408081054688,
"step": 541
},
{
"KL/chosen_KL_mean": -606.02099609375,
"KL/mean": -850.662841796875,
"KL/rejected_KL_mean": -1095.3046875,
"KL/std": 556.0362548828125,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.000812588375993073,
"fcm_dpo/delta": -0.015589557588100433,
"fcm_dpo/margin": 489.2837829589844,
"fcm_dpo/q_t": 0.40638577938079834,
"grad_norm": 43.08638000488281,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": -1.0107741355895996,
"logits/rejected": -1.045440435409546,
"logps/chosen": -663.5059814453125,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -1191.77978515625,
"loss": 1.1037,
"margin_dpo/margin_mean": 489.2837829589844,
"margin_dpo/margin_std": 664.2471313476562,
"step": 542
},
{
"KL/chosen_KL_mean": -686.9749145507812,
"KL/mean": -857.873046875,
"KL/rejected_KL_mean": -1028.7711181640625,
"KL/std": 652.3204956054688,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.0009889693465083838,
"fcm_dpo/delta": 0.09416899085044861,
"fcm_dpo/margin": 341.7962646484375,
"fcm_dpo/q_t": 0.43743741512298584,
"grad_norm": 41.49184799194336,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": -0.9908128976821899,
"logits/rejected": -1.023362398147583,
"logps/chosen": -747.5924072265625,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -1134.3701171875,
"loss": 1.2031,
"margin_dpo/margin_mean": 341.7962646484375,
"margin_dpo/margin_std": 781.2564086914062,
"step": 543
},
{
"KL/chosen_KL_mean": -705.4442749023438,
"KL/mean": -904.148193359375,
"KL/rejected_KL_mean": -1102.85205078125,
"KL/std": 532.5914306640625,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.0010228848550468683,
"fcm_dpo/delta": -0.011566650122404099,
"fcm_dpo/margin": 397.4078369140625,
"fcm_dpo/q_t": 0.40789809823036194,
"grad_norm": 28.750972747802734,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": -0.9685473442077637,
"logits/rejected": -0.9889190196990967,
"logps/chosen": -765.0865478515625,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -1203.8067626953125,
"loss": 1.1104,
"margin_dpo/margin_mean": 397.4078369140625,
"margin_dpo/margin_std": 614.56640625,
"step": 544
},
{
"KL/chosen_KL_mean": -649.4158935546875,
"KL/mean": -885.8656616210938,
"KL/rejected_KL_mean": -1122.3154296875,
"KL/std": 538.6456298828125,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0009213591692969203,
"fcm_dpo/delta": -0.06585489213466644,
"fcm_dpo/margin": 472.8994140625,
"fcm_dpo/q_t": 0.3961293399333954,
"grad_norm": 26.994535446166992,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": -0.8998408913612366,
"logits/rejected": -0.9357277154922485,
"logps/chosen": -717.0645141601562,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -1218.223388671875,
"loss": 1.0743,
"margin_dpo/margin_mean": 472.8994140625,
"margin_dpo/margin_std": 626.8919677734375,
"step": 545
},
{
"KL/chosen_KL_mean": -610.9400024414062,
"KL/mean": -767.9046630859375,
"KL/rejected_KL_mean": -924.8692626953125,
"KL/std": 476.8546142578125,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0010576581116765738,
"fcm_dpo/delta": 0.0986643135547638,
"fcm_dpo/margin": 313.9293212890625,
"fcm_dpo/q_t": 0.4284512400627136,
"grad_norm": 36.017555236816406,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": -0.9738471508026123,
"logits/rejected": -0.9590544104576111,
"logps/chosen": -661.6842041015625,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -1006.7355346679688,
"loss": 1.1519,
"margin_dpo/margin_mean": 313.9293518066406,
"margin_dpo/margin_std": 505.626220703125,
"step": 546
},
{
"KL/chosen_KL_mean": -610.9646606445312,
"KL/mean": -821.750732421875,
"KL/rejected_KL_mean": -1032.536865234375,
"KL/std": 533.4896240234375,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.001024933997541666,
"fcm_dpo/delta": -0.056229621171951294,
"fcm_dpo/margin": 421.57220458984375,
"fcm_dpo/q_t": 0.3993460237979889,
"grad_norm": 39.7552490234375,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": -0.9101555943489075,
"logits/rejected": -0.9197220802307129,
"logps/chosen": -684.65234375,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -1123.29833984375,
"loss": 1.0807,
"margin_dpo/margin_mean": 421.57220458984375,
"margin_dpo/margin_std": 582.0226440429688,
"step": 547
},
{
"KL/chosen_KL_mean": -647.4964599609375,
"KL/mean": -836.765869140625,
"KL/rejected_KL_mean": -1026.035400390625,
"KL/std": 553.211181640625,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.0010135022457689047,
"fcm_dpo/delta": 0.026592375710606575,
"fcm_dpo/margin": 378.53887939453125,
"fcm_dpo/q_t": 0.4186980724334717,
"grad_norm": 30.77254295349121,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": -0.9787480235099792,
"logits/rejected": -0.9837968349456787,
"logps/chosen": -712.7427978515625,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -1120.1534423828125,
"loss": 1.124,
"margin_dpo/margin_mean": 378.53887939453125,
"margin_dpo/margin_std": 625.75048828125,
"step": 548
},
{
"KL/chosen_KL_mean": -664.22216796875,
"KL/mean": -802.9415283203125,
"KL/rejected_KL_mean": -941.660888671875,
"KL/std": 461.15252685546875,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.0010441699996590614,
"fcm_dpo/delta": 0.008397620171308517,
"fcm_dpo/margin": 277.43865966796875,
"fcm_dpo/q_t": 0.4350124001502991,
"grad_norm": 47.32408142089844,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": -1.0104793310165405,
"logits/rejected": -1.0005792379379272,
"logps/chosen": -713.4345703125,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -1015.5711669921875,
"loss": 1.1953,
"margin_dpo/margin_mean": 277.43865966796875,
"margin_dpo/margin_std": 547.7349243164062,
"step": 549
},
{
"KL/chosen_KL_mean": -645.0916748046875,
"KL/mean": -818.9498291015625,
"KL/rejected_KL_mean": -992.80810546875,
"KL/std": 506.2775573730469,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.0010925406822934747,
"fcm_dpo/delta": 0.03250662982463837,
"fcm_dpo/margin": 347.7164001464844,
"fcm_dpo/q_t": 0.41644904017448425,
"grad_norm": 42.81041717529297,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": -0.9073216915130615,
"logits/rejected": -0.9299839735031128,
"logps/chosen": -701.8985595703125,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -1087.933837890625,
"loss": 1.124,
"margin_dpo/margin_mean": 347.7164306640625,
"margin_dpo/margin_std": 549.0792846679688,
"step": 550
},
{
"KL/chosen_KL_mean": -558.291259765625,
"KL/mean": -833.9419555664062,
"KL/rejected_KL_mean": -1109.5927734375,
"KL/std": 536.7257080078125,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0008524754084646702,
"fcm_dpo/delta": -0.15517154335975647,
"fcm_dpo/margin": 551.3014526367188,
"fcm_dpo/q_t": 0.37387317419052124,
"grad_norm": 49.040679931640625,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": -0.8993926048278809,
"logits/rejected": -0.954781174659729,
"logps/chosen": -617.3975830078125,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -1221.2655029296875,
"loss": 1.0186,
"margin_dpo/margin_mean": 551.3014526367188,
"margin_dpo/margin_std": 538.538330078125,
"step": 551
},
{
"KL/chosen_KL_mean": -533.4443359375,
"KL/mean": -838.3875732421875,
"KL/rejected_KL_mean": -1143.330810546875,
"KL/std": 620.9227294921875,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0006882591405883431,
"fcm_dpo/delta": -0.06372275948524475,
"fcm_dpo/margin": 609.8865356445312,
"fcm_dpo/q_t": 0.3962107002735138,
"grad_norm": 35.4857063293457,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": -0.9574573040008545,
"logits/rejected": -0.9972236156463623,
"logps/chosen": -595.7989501953125,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -1247.892822265625,
"loss": 1.0688,
"margin_dpo/margin_mean": 609.8865356445312,
"margin_dpo/margin_std": 702.5086669921875,
"step": 552
},
{
"KL/chosen_KL_mean": -621.122802734375,
"KL/mean": -784.8905029296875,
"KL/rejected_KL_mean": -948.6583251953125,
"KL/std": 512.807861328125,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0009008180350065231,
"fcm_dpo/delta": 0.1481509506702423,
"fcm_dpo/margin": 327.53546142578125,
"fcm_dpo/q_t": 0.44127804040908813,
"grad_norm": 25.92827033996582,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": -0.9558432698249817,
"logits/rejected": -0.9528769254684448,
"logps/chosen": -689.381591796875,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -1046.7554931640625,
"loss": 1.1731,
"margin_dpo/margin_mean": 327.53546142578125,
"margin_dpo/margin_std": 559.0849609375,
"step": 553
},
{
"KL/chosen_KL_mean": -633.1609497070312,
"KL/mean": -847.7535400390625,
"KL/rejected_KL_mean": -1062.3460693359375,
"KL/std": 536.1700439453125,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0009824573062360287,
"fcm_dpo/delta": -0.04598114266991615,
"fcm_dpo/margin": 429.18511962890625,
"fcm_dpo/q_t": 0.4033704102039337,
"grad_norm": 57.81589126586914,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": -0.9455916285514832,
"logits/rejected": -0.9259899258613586,
"logps/chosen": -701.108642578125,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -1152.1287841796875,
"loss": 1.1217,
"margin_dpo/margin_mean": 429.18511962890625,
"margin_dpo/margin_std": 682.432861328125,
"step": 554
},
{
"KL/chosen_KL_mean": -634.130615234375,
"KL/mean": -891.5674438476562,
"KL/rejected_KL_mean": -1149.0042724609375,
"KL/std": 582.853515625,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.000833613914437592,
"fcm_dpo/delta": -0.051001857966184616,
"fcm_dpo/margin": 514.8737182617188,
"fcm_dpo/q_t": 0.40110278129577637,
"grad_norm": 41.96858215332031,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": -0.9838435649871826,
"logits/rejected": -1.0397891998291016,
"logps/chosen": -687.4610595703125,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -1257.483642578125,
"loss": 1.093,
"margin_dpo/margin_mean": 514.8736572265625,
"margin_dpo/margin_std": 755.9553833007812,
"step": 555
},
{
"KL/chosen_KL_mean": -619.962646484375,
"KL/mean": -767.575439453125,
"KL/rejected_KL_mean": -915.1883544921875,
"KL/std": 455.46258544921875,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0010035810992121696,
"fcm_dpo/delta": 0.14432799816131592,
"fcm_dpo/margin": 295.22576904296875,
"fcm_dpo/q_t": 0.4395366907119751,
"grad_norm": 32.15121841430664,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": -0.9909765720367432,
"logits/rejected": -0.9854000210762024,
"logps/chosen": -678.6070556640625,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -1016.5287475585938,
"loss": 1.163,
"margin_dpo/margin_mean": 295.22576904296875,
"margin_dpo/margin_std": 462.01824951171875,
"step": 556
},
{
"KL/chosen_KL_mean": -678.8939208984375,
"KL/mean": -913.603759765625,
"KL/rejected_KL_mean": -1148.3134765625,
"KL/std": 606.7928466796875,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.0009204911766573787,
"fcm_dpo/delta": -0.06962708383798599,
"fcm_dpo/margin": 469.4195556640625,
"fcm_dpo/q_t": 0.39773887395858765,
"grad_norm": 51.71062088012695,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": -0.9994686841964722,
"logits/rejected": -1.0355925559997559,
"logps/chosen": -746.734619140625,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -1258.253173828125,
"loss": 1.0935,
"margin_dpo/margin_mean": 469.4195556640625,
"margin_dpo/margin_std": 688.9013671875,
"step": 557
},
{
"KL/chosen_KL_mean": -629.8007202148438,
"KL/mean": -937.6903076171875,
"KL/rejected_KL_mean": -1245.579833984375,
"KL/std": 600.0916748046875,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0007529731374233961,
"fcm_dpo/delta": -0.12873995304107666,
"fcm_dpo/margin": 615.7791748046875,
"fcm_dpo/q_t": 0.37874358892440796,
"grad_norm": 26.18462562561035,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": -1.0540165901184082,
"logits/rejected": -1.0735430717468262,
"logps/chosen": -692.1689453125,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -1347.740966796875,
"loss": 1.037,
"margin_dpo/margin_mean": 615.7791748046875,
"margin_dpo/margin_std": 673.7421264648438,
"step": 558
},
{
"KL/chosen_KL_mean": -701.1896362304688,
"KL/mean": -954.133544921875,
"KL/rejected_KL_mean": -1207.0775146484375,
"KL/std": 592.470458984375,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0007604295969940722,
"fcm_dpo/delta": 0.023233801126480103,
"fcm_dpo/margin": 505.8878479003906,
"fcm_dpo/q_t": 0.41252970695495605,
"grad_norm": 24.069955825805664,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": -1.1368141174316406,
"logits/rejected": -1.1402065753936768,
"logps/chosen": -761.9420166015625,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -1300.519775390625,
"loss": 1.0951,
"margin_dpo/margin_mean": 505.8878479003906,
"margin_dpo/margin_std": 648.712158203125,
"step": 559
},
{
"KL/chosen_KL_mean": -657.3939208984375,
"KL/mean": -828.5643310546875,
"KL/rejected_KL_mean": -999.7348022460938,
"KL/std": 511.27996826171875,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0008098391117528081,
"fcm_dpo/delta": 0.034772761166095734,
"fcm_dpo/margin": 342.3409118652344,
"fcm_dpo/q_t": 0.437608540058136,
"grad_norm": 26.729141235351562,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": -0.9040641784667969,
"logits/rejected": -0.8838648796081543,
"logps/chosen": -715.4976806640625,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -1079.72607421875,
"loss": 1.1829,
"margin_dpo/margin_mean": 342.3409423828125,
"margin_dpo/margin_std": 584.12548828125,
"step": 560
},
{
"KL/chosen_KL_mean": -759.172119140625,
"KL/mean": -920.302490234375,
"KL/rejected_KL_mean": -1081.432861328125,
"KL/std": 530.906494140625,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0009913041722029448,
"fcm_dpo/delta": 0.1052679792046547,
"fcm_dpo/margin": 322.26068115234375,
"fcm_dpo/q_t": 0.43175017833709717,
"grad_norm": 45.66500473022461,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": -1.0488073825836182,
"logits/rejected": -1.0699024200439453,
"logps/chosen": -810.0850219726562,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -1180.50146484375,
"loss": 1.1919,
"margin_dpo/margin_mean": 322.2607116699219,
"margin_dpo/margin_std": 664.1737060546875,
"step": 561
},
{
"KL/chosen_KL_mean": -713.2972412109375,
"KL/mean": -922.2352905273438,
"KL/rejected_KL_mean": -1131.1732177734375,
"KL/std": 585.4476928710938,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.000937954755499959,
"fcm_dpo/delta": 0.0012168213725090027,
"fcm_dpo/margin": 417.87603759765625,
"fcm_dpo/q_t": 0.4111784100532532,
"grad_norm": 32.701107025146484,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": -1.0261307954788208,
"logits/rejected": -1.029954433441162,
"logps/chosen": -772.7616577148438,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -1227.7158203125,
"loss": 1.1174,
"margin_dpo/margin_mean": 417.87603759765625,
"margin_dpo/margin_std": 627.3534545898438,
"step": 562
},
{
"KL/chosen_KL_mean": -804.9948120117188,
"KL/mean": -967.0625,
"KL/rejected_KL_mean": -1129.1302490234375,
"KL/std": 637.6392211914062,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.0009516784339211881,
"fcm_dpo/delta": -0.052980802953243256,
"fcm_dpo/margin": 324.13543701171875,
"fcm_dpo/q_t": 0.42142850160598755,
"grad_norm": 52.949893951416016,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": -1.0193402767181396,
"logits/rejected": -1.0229434967041016,
"logps/chosen": -865.60302734375,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -1223.697998046875,
"loss": 1.2444,
"margin_dpo/margin_mean": 324.13543701171875,
"margin_dpo/margin_std": 768.7203369140625,
"step": 563
},
{
"KL/chosen_KL_mean": -731.5855712890625,
"KL/mean": -927.5992431640625,
"KL/rejected_KL_mean": -1123.6129150390625,
"KL/std": 567.4203491210938,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.000967438449151814,
"fcm_dpo/delta": 0.030362479388713837,
"fcm_dpo/margin": 392.0273132324219,
"fcm_dpo/q_t": 0.41628676652908325,
"grad_norm": 39.21650695800781,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": -1.102457880973816,
"logits/rejected": -1.0930469036102295,
"logps/chosen": -805.8539428710938,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -1216.85107421875,
"loss": 1.1161,
"margin_dpo/margin_mean": 392.0273132324219,
"margin_dpo/margin_std": 574.1221923828125,
"step": 564
},
{
"KL/chosen_KL_mean": -767.8232421875,
"KL/mean": -991.4110107421875,
"KL/rejected_KL_mean": -1214.998779296875,
"KL/std": 661.7870483398438,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0009264518739655614,
"fcm_dpo/delta": -0.024389147758483887,
"fcm_dpo/margin": 447.1756286621094,
"fcm_dpo/q_t": 0.4096825420856476,
"grad_norm": 40.5957145690918,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": -1.025184154510498,
"logits/rejected": -1.0267276763916016,
"logps/chosen": -836.8431396484375,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -1300.77783203125,
"loss": 1.1389,
"margin_dpo/margin_mean": 447.1756286621094,
"margin_dpo/margin_std": 811.365234375,
"step": 565
},
{
"KL/chosen_KL_mean": -710.8924560546875,
"KL/mean": -951.005615234375,
"KL/rejected_KL_mean": -1191.1187744140625,
"KL/std": 655.048828125,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.0008984344312921166,
"fcm_dpo/delta": -0.06253774464130402,
"fcm_dpo/margin": 480.226318359375,
"fcm_dpo/q_t": 0.3989356458187103,
"grad_norm": 32.377769470214844,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": -0.997527003288269,
"logits/rejected": -1.0165774822235107,
"logps/chosen": -777.4378051757812,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -1294.988037109375,
"loss": 1.1279,
"margin_dpo/margin_mean": 480.226318359375,
"margin_dpo/margin_std": 777.289794921875,
"step": 566
},
{
"KL/chosen_KL_mean": -701.48583984375,
"KL/mean": -886.89306640625,
"KL/rejected_KL_mean": -1072.3001708984375,
"KL/std": 463.2765197753906,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.000920231977943331,
"fcm_dpo/delta": 0.0880487933754921,
"fcm_dpo/margin": 370.814453125,
"fcm_dpo/q_t": 0.42590731382369995,
"grad_norm": 27.92223358154297,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": -1.1106657981872559,
"logits/rejected": -1.1169605255126953,
"logps/chosen": -754.3441162109375,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -1157.671142578125,
"loss": 1.1274,
"margin_dpo/margin_mean": 370.814453125,
"margin_dpo/margin_std": 517.0704345703125,
"step": 567
},
{
"KL/chosen_KL_mean": -680.7457885742188,
"KL/mean": -934.899658203125,
"KL/rejected_KL_mean": -1189.053466796875,
"KL/std": 556.766845703125,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0008575194515287876,
"fcm_dpo/delta": -0.06648175418376923,
"fcm_dpo/margin": 508.3077087402344,
"fcm_dpo/q_t": 0.3927525281906128,
"grad_norm": 30.725919723510742,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": -0.9947052001953125,
"logits/rejected": -1.032778024673462,
"logps/chosen": -725.9381713867188,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -1278.14599609375,
"loss": 1.0558,
"margin_dpo/margin_mean": 508.3077392578125,
"margin_dpo/margin_std": 588.199951171875,
"step": 568
},
{
"KL/chosen_KL_mean": -766.2860717773438,
"KL/mean": -974.7943115234375,
"KL/rejected_KL_mean": -1183.3026123046875,
"KL/std": 686.7083740234375,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0009196175960823894,
"fcm_dpo/delta": 0.012853320688009262,
"fcm_dpo/margin": 417.0164794921875,
"fcm_dpo/q_t": 0.41270917654037476,
"grad_norm": 42.41626739501953,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": -1.0407395362854004,
"logits/rejected": -1.0457051992416382,
"logps/chosen": -830.0066528320312,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -1262.40576171875,
"loss": 1.1548,
"margin_dpo/margin_mean": 417.0164794921875,
"margin_dpo/margin_std": 728.3641357421875,
"step": 569
},
{
"KL/chosen_KL_mean": -706.3309326171875,
"KL/mean": -890.91357421875,
"KL/rejected_KL_mean": -1075.49609375,
"KL/std": 532.2640380859375,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.0009554984280839562,
"fcm_dpo/delta": 0.07299959659576416,
"fcm_dpo/margin": 369.1651306152344,
"fcm_dpo/q_t": 0.424812376499176,
"grad_norm": 25.38414764404297,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": -1.12546968460083,
"logits/rejected": -1.1144479513168335,
"logps/chosen": -767.945556640625,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -1157.637939453125,
"loss": 1.1375,
"margin_dpo/margin_mean": 369.1651306152344,
"margin_dpo/margin_std": 582.8961181640625,
"step": 570
},
{
"KL/chosen_KL_mean": -729.112548828125,
"KL/mean": -953.21875,
"KL/rejected_KL_mean": -1177.324951171875,
"KL/std": 557.555908203125,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.0009227694245055318,
"fcm_dpo/delta": -0.02847745269536972,
"fcm_dpo/margin": 448.21246337890625,
"fcm_dpo/q_t": 0.4045218527317047,
"grad_norm": 43.35622024536133,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": -0.9972771406173706,
"logits/rejected": -1.0120426416397095,
"logps/chosen": -782.1666259765625,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -1268.661865234375,
"loss": 1.1091,
"margin_dpo/margin_mean": 448.21246337890625,
"margin_dpo/margin_std": 690.9599609375,
"step": 571
},
{
"KL/chosen_KL_mean": -768.8928833007812,
"KL/mean": -1001.8563232421875,
"KL/rejected_KL_mean": -1234.8197021484375,
"KL/std": 665.1800537109375,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.000880242558196187,
"fcm_dpo/delta": -0.021999845281243324,
"fcm_dpo/margin": 465.9267883300781,
"fcm_dpo/q_t": 0.40960174798965454,
"grad_norm": 27.348243713378906,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": -1.0507442951202393,
"logits/rejected": -1.080924153327942,
"logps/chosen": -817.3521728515625,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -1318.376708984375,
"loss": 1.13,
"margin_dpo/margin_mean": 465.92681884765625,
"margin_dpo/margin_std": 805.666015625,
"step": 572
},
{
"KL/chosen_KL_mean": -752.9931030273438,
"KL/mean": -959.6439208984375,
"KL/rejected_KL_mean": -1166.294677734375,
"KL/std": 583.8150634765625,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.000941689417231828,
"fcm_dpo/delta": 0.016215384006500244,
"fcm_dpo/margin": 413.30157470703125,
"fcm_dpo/q_t": 0.4137089252471924,
"grad_norm": 29.199546813964844,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": -1.084615707397461,
"logits/rejected": -1.095890760421753,
"logps/chosen": -804.61572265625,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -1251.61962890625,
"loss": 1.1048,
"margin_dpo/margin_mean": 413.30157470703125,
"margin_dpo/margin_std": 595.2252197265625,
"step": 573
},
{
"KL/chosen_KL_mean": -693.208984375,
"KL/mean": -919.9398193359375,
"KL/rejected_KL_mean": -1146.670654296875,
"KL/std": 540.618896484375,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0009166327654384077,
"fcm_dpo/delta": -0.027354443445801735,
"fcm_dpo/margin": 453.4616394042969,
"fcm_dpo/q_t": 0.40286535024642944,
"grad_norm": 39.16676712036133,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": -1.1048978567123413,
"logits/rejected": -1.1292717456817627,
"logps/chosen": -744.25341796875,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -1239.47705078125,
"loss": 1.09,
"margin_dpo/margin_mean": 453.46160888671875,
"margin_dpo/margin_std": 616.9259643554688,
"step": 574
},
{
"KL/chosen_KL_mean": -759.01708984375,
"KL/mean": -962.711181640625,
"KL/rejected_KL_mean": -1166.4052734375,
"KL/std": 562.2581787109375,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0007753444951958954,
"fcm_dpo/delta": -0.06450725346803665,
"fcm_dpo/margin": 407.38818359375,
"fcm_dpo/q_t": 0.42131945490837097,
"grad_norm": 27.12775993347168,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": -0.9311203360557556,
"logits/rejected": -0.9386662244796753,
"logps/chosen": -830.8072509765625,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -1261.79150390625,
"loss": 1.1753,
"margin_dpo/margin_mean": 407.3881530761719,
"margin_dpo/margin_std": 729.4576416015625,
"step": 575
},
{
"KL/chosen_KL_mean": -730.3524169921875,
"KL/mean": -889.8451538085938,
"KL/rejected_KL_mean": -1049.3380126953125,
"KL/std": 508.9241943359375,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.0008465542923659086,
"fcm_dpo/delta": 0.043933454900979996,
"fcm_dpo/margin": 318.985595703125,
"fcm_dpo/q_t": 0.4400612413883209,
"grad_norm": 28.7397403717041,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": -1.0479196310043335,
"logits/rejected": -1.076101541519165,
"logps/chosen": -784.6153564453125,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -1150.09228515625,
"loss": 1.1968,
"margin_dpo/margin_mean": 318.9855651855469,
"margin_dpo/margin_std": 597.7258911132812,
"step": 576
},
{
"KL/chosen_KL_mean": -645.6124877929688,
"KL/mean": -855.6819458007812,
"KL/rejected_KL_mean": -1065.75146484375,
"KL/std": 571.1446533203125,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0008858378860168159,
"fcm_dpo/delta": 0.04430598393082619,
"fcm_dpo/margin": 420.138916015625,
"fcm_dpo/q_t": 0.4191765785217285,
"grad_norm": 24.229158401489258,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": -1.0603196620941162,
"logits/rejected": -1.0447897911071777,
"logps/chosen": -707.522216796875,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -1149.8221435546875,
"loss": 1.1276,
"margin_dpo/margin_mean": 420.138916015625,
"margin_dpo/margin_std": 631.5208740234375,
"step": 577
},
{
"KL/chosen_KL_mean": -638.2508544921875,
"KL/mean": -869.6015014648438,
"KL/rejected_KL_mean": -1100.9521484375,
"KL/std": 571.8765258789062,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0008566380711272359,
"fcm_dpo/delta": -0.007108829915523529,
"fcm_dpo/margin": 462.70135498046875,
"fcm_dpo/q_t": 0.40710175037384033,
"grad_norm": 31.410873413085938,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": -0.9340611696243286,
"logits/rejected": -0.968732476234436,
"logps/chosen": -687.5145263671875,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -1184.388427734375,
"loss": 1.0938,
"margin_dpo/margin_mean": 462.70135498046875,
"margin_dpo/margin_std": 587.1175537109375,
"step": 578
},
{
"KL/chosen_KL_mean": -741.0704345703125,
"KL/mean": -876.5369873046875,
"KL/rejected_KL_mean": -1012.0033569335938,
"KL/std": 570.01953125,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0010091912699863315,
"fcm_dpo/delta": 0.05066795274615288,
"fcm_dpo/margin": 270.932861328125,
"fcm_dpo/q_t": 0.44031286239624023,
"grad_norm": 65.59943389892578,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": -0.9676034450531006,
"logits/rejected": -0.9647072553634644,
"logps/chosen": -799.956298828125,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -1106.791015625,
"loss": 1.2276,
"margin_dpo/margin_mean": 270.93292236328125,
"margin_dpo/margin_std": 630.8173217773438,
"step": 579
},
{
"KL/chosen_KL_mean": -552.2589721679688,
"KL/mean": -812.1983642578125,
"KL/rejected_KL_mean": -1072.1376953125,
"KL/std": 636.0146484375,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0008385708788409829,
"fcm_dpo/delta": -0.08361184597015381,
"fcm_dpo/margin": 519.8788452148438,
"fcm_dpo/q_t": 0.3960821032524109,
"grad_norm": 34.06106185913086,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": -0.8970617055892944,
"logits/rejected": -0.9277414679527283,
"logps/chosen": -600.9658203125,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -1153.896240234375,
"loss": 1.0825,
"margin_dpo/margin_mean": 519.8788452148438,
"margin_dpo/margin_std": 744.65966796875,
"step": 580
},
{
"KL/chosen_KL_mean": -730.3821411132812,
"KL/mean": -920.6229858398438,
"KL/rejected_KL_mean": -1110.86376953125,
"KL/std": 590.761962890625,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.0009422561852261424,
"fcm_dpo/delta": 0.06384095549583435,
"fcm_dpo/margin": 380.4817199707031,
"fcm_dpo/q_t": 0.4236143231391907,
"grad_norm": 37.83233642578125,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": -1.045546054840088,
"logits/rejected": -1.0597128868103027,
"logps/chosen": -792.6044921875,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -1202.599609375,
"loss": 1.1464,
"margin_dpo/margin_mean": 380.48175048828125,
"margin_dpo/margin_std": 654.8671875,
"step": 581
},
{
"KL/chosen_KL_mean": -663.98291015625,
"KL/mean": -884.2652587890625,
"KL/rejected_KL_mean": -1104.547607421875,
"KL/std": 535.1427001953125,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0009837101679295301,
"fcm_dpo/delta": -0.08480895310640335,
"fcm_dpo/margin": 440.5647277832031,
"fcm_dpo/q_t": 0.3922159671783447,
"grad_norm": 33.73926544189453,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": -1.0071823596954346,
"logits/rejected": -1.0349555015563965,
"logps/chosen": -722.2115478515625,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -1214.6171875,
"loss": 1.1097,
"margin_dpo/margin_mean": 440.56475830078125,
"margin_dpo/margin_std": 578.998046875,
"step": 582
},
{
"KL/chosen_KL_mean": -755.7252197265625,
"KL/mean": -893.9094848632812,
"KL/rejected_KL_mean": -1032.0936279296875,
"KL/std": 595.7422485351562,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0008164974860846996,
"fcm_dpo/delta": -0.0013605619315057993,
"fcm_dpo/margin": 276.36834716796875,
"fcm_dpo/q_t": 0.4494253695011139,
"grad_norm": 38.58605194091797,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": -0.9783551096916199,
"logits/rejected": -0.9746303558349609,
"logps/chosen": -812.588134765625,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -1120.49755859375,
"loss": 1.2611,
"margin_dpo/margin_mean": 276.36834716796875,
"margin_dpo/margin_std": 718.41357421875,
"step": 583
},
{
"KL/chosen_KL_mean": -633.4464721679688,
"KL/mean": -902.180908203125,
"KL/rejected_KL_mean": -1170.915283203125,
"KL/std": 608.067626953125,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0007747331401333213,
"fcm_dpo/delta": -0.028918039053678513,
"fcm_dpo/margin": 537.46875,
"fcm_dpo/q_t": 0.4025737941265106,
"grad_norm": 19.63285255432129,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": -0.986220121383667,
"logits/rejected": -1.0069942474365234,
"logps/chosen": -690.34716796875,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -1268.55126953125,
"loss": 1.0713,
"margin_dpo/margin_mean": 537.46875,
"margin_dpo/margin_std": 643.06298828125,
"step": 584
},
{
"KL/chosen_KL_mean": -779.0941772460938,
"KL/mean": -1014.2800903320312,
"KL/rejected_KL_mean": -1249.466064453125,
"KL/std": 677.1864013671875,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0007966947159729898,
"fcm_dpo/delta": 0.03990716487169266,
"fcm_dpo/margin": 470.371826171875,
"fcm_dpo/q_t": 0.4176916182041168,
"grad_norm": 31.820642471313477,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": -1.07651948928833,
"logits/rejected": -1.104307770729065,
"logps/chosen": -840.068115234375,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -1333.635498046875,
"loss": 1.1456,
"margin_dpo/margin_mean": 470.371826171875,
"margin_dpo/margin_std": 820.4464111328125,
"step": 585
},
{
"KL/chosen_KL_mean": -788.8521728515625,
"KL/mean": -943.93310546875,
"KL/rejected_KL_mean": -1099.013916015625,
"KL/std": 580.5469970703125,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0010007360251620412,
"fcm_dpo/delta": 0.1305122673511505,
"fcm_dpo/margin": 310.16180419921875,
"fcm_dpo/q_t": 0.43785321712493896,
"grad_norm": 55.22218322753906,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": -1.0562876462936401,
"logits/rejected": -1.0779566764831543,
"logps/chosen": -845.707763671875,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -1190.816650390625,
"loss": 1.1985,
"margin_dpo/margin_mean": 310.16180419921875,
"margin_dpo/margin_std": 638.7469482421875,
"step": 586
},
{
"KL/chosen_KL_mean": -554.4176635742188,
"KL/mean": -765.8219604492188,
"KL/rejected_KL_mean": -977.2261962890625,
"KL/std": 601.9225463867188,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.0010241747368127108,
"fcm_dpo/delta": -0.05821375921368599,
"fcm_dpo/margin": 422.8085632324219,
"fcm_dpo/q_t": 0.39820683002471924,
"grad_norm": 31.542490005493164,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": -1.0706478357315063,
"logits/rejected": -1.0984125137329102,
"logps/chosen": -599.1092529296875,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -1059.85009765625,
"loss": 1.0901,
"margin_dpo/margin_mean": 422.80853271484375,
"margin_dpo/margin_std": 611.4573974609375,
"step": 587
},
{
"KL/chosen_KL_mean": -721.3621826171875,
"KL/mean": -904.2818603515625,
"KL/rejected_KL_mean": -1087.2015380859375,
"KL/std": 524.9571533203125,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.0009967966470867395,
"fcm_dpo/delta": 0.05296815186738968,
"fcm_dpo/margin": 365.8393859863281,
"fcm_dpo/q_t": 0.41959983110427856,
"grad_norm": 32.92317581176758,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": -1.0092002153396606,
"logits/rejected": -1.0338335037231445,
"logps/chosen": -771.6571044921875,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -1194.571533203125,
"loss": 1.1352,
"margin_dpo/margin_mean": 365.83941650390625,
"margin_dpo/margin_std": 559.2975463867188,
"step": 588
},
{
"KL/chosen_KL_mean": -717.5643920898438,
"KL/mean": -957.2388916015625,
"KL/rejected_KL_mean": -1196.9134521484375,
"KL/std": 576.5167846679688,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.0009336763177998364,
"fcm_dpo/delta": -0.08870557695627213,
"fcm_dpo/margin": 479.34906005859375,
"fcm_dpo/q_t": 0.39069461822509766,
"grad_norm": 31.190114974975586,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": -0.9594367146492004,
"logits/rejected": -0.968762993812561,
"logps/chosen": -777.4942626953125,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -1308.56884765625,
"loss": 1.0654,
"margin_dpo/margin_mean": 479.34906005859375,
"margin_dpo/margin_std": 618.2476806640625,
"step": 589
},
{
"KL/chosen_KL_mean": -610.7032470703125,
"KL/mean": -851.4964599609375,
"KL/rejected_KL_mean": -1092.28955078125,
"KL/std": 560.0525512695312,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.0008607573690824211,
"fcm_dpo/delta": -0.025180064141750336,
"fcm_dpo/margin": 481.5863037109375,
"fcm_dpo/q_t": 0.40354564785957336,
"grad_norm": 23.0913028717041,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": -0.9822874069213867,
"logits/rejected": -1.0102424621582031,
"logps/chosen": -666.5130615234375,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -1198.3524169921875,
"loss": 1.0802,
"margin_dpo/margin_mean": 481.5863037109375,
"margin_dpo/margin_std": 624.2484741210938,
"step": 590
},
{
"KL/chosen_KL_mean": -666.4302978515625,
"KL/mean": -876.166259765625,
"KL/rejected_KL_mean": -1085.902099609375,
"KL/std": 542.080322265625,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0008978393743745983,
"fcm_dpo/delta": 0.03757140040397644,
"fcm_dpo/margin": 419.4718017578125,
"fcm_dpo/q_t": 0.41537439823150635,
"grad_norm": 31.897592544555664,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": -1.0660505294799805,
"logits/rejected": -1.0602378845214844,
"logps/chosen": -722.6708984375,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -1169.6884765625,
"loss": 1.1202,
"margin_dpo/margin_mean": 419.4718017578125,
"margin_dpo/margin_std": 628.8763427734375,
"step": 591
},
{
"KL/chosen_KL_mean": -694.1058349609375,
"KL/mean": -893.2886962890625,
"KL/rejected_KL_mean": -1092.4716796875,
"KL/std": 556.25927734375,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.0009542823536321521,
"fcm_dpo/delta": 0.03209465742111206,
"fcm_dpo/margin": 398.3658447265625,
"fcm_dpo/q_t": 0.4169124960899353,
"grad_norm": 41.34512710571289,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": -1.0209119319915771,
"logits/rejected": -1.0272916555404663,
"logps/chosen": -742.0460815429688,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -1168.20458984375,
"loss": 1.1721,
"margin_dpo/margin_mean": 398.3658447265625,
"margin_dpo/margin_std": 758.87109375,
"step": 592
},
{
"KL/chosen_KL_mean": -679.8154296875,
"KL/mean": -824.459228515625,
"KL/rejected_KL_mean": -969.10302734375,
"KL/std": 587.4564208984375,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.001031426596455276,
"fcm_dpo/delta": 0.024800950661301613,
"fcm_dpo/margin": 289.28753662109375,
"fcm_dpo/q_t": 0.4358917474746704,
"grad_norm": 62.813846588134766,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": -0.9074722528457642,
"logits/rejected": -0.9004065990447998,
"logps/chosen": -728.5061645507812,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -1030.010986328125,
"loss": 1.2174,
"margin_dpo/margin_mean": 289.28753662109375,
"margin_dpo/margin_std": 654.1934204101562,
"step": 593
},
{
"KL/chosen_KL_mean": -626.3131103515625,
"KL/mean": -814.7869873046875,
"KL/rejected_KL_mean": -1003.2608642578125,
"KL/std": 552.5802001953125,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0010870166588574648,
"fcm_dpo/delta": -0.03803686052560806,
"fcm_dpo/margin": 376.94769287109375,
"fcm_dpo/q_t": 0.4029105305671692,
"grad_norm": 54.68209457397461,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": -1.0033159255981445,
"logits/rejected": -0.9932465553283691,
"logps/chosen": -681.248046875,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -1089.360595703125,
"loss": 1.1364,
"margin_dpo/margin_mean": 376.94769287109375,
"margin_dpo/margin_std": 580.914794921875,
"step": 594
},
{
"KL/chosen_KL_mean": -590.2716674804688,
"KL/mean": -798.0963134765625,
"KL/rejected_KL_mean": -1005.9209594726562,
"KL/std": 527.621826171875,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.0009453489328734577,
"fcm_dpo/delta": 0.01016305387020111,
"fcm_dpo/margin": 415.6493225097656,
"fcm_dpo/q_t": 0.41294392943382263,
"grad_norm": 53.49211120605469,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": -0.970264196395874,
"logits/rejected": -0.9880306124687195,
"logps/chosen": -639.6920776367188,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -1086.54833984375,
"loss": 1.1058,
"margin_dpo/margin_mean": 415.6493225097656,
"margin_dpo/margin_std": 597.9583129882812,
"step": 595
},
{
"KL/chosen_KL_mean": -709.765625,
"KL/mean": -855.121337890625,
"KL/rejected_KL_mean": -1000.47705078125,
"KL/std": 552.8822021484375,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.0011489235330373049,
"fcm_dpo/delta": 0.09330044686794281,
"fcm_dpo/margin": 290.71142578125,
"fcm_dpo/q_t": 0.4343324303627014,
"grad_norm": 93.66545867919922,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": -0.9842763543128967,
"logits/rejected": -0.9520334005355835,
"logps/chosen": -769.5573120117188,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -1080.568115234375,
"loss": 1.2066,
"margin_dpo/margin_mean": 290.71142578125,
"margin_dpo/margin_std": 659.2149658203125,
"step": 596
},
{
"KL/chosen_KL_mean": -605.57763671875,
"KL/mean": -864.177001953125,
"KL/rejected_KL_mean": -1122.7763671875,
"KL/std": 595.9703369140625,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.0008297483436763287,
"fcm_dpo/delta": -0.1401767134666443,
"fcm_dpo/margin": 517.19873046875,
"fcm_dpo/q_t": 0.3865550458431244,
"grad_norm": 30.938121795654297,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": -0.8776600956916809,
"logits/rejected": -0.9235655069351196,
"logps/chosen": -662.83837890625,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -1223.469970703125,
"loss": 1.0834,
"margin_dpo/margin_mean": 517.1986694335938,
"margin_dpo/margin_std": 668.316162109375,
"step": 597
},
{
"KL/chosen_KL_mean": -645.602294921875,
"KL/mean": -850.23193359375,
"KL/rejected_KL_mean": -1054.861572265625,
"KL/std": 531.68505859375,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0008029807358980179,
"fcm_dpo/delta": -0.050433676689863205,
"fcm_dpo/margin": 409.2593994140625,
"fcm_dpo/q_t": 0.41770249605178833,
"grad_norm": 26.686721801757812,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": -0.9931685328483582,
"logits/rejected": -1.0055735111236572,
"logps/chosen": -698.1207885742188,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -1144.3055419921875,
"loss": 1.1418,
"margin_dpo/margin_mean": 409.2593994140625,
"margin_dpo/margin_std": 592.2933349609375,
"step": 598
},
{
"KL/chosen_KL_mean": -626.4500732421875,
"KL/mean": -800.4187622070312,
"KL/rejected_KL_mean": -974.387451171875,
"KL/std": 483.94842529296875,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0007874942384660244,
"fcm_dpo/delta": -0.009737352840602398,
"fcm_dpo/margin": 347.93743896484375,
"fcm_dpo/q_t": 0.4326390027999878,
"grad_norm": 21.70595932006836,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": -0.8962666988372803,
"logits/rejected": -0.9054356813430786,
"logps/chosen": -676.2527465820312,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -1057.365966796875,
"loss": 1.168,
"margin_dpo/margin_mean": 347.93743896484375,
"margin_dpo/margin_std": 467.16839599609375,
"step": 599
},
{
"KL/chosen_KL_mean": -702.6114501953125,
"KL/mean": -863.553955078125,
"KL/rejected_KL_mean": -1024.496337890625,
"KL/std": 520.333251953125,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0009568602545186877,
"fcm_dpo/delta": 0.1319303810596466,
"fcm_dpo/margin": 321.88494873046875,
"fcm_dpo/q_t": 0.43882617354393005,
"grad_norm": 38.444881439208984,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": -1.0397906303405762,
"logits/rejected": -1.0333709716796875,
"logps/chosen": -769.04638671875,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -1109.952880859375,
"loss": 1.1911,
"margin_dpo/margin_mean": 321.88494873046875,
"margin_dpo/margin_std": 643.2849731445312,
"step": 600
},
{
"KL/chosen_KL_mean": -702.807373046875,
"KL/mean": -912.3177490234375,
"KL/rejected_KL_mean": -1121.828125,
"KL/std": 553.8662109375,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0009801883716136217,
"fcm_dpo/delta": -0.01944911666214466,
"fcm_dpo/margin": 419.02081298828125,
"fcm_dpo/q_t": 0.40293174982070923,
"grad_norm": 32.09441375732422,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": -0.9758914113044739,
"logits/rejected": -0.9933589696884155,
"logps/chosen": -761.9410400390625,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -1216.5191650390625,
"loss": 1.0856,
"margin_dpo/margin_mean": 419.02081298828125,
"margin_dpo/margin_std": 550.6053466796875,
"step": 601
},
{
"KL/chosen_KL_mean": -479.7313232421875,
"KL/mean": -729.894287109375,
"KL/rejected_KL_mean": -980.057373046875,
"KL/std": 495.54962158203125,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0009083700715564191,
"fcm_dpo/delta": -0.10118807107210159,
"fcm_dpo/margin": 500.3260192871094,
"fcm_dpo/q_t": 0.3865256905555725,
"grad_norm": 69.2566909790039,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": -0.9927771091461182,
"logits/rejected": -1.0256961584091187,
"logps/chosen": -528.3248291015625,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -1067.725830078125,
"loss": 1.0464,
"margin_dpo/margin_mean": 500.3260192871094,
"margin_dpo/margin_std": 536.3623046875,
"step": 602
},
{
"KL/chosen_KL_mean": -610.1720581054688,
"KL/mean": -848.7678833007812,
"KL/rejected_KL_mean": -1087.3636474609375,
"KL/std": 582.8988037109375,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0008255116990767419,
"fcm_dpo/delta": 0.009567895904183388,
"fcm_dpo/margin": 477.191650390625,
"fcm_dpo/q_t": 0.41279083490371704,
"grad_norm": 32.93044662475586,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": -0.9903879165649414,
"logits/rejected": -0.9831830263137817,
"logps/chosen": -680.586669921875,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -1187.689208984375,
"loss": 1.1056,
"margin_dpo/margin_mean": 477.191650390625,
"margin_dpo/margin_std": 704.5848388671875,
"step": 603
},
{
"KL/chosen_KL_mean": -604.884521484375,
"KL/mean": -850.2882080078125,
"KL/rejected_KL_mean": -1095.69189453125,
"KL/std": 557.6204833984375,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0008146062027662992,
"fcm_dpo/delta": 0.0001351330429315567,
"fcm_dpo/margin": 490.8074035644531,
"fcm_dpo/q_t": 0.40916040539741516,
"grad_norm": 38.27802658081055,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": -0.9702785015106201,
"logits/rejected": -1.004528284072876,
"logps/chosen": -651.3425903320312,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -1187.54638671875,
"loss": 1.1004,
"margin_dpo/margin_mean": 490.8074035644531,
"margin_dpo/margin_std": 693.0052490234375,
"step": 604
},
{
"KL/chosen_KL_mean": -633.1536865234375,
"KL/mean": -830.642578125,
"KL/rejected_KL_mean": -1028.1314697265625,
"KL/std": 524.9966430664062,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0008949829498305917,
"fcm_dpo/delta": 0.07238632440567017,
"fcm_dpo/margin": 394.977783203125,
"fcm_dpo/q_t": 0.42492377758026123,
"grad_norm": 38.80448913574219,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": -0.9616391658782959,
"logits/rejected": -0.9820314645767212,
"logps/chosen": -699.4030151367188,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -1130.4365234375,
"loss": 1.1385,
"margin_dpo/margin_mean": 394.977783203125,
"margin_dpo/margin_std": 627.3665771484375,
"step": 605
},
{
"KL/chosen_KL_mean": -626.211181640625,
"KL/mean": -838.0142211914062,
"KL/rejected_KL_mean": -1049.8173828125,
"KL/std": 531.291748046875,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0009638095507398248,
"fcm_dpo/delta": -0.017124518752098083,
"fcm_dpo/margin": 423.60614013671875,
"fcm_dpo/q_t": 0.4059169888496399,
"grad_norm": 26.27326011657715,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": -0.9585670232772827,
"logits/rejected": -0.9715864658355713,
"logps/chosen": -681.0302734375,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -1148.188720703125,
"loss": 1.0882,
"margin_dpo/margin_mean": 423.60614013671875,
"margin_dpo/margin_std": 562.107666015625,
"step": 606
},
{
"KL/chosen_KL_mean": -664.5488891601562,
"KL/mean": -828.581298828125,
"KL/rejected_KL_mean": -992.6136474609375,
"KL/std": 521.07080078125,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.0010647273156791925,
"fcm_dpo/delta": 0.06575503945350647,
"fcm_dpo/margin": 328.06475830078125,
"fcm_dpo/q_t": 0.42434990406036377,
"grad_norm": 34.5279655456543,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": -0.9723426103591919,
"logits/rejected": -0.9582047462463379,
"logps/chosen": -722.6329345703125,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -1072.3907470703125,
"loss": 1.1543,
"margin_dpo/margin_mean": 328.06475830078125,
"margin_dpo/margin_std": 572.1151123046875,
"step": 607
},
{
"KL/chosen_KL_mean": -599.10595703125,
"KL/mean": -798.927978515625,
"KL/rejected_KL_mean": -998.75,
"KL/std": 486.12176513671875,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.001036277157254517,
"fcm_dpo/delta": -0.02508939430117607,
"fcm_dpo/margin": 399.6440734863281,
"fcm_dpo/q_t": 0.4025970697402954,
"grad_norm": 39.96210479736328,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": -1.0201570987701416,
"logits/rejected": -1.0304514169692993,
"logps/chosen": -656.5568237304688,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -1093.5234375,
"loss": 1.0788,
"margin_dpo/margin_mean": 399.64404296875,
"margin_dpo/margin_std": 502.84564208984375,
"step": 608
},
{
"KL/chosen_KL_mean": -625.5040283203125,
"KL/mean": -888.441162109375,
"KL/rejected_KL_mean": -1151.378173828125,
"KL/std": 666.3944091796875,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0008594167302362621,
"fcm_dpo/delta": -0.09702034294605255,
"fcm_dpo/margin": 525.8741455078125,
"fcm_dpo/q_t": 0.3949633240699768,
"grad_norm": 29.249309539794922,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": -1.0417685508728027,
"logits/rejected": -1.0658833980560303,
"logps/chosen": -684.3094482421875,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -1240.1943359375,
"loss": 1.0786,
"margin_dpo/margin_mean": 525.8742065429688,
"margin_dpo/margin_std": 787.166748046875,
"step": 609
},
{
"KL/chosen_KL_mean": -631.95361328125,
"KL/mean": -791.3546142578125,
"KL/rejected_KL_mean": -950.7557373046875,
"KL/std": 494.2914733886719,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0009960741735994816,
"fcm_dpo/delta": 0.11386167258024216,
"fcm_dpo/margin": 318.8021240234375,
"fcm_dpo/q_t": 0.4325554370880127,
"grad_norm": 40.697330474853516,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": -0.9712044596672058,
"logits/rejected": -0.9452321529388428,
"logps/chosen": -697.648681640625,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -1034.1611328125,
"loss": 1.1692,
"margin_dpo/margin_mean": 318.8021240234375,
"margin_dpo/margin_std": 572.7630615234375,
"step": 610
},
{
"KL/chosen_KL_mean": -635.2578125,
"KL/mean": -909.711181640625,
"KL/rejected_KL_mean": -1184.1644287109375,
"KL/std": 663.8045654296875,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.0008432117174379528,
"fcm_dpo/delta": -0.12401486933231354,
"fcm_dpo/margin": 548.9066162109375,
"fcm_dpo/q_t": 0.38857901096343994,
"grad_norm": 30.683746337890625,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": -1.0351388454437256,
"logits/rejected": -1.0823793411254883,
"logps/chosen": -687.8572998046875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -1270.495361328125,
"loss": 1.0702,
"margin_dpo/margin_mean": 548.9066162109375,
"margin_dpo/margin_std": 768.596923828125,
"step": 611
},
{
"KL/chosen_KL_mean": -690.9043579101562,
"KL/mean": -900.0711669921875,
"KL/rejected_KL_mean": -1109.238037109375,
"KL/std": 532.022705078125,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.0008869701996445656,
"fcm_dpo/delta": 0.03892756253480911,
"fcm_dpo/margin": 418.3336181640625,
"fcm_dpo/q_t": 0.4165932536125183,
"grad_norm": 28.735475540161133,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": -1.0041875839233398,
"logits/rejected": -1.0075181722640991,
"logps/chosen": -750.2280883789062,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -1197.55029296875,
"loss": 1.1135,
"margin_dpo/margin_mean": 418.3336181640625,
"margin_dpo/margin_std": 586.0115356445312,
"step": 612
},
{
"KL/chosen_KL_mean": -596.6041259765625,
"KL/mean": -865.2132568359375,
"KL/rejected_KL_mean": -1133.8223876953125,
"KL/std": 586.6395874023438,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0007895183516666293,
"fcm_dpo/delta": -0.046130187809467316,
"fcm_dpo/margin": 537.21826171875,
"fcm_dpo/q_t": 0.3976578116416931,
"grad_norm": 28.2447509765625,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": -1.0270159244537354,
"logits/rejected": -1.0836434364318848,
"logps/chosen": -656.3341064453125,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -1238.929931640625,
"loss": 1.0612,
"margin_dpo/margin_mean": 537.21826171875,
"margin_dpo/margin_std": 617.8411865234375,
"step": 613
},
{
"KL/chosen_KL_mean": -683.5602416992188,
"KL/mean": -936.4262084960938,
"KL/rejected_KL_mean": -1189.292236328125,
"KL/std": 602.0802001953125,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.0007989507867023349,
"fcm_dpo/delta": -0.007680162787437439,
"fcm_dpo/margin": 505.73199462890625,
"fcm_dpo/q_t": 0.40751612186431885,
"grad_norm": 35.26876449584961,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": -0.9546043276786804,
"logits/rejected": -1.0301423072814941,
"logps/chosen": -736.4992065429688,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -1293.9715576171875,
"loss": 1.101,
"margin_dpo/margin_mean": 505.73199462890625,
"margin_dpo/margin_std": 729.1088256835938,
"step": 614
},
{
"KL/chosen_KL_mean": -640.28759765625,
"KL/mean": -844.96484375,
"KL/rejected_KL_mean": -1049.6419677734375,
"KL/std": 611.680419921875,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.000880036037415266,
"fcm_dpo/delta": 0.05419084057211876,
"fcm_dpo/margin": 409.35443115234375,
"fcm_dpo/q_t": 0.4209446310997009,
"grad_norm": 33.03697204589844,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": -0.9839037656784058,
"logits/rejected": -1.0060296058654785,
"logps/chosen": -706.1048583984375,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -1144.8194580078125,
"loss": 1.1391,
"margin_dpo/margin_mean": 409.35443115234375,
"margin_dpo/margin_std": 675.0528564453125,
"step": 615
},
{
"KL/chosen_KL_mean": -780.3349609375,
"KL/mean": -949.9927368164062,
"KL/rejected_KL_mean": -1119.6505126953125,
"KL/std": 530.0982055664062,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0010057191830128431,
"fcm_dpo/delta": 0.08487111330032349,
"fcm_dpo/margin": 339.3155517578125,
"fcm_dpo/q_t": 0.4275793731212616,
"grad_norm": 35.978885650634766,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": -1.0499340295791626,
"logits/rejected": -1.0310404300689697,
"logps/chosen": -845.4678344726562,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -1194.35107421875,
"loss": 1.1523,
"margin_dpo/margin_mean": 339.3155517578125,
"margin_dpo/margin_std": 571.7545166015625,
"step": 616
},
{
"KL/chosen_KL_mean": -708.6893310546875,
"KL/mean": -836.7636108398438,
"KL/rejected_KL_mean": -964.837890625,
"KL/std": 481.6317443847656,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0012495677219703794,
"fcm_dpo/delta": 0.1090506836771965,
"fcm_dpo/margin": 256.1485595703125,
"fcm_dpo/q_t": 0.4353943467140198,
"grad_norm": 90.08831024169922,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": -0.9055740833282471,
"logits/rejected": -0.8715794086456299,
"logps/chosen": -771.6948852539062,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -1029.072021484375,
"loss": 1.208,
"margin_dpo/margin_mean": 256.1485290527344,
"margin_dpo/margin_std": 568.8765869140625,
"step": 617
},
{
"KL/chosen_KL_mean": -740.5645141601562,
"KL/mean": -1015.2177124023438,
"KL/rejected_KL_mean": -1289.870849609375,
"KL/std": 659.4688720703125,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.0009442686568945646,
"fcm_dpo/delta": -0.24364520609378815,
"fcm_dpo/margin": 549.306396484375,
"fcm_dpo/q_t": 0.3701603412628174,
"grad_norm": 38.955535888671875,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": -1.0169545412063599,
"logits/rejected": -1.0575779676437378,
"logps/chosen": -807.6658935546875,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -1382.0242919921875,
"loss": 1.0903,
"margin_dpo/margin_mean": 549.306396484375,
"margin_dpo/margin_std": 852.4629516601562,
"step": 618
},
{
"KL/chosen_KL_mean": -735.8086547851562,
"KL/mean": -945.514892578125,
"KL/rejected_KL_mean": -1155.22119140625,
"KL/std": 635.7939453125,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.0008637189166620374,
"fcm_dpo/delta": 0.05800282955169678,
"fcm_dpo/margin": 419.4125671386719,
"fcm_dpo/q_t": 0.4264986217021942,
"grad_norm": 39.2636604309082,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": -1.0181564092636108,
"logits/rejected": -1.0320208072662354,
"logps/chosen": -791.786865234375,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -1248.4066162109375,
"loss": 1.1748,
"margin_dpo/margin_mean": 419.41259765625,
"margin_dpo/margin_std": 845.4202880859375,
"step": 619
},
{
"KL/chosen_KL_mean": -677.5810546875,
"KL/mean": -869.0966796875,
"KL/rejected_KL_mean": -1060.6123046875,
"KL/std": 539.739990234375,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0009379057446494699,
"fcm_dpo/delta": 0.06312977522611618,
"fcm_dpo/margin": 383.0312194824219,
"fcm_dpo/q_t": 0.4227410554885864,
"grad_norm": 29.400907516479492,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": -1.0447208881378174,
"logits/rejected": -1.05232572555542,
"logps/chosen": -737.3785400390625,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -1139.02294921875,
"loss": 1.1375,
"margin_dpo/margin_mean": 383.0312194824219,
"margin_dpo/margin_std": 603.0001220703125,
"step": 620
},
{
"KL/chosen_KL_mean": -654.668701171875,
"KL/mean": -947.2188110351562,
"KL/rejected_KL_mean": -1239.768798828125,
"KL/std": 664.20166015625,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.0008394881151616573,
"fcm_dpo/delta": -0.17594929039478302,
"fcm_dpo/margin": 585.100341796875,
"fcm_dpo/q_t": 0.37430325150489807,
"grad_norm": 41.29154968261719,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": -1.0949594974517822,
"logits/rejected": -1.1471253633499146,
"logps/chosen": -708.6024169921875,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -1328.138427734375,
"loss": 1.0593,
"margin_dpo/margin_mean": 585.1002807617188,
"margin_dpo/margin_std": 730.66455078125,
"step": 621
},
{
"KL/chosen_KL_mean": -661.12158203125,
"KL/mean": -863.3720703125,
"KL/rejected_KL_mean": -1065.6224365234375,
"KL/std": 502.6478271484375,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0008088588365353644,
"fcm_dpo/delta": 0.10930877178907394,
"fcm_dpo/margin": 404.5008544921875,
"fcm_dpo/q_t": 0.4323442876338959,
"grad_norm": 22.646230697631836,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": -0.941389799118042,
"logits/rejected": -0.9269174337387085,
"logps/chosen": -721.407470703125,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -1151.1412353515625,
"loss": 1.1511,
"margin_dpo/margin_mean": 404.5008544921875,
"margin_dpo/margin_std": 631.4569091796875,
"step": 622
},
{
"KL/chosen_KL_mean": -722.4276733398438,
"KL/mean": -880.337158203125,
"KL/rejected_KL_mean": -1038.2467041015625,
"KL/std": 528.0697631835938,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0010385662317276,
"fcm_dpo/delta": 0.10328055918216705,
"fcm_dpo/margin": 315.8189697265625,
"fcm_dpo/q_t": 0.4342414438724518,
"grad_norm": 46.649322509765625,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": -1.0891550779342651,
"logits/rejected": -1.091712474822998,
"logps/chosen": -786.5845947265625,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -1123.3297119140625,
"loss": 1.186,
"margin_dpo/margin_mean": 315.8189697265625,
"margin_dpo/margin_std": 645.6528930664062,
"step": 623
},
{
"KL/chosen_KL_mean": -704.1187133789062,
"KL/mean": -950.7569580078125,
"KL/rejected_KL_mean": -1197.3951416015625,
"KL/std": 544.427490234375,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0009231247240677476,
"fcm_dpo/delta": -0.10394740104675293,
"fcm_dpo/margin": 493.2764892578125,
"fcm_dpo/q_t": 0.38469332456588745,
"grad_norm": 31.07668685913086,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": -1.077886700630188,
"logits/rejected": -1.079150676727295,
"logps/chosen": -776.037353515625,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -1294.5272216796875,
"loss": 1.0515,
"margin_dpo/margin_mean": 493.2764892578125,
"margin_dpo/margin_std": 586.5498046875,
"step": 624
},
{
"KL/chosen_KL_mean": -672.1764526367188,
"KL/mean": -935.4581298828125,
"KL/rejected_KL_mean": -1198.7398681640625,
"KL/std": 594.755615234375,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.0008218766888603568,
"fcm_dpo/delta": -0.057466380298137665,
"fcm_dpo/margin": 526.5632934570312,
"fcm_dpo/q_t": 0.39506271481513977,
"grad_norm": 63.15570068359375,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": -0.9775293469429016,
"logits/rejected": -1.0035572052001953,
"logps/chosen": -730.5185546875,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -1284.8302001953125,
"loss": 1.0508,
"margin_dpo/margin_mean": 526.5633544921875,
"margin_dpo/margin_std": 566.92041015625,
"step": 625
},
{
"KL/chosen_KL_mean": -812.67529296875,
"KL/mean": -959.8338623046875,
"KL/rejected_KL_mean": -1106.9925537109375,
"KL/std": 628.1944580078125,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.0009906254708766937,
"fcm_dpo/delta": 0.14670009911060333,
"fcm_dpo/margin": 294.3172607421875,
"fcm_dpo/q_t": 0.43976619839668274,
"grad_norm": 35.10493469238281,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": -0.9490357637405396,
"logits/rejected": -0.9404639005661011,
"logps/chosen": -887.7879028320312,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -1206.1812744140625,
"loss": 1.2132,
"margin_dpo/margin_mean": 294.3172607421875,
"margin_dpo/margin_std": 620.13720703125,
"step": 626
},
{
"KL/chosen_KL_mean": -542.8727416992188,
"KL/mean": -843.132080078125,
"KL/rejected_KL_mean": -1143.3914794921875,
"KL/std": 703.228271484375,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.0008381834486499429,
"fcm_dpo/delta": -0.20297735929489136,
"fcm_dpo/margin": 600.518798828125,
"fcm_dpo/q_t": 0.3782247304916382,
"grad_norm": 29.68364715576172,
"learning_rate": 9.897955805412e-09,
"logits/chosen": -0.9200704097747803,
"logits/rejected": -0.9946620464324951,
"logps/chosen": -590.6159057617188,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -1250.14599609375,
"loss": 1.0535,
"margin_dpo/margin_mean": 600.518798828125,
"margin_dpo/margin_std": 814.1729736328125,
"step": 627
},
{
"KL/chosen_KL_mean": -741.293701171875,
"KL/mean": -966.7863159179688,
"KL/rejected_KL_mean": -1192.279052734375,
"KL/std": 596.0997924804688,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0007803476182743907,
"fcm_dpo/delta": 0.07325306534767151,
"fcm_dpo/margin": 450.9853515625,
"fcm_dpo/q_t": 0.423714816570282,
"grad_norm": 29.094398498535156,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": -1.0479551553726196,
"logits/rejected": -1.0646214485168457,
"logps/chosen": -801.4765625,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -1293.833740234375,
"loss": 1.1308,
"margin_dpo/margin_mean": 450.9853515625,
"margin_dpo/margin_std": 675.803466796875,
"step": 628
},
{
"KL/chosen_KL_mean": -719.9290771484375,
"KL/mean": -929.900634765625,
"KL/rejected_KL_mean": -1139.872314453125,
"KL/std": 583.1702270507812,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.0008786619873717427,
"fcm_dpo/delta": 0.04738582670688629,
"fcm_dpo/margin": 419.9432373046875,
"fcm_dpo/q_t": 0.4166840612888336,
"grad_norm": 32.80850601196289,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": -1.0385822057724,
"logits/rejected": -1.0577049255371094,
"logps/chosen": -784.142578125,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -1231.52587890625,
"loss": 1.112,
"margin_dpo/margin_mean": 419.9432373046875,
"margin_dpo/margin_std": 578.53662109375,
"step": 629
},
{
"KL/chosen_KL_mean": -647.0543212890625,
"KL/mean": -764.2266845703125,
"KL/rejected_KL_mean": -881.39892578125,
"KL/std": 556.714599609375,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.0009413863299414515,
"fcm_dpo/delta": 0.056776098906993866,
"fcm_dpo/margin": 234.34454345703125,
"fcm_dpo/q_t": 0.4554150104522705,
"grad_norm": 49.36244201660156,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": -0.9865130186080933,
"logits/rejected": -0.9569113254547119,
"logps/chosen": -706.3453369140625,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -964.9971923828125,
"loss": 1.2685,
"margin_dpo/margin_mean": 234.34454345703125,
"margin_dpo/margin_std": 668.56982421875,
"step": 630
},
{
"KL/chosen_KL_mean": -776.7286987304688,
"KL/mean": -965.1847534179688,
"KL/rejected_KL_mean": -1153.6407470703125,
"KL/std": 581.8084106445312,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.0010326361516490579,
"fcm_dpo/delta": 0.016902513802051544,
"fcm_dpo/margin": 376.9120788574219,
"fcm_dpo/q_t": 0.4136263132095337,
"grad_norm": 44.76034164428711,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": -1.0995469093322754,
"logits/rejected": -1.1115614175796509,
"logps/chosen": -836.1823120117188,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -1234.59228515625,
"loss": 1.1518,
"margin_dpo/margin_mean": 376.9120788574219,
"margin_dpo/margin_std": 690.0167236328125,
"step": 631
},
{
"KL/chosen_KL_mean": -697.9989013671875,
"KL/mean": -890.870361328125,
"KL/rejected_KL_mean": -1083.741943359375,
"KL/std": 523.5582275390625,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.000998594332486391,
"fcm_dpo/delta": 0.01689380779862404,
"fcm_dpo/margin": 385.7430114746094,
"fcm_dpo/q_t": 0.41343018412590027,
"grad_norm": 48.563560485839844,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": -1.047680139541626,
"logits/rejected": -1.0479519367218018,
"logps/chosen": -759.3504638671875,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -1169.902099609375,
"loss": 1.1447,
"margin_dpo/margin_mean": 385.74298095703125,
"margin_dpo/margin_std": 662.6145629882812,
"step": 632
},
{
"KL/chosen_KL_mean": -745.3228149414062,
"KL/mean": -884.4315795898438,
"KL/rejected_KL_mean": -1023.540283203125,
"KL/std": 512.197509765625,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.0012451799120754004,
"fcm_dpo/delta": 0.06800185889005661,
"fcm_dpo/margin": 278.2175598144531,
"fcm_dpo/q_t": 0.4261208176612854,
"grad_norm": 68.927978515625,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": -1.06679105758667,
"logits/rejected": -1.0765254497528076,
"logps/chosen": -802.6009521484375,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -1115.124267578125,
"loss": 1.2008,
"margin_dpo/margin_mean": 278.2175598144531,
"margin_dpo/margin_std": 601.500732421875,
"step": 633
},
{
"KL/chosen_KL_mean": -832.7714233398438,
"KL/mean": -965.0292358398438,
"KL/rejected_KL_mean": -1097.2869873046875,
"KL/std": 614.1533203125,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.0013293168740347028,
"fcm_dpo/delta": 0.07503058761358261,
"fcm_dpo/margin": 264.5155334472656,
"fcm_dpo/q_t": 0.4308067560195923,
"grad_norm": 51.24019241333008,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": -0.9571331739425659,
"logits/rejected": -0.9513822197914124,
"logps/chosen": -899.390380859375,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -1204.41259765625,
"loss": 1.2102,
"margin_dpo/margin_mean": 264.5155334472656,
"margin_dpo/margin_std": 607.553466796875,
"step": 634
},
{
"KL/chosen_KL_mean": -726.7911376953125,
"KL/mean": -909.9793090820312,
"KL/rejected_KL_mean": -1093.16748046875,
"KL/std": 651.4637451171875,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.0012207648251205683,
"fcm_dpo/delta": -0.08999871462583542,
"fcm_dpo/margin": 366.37628173828125,
"fcm_dpo/q_t": 0.3993282914161682,
"grad_norm": 50.3961067199707,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": -1.0328423976898193,
"logits/rejected": -1.0168393850326538,
"logps/chosen": -800.7467041015625,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -1175.66796875,
"loss": 1.1187,
"margin_dpo/margin_mean": 366.37628173828125,
"margin_dpo/margin_std": 644.3758544921875,
"step": 635
},
{
"KL/chosen_KL_mean": -682.0125732421875,
"KL/mean": -899.14404296875,
"KL/rejected_KL_mean": -1116.275634765625,
"KL/std": 621.2393798828125,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.0010760778095573187,
"fcm_dpo/delta": -0.12986694276332855,
"fcm_dpo/margin": 434.2630615234375,
"fcm_dpo/q_t": 0.38959363102912903,
"grad_norm": 29.961071014404297,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": -0.9651399254798889,
"logits/rejected": -0.9689005613327026,
"logps/chosen": -741.6414794921875,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -1198.25439453125,
"loss": 1.0948,
"margin_dpo/margin_mean": 434.2630310058594,
"margin_dpo/margin_std": 650.568603515625,
"step": 636
},
{
"KL/chosen_KL_mean": -688.02197265625,
"KL/mean": -933.2636108398438,
"KL/rejected_KL_mean": -1178.505126953125,
"KL/std": 643.35302734375,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.0008327056420966983,
"fcm_dpo/delta": -0.024866830557584763,
"fcm_dpo/margin": 490.4832458496094,
"fcm_dpo/q_t": 0.4043170213699341,
"grad_norm": 34.2825813293457,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": -1.08000648021698,
"logits/rejected": -1.127310872077942,
"logps/chosen": -737.6746826171875,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -1276.910400390625,
"loss": 1.0877,
"margin_dpo/margin_mean": 490.48321533203125,
"margin_dpo/margin_std": 645.675537109375,
"step": 637
},
{
"KL/chosen_KL_mean": -676.4507446289062,
"KL/mean": -868.2711791992188,
"KL/rejected_KL_mean": -1060.0916748046875,
"KL/std": 593.851318359375,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.0008206696948036551,
"fcm_dpo/delta": -0.058288540691137314,
"fcm_dpo/margin": 383.64093017578125,
"fcm_dpo/q_t": 0.419683039188385,
"grad_norm": 30.65635871887207,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": -0.9378705024719238,
"logits/rejected": -0.9226667881011963,
"logps/chosen": -734.607421875,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -1139.39306640625,
"loss": 1.1863,
"margin_dpo/margin_mean": 383.64093017578125,
"margin_dpo/margin_std": 691.0709228515625,
"step": 638
},
{
"KL/chosen_KL_mean": -905.7219848632812,
"KL/mean": -986.7338256835938,
"KL/rejected_KL_mean": -1067.7457275390625,
"KL/std": 534.1470336914062,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.0007728881319053471,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 162.023681640625,
"fcm_dpo/q_t": 0.47046107053756714,
"grad_norm": 76.31160736083984,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": -0.9101752042770386,
"logits/rejected": -0.8493287563323975,
"logps/chosen": -978.045166015625,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -1142.0206298828125,
"loss": 1.3279,
"margin_dpo/margin_mean": 162.023681640625,
"margin_dpo/margin_std": 657.9573364257812,
"step": 639
},
{
"KL/chosen_KL_mean": -690.0206298828125,
"KL/mean": -973.13671875,
"KL/rejected_KL_mean": -1256.2529296875,
"KL/std": 648.8939819335938,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0007562098908238113,
"fcm_dpo/delta": -0.05274926871061325,
"fcm_dpo/margin": 566.2322387695312,
"fcm_dpo/q_t": 0.3993333876132965,
"grad_norm": 46.08436965942383,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": -0.9405654668807983,
"logits/rejected": -0.9669902920722961,
"logps/chosen": -746.1549682617188,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -1364.85302734375,
"loss": 1.0815,
"margin_dpo/margin_mean": 566.2322387695312,
"margin_dpo/margin_std": 748.8364868164062,
"step": 640
},
{
"KL/chosen_KL_mean": -803.405517578125,
"KL/mean": -974.4388427734375,
"KL/rejected_KL_mean": -1145.47216796875,
"KL/std": 547.9729614257812,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0008681797189638019,
"fcm_dpo/delta": 0.14330008625984192,
"fcm_dpo/margin": 342.0666809082031,
"fcm_dpo/q_t": 0.4413103461265564,
"grad_norm": 27.740734100341797,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": -1.0207037925720215,
"logits/rejected": -1.02461838722229,
"logps/chosen": -868.40234375,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -1232.4644775390625,
"loss": 1.2043,
"margin_dpo/margin_mean": 342.06671142578125,
"margin_dpo/margin_std": 712.2955932617188,
"step": 641
},
{
"KL/chosen_KL_mean": -779.18896484375,
"KL/mean": -998.7464599609375,
"KL/rejected_KL_mean": -1218.3040771484375,
"KL/std": 729.4473266601562,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0009394697844982147,
"fcm_dpo/delta": -0.026245135813951492,
"fcm_dpo/margin": 439.1151123046875,
"fcm_dpo/q_t": 0.41486814618110657,
"grad_norm": 41.30564880371094,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": -0.9439425468444824,
"logits/rejected": -0.9753347635269165,
"logps/chosen": -844.878173828125,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -1328.546142578125,
"loss": 1.1663,
"margin_dpo/margin_mean": 439.1151428222656,
"margin_dpo/margin_std": 900.11669921875,
"step": 642
},
{
"KL/chosen_KL_mean": -674.6640625,
"KL/mean": -851.1025390625,
"KL/rejected_KL_mean": -1027.5411376953125,
"KL/std": 530.1317138671875,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0008457102812826633,
"fcm_dpo/delta": -0.019284440204501152,
"fcm_dpo/margin": 352.87701416015625,
"fcm_dpo/q_t": 0.4285130500793457,
"grad_norm": 34.299949645996094,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": -1.0501961708068848,
"logits/rejected": -1.0501909255981445,
"logps/chosen": -726.614013671875,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -1115.009521484375,
"loss": 1.1622,
"margin_dpo/margin_mean": 352.87701416015625,
"margin_dpo/margin_std": 535.4759521484375,
"step": 643
},
{
"KL/chosen_KL_mean": -669.6585693359375,
"KL/mean": -843.1557006835938,
"KL/rejected_KL_mean": -1016.65283203125,
"KL/std": 622.4733276367188,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.0009780020918697119,
"fcm_dpo/delta": 0.08886226266622543,
"fcm_dpo/margin": 346.9942932128906,
"fcm_dpo/q_t": 0.43022337555885315,
"grad_norm": 48.09732437133789,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": -0.9770164489746094,
"logits/rejected": -0.9840372800827026,
"logps/chosen": -728.67626953125,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -1103.78955078125,
"loss": 1.1865,
"margin_dpo/margin_mean": 346.9942626953125,
"margin_dpo/margin_std": 714.44580078125,
"step": 644
},
{
"KL/chosen_KL_mean": -693.9857788085938,
"KL/mean": -786.16162109375,
"KL/rejected_KL_mean": -878.3374633789062,
"KL/std": 501.67999267578125,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0011059939861297607,
"fcm_dpo/delta": 0.04529920592904091,
"fcm_dpo/margin": 184.3516387939453,
"fcm_dpo/q_t": 0.45926517248153687,
"grad_norm": 131.60787963867188,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": -0.989364743232727,
"logits/rejected": -0.97637939453125,
"logps/chosen": -749.86181640625,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -976.1182861328125,
"loss": 1.3317,
"margin_dpo/margin_mean": 184.3516387939453,
"margin_dpo/margin_std": 676.8275146484375,
"step": 645
},
{
"KL/chosen_KL_mean": -658.73046875,
"KL/mean": -810.00341796875,
"KL/rejected_KL_mean": -961.2762451171875,
"KL/std": 463.8369140625,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.0011796164326369762,
"fcm_dpo/delta": 0.06660034507513046,
"fcm_dpo/margin": 302.5458068847656,
"fcm_dpo/q_t": 0.42494145035743713,
"grad_norm": 58.692771911621094,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": -0.9866272211074829,
"logits/rejected": -0.9638226628303528,
"logps/chosen": -720.0062255859375,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -1038.7821044921875,
"loss": 1.1639,
"margin_dpo/margin_mean": 302.54583740234375,
"margin_dpo/margin_std": 556.3814697265625,
"step": 646
},
{
"KL/chosen_KL_mean": -595.4979248046875,
"KL/mean": -783.5799560546875,
"KL/rejected_KL_mean": -971.6620483398438,
"KL/std": 541.9985961914062,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.001148381270468235,
"fcm_dpo/delta": -0.05802955478429794,
"fcm_dpo/margin": 376.1640319824219,
"fcm_dpo/q_t": 0.3967111110687256,
"grad_norm": 51.125606536865234,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": -0.9212703704833984,
"logits/rejected": -0.9506291151046753,
"logps/chosen": -650.350341796875,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -1065.181396484375,
"loss": 1.0894,
"margin_dpo/margin_mean": 376.1640319824219,
"margin_dpo/margin_std": 538.5177001953125,
"step": 647
},
{
"KL/chosen_KL_mean": -622.4876708984375,
"KL/mean": -907.7584228515625,
"KL/rejected_KL_mean": -1193.0291748046875,
"KL/std": 659.01708984375,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.0008362603839486837,
"fcm_dpo/delta": -0.17148783802986145,
"fcm_dpo/margin": 570.54150390625,
"fcm_dpo/q_t": 0.3812508285045624,
"grad_norm": 28.91020965576172,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": -1.062262773513794,
"logits/rejected": -1.119946002960205,
"logps/chosen": -676.6591186523438,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -1291.741943359375,
"loss": 1.0583,
"margin_dpo/margin_mean": 570.5415649414062,
"margin_dpo/margin_std": 795.2802734375,
"step": 648
},
{
"KL/chosen_KL_mean": -647.7181396484375,
"KL/mean": -777.0692138671875,
"KL/rejected_KL_mean": -906.4202880859375,
"KL/std": 489.11322021484375,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0008656082791276276,
"fcm_dpo/delta": 0.07774749398231506,
"fcm_dpo/margin": 258.7022399902344,
"fcm_dpo/q_t": 0.4531601071357727,
"grad_norm": 29.970624923706055,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": -1.0473252534866333,
"logits/rejected": -1.0361632108688354,
"logps/chosen": -710.198486328125,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -986.4974975585938,
"loss": 1.2466,
"margin_dpo/margin_mean": 258.7022399902344,
"margin_dpo/margin_std": 601.4564208984375,
"step": 649
},
{
"KL/chosen_KL_mean": -711.3486328125,
"KL/mean": -921.720703125,
"KL/rejected_KL_mean": -1132.0926513671875,
"KL/std": 623.22998046875,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.0009292041650041938,
"fcm_dpo/delta": 0.013731414452195168,
"fcm_dpo/margin": 420.7440185546875,
"fcm_dpo/q_t": 0.41418665647506714,
"grad_norm": 31.185792922973633,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": -0.9931234121322632,
"logits/rejected": -1.013329029083252,
"logps/chosen": -767.4414672851562,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -1230.357421875,
"loss": 1.1316,
"margin_dpo/margin_mean": 420.7440185546875,
"margin_dpo/margin_std": 705.7841796875,
"step": 650
},
{
"KL/chosen_KL_mean": -472.3923034667969,
"KL/mean": -752.0784912109375,
"KL/rejected_KL_mean": -1031.7647705078125,
"KL/std": 590.5972900390625,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0008050062460824847,
"fcm_dpo/delta": -0.09981894493103027,
"fcm_dpo/margin": 559.3723754882812,
"fcm_dpo/q_t": 0.3874790072441101,
"grad_norm": 33.94593048095703,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": -1.0268137454986572,
"logits/rejected": -1.0827641487121582,
"logps/chosen": -515.8177490234375,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -1131.72265625,
"loss": 1.0412,
"margin_dpo/margin_mean": 559.3724365234375,
"margin_dpo/margin_std": 614.9459228515625,
"step": 651
},
{
"KL/chosen_KL_mean": -589.6246337890625,
"KL/mean": -800.077392578125,
"KL/rejected_KL_mean": -1010.5301513671875,
"KL/std": 563.1753540039062,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0008659152081236243,
"fcm_dpo/delta": 0.053139444440603256,
"fcm_dpo/margin": 420.905517578125,
"fcm_dpo/q_t": 0.42015212774276733,
"grad_norm": 43.597076416015625,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": -1.026230812072754,
"logits/rejected": -1.0473260879516602,
"logps/chosen": -652.201416015625,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -1122.2979736328125,
"loss": 1.1242,
"margin_dpo/margin_mean": 420.905517578125,
"margin_dpo/margin_std": 636.0498046875,
"step": 652
},
{
"KL/chosen_KL_mean": -745.919677734375,
"KL/mean": -946.8770751953125,
"KL/rejected_KL_mean": -1147.834228515625,
"KL/std": 630.18896484375,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0009330883622169495,
"fcm_dpo/delta": 0.03911864012479782,
"fcm_dpo/margin": 401.91461181640625,
"fcm_dpo/q_t": 0.4190702438354492,
"grad_norm": 33.804931640625,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": -1.0814123153686523,
"logits/rejected": -1.107730507850647,
"logps/chosen": -807.03271484375,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -1251.083984375,
"loss": 1.1333,
"margin_dpo/margin_mean": 401.9146423339844,
"margin_dpo/margin_std": 663.3088989257812,
"step": 653
},
{
"KL/chosen_KL_mean": -651.044677734375,
"KL/mean": -828.8569946289062,
"KL/rejected_KL_mean": -1006.66943359375,
"KL/std": 504.73419189453125,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.0010234671644866467,
"fcm_dpo/delta": 0.05655752867460251,
"fcm_dpo/margin": 355.62469482421875,
"fcm_dpo/q_t": 0.42311620712280273,
"grad_norm": 37.25580978393555,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": -1.0293757915496826,
"logits/rejected": -1.0464175939559937,
"logps/chosen": -712.7728271484375,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -1105.4432373046875,
"loss": 1.141,
"margin_dpo/margin_mean": 355.62469482421875,
"margin_dpo/margin_std": 592.0263671875,
"step": 654
},
{
"KL/chosen_KL_mean": -607.3045654296875,
"KL/mean": -795.6822509765625,
"KL/rejected_KL_mean": -984.0599365234375,
"KL/std": 513.406005859375,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0010560275986790657,
"fcm_dpo/delta": 0.003467606846243143,
"fcm_dpo/margin": 376.75543212890625,
"fcm_dpo/q_t": 0.4126419126987457,
"grad_norm": 35.243900299072266,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": -0.9929590225219727,
"logits/rejected": -1.032914638519287,
"logps/chosen": -656.88134765625,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -1082.351806640625,
"loss": 1.1245,
"margin_dpo/margin_mean": 376.75543212890625,
"margin_dpo/margin_std": 618.8502197265625,
"step": 655
},
{
"KL/chosen_KL_mean": -648.4987182617188,
"KL/mean": -975.326904296875,
"KL/rejected_KL_mean": -1302.1551513671875,
"KL/std": 691.7890014648438,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0007785690249875188,
"fcm_dpo/delta": -0.22655422985553741,
"fcm_dpo/margin": 653.6564331054688,
"fcm_dpo/q_t": 0.36488282680511475,
"grad_norm": 64.71890258789062,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": -0.899747371673584,
"logits/rejected": -0.968382716178894,
"logps/chosen": -701.0481567382812,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -1415.829833984375,
"loss": 1.013,
"margin_dpo/margin_mean": 653.6564331054688,
"margin_dpo/margin_std": 684.8790283203125,
"step": 656
},
{
"KL/chosen_KL_mean": -607.3179931640625,
"KL/mean": -871.7550659179688,
"KL/rejected_KL_mean": -1136.192138671875,
"KL/std": 643.2208862304688,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.0007102171657606959,
"fcm_dpo/delta": 0.039193179458379745,
"fcm_dpo/margin": 528.8741455078125,
"fcm_dpo/q_t": 0.415811151266098,
"grad_norm": 30.748796463012695,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": -1.0481717586517334,
"logits/rejected": -1.0999813079833984,
"logps/chosen": -654.0185546875,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -1234.1070556640625,
"loss": 1.0999,
"margin_dpo/margin_mean": 528.8741455078125,
"margin_dpo/margin_std": 680.64990234375,
"step": 657
},
{
"KL/chosen_KL_mean": -668.5402221679688,
"KL/mean": -876.7470703125,
"KL/rejected_KL_mean": -1084.9539794921875,
"KL/std": 520.5706176757812,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0008155145333148539,
"fcm_dpo/delta": 0.0921507477760315,
"fcm_dpo/margin": 416.4137268066406,
"fcm_dpo/q_t": 0.4271154999732971,
"grad_norm": 30.934783935546875,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": -0.9990421533584595,
"logits/rejected": -1.0265355110168457,
"logps/chosen": -729.4984130859375,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -1180.8934326171875,
"loss": 1.1332,
"margin_dpo/margin_mean": 416.4136962890625,
"margin_dpo/margin_std": 572.531494140625,
"step": 658
},
{
"KL/chosen_KL_mean": -589.103515625,
"KL/mean": -787.4786376953125,
"KL/rejected_KL_mean": -985.8536376953125,
"KL/std": 501.25604248046875,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0009247631533071399,
"fcm_dpo/delta": 0.0519876554608345,
"fcm_dpo/margin": 396.7500915527344,
"fcm_dpo/q_t": 0.42018935084342957,
"grad_norm": 30.552549362182617,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": -0.9165958762168884,
"logits/rejected": -0.885309100151062,
"logps/chosen": -665.8465576171875,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -1073.3245849609375,
"loss": 1.1172,
"margin_dpo/margin_mean": 396.7500915527344,
"margin_dpo/margin_std": 562.6476440429688,
"step": 659
},
{
"KL/chosen_KL_mean": -651.0494384765625,
"KL/mean": -908.062744140625,
"KL/rejected_KL_mean": -1165.076171875,
"KL/std": 607.6834716796875,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0008679507300257683,
"fcm_dpo/delta": -0.08382175117731094,
"fcm_dpo/margin": 514.0266723632812,
"fcm_dpo/q_t": 0.39148008823394775,
"grad_norm": 46.10919952392578,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": -1.017472505569458,
"logits/rejected": -1.033474326133728,
"logps/chosen": -710.0972900390625,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -1241.0361328125,
"loss": 1.0632,
"margin_dpo/margin_mean": 514.0266723632812,
"margin_dpo/margin_std": 650.8546142578125,
"step": 660
},
{
"KL/chosen_KL_mean": -573.46484375,
"KL/mean": -808.244140625,
"KL/rejected_KL_mean": -1043.0234375,
"KL/std": 656.3875732421875,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.0008553254883736372,
"fcm_dpo/delta": -0.0076020583510398865,
"fcm_dpo/margin": 469.55877685546875,
"fcm_dpo/q_t": 0.408011257648468,
"grad_norm": 39.97681427001953,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": -1.0107231140136719,
"logits/rejected": -1.019051194190979,
"logps/chosen": -624.1387939453125,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -1129.0291748046875,
"loss": 1.0917,
"margin_dpo/margin_mean": 469.5587463378906,
"margin_dpo/margin_std": 631.5257568359375,
"step": 661
},
{
"KL/chosen_KL_mean": -674.645263671875,
"KL/mean": -845.860595703125,
"KL/rejected_KL_mean": -1017.075927734375,
"KL/std": 547.025146484375,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0009593941504135728,
"fcm_dpo/delta": 0.10353372991085052,
"fcm_dpo/margin": 342.43072509765625,
"fcm_dpo/q_t": 0.43190035223960876,
"grad_norm": 30.81041717529297,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": -0.9726383090019226,
"logits/rejected": -0.969027042388916,
"logps/chosen": -743.9063110351562,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -1106.1318359375,
"loss": 1.1861,
"margin_dpo/margin_mean": 342.43072509765625,
"margin_dpo/margin_std": 687.1112060546875,
"step": 662
},
{
"KL/chosen_KL_mean": -622.2049560546875,
"KL/mean": -859.8319091796875,
"KL/rejected_KL_mean": -1097.458740234375,
"KL/std": 645.55224609375,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.000906460452824831,
"fcm_dpo/delta": -0.05661872774362564,
"fcm_dpo/margin": 475.2537536621094,
"fcm_dpo/q_t": 0.40141403675079346,
"grad_norm": 25.596982955932617,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": -1.029599666595459,
"logits/rejected": -1.055091381072998,
"logps/chosen": -687.0839233398438,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -1211.3841552734375,
"loss": 1.1023,
"margin_dpo/margin_mean": 475.25372314453125,
"margin_dpo/margin_std": 764.5504760742188,
"step": 663
},
{
"KL/chosen_KL_mean": -656.9991455078125,
"KL/mean": -909.43798828125,
"KL/rejected_KL_mean": -1161.876708984375,
"KL/std": 617.7022705078125,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.0008173753740265965,
"fcm_dpo/delta": -0.0294804647564888,
"fcm_dpo/margin": 504.87762451171875,
"fcm_dpo/q_t": 0.40310847759246826,
"grad_norm": 22.38481330871582,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": -0.987866997718811,
"logits/rejected": -1.0178110599517822,
"logps/chosen": -717.8876342773438,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -1267.3984375,
"loss": 1.0872,
"margin_dpo/margin_mean": 504.87762451171875,
"margin_dpo/margin_std": 675.4814453125,
"step": 664
},
{
"KL/chosen_KL_mean": -591.2396240234375,
"KL/mean": -797.2940673828125,
"KL/rejected_KL_mean": -1003.3485717773438,
"KL/std": 507.349609375,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.0009045639308169484,
"fcm_dpo/delta": 0.04246987774968147,
"fcm_dpo/margin": 412.10906982421875,
"fcm_dpo/q_t": 0.4166688919067383,
"grad_norm": 44.036556243896484,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": -1.0652823448181152,
"logits/rejected": -1.0855448246002197,
"logps/chosen": -651.8037109375,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -1088.157470703125,
"loss": 1.099,
"margin_dpo/margin_mean": 412.10906982421875,
"margin_dpo/margin_std": 510.36553955078125,
"step": 665
},
{
"KL/chosen_KL_mean": -635.3843994140625,
"KL/mean": -849.865234375,
"KL/rejected_KL_mean": -1064.3460693359375,
"KL/std": 543.2855224609375,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0009157652966678143,
"fcm_dpo/delta": 0.011229622177779675,
"fcm_dpo/margin": 428.96173095703125,
"fcm_dpo/q_t": 0.4111603796482086,
"grad_norm": 29.72758674621582,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": -0.9271786212921143,
"logits/rejected": -0.9376795291900635,
"logps/chosen": -699.8043212890625,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -1160.23779296875,
"loss": 1.1035,
"margin_dpo/margin_mean": 428.96173095703125,
"margin_dpo/margin_std": 589.12890625,
"step": 666
},
{
"KL/chosen_KL_mean": -728.5592041015625,
"KL/mean": -976.642822265625,
"KL/rejected_KL_mean": -1224.7265625,
"KL/std": 594.4561767578125,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.0008521339623257518,
"fcm_dpo/delta": -0.04440337419509888,
"fcm_dpo/margin": 496.16729736328125,
"fcm_dpo/q_t": 0.39933979511260986,
"grad_norm": 36.45888137817383,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": -0.9821093082427979,
"logits/rejected": -0.9815555810928345,
"logps/chosen": -797.8362426757812,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -1312.56201171875,
"loss": 1.0793,
"margin_dpo/margin_mean": 496.1672668457031,
"margin_dpo/margin_std": 653.2168579101562,
"step": 667
},
{
"KL/chosen_KL_mean": -755.283447265625,
"KL/mean": -929.7744140625,
"KL/rejected_KL_mean": -1104.265380859375,
"KL/std": 615.79931640625,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0010012466227635741,
"fcm_dpo/delta": 0.061918459832668304,
"fcm_dpo/margin": 348.9818420410156,
"fcm_dpo/q_t": 0.4277912974357605,
"grad_norm": 45.25949478149414,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": -1.0155105590820312,
"logits/rejected": -1.0245213508605957,
"logps/chosen": -827.887451171875,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -1208.00439453125,
"loss": 1.1889,
"margin_dpo/margin_mean": 348.9818420410156,
"margin_dpo/margin_std": 726.23681640625,
"step": 668
},
{
"KL/chosen_KL_mean": -597.3612060546875,
"KL/mean": -849.0230102539062,
"KL/rejected_KL_mean": -1100.6846923828125,
"KL/std": 599.0240478515625,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0008747231913730502,
"fcm_dpo/delta": -0.07254499942064285,
"fcm_dpo/margin": 503.323486328125,
"fcm_dpo/q_t": 0.393608957529068,
"grad_norm": 22.192842483520508,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": -0.9759036302566528,
"logits/rejected": -0.9822956323623657,
"logps/chosen": -643.4776611328125,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -1178.609130859375,
"loss": 1.0742,
"margin_dpo/margin_mean": 503.323486328125,
"margin_dpo/margin_std": 677.1585693359375,
"step": 669
},
{
"KL/chosen_KL_mean": -544.9779052734375,
"KL/mean": -801.1505737304688,
"KL/rejected_KL_mean": -1057.3232421875,
"KL/std": 542.9509887695312,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.0008237832807935774,
"fcm_dpo/delta": -0.039803534746170044,
"fcm_dpo/margin": 512.3453369140625,
"fcm_dpo/q_t": 0.3997776508331299,
"grad_norm": 40.426666259765625,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": -0.9458330273628235,
"logits/rejected": -0.9657979011535645,
"logps/chosen": -607.3236694335938,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -1154.2637939453125,
"loss": 1.0816,
"margin_dpo/margin_mean": 512.3453369140625,
"margin_dpo/margin_std": 667.56640625,
"step": 670
},
{
"KL/chosen_KL_mean": -693.818603515625,
"KL/mean": -904.692626953125,
"KL/rejected_KL_mean": -1115.5667724609375,
"KL/std": 550.0133056640625,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0008690550457686186,
"fcm_dpo/delta": 0.04318936541676521,
"fcm_dpo/margin": 421.74822998046875,
"fcm_dpo/q_t": 0.41694390773773193,
"grad_norm": 26.835704803466797,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": -1.0066074132919312,
"logits/rejected": -1.0331590175628662,
"logps/chosen": -741.8187255859375,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -1199.3861083984375,
"loss": 1.1341,
"margin_dpo/margin_mean": 421.74822998046875,
"margin_dpo/margin_std": 664.017333984375,
"step": 671
},
{
"KL/chosen_KL_mean": -766.8887329101562,
"KL/mean": -968.2491455078125,
"KL/rejected_KL_mean": -1169.6094970703125,
"KL/std": 652.8719482421875,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.0009030759101733565,
"fcm_dpo/delta": 0.057317838072776794,
"fcm_dpo/margin": 402.7208251953125,
"fcm_dpo/q_t": 0.4237225651741028,
"grad_norm": 62.17927932739258,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": -1.085627794265747,
"logits/rejected": -1.0907000303268433,
"logps/chosen": -825.4720458984375,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -1262.749755859375,
"loss": 1.1636,
"margin_dpo/margin_mean": 402.7208251953125,
"margin_dpo/margin_std": 737.9989013671875,
"step": 672
},
{
"KL/chosen_KL_mean": -674.4365234375,
"KL/mean": -871.0324096679688,
"KL/rejected_KL_mean": -1067.62841796875,
"KL/std": 548.711181640625,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0009767746087163687,
"fcm_dpo/delta": 0.02593318186700344,
"fcm_dpo/margin": 393.1917724609375,
"fcm_dpo/q_t": 0.41595256328582764,
"grad_norm": 40.66542053222656,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": -1.0082206726074219,
"logits/rejected": -1.0116488933563232,
"logps/chosen": -721.15966796875,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -1152.924560546875,
"loss": 1.119,
"margin_dpo/margin_mean": 393.1917724609375,
"margin_dpo/margin_std": 624.88330078125,
"step": 673
},
{
"KL/chosen_KL_mean": -571.8458862304688,
"KL/mean": -831.9334716796875,
"KL/rejected_KL_mean": -1092.0211181640625,
"KL/std": 578.303466796875,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0008713127463124692,
"fcm_dpo/delta": -0.09795575588941574,
"fcm_dpo/margin": 520.1752319335938,
"fcm_dpo/q_t": 0.39291104674339294,
"grad_norm": 35.70085144042969,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": -0.9462261199951172,
"logits/rejected": -0.9662339091300964,
"logps/chosen": -617.2913818359375,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -1162.067138671875,
"loss": 1.0669,
"margin_dpo/margin_mean": 520.1752319335938,
"margin_dpo/margin_std": 695.502685546875,
"step": 674
},
{
"KL/chosen_KL_mean": -642.9573974609375,
"KL/mean": -909.2418212890625,
"KL/rejected_KL_mean": -1175.526123046875,
"KL/std": 614.2586059570312,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.0007445673691108823,
"fcm_dpo/delta": -0.013315960764884949,
"fcm_dpo/margin": 532.56884765625,
"fcm_dpo/q_t": 0.4076732397079468,
"grad_norm": 23.048574447631836,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": -1.0072154998779297,
"logits/rejected": -1.0248122215270996,
"logps/chosen": -687.1336669921875,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -1249.6181640625,
"loss": 1.0911,
"margin_dpo/margin_mean": 532.56884765625,
"margin_dpo/margin_std": 652.190185546875,
"step": 675
},
{
"KL/chosen_KL_mean": -695.682373046875,
"KL/mean": -932.237060546875,
"KL/rejected_KL_mean": -1168.7918701171875,
"KL/std": 565.1691284179688,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0008274284191429615,
"fcm_dpo/delta": 0.013139697723090649,
"fcm_dpo/margin": 473.10943603515625,
"fcm_dpo/q_t": 0.409980833530426,
"grad_norm": 23.929630279541016,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": -0.9380951523780823,
"logits/rejected": -0.9486560821533203,
"logps/chosen": -767.0808715820312,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -1257.150634765625,
"loss": 1.086,
"margin_dpo/margin_mean": 473.1094665527344,
"margin_dpo/margin_std": 570.78515625,
"step": 676
},
{
"KL/chosen_KL_mean": -701.174560546875,
"KL/mean": -933.6365356445312,
"KL/rejected_KL_mean": -1166.0985107421875,
"KL/std": 588.0880126953125,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0008385643595829606,
"fcm_dpo/delta": 0.01662612520158291,
"fcm_dpo/margin": 464.9239501953125,
"fcm_dpo/q_t": 0.4153476655483246,
"grad_norm": 24.641782760620117,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": -1.0368056297302246,
"logits/rejected": -1.0287206172943115,
"logps/chosen": -757.7020263671875,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -1244.324951171875,
"loss": 1.1094,
"margin_dpo/margin_mean": 464.9239196777344,
"margin_dpo/margin_std": 704.014892578125,
"step": 677
},
{
"KL/chosen_KL_mean": -589.749267578125,
"KL/mean": -871.703369140625,
"KL/rejected_KL_mean": -1153.657470703125,
"KL/std": 666.5526123046875,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.0007895482704043388,
"fcm_dpo/delta": -0.08192657679319382,
"fcm_dpo/margin": 563.908203125,
"fcm_dpo/q_t": 0.39262598752975464,
"grad_norm": 28.306751251220703,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": -0.9746444821357727,
"logits/rejected": -1.0132948160171509,
"logps/chosen": -635.8837280273438,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -1234.2620849609375,
"loss": 1.0705,
"margin_dpo/margin_mean": 563.9081420898438,
"margin_dpo/margin_std": 737.7059326171875,
"step": 678
},
{
"KL/chosen_KL_mean": -681.0501708984375,
"KL/mean": -873.888916015625,
"KL/rejected_KL_mean": -1066.727783203125,
"KL/std": 521.9443359375,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0008484461577609181,
"fcm_dpo/delta": 0.10628005862236023,
"fcm_dpo/margin": 385.67755126953125,
"fcm_dpo/q_t": 0.43186846375465393,
"grad_norm": 29.822158813476562,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": -0.9917019605636597,
"logits/rejected": -0.9838023781776428,
"logps/chosen": -731.3450927734375,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -1143.3258056640625,
"loss": 1.1502,
"margin_dpo/margin_mean": 385.67755126953125,
"margin_dpo/margin_std": 617.3707275390625,
"step": 679
},
{
"KL/chosen_KL_mean": -673.279541015625,
"KL/mean": -923.5628051757812,
"KL/rejected_KL_mean": -1173.8460693359375,
"KL/std": 654.016845703125,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.0008149506174959242,
"fcm_dpo/delta": -0.02367626130580902,
"fcm_dpo/margin": 500.56646728515625,
"fcm_dpo/q_t": 0.4036046266555786,
"grad_norm": 44.08322525024414,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": -0.9790595173835754,
"logits/rejected": -1.0117442607879639,
"logps/chosen": -750.1953125,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -1286.2308349609375,
"loss": 1.1057,
"margin_dpo/margin_mean": 500.56646728515625,
"margin_dpo/margin_std": 737.4908447265625,
"step": 680
},
{
"KL/chosen_KL_mean": -678.2127685546875,
"KL/mean": -878.902587890625,
"KL/rejected_KL_mean": -1079.592529296875,
"KL/std": 559.0294799804688,
"epoch": 1.0,
"fcm_dpo/beta": 0.0008184043690562248,
"fcm_dpo/delta": -0.0384669154882431,
"fcm_dpo/margin": 401.3796691894531,
"fcm_dpo/q_t": 0.4222037196159363,
"grad_norm": 26.581893920898438,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": -1.0419981479644775,
"logits/rejected": -1.0603480339050293,
"logps/chosen": -739.1700439453125,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -1168.150390625,
"loss": 1.1543,
"margin_dpo/margin_mean": 401.379638671875,
"margin_dpo/margin_std": 625.6992797851562,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 1.1280472663705745,
"train_runtime": 1739.8346,
"train_samples_per_second": 25.059,
"train_steps_per_second": 0.391
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink