W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.8-20260428-045924
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
llama-3-8b-base-new-dpo-hh-…/trainer_state.json
ModelHub XC bea959d0dc 初始化项目，由ModelHub XC社区提供模型 
Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-eta-0.1-s_star-0.8-20260428-045924
Source: Original Platform
2026-05-22 20:39:35 +08:00

15707 lines
590 KiB
JSON
Raw Permalink Blame History

{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 200,
"global_step": 681,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"KL/chosen_KL_mean": 0.00527191162109375,
"KL/mean": 0.016706019639968872,
"KL/rejected_KL_mean": 0.028141021728515625,
"KL/std": 0.272699236869812,
"epoch": 0.0014684287812041115,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.02287006378173828,
"fcm_dpo/q_t": 0.5005706548690796,
"grad_norm": 83.5241928100586,
"learning_rate": 0.0,
"logits/chosen": -0.4974287748336792,
"logits/rejected": -0.43299180269241333,
"logps/chosen": -50.1435661315918,
"logps/ref_chosen": -50.14883804321289,
"logps/ref_rejected": -74.1280517578125,
"logps/rejected": -74.09991455078125,
"loss": 1.389,
"margin_dpo/margin_mean": -0.02287048101425171,
"margin_dpo/margin_std": 0.41920793056488037,
"step": 1
},
{
"KL/chosen_KL_mean": -0.03498649597167969,
"KL/mean": -0.00212840735912323,
"KL/rejected_KL_mean": 0.030735015869140625,
"KL/std": 0.24797174334526062,
"epoch": 0.002936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06572261452674866,
"fcm_dpo/q_t": 0.5016425848007202,
"grad_norm": 72.2212142944336,
"learning_rate": 7.246376811594203e-09,
"logits/chosen": -0.49536412954330444,
"logits/rejected": -0.4594460427761078,
"logps/chosen": -52.65568923950195,
"logps/ref_chosen": -52.620704650878906,
"logps/ref_rejected": -75.30413818359375,
"logps/rejected": -75.27340698242188,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06572240591049194,
"margin_dpo/margin_std": 0.35048407316207886,
"step": 2
},
{
"KL/chosen_KL_mean": -0.001789093017578125,
"KL/mean": 0.018673986196517944,
"KL/rejected_KL_mean": 0.03913688659667969,
"KL/std": 0.2510424256324768,
"epoch": 0.004405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.04093104600906372,
"fcm_dpo/q_t": 0.5010229349136353,
"grad_norm": 71.08148193359375,
"learning_rate": 1.4492753623188406e-08,
"logits/chosen": -0.48160529136657715,
"logits/rejected": -0.4421182870864868,
"logps/chosen": -60.98338317871094,
"logps/ref_chosen": -60.981597900390625,
"logps/ref_rejected": -68.67259216308594,
"logps/rejected": -68.63346099853516,
"loss": 1.3908,
"margin_dpo/margin_mean": -0.04093062877655029,
"margin_dpo/margin_std": 0.3883278965950012,
"step": 3
},
{
"KL/chosen_KL_mean": -0.035938262939453125,
"KL/mean": -0.004994198679924011,
"KL/rejected_KL_mean": 0.02594757080078125,
"KL/std": 0.25746771693229675,
"epoch": 0.005873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06188991665840149,
"fcm_dpo/q_t": 0.5015468001365662,
"grad_norm": 72.06177520751953,
"learning_rate": 2.1739130434782606e-08,
"logits/chosen": -0.4687862992286682,
"logits/rejected": -0.4411006569862366,
"logps/chosen": -56.80364990234375,
"logps/ref_chosen": -56.7677116394043,
"logps/ref_rejected": -86.64710998535156,
"logps/rejected": -86.62116241455078,
"loss": 1.3928,
"margin_dpo/margin_mean": -0.06189015507698059,
"margin_dpo/margin_std": 0.36542826890945435,
"step": 4
},
{
"KL/chosen_KL_mean": 0.041553497314453125,
"KL/mean": 0.02368500828742981,
"KL/rejected_KL_mean": 0.00582122802734375,
"KL/std": 0.25065213441848755,
"epoch": 0.007342143906020558,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.03573501110076904,
"fcm_dpo/q_t": 0.4991065561771393,
"grad_norm": 89.9738998413086,
"learning_rate": 2.898550724637681e-08,
"logits/chosen": -0.514451265335083,
"logits/rejected": -0.4707820415496826,
"logps/chosen": -53.81782150268555,
"logps/ref_chosen": -53.859375,
"logps/ref_rejected": -84.14918518066406,
"logps/rejected": -84.14336395263672,
"loss": 1.383,
"margin_dpo/margin_mean": 0.035734742879867554,
"margin_dpo/margin_std": 0.3362279534339905,
"step": 5
},
{
"KL/chosen_KL_mean": -0.03502845764160156,
"KL/mean": -0.045771896839141846,
"KL/rejected_KL_mean": -0.0565185546875,
"KL/std": 0.302729070186615,
"epoch": 0.00881057268722467,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.021490395069122314,
"fcm_dpo/q_t": 0.4994634687900543,
"grad_norm": 91.83248138427734,
"learning_rate": 3.6231884057971014e-08,
"logits/chosen": -0.4969807267189026,
"logits/rejected": -0.4538990259170532,
"logps/chosen": -63.042510986328125,
"logps/ref_chosen": -63.007484436035156,
"logps/ref_rejected": -92.64534759521484,
"logps/rejected": -92.70186614990234,
"loss": 1.3846,
"margin_dpo/margin_mean": 0.02148991823196411,
"margin_dpo/margin_std": 0.44182121753692627,
"step": 6
},
{
"KL/chosen_KL_mean": 0.03751373291015625,
"KL/mean": 0.04191398620605469,
"KL/rejected_KL_mean": 0.04631805419921875,
"KL/std": 0.2821422517299652,
"epoch": 0.010279001468428781,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008798539638519287,
"fcm_dpo/q_t": 0.5002202987670898,
"grad_norm": 82.35990142822266,
"learning_rate": 4.347826086956521e-08,
"logits/chosen": -0.5038993954658508,
"logits/rejected": -0.47004765272140503,
"logps/chosen": -57.7373046875,
"logps/ref_chosen": -57.774818420410156,
"logps/ref_rejected": -103.92059326171875,
"logps/rejected": -103.87427520751953,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.00879824161529541,
"margin_dpo/margin_std": 0.41485100984573364,
"step": 7
},
{
"KL/chosen_KL_mean": 0.012628555297851562,
"KL/mean": -0.0013420134782791138,
"KL/rejected_KL_mean": -0.01531219482421875,
"KL/std": 0.3049571216106415,
"epoch": 0.011747430249632892,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.027936726808547974,
"fcm_dpo/q_t": 0.49930089712142944,
"grad_norm": 78.54287719726562,
"learning_rate": 5.0724637681159424e-08,
"logits/chosen": -0.49774545431137085,
"logits/rejected": -0.4704884886741638,
"logps/chosen": -58.703407287597656,
"logps/ref_chosen": -58.716033935546875,
"logps/ref_rejected": -79.3114242553711,
"logps/rejected": -79.32673645019531,
"loss": 1.3839,
"margin_dpo/margin_mean": 0.027936339378356934,
"margin_dpo/margin_std": 0.38197433948516846,
"step": 8
},
{
"KL/chosen_KL_mean": 0.00943756103515625,
"KL/mean": 0.014243200421333313,
"KL/rejected_KL_mean": 0.019046783447265625,
"KL/std": 0.28038617968559265,
"epoch": 0.013215859030837005,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.009610921144485474,
"fcm_dpo/q_t": 0.5002400875091553,
"grad_norm": 85.46974182128906,
"learning_rate": 5.797101449275362e-08,
"logits/chosen": -0.5015987753868103,
"logits/rejected": -0.4552796483039856,
"logps/chosen": -69.85740661621094,
"logps/ref_chosen": -69.8668441772461,
"logps/ref_rejected": -99.6026611328125,
"logps/rejected": -99.5836181640625,
"loss": 1.3876,
"margin_dpo/margin_mean": -0.009611576795578003,
"margin_dpo/margin_std": 0.3819504380226135,
"step": 9
},
{
"KL/chosen_KL_mean": 0.03025054931640625,
"KL/mean": 0.027018234133720398,
"KL/rejected_KL_mean": 0.0237884521484375,
"KL/std": 0.2487342655658722,
"epoch": 0.014684287812041116,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.006460636854171753,
"fcm_dpo/q_t": 0.4998389482498169,
"grad_norm": 70.36607360839844,
"learning_rate": 6.521739130434782e-08,
"logits/chosen": -0.4691097140312195,
"logits/rejected": -0.4247715175151825,
"logps/chosen": -48.32743453979492,
"logps/ref_chosen": -48.35768508911133,
"logps/ref_rejected": -80.37206268310547,
"logps/rejected": -80.34827423095703,
"loss": 1.386,
"margin_dpo/margin_mean": 0.006460994482040405,
"margin_dpo/margin_std": 0.37741619348526,
"step": 10
},
{
"KL/chosen_KL_mean": -0.013975143432617188,
"KL/mean": -0.009909212589263916,
"KL/rejected_KL_mean": -0.005840301513671875,
"KL/std": 0.25705647468566895,
"epoch": 0.016152716593245228,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.008137047290802002,
"fcm_dpo/q_t": 0.5002034902572632,
"grad_norm": 68.1004867553711,
"learning_rate": 7.246376811594203e-08,
"logits/chosen": -0.4795861840248108,
"logits/rejected": -0.4555514454841614,
"logps/chosen": -53.03083038330078,
"logps/ref_chosen": -53.01685333251953,
"logps/ref_rejected": -87.78038024902344,
"logps/rejected": -87.78621673583984,
"loss": 1.3875,
"margin_dpo/margin_mean": -0.008136779069900513,
"margin_dpo/margin_std": 0.36586588621139526,
"step": 11
},
{
"KL/chosen_KL_mean": -0.026020050048828125,
"KL/mean": -0.01642027497291565,
"KL/rejected_KL_mean": -0.00681304931640625,
"KL/std": 0.29991090297698975,
"epoch": 0.01762114537444934,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.019200026988983154,
"fcm_dpo/q_t": 0.5004792213439941,
"grad_norm": 90.451904296875,
"learning_rate": 7.971014492753623e-08,
"logits/chosen": -0.523429811000824,
"logits/rejected": -0.4867948889732361,
"logps/chosen": -61.831459045410156,
"logps/ref_chosen": -61.80543518066406,
"logps/ref_rejected": -104.8582763671875,
"logps/rejected": -104.86508178710938,
"loss": 1.3886,
"margin_dpo/margin_mean": -0.019200772047042847,
"margin_dpo/margin_std": 0.40876132249832153,
"step": 12
},
{
"KL/chosen_KL_mean": 0.00020599365234375,
"KL/mean": 0.01627969741821289,
"KL/rejected_KL_mean": 0.03235626220703125,
"KL/std": 0.30039823055267334,
"epoch": 0.01908957415565345,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.03214287757873535,
"fcm_dpo/q_t": 0.5008050203323364,
"grad_norm": 79.43994903564453,
"learning_rate": 8.695652173913042e-08,
"logits/chosen": -0.46810948848724365,
"logits/rejected": -0.43907326459884644,
"logps/chosen": -64.26014709472656,
"logps/ref_chosen": -64.2603530883789,
"logps/ref_rejected": -87.20307922363281,
"logps/rejected": -87.17073059082031,
"loss": 1.39,
"margin_dpo/margin_mean": -0.03214350342750549,
"margin_dpo/margin_std": 0.41335833072662354,
"step": 13
},
{
"KL/chosen_KL_mean": -0.05523681640625,
"KL/mean": -0.02267158031463623,
"KL/rejected_KL_mean": 0.0098876953125,
"KL/std": 0.2773570418357849,
"epoch": 0.020558002936857563,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.06512236595153809,
"fcm_dpo/q_t": 0.5016275644302368,
"grad_norm": 85.54232025146484,
"learning_rate": 9.420289855072464e-08,
"logits/chosen": -0.461495578289032,
"logits/rejected": -0.4215831756591797,
"logps/chosen": -58.16544723510742,
"logps/ref_chosen": -58.11021041870117,
"logps/ref_rejected": -104.04708099365234,
"logps/rejected": -104.03719329833984,
"loss": 1.3932,
"margin_dpo/margin_mean": -0.06512203812599182,
"margin_dpo/margin_std": 0.40178561210632324,
"step": 14
},
{
"KL/chosen_KL_mean": 0.008823394775390625,
"KL/mean": -0.01242247223854065,
"KL/rejected_KL_mean": -0.03366851806640625,
"KL/std": 0.24548175930976868,
"epoch": 0.022026431718061675,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.04249519109725952,
"fcm_dpo/q_t": 0.4989377558231354,
"grad_norm": 63.90875244140625,
"learning_rate": 1.0144927536231885e-07,
"logits/chosen": -0.4897874891757965,
"logits/rejected": -0.47086426615715027,
"logps/chosen": -56.95808410644531,
"logps/ref_chosen": -56.96691131591797,
"logps/ref_rejected": -80.80863952636719,
"logps/rejected": -80.84231567382812,
"loss": 1.3825,
"margin_dpo/margin_mean": 0.04249492287635803,
"margin_dpo/margin_std": 0.40503597259521484,
"step": 15
},
{
"KL/chosen_KL_mean": -0.04467201232910156,
"KL/mean": -0.026810422539711,
"KL/rejected_KL_mean": -0.008953094482421875,
"KL/std": 0.2630922496318817,
"epoch": 0.023494860499265784,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": -0.0357193648815155,
"fcm_dpo/q_t": 0.500891923904419,
"grad_norm": 84.26193237304688,
"learning_rate": 1.0869565217391303e-07,
"logits/chosen": -0.5425084233283997,
"logits/rejected": -0.5031050443649292,
"logps/chosen": -61.78456497192383,
"logps/ref_chosen": -61.739891052246094,
"logps/ref_rejected": -84.36947631835938,
"logps/rejected": -84.37842559814453,
"loss": 1.3902,
"margin_dpo/margin_mean": -0.03571975231170654,
"margin_dpo/margin_std": 0.38462013006210327,
"step": 16
},
{
"KL/chosen_KL_mean": 0.0346221923828125,
"KL/mean": -0.005598008632659912,
"KL/rejected_KL_mean": -0.045818328857421875,
"KL/std": 0.26897895336151123,
"epoch": 0.024963289280469897,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08044159412384033,
"fcm_dpo/q_t": 0.4979892373085022,
"grad_norm": 78.46760559082031,
"learning_rate": 1.1594202898550725e-07,
"logits/chosen": -0.5154045820236206,
"logits/rejected": -0.480136513710022,
"logps/chosen": -67.67571258544922,
"logps/ref_chosen": -67.71033477783203,
"logps/ref_rejected": -85.37865447998047,
"logps/rejected": -85.42446899414062,
"loss": 1.3786,
"margin_dpo/margin_mean": 0.08044150471687317,
"margin_dpo/margin_std": 0.3672877550125122,
"step": 17
},
{
"KL/chosen_KL_mean": -0.008848190307617188,
"KL/mean": -0.022856131196022034,
"KL/rejected_KL_mean": -0.036865234375,
"KL/std": 0.2776832580566406,
"epoch": 0.02643171806167401,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.028015047311782837,
"fcm_dpo/q_t": 0.4992990791797638,
"grad_norm": 82.00312042236328,
"learning_rate": 1.2318840579710146e-07,
"logits/chosen": -0.516010582447052,
"logits/rejected": -0.4631628394126892,
"logps/chosen": -47.74833679199219,
"logps/ref_chosen": -47.7394905090332,
"logps/ref_rejected": -75.4722900390625,
"logps/rejected": -75.5091552734375,
"loss": 1.3838,
"margin_dpo/margin_mean": 0.028014779090881348,
"margin_dpo/margin_std": 0.3619588017463684,
"step": 18
},
{
"KL/chosen_KL_mean": 0.031284332275390625,
"KL/mean": -0.0030418336391448975,
"KL/rejected_KL_mean": -0.03736114501953125,
"KL/std": 0.23613137006759644,
"epoch": 0.027900146842878122,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06864070892333984,
"fcm_dpo/q_t": 0.49828577041625977,
"grad_norm": 73.23355865478516,
"learning_rate": 1.3043478260869563e-07,
"logits/chosen": -0.48611390590667725,
"logits/rejected": -0.435970276594162,
"logps/chosen": -70.174072265625,
"logps/ref_chosen": -70.20536041259766,
"logps/ref_rejected": -89.7575912475586,
"logps/rejected": -89.79495239257812,
"loss": 1.3798,
"margin_dpo/margin_mean": 0.06864088773727417,
"margin_dpo/margin_std": 0.35450613498687744,
"step": 19
},
{
"KL/chosen_KL_mean": -0.012430191040039062,
"KL/mean": -0.04596352577209473,
"KL/rejected_KL_mean": -0.07949447631835938,
"KL/std": 0.2700064778327942,
"epoch": 0.02936857562408223,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.06706851720809937,
"fcm_dpo/q_t": 0.49832427501678467,
"grad_norm": 73.91919708251953,
"learning_rate": 1.3768115942028986e-07,
"logits/chosen": -0.5568511486053467,
"logits/rejected": -0.5009536743164062,
"logps/chosen": -50.815673828125,
"logps/ref_chosen": -50.80324172973633,
"logps/ref_rejected": -78.82334899902344,
"logps/rejected": -78.90284729003906,
"loss": 1.3799,
"margin_dpo/margin_mean": 0.06706875562667847,
"margin_dpo/margin_std": 0.3464469313621521,
"step": 20
},
{
"KL/chosen_KL_mean": -0.012804031372070312,
"KL/mean": -0.05751395970582962,
"KL/rejected_KL_mean": -0.10222244262695312,
"KL/std": 0.2957477867603302,
"epoch": 0.030837004405286344,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.08941756188869476,
"fcm_dpo/q_t": 0.49776533246040344,
"grad_norm": 77.51765441894531,
"learning_rate": 1.4492753623188405e-07,
"logits/chosen": -0.5174500346183777,
"logits/rejected": -0.4953506886959076,
"logps/chosen": -50.07582092285156,
"logps/ref_chosen": -50.063018798828125,
"logps/ref_rejected": -77.86878967285156,
"logps/rejected": -77.97101593017578,
"loss": 1.3777,
"margin_dpo/margin_mean": 0.08941812813282013,
"margin_dpo/margin_std": 0.3819068670272827,
"step": 21
},
{
"KL/chosen_KL_mean": 0.02753448486328125,
"KL/mean": -0.06719014048576355,
"KL/rejected_KL_mean": -0.16191482543945312,
"KL/std": 0.3151050806045532,
"epoch": 0.032305433186490456,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.18945235013961792,
"fcm_dpo/q_t": 0.49526724219322205,
"grad_norm": 84.16546630859375,
"learning_rate": 1.5217391304347825e-07,
"logits/chosen": -0.4989904761314392,
"logits/rejected": -0.45593979954719543,
"logps/chosen": -59.03010177612305,
"logps/ref_chosen": -59.05763626098633,
"logps/ref_rejected": -97.50466918945312,
"logps/rejected": -97.66658782958984,
"loss": 1.3679,
"margin_dpo/margin_mean": 0.1894513964653015,
"margin_dpo/margin_std": 0.44450777769088745,
"step": 22
},
{
"KL/chosen_KL_mean": 0.08102035522460938,
"KL/mean": -0.04034559428691864,
"KL/rejected_KL_mean": -0.161712646484375,
"KL/std": 0.3124745488166809,
"epoch": 0.033773861967694566,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.24273207783699036,
"fcm_dpo/q_t": 0.49393430352211,
"grad_norm": 80.9083251953125,
"learning_rate": 1.5942028985507245e-07,
"logits/chosen": -0.5055763721466064,
"logits/rejected": -0.48390600085258484,
"logps/chosen": -59.996673583984375,
"logps/ref_chosen": -60.07769775390625,
"logps/ref_rejected": -81.13955688476562,
"logps/rejected": -81.30126953125,
"loss": 1.3626,
"margin_dpo/margin_mean": 0.24273264408111572,
"margin_dpo/margin_std": 0.39859479665756226,
"step": 23
},
{
"KL/chosen_KL_mean": 0.04727935791015625,
"KL/mean": -0.07922174781560898,
"KL/rejected_KL_mean": -0.20572662353515625,
"KL/std": 0.33435285091400146,
"epoch": 0.03524229074889868,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.25300681591033936,
"fcm_dpo/q_t": 0.4936782717704773,
"grad_norm": 82.37586212158203,
"learning_rate": 1.6666666666666665e-07,
"logits/chosen": -0.5237699747085571,
"logits/rejected": -0.5069272518157959,
"logps/chosen": -44.24375534057617,
"logps/ref_chosen": -44.29103469848633,
"logps/ref_rejected": -99.12521362304688,
"logps/rejected": -99.33094024658203,
"loss": 1.3617,
"margin_dpo/margin_mean": 0.2530064284801483,
"margin_dpo/margin_std": 0.4634859561920166,
"step": 24
},
{
"KL/chosen_KL_mean": 0.016876220703125,
"KL/mean": -0.10173620283603668,
"KL/rejected_KL_mean": -0.22034454345703125,
"KL/std": 0.35340842604637146,
"epoch": 0.03671071953010279,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.23721817135810852,
"fcm_dpo/q_t": 0.4940733313560486,
"grad_norm": 74.06179809570312,
"learning_rate": 1.7391304347826085e-07,
"logits/chosen": -0.5053163170814514,
"logits/rejected": -0.4756869077682495,
"logps/chosen": -52.52017593383789,
"logps/ref_chosen": -52.537052154541016,
"logps/ref_rejected": -89.34219360351562,
"logps/rejected": -89.56253051757812,
"loss": 1.3632,
"margin_dpo/margin_mean": 0.2372182309627533,
"margin_dpo/margin_std": 0.4329448938369751,
"step": 25
},
{
"KL/chosen_KL_mean": 0.08753395080566406,
"KL/mean": -0.11949370801448822,
"KL/rejected_KL_mean": -0.3265228271484375,
"KL/std": 0.44881671667099,
"epoch": 0.0381791483113069,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4140579402446747,
"fcm_dpo/q_t": 0.4896630048751831,
"grad_norm": 87.02017974853516,
"learning_rate": 1.8115942028985507e-07,
"logits/chosen": -0.5420371294021606,
"logits/rejected": -0.5105117559432983,
"logps/chosen": -53.83527374267578,
"logps/ref_chosen": -53.92280578613281,
"logps/ref_rejected": -103.35971069335938,
"logps/rejected": -103.68623352050781,
"loss": 1.3462,
"margin_dpo/margin_mean": 0.4140591323375702,
"margin_dpo/margin_std": 0.555227518081665,
"step": 26
},
{
"KL/chosen_KL_mean": 0.07442474365234375,
"KL/mean": -0.15376684069633484,
"KL/rejected_KL_mean": -0.3819580078125,
"KL/std": 0.41409561038017273,
"epoch": 0.039647577092511016,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.45638173818588257,
"fcm_dpo/q_t": 0.4886014759540558,
"grad_norm": 94.21731567382812,
"learning_rate": 1.8840579710144927e-07,
"logits/chosen": -0.5366708040237427,
"logits/rejected": -0.500505805015564,
"logps/chosen": -42.82410430908203,
"logps/ref_chosen": -42.898529052734375,
"logps/ref_rejected": -98.72419738769531,
"logps/rejected": -99.10615539550781,
"loss": 1.3419,
"margin_dpo/margin_mean": 0.4563814401626587,
"margin_dpo/margin_std": 0.5116335153579712,
"step": 27
},
{
"KL/chosen_KL_mean": 0.017200469970703125,
"KL/mean": -0.14496609568595886,
"KL/rejected_KL_mean": -0.3071327209472656,
"KL/std": 0.4189748764038086,
"epoch": 0.041116005873715125,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.3243354558944702,
"fcm_dpo/q_t": 0.4918993413448334,
"grad_norm": 75.2711410522461,
"learning_rate": 1.9565217391304347e-07,
"logits/chosen": -0.5030197501182556,
"logits/rejected": -0.4474295377731323,
"logps/chosen": -60.53929901123047,
"logps/ref_chosen": -60.55650329589844,
"logps/ref_rejected": -91.40111541748047,
"logps/rejected": -91.708251953125,
"loss": 1.3549,
"margin_dpo/margin_mean": 0.32433581352233887,
"margin_dpo/margin_std": 0.5495618581771851,
"step": 28
},
{
"KL/chosen_KL_mean": 0.10011100769042969,
"KL/mean": -0.1491682380437851,
"KL/rejected_KL_mean": -0.3984489440917969,
"KL/std": 0.4559759795665741,
"epoch": 0.042584434654919234,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.4985598027706146,
"fcm_dpo/q_t": 0.4875490665435791,
"grad_norm": 90.70254516601562,
"learning_rate": 2.028985507246377e-07,
"logits/chosen": -0.5579368472099304,
"logits/rejected": -0.5119800567626953,
"logps/chosen": -57.707672119140625,
"logps/ref_chosen": -57.80778503417969,
"logps/ref_rejected": -97.39434814453125,
"logps/rejected": -97.79280090332031,
"loss": 1.3378,
"margin_dpo/margin_mean": 0.4985605478286743,
"margin_dpo/margin_std": 0.5526378154754639,
"step": 29
},
{
"KL/chosen_KL_mean": 0.13415145874023438,
"KL/mean": -0.2075020968914032,
"KL/rejected_KL_mean": -0.5491600036621094,
"KL/std": 0.5850201845169067,
"epoch": 0.04405286343612335,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6833061575889587,
"fcm_dpo/q_t": 0.4829457402229309,
"grad_norm": 87.75554656982422,
"learning_rate": 2.1014492753623187e-07,
"logits/chosen": -0.4961678981781006,
"logits/rejected": -0.46552735567092896,
"logps/chosen": -52.44322204589844,
"logps/ref_chosen": -52.577369689941406,
"logps/ref_rejected": -98.48920440673828,
"logps/rejected": -99.03836059570312,
"loss": 1.3202,
"margin_dpo/margin_mean": 0.683306097984314,
"margin_dpo/margin_std": 0.6441049575805664,
"step": 30
},
{
"KL/chosen_KL_mean": 0.11783027648925781,
"KL/mean": -0.1323196440935135,
"KL/rejected_KL_mean": -0.3824729919433594,
"KL/std": 0.5714821219444275,
"epoch": 0.04552129221732746,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.5003038644790649,
"fcm_dpo/q_t": 0.4875211715698242,
"grad_norm": 67.77152252197266,
"learning_rate": 2.1739130434782607e-07,
"logits/chosen": -0.4899521470069885,
"logits/rejected": -0.44324806332588196,
"logps/chosen": -63.689090728759766,
"logps/ref_chosen": -63.806922912597656,
"logps/ref_rejected": -72.89400482177734,
"logps/rejected": -73.27647399902344,
"loss": 1.3382,
"margin_dpo/margin_mean": 0.5003036260604858,
"margin_dpo/margin_std": 0.7216464877128601,
"step": 31
},
{
"KL/chosen_KL_mean": 0.15192794799804688,
"KL/mean": -0.22155286371707916,
"KL/rejected_KL_mean": -0.59503173828125,
"KL/std": 0.7016799449920654,
"epoch": 0.04698972099853157,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.7469567060470581,
"fcm_dpo/q_t": 0.4813820719718933,
"grad_norm": 82.0534896850586,
"learning_rate": 2.2463768115942027e-07,
"logits/chosen": -0.5268591046333313,
"logits/rejected": -0.48644328117370605,
"logps/chosen": -62.58759689331055,
"logps/ref_chosen": -62.739524841308594,
"logps/ref_rejected": -89.3175048828125,
"logps/rejected": -89.91253662109375,
"loss": 1.3148,
"margin_dpo/margin_mean": 0.7469568252563477,
"margin_dpo/margin_std": 0.8592814803123474,
"step": 32
},
{
"KL/chosen_KL_mean": 0.10763168334960938,
"KL/mean": -0.20983782410621643,
"KL/rejected_KL_mean": -0.5273056030273438,
"KL/std": 0.5811012983322144,
"epoch": 0.048458149779735685,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.6349318623542786,
"fcm_dpo/q_t": 0.4841501712799072,
"grad_norm": 72.05321502685547,
"learning_rate": 2.318840579710145e-07,
"logits/chosen": -0.4794453978538513,
"logits/rejected": -0.45305970311164856,
"logps/chosen": -53.153343200683594,
"logps/ref_chosen": -53.26097106933594,
"logps/ref_rejected": -87.8851318359375,
"logps/rejected": -88.41244506835938,
"loss": 1.3249,
"margin_dpo/margin_mean": 0.6349321603775024,
"margin_dpo/margin_std": 0.6593722701072693,
"step": 33
},
{
"KL/chosen_KL_mean": 0.07497215270996094,
"KL/mean": -0.299121618270874,
"KL/rejected_KL_mean": -0.6732177734375,
"KL/std": 0.7037482857704163,
"epoch": 0.049926578560939794,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 0.748187780380249,
"fcm_dpo/q_t": 0.4813481569290161,
"grad_norm": 77.933837890625,
"learning_rate": 2.391304347826087e-07,
"logits/chosen": -0.5027347803115845,
"logits/rejected": -0.48531198501586914,
"logps/chosen": -50.74235534667969,
"logps/ref_chosen": -50.81732940673828,
"logps/ref_rejected": -101.92184448242188,
"logps/rejected": -102.59506225585938,
"loss": 1.3148,
"margin_dpo/margin_mean": 0.7481868267059326,
"margin_dpo/margin_std": 0.8796716928482056,
"step": 34
},
{
"KL/chosen_KL_mean": 0.14807891845703125,
"KL/mean": -0.4777490496635437,
"KL/rejected_KL_mean": -1.10357666015625,
"KL/std": 0.9910285472869873,
"epoch": 0.0513950073421439,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.2516549825668335,
"fcm_dpo/q_t": 0.4688939154148102,
"grad_norm": 82.58599090576172,
"learning_rate": 2.463768115942029e-07,
"logits/chosen": -0.531462550163269,
"logits/rejected": -0.4944959580898285,
"logps/chosen": -50.87641143798828,
"logps/ref_chosen": -51.02449035644531,
"logps/ref_rejected": -106.82443237304688,
"logps/rejected": -107.92800903320312,
"loss": 1.268,
"margin_dpo/margin_mean": 1.2516546249389648,
"margin_dpo/margin_std": 1.0724127292633057,
"step": 35
},
{
"KL/chosen_KL_mean": 0.06724357604980469,
"KL/mean": -0.5083455443382263,
"KL/rejected_KL_mean": -1.0839347839355469,
"KL/std": 1.0932717323303223,
"epoch": 0.05286343612334802,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.1511791944503784,
"fcm_dpo/q_t": 0.47139599919319153,
"grad_norm": 72.46247100830078,
"learning_rate": 2.536231884057971e-07,
"logits/chosen": -0.5766887664794922,
"logits/rejected": -0.5414172410964966,
"logps/chosen": -51.92424774169922,
"logps/ref_chosen": -51.991493225097656,
"logps/ref_rejected": -86.0406265258789,
"logps/rejected": -87.12455749511719,
"loss": 1.2787,
"margin_dpo/margin_mean": 1.1511785984039307,
"margin_dpo/margin_std": 1.2563061714172363,
"step": 36
},
{
"KL/chosen_KL_mean": 0.0029850006103515625,
"KL/mean": -0.5103100538253784,
"KL/rejected_KL_mean": -1.0236091613769531,
"KL/std": 1.0585589408874512,
"epoch": 0.05433186490455213,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.0265923738479614,
"fcm_dpo/q_t": 0.474545419216156,
"grad_norm": 61.76712417602539,
"learning_rate": 2.6086956521739126e-07,
"logits/chosen": -0.5153898000717163,
"logits/rejected": -0.4721587300300598,
"logps/chosen": -62.80412292480469,
"logps/ref_chosen": -62.807106018066406,
"logps/ref_rejected": -77.89507293701172,
"logps/rejected": -78.91868591308594,
"loss": 1.2909,
"margin_dpo/margin_mean": 1.0265920162200928,
"margin_dpo/margin_std": 1.3249099254608154,
"step": 37
},
{
"KL/chosen_KL_mean": 0.15967178344726562,
"KL/mean": -0.5315895080566406,
"KL/rejected_KL_mean": -1.2228507995605469,
"KL/std": 1.3578490018844604,
"epoch": 0.055800293685756244,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.382524847984314,
"fcm_dpo/q_t": 0.4658650755882263,
"grad_norm": 69.12739562988281,
"learning_rate": 2.681159420289855e-07,
"logits/chosen": -0.5189441442489624,
"logits/rejected": -0.4861838221549988,
"logps/chosen": -48.23085021972656,
"logps/ref_chosen": -48.39051818847656,
"logps/ref_rejected": -97.91244506835938,
"logps/rejected": -99.13529205322266,
"loss": 1.2596,
"margin_dpo/margin_mean": 1.3825253248214722,
"margin_dpo/margin_std": 1.6585068702697754,
"step": 38
},
{
"KL/chosen_KL_mean": 0.07357215881347656,
"KL/mean": -0.7325087785720825,
"KL/rejected_KL_mean": -1.5385856628417969,
"KL/std": 1.2871696949005127,
"epoch": 0.05726872246696035,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.612156629562378,
"fcm_dpo/q_t": 0.4599946141242981,
"grad_norm": 73.47403717041016,
"learning_rate": 2.753623188405797e-07,
"logits/chosen": -0.5452001094818115,
"logits/rejected": -0.5045498013496399,
"logps/chosen": -50.676902770996094,
"logps/ref_chosen": -50.75047302246094,
"logps/ref_rejected": -78.56951141357422,
"logps/rejected": -80.10809326171875,
"loss": 1.2365,
"margin_dpo/margin_mean": 1.612156867980957,
"margin_dpo/margin_std": 1.3819551467895508,
"step": 39
},
{
"KL/chosen_KL_mean": 0.1919574737548828,
"KL/mean": -0.5750719308853149,
"KL/rejected_KL_mean": -1.34210205078125,
"KL/std": 1.3759987354278564,
"epoch": 0.05873715124816446,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.5340617895126343,
"fcm_dpo/q_t": 0.4621471166610718,
"grad_norm": 60.486148834228516,
"learning_rate": 2.8260869565217386e-07,
"logits/chosen": -0.5081291794776917,
"logits/rejected": -0.477234423160553,
"logps/chosen": -57.79310989379883,
"logps/ref_chosen": -57.985069274902344,
"logps/ref_rejected": -74.3000717163086,
"logps/rejected": -75.64217376708984,
"loss": 1.2461,
"margin_dpo/margin_mean": 1.5340622663497925,
"margin_dpo/margin_std": 1.6614691019058228,
"step": 40
},
{
"KL/chosen_KL_mean": 0.04014015197753906,
"KL/mean": -0.9176149368286133,
"KL/rejected_KL_mean": -1.8753662109375,
"KL/std": 1.8172500133514404,
"epoch": 0.06020558002936858,
"fcm_dpo/beta": 0.10000000149011612,
"fcm_dpo/delta": 0.0,
"fcm_dpo/margin": 1.9155058860778809,
"fcm_dpo/q_t": 0.4528723359107971,
"grad_norm": 68.08113098144531,
"learning_rate": 2.898550724637681e-07,
"logits/chosen": -0.510447084903717,
"logits/rejected": -0.4722723066806793,
"logps/chosen": -62.655677795410156,
"logps/ref_chosen": -62.69581604003906,
"logps/ref_rejected": -97.02352905273438,
"logps/rejected": -98.89889526367188,
"loss": 1.2138,
"margin_dpo/margin_mean": 1.9155060052871704,
"margin_dpo/margin_std": 1.9704290628433228,
"step": 41
},
{
"KL/chosen_KL_mean": 0.2155475616455078,
"KL/mean": -1.0993424654006958,
"KL/rejected_KL_mean": -2.4142303466796875,
"KL/std": 2.225177764892578,
"epoch": 0.06167400881057269,
"fcm_dpo/beta": 0.108363576233387,
"fcm_dpo/delta": 0.529624342918396,
"fcm_dpo/margin": 2.629781723022461,
"fcm_dpo/q_t": 0.43431544303894043,
"grad_norm": 83.5115737915039,
"learning_rate": 2.971014492753623e-07,
"logits/chosen": -0.5298647880554199,
"logits/rejected": -0.48321884870529175,
"logps/chosen": -58.750877380371094,
"logps/ref_chosen": -58.966426849365234,
"logps/ref_rejected": -109.90837097167969,
"logps/rejected": -112.32260131835938,
"loss": 1.1397,
"margin_dpo/margin_mean": 2.629781723022461,
"margin_dpo/margin_std": 2.5406675338745117,
"step": 42
},
{
"KL/chosen_KL_mean": 0.53778076171875,
"KL/mean": -0.7240562438964844,
"KL/rejected_KL_mean": -1.98590087890625,
"KL/std": 1.8650085926055908,
"epoch": 0.0631424375917768,
"fcm_dpo/beta": 0.11999893188476562,
"fcm_dpo/delta": 0.5129410028457642,
"fcm_dpo/margin": 2.5236780643463135,
"fcm_dpo/q_t": 0.42989984154701233,
"grad_norm": 82.07591247558594,
"learning_rate": 3.043478260869565e-07,
"logits/chosen": -0.5556699633598328,
"logits/rejected": -0.5314440727233887,
"logps/chosen": -53.61821746826172,
"logps/ref_chosen": -54.15599822998047,
"logps/ref_rejected": -96.48019409179688,
"logps/rejected": -98.46609497070312,
"loss": 1.1214,
"margin_dpo/margin_mean": 2.5236783027648926,
"margin_dpo/margin_std": 2.0528616905212402,
"step": 43
},
{
"KL/chosen_KL_mean": 0.22237205505371094,
"KL/mean": -1.1844167709350586,
"KL/rejected_KL_mean": -2.591205596923828,
"KL/std": 2.265498638153076,
"epoch": 0.06461086637298091,
"fcm_dpo/beta": 0.1317014992237091,
"fcm_dpo/delta": 0.445910781621933,
"fcm_dpo/margin": 2.8135740756988525,
"fcm_dpo/q_t": 0.4144276976585388,
"grad_norm": 95.687255859375,
"learning_rate": 3.115942028985507e-07,
"logits/chosen": -0.4916815161705017,
"logits/rejected": -0.4712473750114441,
"logps/chosen": -49.85612487792969,
"logps/ref_chosen": -50.07849884033203,
"logps/ref_rejected": -108.78376007080078,
"logps/rejected": -111.37496948242188,
"loss": 1.0708,
"margin_dpo/margin_mean": 2.8135738372802734,
"margin_dpo/margin_std": 2.256591796875,
"step": 44
},
{
"KL/chosen_KL_mean": 0.11939048767089844,
"KL/mean": -1.027750015258789,
"KL/rejected_KL_mean": -2.1748924255371094,
"KL/std": 2.073944091796875,
"epoch": 0.06607929515418502,
"fcm_dpo/beta": 0.1451357901096344,
"fcm_dpo/delta": 0.4816334843635559,
"fcm_dpo/margin": 2.2942895889282227,
"fcm_dpo/q_t": 0.42459607124328613,
"grad_norm": 80.60940551757812,
"learning_rate": 3.188405797101449e-07,
"logits/chosen": -0.4741925597190857,
"logits/rejected": -0.4613730311393738,
"logps/chosen": -48.29553985595703,
"logps/ref_chosen": -48.4149284362793,
"logps/ref_rejected": -77.93643188476562,
"logps/rejected": -80.11132049560547,
"loss": 1.1144,
"margin_dpo/margin_mean": 2.2942888736724854,
"margin_dpo/margin_std": 2.5776329040527344,
"step": 45
},
{
"KL/chosen_KL_mean": 0.1797943115234375,
"KL/mean": -1.272498369216919,
"KL/rejected_KL_mean": -2.72479248046875,
"KL/std": 2.6093244552612305,
"epoch": 0.06754772393538913,
"fcm_dpo/beta": 0.15693849325180054,
"fcm_dpo/delta": 0.3592992424964905,
"fcm_dpo/margin": 2.9045841693878174,
"fcm_dpo/q_t": 0.3983314037322998,
"grad_norm": 93.62377166748047,
"learning_rate": 3.260869565217391e-07,
"logits/chosen": -0.549965500831604,
"logits/rejected": -0.5000295639038086,
"logps/chosen": -55.81963348388672,
"logps/ref_chosen": -55.999427795410156,
"logps/ref_rejected": -95.652587890625,
"logps/rejected": -98.37738037109375,
"loss": 1.0358,
"margin_dpo/margin_mean": 2.9045839309692383,
"margin_dpo/margin_std": 3.1410748958587646,
"step": 46
},
{
"KL/chosen_KL_mean": 0.3869171142578125,
"KL/mean": -1.106046199798584,
"KL/rejected_KL_mean": -2.599010467529297,
"KL/std": 2.578082799911499,
"epoch": 0.06901615271659324,
"fcm_dpo/beta": 0.16788116097450256,
"fcm_dpo/delta": 0.3120081424713135,
"fcm_dpo/margin": 2.985929012298584,
"fcm_dpo/q_t": 0.3863651156425476,
"grad_norm": 92.04369354248047,
"learning_rate": 3.333333333333333e-07,
"logits/chosen": -0.5671145915985107,
"logits/rejected": -0.5139440298080444,
"logps/chosen": -57.539161682128906,
"logps/ref_chosen": -57.92607879638672,
"logps/ref_rejected": -94.67920684814453,
"logps/rejected": -97.27821350097656,
"loss": 0.9928,
"margin_dpo/margin_mean": 2.985929489135742,
"margin_dpo/margin_std": 2.6357438564300537,
"step": 47
},
{
"KL/chosen_KL_mean": 0.05410575866699219,
"KL/mean": -1.496936559677124,
"KL/rejected_KL_mean": -3.0479812622070312,
"KL/std": 2.6523523330688477,
"epoch": 0.07048458149779736,
"fcm_dpo/beta": 0.17588907480239868,
"fcm_dpo/delta": 0.2697899341583252,
"fcm_dpo/margin": 3.102083444595337,
"fcm_dpo/q_t": 0.3784920573234558,
"grad_norm": 104.22386932373047,
"learning_rate": 3.4057971014492755e-07,
"logits/chosen": -0.5789310932159424,
"logits/rejected": -0.519860565662384,
"logps/chosen": -57.133968353271484,
"logps/ref_chosen": -57.188072204589844,
"logps/ref_rejected": -88.0166015625,
"logps/rejected": -91.06458282470703,
"loss": 0.9819,
"margin_dpo/margin_mean": 3.102083444595337,
"margin_dpo/margin_std": 2.9072694778442383,
"step": 48
},
{
"KL/chosen_KL_mean": 0.3045520782470703,
"KL/mean": -1.5718982219696045,
"KL/rejected_KL_mean": -3.448345184326172,
"KL/std": 3.377166748046875,
"epoch": 0.07195301027900147,
"fcm_dpo/beta": 0.1832749843597412,
"fcm_dpo/delta": 0.11903807520866394,
"fcm_dpo/margin": 3.752896547317505,
"fcm_dpo/q_t": 0.3552005887031555,
"grad_norm": 87.47305297851562,
"learning_rate": 3.478260869565217e-07,
"logits/chosen": -0.5585795640945435,
"logits/rejected": -0.5012216567993164,
"logps/chosen": -61.38071823120117,
"logps/ref_chosen": -61.685272216796875,
"logps/ref_rejected": -83.76747131347656,
"logps/rejected": -87.2158203125,
"loss": 0.9196,
"margin_dpo/margin_mean": 3.752896308898926,
"margin_dpo/margin_std": 3.921967029571533,
"step": 49
},
{
"KL/chosen_KL_mean": -0.16571998596191406,
"KL/mean": -2.2112300395965576,
"KL/rejected_KL_mean": -4.256736755371094,
"KL/std": 3.468242645263672,
"epoch": 0.07342143906020558,
"fcm_dpo/beta": 0.18442076444625854,
"fcm_dpo/delta": 0.04927580803632736,
"fcm_dpo/margin": 4.091017246246338,
"fcm_dpo/q_t": 0.34040141105651855,
"grad_norm": 90.17442321777344,
"learning_rate": 3.5507246376811595e-07,
"logits/chosen": -0.5344624519348145,
"logits/rejected": -0.49823105335235596,
"logps/chosen": -58.88985824584961,
"logps/ref_chosen": -58.72413635253906,
"logps/ref_rejected": -96.35814666748047,
"logps/rejected": -100.61488342285156,
"loss": 0.8788,
"margin_dpo/margin_mean": 4.091017246246338,
"margin_dpo/margin_std": 3.9831931591033936,
"step": 50
},
{
"KL/chosen_KL_mean": -0.31114959716796875,
"KL/mean": -2.3097453117370605,
"KL/rejected_KL_mean": -4.308345794677734,
"KL/std": 4.11016845703125,
"epoch": 0.07488986784140969,
"fcm_dpo/beta": 0.18537896871566772,
"fcm_dpo/delta": 0.06288561224937439,
"fcm_dpo/margin": 3.997190237045288,
"fcm_dpo/q_t": 0.35509026050567627,
"grad_norm": 76.0972900390625,
"learning_rate": 3.6231884057971015e-07,
"logits/chosen": -0.5231320858001709,
"logits/rejected": -0.49016422033309937,
"logps/chosen": -61.684818267822266,
"logps/ref_chosen": -61.3736686706543,
"logps/ref_rejected": -76.00199890136719,
"logps/rejected": -80.31034851074219,
"loss": 0.9433,
"margin_dpo/margin_mean": 3.997189998626709,
"margin_dpo/margin_std": 5.026128768920898,
"step": 51
},
{
"KL/chosen_KL_mean": 0.3505516052246094,
"KL/mean": -2.698451519012451,
"KL/rejected_KL_mean": -5.747459411621094,
"KL/std": 4.724987030029297,
"epoch": 0.0763582966226138,
"fcm_dpo/beta": 0.17869603633880615,
"fcm_dpo/delta": -0.32520201802253723,
"fcm_dpo/margin": 6.09800910949707,
"fcm_dpo/q_t": 0.27982762455940247,
"grad_norm": 72.83785247802734,
"learning_rate": 3.695652173913043e-07,
"logits/chosen": -0.5447903871536255,
"logits/rejected": -0.4885804057121277,
"logps/chosen": -51.9868049621582,
"logps/ref_chosen": -52.33735656738281,
"logps/ref_rejected": -79.97391510009766,
"logps/rejected": -85.72137451171875,
"loss": 0.7172,
"margin_dpo/margin_mean": 6.09800910949707,
"margin_dpo/margin_std": 5.0404887199401855,
"step": 52
},
{
"KL/chosen_KL_mean": -0.15282440185546875,
"KL/mean": -3.2396836280822754,
"KL/rejected_KL_mean": -6.326541900634766,
"KL/std": 5.3033671379089355,
"epoch": 0.07782672540381791,
"fcm_dpo/beta": 0.17080718278884888,
"fcm_dpo/delta": -0.284501850605011,
"fcm_dpo/margin": 6.173723220825195,
"fcm_dpo/q_t": 0.3035929799079895,
"grad_norm": 79.4716796875,
"learning_rate": 3.7681159420289855e-07,
"logits/chosen": -0.6213881969451904,
"logits/rejected": -0.6000795364379883,
"logps/chosen": -53.46747589111328,
"logps/ref_chosen": -53.31465148925781,
"logps/ref_rejected": -91.78359985351562,
"logps/rejected": -98.11013793945312,
"loss": 0.7851,
"margin_dpo/margin_mean": 6.173722743988037,
"margin_dpo/margin_std": 6.064870834350586,
"step": 53
},
{
"KL/chosen_KL_mean": -0.37441253662109375,
"KL/mean": -3.029232978820801,
"KL/rejected_KL_mean": -5.684051513671875,
"KL/std": 5.031246662139893,
"epoch": 0.07929515418502203,
"fcm_dpo/beta": 0.16453491151332855,
"fcm_dpo/delta": -0.08063030242919922,
"fcm_dpo/margin": 5.309640407562256,
"fcm_dpo/q_t": 0.321298748254776,
"grad_norm": 69.15568542480469,
"learning_rate": 3.8405797101449274e-07,
"logits/chosen": -0.5816128253936768,
"logits/rejected": -0.5275565385818481,
"logps/chosen": -51.063072204589844,
"logps/ref_chosen": -50.68865966796875,
"logps/ref_rejected": -91.71539306640625,
"logps/rejected": -97.39944458007812,
"loss": 0.8212,
"margin_dpo/margin_mean": 5.309639930725098,
"margin_dpo/margin_std": 5.0380449295043945,
"step": 54
},
{
"KL/chosen_KL_mean": -0.8430213928222656,
"KL/mean": -4.0328216552734375,
"KL/rejected_KL_mean": -7.222618103027344,
"KL/std": 6.049294471740723,
"epoch": 0.08076358296622614,
"fcm_dpo/beta": 0.1586906611919403,
"fcm_dpo/delta": -0.23564836382865906,
"fcm_dpo/margin": 6.379594802856445,
"fcm_dpo/q_t": 0.31021779775619507,
"grad_norm": 74.39564514160156,
"learning_rate": 3.9130434782608694e-07,
"logits/chosen": -0.6505610942840576,
"logits/rejected": -0.5886675119400024,
"logps/chosen": -63.458255767822266,
"logps/ref_chosen": -62.615234375,
"logps/ref_rejected": -88.99349975585938,
"logps/rejected": -96.21612548828125,
"loss": 0.8466,
"margin_dpo/margin_mean": 6.379594802856445,
"margin_dpo/margin_std": 7.276340484619141,
"step": 55
},
{
"KL/chosen_KL_mean": -0.6147956848144531,
"KL/mean": -3.5950727462768555,
"KL/rejected_KL_mean": -6.575355529785156,
"KL/std": 5.803818702697754,
"epoch": 0.08223201174743025,
"fcm_dpo/beta": 0.1531292051076889,
"fcm_dpo/delta": -0.12394850701093674,
"fcm_dpo/margin": 5.960559844970703,
"fcm_dpo/q_t": 0.32575082778930664,
"grad_norm": 61.359893798828125,
"learning_rate": 3.9855072463768114e-07,
"logits/chosen": -0.6318885087966919,
"logits/rejected": -0.5902144908905029,
"logps/chosen": -58.547523498535156,
"logps/ref_chosen": -57.9327278137207,
"logps/ref_rejected": -94.1744384765625,
"logps/rejected": -100.74979400634766,
"loss": 0.8844,
"margin_dpo/margin_mean": 5.9605607986450195,
"margin_dpo/margin_std": 7.014354705810547,
"step": 56
},
{
"KL/chosen_KL_mean": -0.6196041107177734,
"KL/mean": -3.9665966033935547,
"KL/rejected_KL_mean": -7.313587188720703,
"KL/std": 5.700008392333984,
"epoch": 0.08370044052863436,
"fcm_dpo/beta": 0.14832691848278046,
"fcm_dpo/delta": -0.2137039601802826,
"fcm_dpo/margin": 6.693978309631348,
"fcm_dpo/q_t": 0.29930901527404785,
"grad_norm": 65.74341583251953,
"learning_rate": 4.057971014492754e-07,
"logits/chosen": -0.5702251195907593,
"logits/rejected": -0.5417561531066895,
"logps/chosen": -71.1148910522461,
"logps/ref_chosen": -70.49528503417969,
"logps/ref_rejected": -95.56546020507812,
"logps/rejected": -102.87904357910156,
"loss": 0.7907,
"margin_dpo/margin_mean": 6.693978309631348,
"margin_dpo/margin_std": 6.242074966430664,
"step": 57
},
{
"KL/chosen_KL_mean": -0.8837299346923828,
"KL/mean": -4.569659233093262,
"KL/rejected_KL_mean": -8.255584716796875,
"KL/std": 6.463259220123291,
"epoch": 0.08516886930983847,
"fcm_dpo/beta": 0.14102259278297424,
"fcm_dpo/delta": -0.26656439900398254,
"fcm_dpo/margin": 7.371853351593018,
"fcm_dpo/q_t": 0.3033452033996582,
"grad_norm": 75.09760284423828,
"learning_rate": 4.1304347826086954e-07,
"logits/chosen": -0.5813875794410706,
"logits/rejected": -0.5039342641830444,
"logps/chosen": -63.01667022705078,
"logps/ref_chosen": -62.13294219970703,
"logps/ref_rejected": -84.61729431152344,
"logps/rejected": -92.87287902832031,
"loss": 0.8067,
"margin_dpo/margin_mean": 7.371853828430176,
"margin_dpo/margin_std": 7.66532039642334,
"step": 58
},
{
"KL/chosen_KL_mean": -1.2969722747802734,
"KL/mean": -5.302949905395508,
"KL/rejected_KL_mean": -9.30892562866211,
"KL/std": 7.01795768737793,
"epoch": 0.08663729809104258,
"fcm_dpo/beta": 0.13059750199317932,
"fcm_dpo/delta": -0.2830684185028076,
"fcm_dpo/margin": 8.011954307556152,
"fcm_dpo/q_t": 0.3002108633518219,
"grad_norm": 64.80830383300781,
"learning_rate": 4.2028985507246374e-07,
"logits/chosen": -0.6242841482162476,
"logits/rejected": -0.5837790966033936,
"logps/chosen": -53.22949981689453,
"logps/ref_chosen": -51.932525634765625,
"logps/ref_rejected": -88.88520050048828,
"logps/rejected": -98.19412231445312,
"loss": 0.7975,
"margin_dpo/margin_mean": 8.011954307556152,
"margin_dpo/margin_std": 8.137838363647461,
"step": 59
},
{
"KL/chosen_KL_mean": -2.28204345703125,
"KL/mean": -5.468286514282227,
"KL/rejected_KL_mean": -8.654525756835938,
"KL/std": 6.25811767578125,
"epoch": 0.0881057268722467,
"fcm_dpo/beta": 0.12927772104740143,
"fcm_dpo/delta": -0.026638053357601166,
"fcm_dpo/margin": 6.372478485107422,
"fcm_dpo/q_t": 0.3292919993400574,
"grad_norm": 78.46163177490234,
"learning_rate": 4.2753623188405794e-07,
"logits/chosen": -0.6169058084487915,
"logits/rejected": -0.5573669672012329,
"logps/chosen": -63.2242317199707,
"logps/ref_chosen": -60.94218826293945,
"logps/ref_rejected": -85.39340209960938,
"logps/rejected": -94.04792785644531,
"loss": 0.9026,
"margin_dpo/margin_mean": 6.3724775314331055,
"margin_dpo/margin_std": 7.160068035125732,
"step": 60
},
{
"KL/chosen_KL_mean": -1.1401500701904297,
"KL/mean": -5.068685531616211,
"KL/rejected_KL_mean": -8.997222900390625,
"KL/std": 8.527802467346191,
"epoch": 0.08957415565345081,
"fcm_dpo/beta": 0.1250603199005127,
"fcm_dpo/delta": -0.20481759309768677,
"fcm_dpo/margin": 7.857068061828613,
"fcm_dpo/q_t": 0.326485812664032,
"grad_norm": 63.45138931274414,
"learning_rate": 4.3478260869565214e-07,
"logits/chosen": -0.6488014459609985,
"logits/rejected": -0.6157772541046143,
"logps/chosen": -61.77367401123047,
"logps/ref_chosen": -60.633522033691406,
"logps/ref_rejected": -89.85249328613281,
"logps/rejected": -98.84971618652344,
"loss": 0.891,
"margin_dpo/margin_mean": 7.857068061828613,
"margin_dpo/margin_std": 10.577825546264648,
"step": 61
},
{
"KL/chosen_KL_mean": -1.325113296508789,
"KL/mean": -4.2535576820373535,
"KL/rejected_KL_mean": -7.181999206542969,
"KL/std": 6.108260154724121,
"epoch": 0.09104258443465492,
"fcm_dpo/beta": 0.1265747994184494,
"fcm_dpo/delta": 0.06192735210061073,
"fcm_dpo/margin": 5.856889724731445,
"fcm_dpo/q_t": 0.353201687335968,
"grad_norm": 66.93692779541016,
"learning_rate": 4.420289855072464e-07,
"logits/chosen": -0.6124836206436157,
"logits/rejected": -0.5789198875427246,
"logps/chosen": -57.47588348388672,
"logps/ref_chosen": -56.15077209472656,
"logps/ref_rejected": -75.56619262695312,
"logps/rejected": -82.74819946289062,
"loss": 0.9536,
"margin_dpo/margin_mean": 5.856890678405762,
"margin_dpo/margin_std": 7.419264316558838,
"step": 62
},
{
"KL/chosen_KL_mean": -2.2093944549560547,
"KL/mean": -6.111651420593262,
"KL/rejected_KL_mean": -10.013912200927734,
"KL/std": 7.733986854553223,
"epoch": 0.09251101321585903,
"fcm_dpo/beta": 0.12191849946975708,
"fcm_dpo/delta": -0.17046590149402618,
"fcm_dpo/margin": 7.8045125007629395,
"fcm_dpo/q_t": 0.30967673659324646,
"grad_norm": 63.06834411621094,
"learning_rate": 4.4927536231884053e-07,
"logits/chosen": -0.5985692739486694,
"logits/rejected": -0.5528501272201538,
"logps/chosen": -75.3567886352539,
"logps/ref_chosen": -73.14739227294922,
"logps/ref_rejected": -97.61006164550781,
"logps/rejected": -107.62397766113281,
"loss": 0.8395,
"margin_dpo/margin_mean": 7.804512023925781,
"margin_dpo/margin_std": 8.159123420715332,
"step": 63
},
{
"KL/chosen_KL_mean": -0.6886730194091797,
"KL/mean": -5.2321319580078125,
"KL/rejected_KL_mean": -9.77560043334961,
"KL/std": 8.251523971557617,
"epoch": 0.09397944199706314,
"fcm_dpo/beta": 0.11495494097471237,
"fcm_dpo/delta": -0.28372639417648315,
"fcm_dpo/margin": 9.086919784545898,
"fcm_dpo/q_t": 0.3031163811683655,
"grad_norm": 56.18279266357422,
"learning_rate": 4.5652173913043473e-07,
"logits/chosen": -0.563762366771698,
"logits/rejected": -0.5307266712188721,
"logps/chosen": -54.68727111816406,
"logps/ref_chosen": -53.998600006103516,
"logps/ref_rejected": -93.53019714355469,
"logps/rejected": -103.30580139160156,
"loss": 0.8288,
"margin_dpo/margin_mean": 9.086919784545898,
"margin_dpo/margin_std": 10.110519409179688,
"step": 64
},
{
"KL/chosen_KL_mean": -2.598611831665039,
"KL/mean": -7.124964237213135,
"KL/rejected_KL_mean": -11.651321411132812,
"KL/std": 9.134474754333496,
"epoch": 0.09544787077826726,
"fcm_dpo/beta": 0.11054727435112,
"fcm_dpo/delta": -0.22596801817417145,
"fcm_dpo/margin": 9.052715301513672,
"fcm_dpo/q_t": 0.3050973117351532,
"grad_norm": 57.133758544921875,
"learning_rate": 4.63768115942029e-07,
"logits/chosen": -0.6590486764907837,
"logits/rejected": -0.6450433731079102,
"logps/chosen": -67.43461608886719,
"logps/ref_chosen": -64.83599853515625,
"logps/ref_rejected": -109.94645690917969,
"logps/rejected": -121.5977783203125,
"loss": 0.8482,
"margin_dpo/margin_mean": 9.052714347839355,
"margin_dpo/margin_std": 9.923721313476562,
"step": 65
},
{
"KL/chosen_KL_mean": -2.3348751068115234,
"KL/mean": -6.542342185974121,
"KL/rejected_KL_mean": -10.749805450439453,
"KL/std": 8.355534553527832,
"epoch": 0.09691629955947137,
"fcm_dpo/beta": 0.10701608657836914,
"fcm_dpo/delta": -0.11439692974090576,
"fcm_dpo/margin": 8.41492748260498,
"fcm_dpo/q_t": 0.3284803032875061,
"grad_norm": 53.33970260620117,
"learning_rate": 4.7101449275362313e-07,
"logits/chosen": -0.6295303702354431,
"logits/rejected": -0.5955126285552979,
"logps/chosen": -53.77840042114258,
"logps/ref_chosen": -51.44352722167969,
"logps/ref_rejected": -75.63629913330078,
"logps/rejected": -86.3861083984375,
"loss": 0.8783,
"margin_dpo/margin_mean": 8.414926528930664,
"margin_dpo/margin_std": 10.007061004638672,
"step": 66
},
{
"KL/chosen_KL_mean": -1.9881420135498047,
"KL/mean": -6.224081039428711,
"KL/rejected_KL_mean": -10.46002197265625,
"KL/std": 8.115812301635742,
"epoch": 0.09838472834067548,
"fcm_dpo/beta": 0.1063048467040062,
"fcm_dpo/delta": -0.11047748476266861,
"fcm_dpo/margin": 8.47187614440918,
"fcm_dpo/q_t": 0.3267403841018677,
"grad_norm": 56.267860412597656,
"learning_rate": 4.782608695652174e-07,
"logits/chosen": -0.5961214303970337,
"logits/rejected": -0.5539319515228271,
"logps/chosen": -61.328948974609375,
"logps/ref_chosen": -59.34080505371094,
"logps/ref_rejected": -72.78728485107422,
"logps/rejected": -83.24730682373047,
"loss": 0.8694,
"margin_dpo/margin_mean": 8.47187614440918,
"margin_dpo/margin_std": 9.791044235229492,
"step": 67
},
{
"KL/chosen_KL_mean": -2.0880203247070312,
"KL/mean": -6.178552627563477,
"KL/rejected_KL_mean": -10.269084930419922,
"KL/std": 7.334725856781006,
"epoch": 0.09985315712187959,
"fcm_dpo/beta": 0.10470952093601227,
"fcm_dpo/delta": -0.062055252492427826,
"fcm_dpo/margin": 8.181063652038574,
"fcm_dpo/q_t": 0.32555100321769714,
"grad_norm": 51.56489181518555,
"learning_rate": 4.855072463768116e-07,
"logits/chosen": -0.6386753916740417,
"logits/rejected": -0.5816385746002197,
"logps/chosen": -67.29385375976562,
"logps/ref_chosen": -65.2058334350586,
"logps/ref_rejected": -77.20724487304688,
"logps/rejected": -87.47633361816406,
"loss": 0.844,
"margin_dpo/margin_mean": 8.181063652038574,
"margin_dpo/margin_std": 8.15043830871582,
"step": 68
},
{
"KL/chosen_KL_mean": -2.4980125427246094,
"KL/mean": -7.213038444519043,
"KL/rejected_KL_mean": -11.92806625366211,
"KL/std": 8.368502616882324,
"epoch": 0.1013215859030837,
"fcm_dpo/beta": 0.10073809325695038,
"fcm_dpo/delta": -0.16765105724334717,
"fcm_dpo/margin": 9.430049896240234,
"fcm_dpo/q_t": 0.3103695511817932,
"grad_norm": 52.50539779663086,
"learning_rate": 4.927536231884058e-07,
"logits/chosen": -0.5846946239471436,
"logits/rejected": -0.5604379177093506,
"logps/chosen": -62.31725311279297,
"logps/ref_chosen": -59.81924057006836,
"logps/ref_rejected": -103.38886260986328,
"logps/rejected": -115.31692504882812,
"loss": 0.7993,
"margin_dpo/margin_mean": 9.430049896240234,
"margin_dpo/margin_std": 9.154158592224121,
"step": 69
},
{
"KL/chosen_KL_mean": -3.4951820373535156,
"KL/mean": -8.612916946411133,
"KL/rejected_KL_mean": -13.73065185546875,
"KL/std": 9.930105209350586,
"epoch": 0.1027900146842878,
"fcm_dpo/beta": 0.09773986041545868,
"fcm_dpo/delta": -0.22211110591888428,
"fcm_dpo/margin": 10.235469818115234,
"fcm_dpo/q_t": 0.3081403374671936,
"grad_norm": 54.948020935058594,
"learning_rate": 5e-07,
"logits/chosen": -0.6405221819877625,
"logits/rejected": -0.6063104867935181,
"logps/chosen": -65.42582702636719,
"logps/ref_chosen": -61.930641174316406,
"logps/ref_rejected": -91.06078338623047,
"logps/rejected": -104.79143524169922,
"loss": 0.811,
"margin_dpo/margin_mean": 10.23547077178955,
"margin_dpo/margin_std": 10.585088729858398,
"step": 70
},
{
"KL/chosen_KL_mean": -2.8669185638427734,
"KL/mean": -8.555381774902344,
"KL/rejected_KL_mean": -14.243846893310547,
"KL/std": 9.787435531616211,
"epoch": 0.10425844346549193,
"fcm_dpo/beta": 0.09209594875574112,
"fcm_dpo/delta": -0.2774484157562256,
"fcm_dpo/margin": 11.37692642211914,
"fcm_dpo/q_t": 0.29503485560417175,
"grad_norm": 47.97392272949219,
"learning_rate": 4.999967061337492e-07,
"logits/chosen": -0.6438924074172974,
"logits/rejected": -0.6015328168869019,
"logps/chosen": -64.61725616455078,
"logps/ref_chosen": -61.750335693359375,
"logps/ref_rejected": -97.33662414550781,
"logps/rejected": -111.5804672241211,
"loss": 0.7634,
"margin_dpo/margin_mean": 11.37692642211914,
"margin_dpo/margin_std": 10.718829154968262,
"step": 71
},
{
"KL/chosen_KL_mean": -2.9292354583740234,
"KL/mean": -8.912168502807617,
"KL/rejected_KL_mean": -14.895095825195312,
"KL/std": 10.194162368774414,
"epoch": 0.10572687224669604,
"fcm_dpo/beta": 0.08702956140041351,
"fcm_dpo/delta": -0.27092522382736206,
"fcm_dpo/margin": 11.965860366821289,
"fcm_dpo/q_t": 0.2947954833507538,
"grad_norm": 52.500999450683594,
"learning_rate": 4.999868246217933e-07,
"logits/chosen": -0.6367168426513672,
"logits/rejected": -0.6010754704475403,
"logps/chosen": -68.98265075683594,
"logps/ref_chosen": -66.05341339111328,
"logps/ref_rejected": -95.2869873046875,
"logps/rejected": -110.18208312988281,
"loss": 0.7728,
"margin_dpo/margin_mean": 11.965860366821289,
"margin_dpo/margin_std": 11.45088005065918,
"step": 72
},
{
"KL/chosen_KL_mean": -4.294904708862305,
"KL/mean": -10.067806243896484,
"KL/rejected_KL_mean": -15.840709686279297,
"KL/std": 12.866714477539062,
"epoch": 0.10719530102790015,
"fcm_dpo/beta": 0.08335524797439575,
"fcm_dpo/delta": -0.1816135048866272,
"fcm_dpo/margin": 11.545804977416992,
"fcm_dpo/q_t": 0.330466091632843,
"grad_norm": 52.436309814453125,
"learning_rate": 4.999703557245192e-07,
"logits/chosen": -0.6677882671356201,
"logits/rejected": -0.6248770952224731,
"logps/chosen": -70.55117797851562,
"logps/ref_chosen": -66.25627136230469,
"logps/ref_rejected": -90.45613098144531,
"logps/rejected": -106.29684448242188,
"loss": 0.9326,
"margin_dpo/margin_mean": 11.545804977416992,
"margin_dpo/margin_std": 15.709431648254395,
"step": 73
},
{
"KL/chosen_KL_mean": -4.670244216918945,
"KL/mean": -10.651287078857422,
"KL/rejected_KL_mean": -16.63233184814453,
"KL/std": 13.789833068847656,
"epoch": 0.10866372980910426,
"fcm_dpo/beta": 0.08051745593547821,
"fcm_dpo/delta": -0.18177367746829987,
"fcm_dpo/margin": 11.96208381652832,
"fcm_dpo/q_t": 0.32404544949531555,
"grad_norm": 55.107704162597656,
"learning_rate": 4.999472998758977e-07,
"logits/chosen": -0.631071925163269,
"logits/rejected": -0.6169995069503784,
"logps/chosen": -58.09512710571289,
"logps/ref_chosen": -53.42488098144531,
"logps/ref_rejected": -95.94693756103516,
"logps/rejected": -112.57926940917969,
"loss": 0.9081,
"margin_dpo/margin_mean": 11.96208381652832,
"margin_dpo/margin_std": 16.845170974731445,
"step": 74
},
{
"KL/chosen_KL_mean": -4.217048645019531,
"KL/mean": -12.636398315429688,
"KL/rejected_KL_mean": -21.055740356445312,
"KL/std": 14.347799301147461,
"epoch": 0.11013215859030837,
"fcm_dpo/beta": 0.0748315155506134,
"fcm_dpo/delta": -0.5235542058944702,
"fcm_dpo/margin": 16.838699340820312,
"fcm_dpo/q_t": 0.2691539525985718,
"grad_norm": 42.937564849853516,
"learning_rate": 4.999176576834721e-07,
"logits/chosen": -0.6606135368347168,
"logits/rejected": -0.6492525339126587,
"logps/chosen": -56.078712463378906,
"logps/ref_chosen": -51.861663818359375,
"logps/ref_rejected": -111.25398254394531,
"logps/rejected": -132.30972290039062,
"loss": 0.709,
"margin_dpo/margin_mean": 16.838699340820312,
"margin_dpo/margin_std": 15.839242935180664,
"step": 75
},
{
"KL/chosen_KL_mean": -5.221561431884766,
"KL/mean": -10.881424903869629,
"KL/rejected_KL_mean": -16.541290283203125,
"KL/std": 11.600919723510742,
"epoch": 0.11160058737151249,
"fcm_dpo/beta": 0.07199326157569885,
"fcm_dpo/delta": -0.016267387196421623,
"fcm_dpo/margin": 11.319733619689941,
"fcm_dpo/q_t": 0.3303011655807495,
"grad_norm": 45.18712615966797,
"learning_rate": 4.998814299283415e-07,
"logits/chosen": -0.6939696073532104,
"logits/rejected": -0.6507217884063721,
"logps/chosen": -58.48760223388672,
"logps/ref_chosen": -53.26603698730469,
"logps/ref_rejected": -78.21662902832031,
"logps/rejected": -94.75791931152344,
"loss": 0.8827,
"margin_dpo/margin_mean": 11.319733619689941,
"margin_dpo/margin_std": 12.374711990356445,
"step": 76
},
{
"KL/chosen_KL_mean": -4.481769561767578,
"KL/mean": -13.34714126586914,
"KL/rejected_KL_mean": -22.21251678466797,
"KL/std": 15.9805908203125,
"epoch": 0.1130690161527166,
"fcm_dpo/beta": 0.06670539081096649,
"fcm_dpo/delta": -0.4409904479980469,
"fcm_dpo/margin": 17.730743408203125,
"fcm_dpo/q_t": 0.27410849928855896,
"grad_norm": 48.95978927612305,
"learning_rate": 4.998386175651409e-07,
"logits/chosen": -0.6710631847381592,
"logits/rejected": -0.6282619833946228,
"logps/chosen": -62.57844924926758,
"logps/ref_chosen": -58.0966796875,
"logps/ref_rejected": -93.77361297607422,
"logps/rejected": -115.98612976074219,
"loss": 0.749,
"margin_dpo/margin_mean": 17.730743408203125,
"margin_dpo/margin_std": 16.987913131713867,
"step": 77
},
{
"KL/chosen_KL_mean": -4.559408187866211,
"KL/mean": -11.559930801391602,
"KL/rejected_KL_mean": -18.56045150756836,
"KL/std": 12.783736228942871,
"epoch": 0.1145374449339207,
"fcm_dpo/beta": 0.06502692401409149,
"fcm_dpo/delta": -0.12146870791912079,
"fcm_dpo/margin": 14.001038551330566,
"fcm_dpo/q_t": 0.31776103377342224,
"grad_norm": 43.656578063964844,
"learning_rate": 4.997892217220159e-07,
"logits/chosen": -0.6281476020812988,
"logits/rejected": -0.598828911781311,
"logps/chosen": -60.173194885253906,
"logps/ref_chosen": -55.61378479003906,
"logps/ref_rejected": -84.93436431884766,
"logps/rejected": -103.49481201171875,
"loss": 0.8351,
"margin_dpo/margin_mean": 14.001038551330566,
"margin_dpo/margin_std": 14.177305221557617,
"step": 78
},
{
"KL/chosen_KL_mean": -4.683469772338867,
"KL/mean": -12.436279296875,
"KL/rejected_KL_mean": -20.1890869140625,
"KL/std": 14.876789093017578,
"epoch": 0.11600587371512482,
"fcm_dpo/beta": 0.06247137486934662,
"fcm_dpo/delta": -0.18688446283340454,
"fcm_dpo/margin": 15.505611419677734,
"fcm_dpo/q_t": 0.3226276934146881,
"grad_norm": 37.53826141357422,
"learning_rate": 4.997332437005931e-07,
"logits/chosen": -0.6610733270645142,
"logits/rejected": -0.6301044225692749,
"logps/chosen": -60.13395690917969,
"logps/ref_chosen": -55.45048522949219,
"logps/ref_rejected": -87.64756774902344,
"logps/rejected": -107.83665466308594,
"loss": 0.8644,
"margin_dpo/margin_mean": 15.505611419677734,
"margin_dpo/margin_std": 18.106834411621094,
"step": 79
},
{
"KL/chosen_KL_mean": -6.907768249511719,
"KL/mean": -13.883707046508789,
"KL/rejected_KL_mean": -20.85964584350586,
"KL/std": 15.121198654174805,
"epoch": 0.11747430249632893,
"fcm_dpo/beta": 0.060649387538433075,
"fcm_dpo/delta": -0.05435388535261154,
"fcm_dpo/margin": 13.95186996459961,
"fcm_dpo/q_t": 0.34249839186668396,
"grad_norm": 40.209007263183594,
"learning_rate": 4.996706849759452e-07,
"logits/chosen": -0.7091320753097534,
"logits/rejected": -0.6612462401390076,
"logps/chosen": -65.42706298828125,
"logps/ref_chosen": -58.519290924072266,
"logps/ref_rejected": -87.54750061035156,
"logps/rejected": -108.40715026855469,
"loss": 0.9236,
"margin_dpo/margin_mean": 13.951870918273926,
"margin_dpo/margin_std": 17.254470825195312,
"step": 80
},
{
"KL/chosen_KL_mean": -5.891490936279297,
"KL/mean": -15.477046966552734,
"KL/rejected_KL_mean": -25.062599182128906,
"KL/std": 18.107501983642578,
"epoch": 0.11894273127753303,
"fcm_dpo/beta": 0.05760374665260315,
"fcm_dpo/delta": -0.3476817011833191,
"fcm_dpo/margin": 19.171104431152344,
"fcm_dpo/q_t": 0.2975410223007202,
"grad_norm": 43.146236419677734,
"learning_rate": 4.996015471965529e-07,
"logits/chosen": -0.7024596333503723,
"logits/rejected": -0.6684285402297974,
"logps/chosen": -72.34036254882812,
"logps/ref_chosen": -66.44886779785156,
"logps/ref_rejected": -129.66270446777344,
"logps/rejected": -154.72531127929688,
"loss": 0.8002,
"margin_dpo/margin_mean": 19.171104431152344,
"margin_dpo/margin_std": 21.296463012695312,
"step": 81
},
{
"KL/chosen_KL_mean": -8.008272171020508,
"KL/mean": -15.765302658081055,
"KL/rejected_KL_mean": -23.522335052490234,
"KL/std": 16.664772033691406,
"epoch": 0.12041116005873716,
"fcm_dpo/beta": 0.05717352032661438,
"fcm_dpo/delta": -0.09888464212417603,
"fcm_dpo/margin": 15.514065742492676,
"fcm_dpo/q_t": 0.3363991379737854,
"grad_norm": 52.070457458496094,
"learning_rate": 4.995258321842611e-07,
"logits/chosen": -0.637446403503418,
"logits/rejected": -0.6206815838813782,
"logps/chosen": -60.240657806396484,
"logps/ref_chosen": -52.232383728027344,
"logps/ref_rejected": -90.74325561523438,
"logps/rejected": -114.26558685302734,
"loss": 0.9861,
"margin_dpo/margin_mean": 15.514065742492676,
"margin_dpo/margin_std": 22.472978591918945,
"step": 82
},
{
"KL/chosen_KL_mean": -8.266363143920898,
"KL/mean": -17.189800262451172,
"KL/rejected_KL_mean": -26.113239288330078,
"KL/std": 17.502941131591797,
"epoch": 0.12187958883994127,
"fcm_dpo/beta": 0.05427241325378418,
"fcm_dpo/delta": -0.18765102326869965,
"fcm_dpo/margin": 17.846874237060547,
"fcm_dpo/q_t": 0.3166254460811615,
"grad_norm": 39.94942092895508,
"learning_rate": 4.994435419342304e-07,
"logits/chosen": -0.6970843076705933,
"logits/rejected": -0.6541832685470581,
"logps/chosen": -64.09375,
"logps/ref_chosen": -55.82738494873047,
"logps/ref_rejected": -103.71589660644531,
"logps/rejected": -129.82913208007812,
"loss": 0.8555,
"margin_dpo/margin_mean": 17.846874237060547,
"margin_dpo/margin_std": 20.124860763549805,
"step": 83
},
{
"KL/chosen_KL_mean": -6.939172744750977,
"KL/mean": -15.054475784301758,
"KL/rejected_KL_mean": -23.16977310180664,
"KL/std": 15.09716796875,
"epoch": 0.12334801762114538,
"fcm_dpo/beta": 0.05286204814910889,
"fcm_dpo/delta": -0.06635652482509613,
"fcm_dpo/margin": 16.23060417175293,
"fcm_dpo/q_t": 0.3202790319919586,
"grad_norm": 38.52730941772461,
"learning_rate": 4.993546786148857e-07,
"logits/chosen": -0.6437501311302185,
"logits/rejected": -0.604649007320404,
"logps/chosen": -74.11534118652344,
"logps/ref_chosen": -67.1761703491211,
"logps/ref_rejected": -87.29859924316406,
"logps/rejected": -110.46837615966797,
"loss": 0.8436,
"margin_dpo/margin_mean": 16.23060417175293,
"margin_dpo/margin_std": 15.421857833862305,
"step": 84
},
{
"KL/chosen_KL_mean": -7.799615859985352,
"KL/mean": -15.955020904541016,
"KL/rejected_KL_mean": -24.110427856445312,
"KL/std": 15.60763931274414,
"epoch": 0.12481644640234948,
"fcm_dpo/beta": 0.053284820169210434,
"fcm_dpo/delta": -0.07798822969198227,
"fcm_dpo/margin": 16.310810089111328,
"fcm_dpo/q_t": 0.33197450637817383,
"grad_norm": 40.09149169921875,
"learning_rate": 4.992592445678582e-07,
"logits/chosen": -0.6409064531326294,
"logits/rejected": -0.6097261905670166,
"logps/chosen": -66.20623779296875,
"logps/ref_chosen": -58.4066162109375,
"logps/ref_rejected": -78.63880157470703,
"logps/rejected": -102.74922943115234,
"loss": 0.8927,
"margin_dpo/margin_mean": 16.31081199645996,
"margin_dpo/margin_std": 18.370267868041992,
"step": 85
},
{
"KL/chosen_KL_mean": -10.027334213256836,
"KL/mean": -18.718555450439453,
"KL/rejected_KL_mean": -27.409774780273438,
"KL/std": 19.970577239990234,
"epoch": 0.1262848751835536,
"fcm_dpo/beta": 0.0518500953912735,
"fcm_dpo/delta": -0.11137088388204575,
"fcm_dpo/margin": 17.3824462890625,
"fcm_dpo/q_t": 0.34738343954086304,
"grad_norm": 48.6260871887207,
"learning_rate": 4.991572423079235e-07,
"logits/chosen": -0.7073214650154114,
"logits/rejected": -0.6920993328094482,
"logps/chosen": -66.164794921875,
"logps/ref_chosen": -56.13746643066406,
"logps/ref_rejected": -88.12165069580078,
"logps/rejected": -115.53142547607422,
"loss": 0.9935,
"margin_dpo/margin_mean": 17.3824462890625,
"margin_dpo/margin_std": 26.812076568603516,
"step": 86
},
{
"KL/chosen_KL_mean": -9.976259231567383,
"KL/mean": -19.749847412109375,
"KL/rejected_KL_mean": -29.5234375,
"KL/std": 20.053821563720703,
"epoch": 0.1277533039647577,
"fcm_dpo/beta": 0.049444474279880524,
"fcm_dpo/delta": -0.1872720569372177,
"fcm_dpo/margin": 19.547176361083984,
"fcm_dpo/q_t": 0.3172088861465454,
"grad_norm": 38.684879302978516,
"learning_rate": 4.990486745229364e-07,
"logits/chosen": -0.7268928289413452,
"logits/rejected": -0.6944937705993652,
"logps/chosen": -65.61235046386719,
"logps/ref_chosen": -55.63609313964844,
"logps/ref_rejected": -95.46757507324219,
"logps/rejected": -124.99101257324219,
"loss": 0.8925,
"margin_dpo/margin_mean": 19.547176361083984,
"margin_dpo/margin_std": 23.78310775756836,
"step": 87
},
{
"KL/chosen_KL_mean": -12.57611083984375,
"KL/mean": -20.00674057006836,
"KL/rejected_KL_mean": -27.437366485595703,
"KL/std": 20.11369514465332,
"epoch": 0.12922173274596183,
"fcm_dpo/beta": 0.049487993121147156,
"fcm_dpo/delta": 0.06895022094249725,
"fcm_dpo/margin": 14.86125373840332,
"fcm_dpo/q_t": 0.35909318923950195,
"grad_norm": 45.062110900878906,
"learning_rate": 4.989335440737586e-07,
"logits/chosen": -0.6849937438964844,
"logits/rejected": -0.6724286079406738,
"logps/chosen": -86.24726104736328,
"logps/ref_chosen": -73.67115020751953,
"logps/ref_rejected": -106.70849609375,
"logps/rejected": -134.1458740234375,
"loss": 1.0091,
"margin_dpo/margin_mean": 14.861252784729004,
"margin_dpo/margin_std": 20.61050796508789,
"step": 88
},
{
"KL/chosen_KL_mean": -7.28764533996582,
"KL/mean": -15.397772789001465,
"KL/rejected_KL_mean": -23.507896423339844,
"KL/std": 17.05303382873535,
"epoch": 0.13069016152716592,
"fcm_dpo/beta": 0.050317272543907166,
"fcm_dpo/delta": -0.018804259598255157,
"fcm_dpo/margin": 16.22024917602539,
"fcm_dpo/q_t": 0.33643651008605957,
"grad_norm": 36.859100341796875,
"learning_rate": 4.988118539941847e-07,
"logits/chosen": -0.7560995817184448,
"logits/rejected": -0.7220498323440552,
"logps/chosen": -67.91255950927734,
"logps/ref_chosen": -60.624916076660156,
"logps/ref_rejected": -82.08354949951172,
"logps/rejected": -105.59144592285156,
"loss": 0.8918,
"margin_dpo/margin_mean": 16.220251083374023,
"margin_dpo/margin_std": 18.740238189697266,
"step": 89
},
{
"KL/chosen_KL_mean": -9.896280288696289,
"KL/mean": -20.272663116455078,
"KL/rejected_KL_mean": -30.649044036865234,
"KL/std": 22.79068374633789,
"epoch": 0.13215859030837004,
"fcm_dpo/beta": 0.048810940235853195,
"fcm_dpo/delta": -0.23755799233913422,
"fcm_dpo/margin": 20.752761840820312,
"fcm_dpo/q_t": 0.32720205187797546,
"grad_norm": 40.190128326416016,
"learning_rate": 4.986836074908615e-07,
"logits/chosen": -0.6363822221755981,
"logits/rejected": -0.6379286050796509,
"logps/chosen": -63.18159103393555,
"logps/ref_chosen": -53.285308837890625,
"logps/ref_rejected": -111.54470825195312,
"logps/rejected": -142.19375610351562,
"loss": 0.9185,
"margin_dpo/margin_mean": 20.752761840820312,
"margin_dpo/margin_std": 27.751495361328125,
"step": 90
},
{
"KL/chosen_KL_mean": -9.575201034545898,
"KL/mean": -19.17307472229004,
"KL/rejected_KL_mean": -28.770950317382812,
"KL/std": 19.327835083007812,
"epoch": 0.13362701908957417,
"fcm_dpo/beta": 0.047079045325517654,
"fcm_dpo/delta": -0.11422688513994217,
"fcm_dpo/margin": 19.195751190185547,
"fcm_dpo/q_t": 0.3320731520652771,
"grad_norm": 40.31279373168945,
"learning_rate": 4.985488079432037e-07,
"logits/chosen": -0.7156394720077515,
"logits/rejected": -0.6806433796882629,
"logps/chosen": -71.37815856933594,
"logps/ref_chosen": -61.802955627441406,
"logps/ref_rejected": -87.87395477294922,
"logps/rejected": -116.64490509033203,
"loss": 0.9117,
"margin_dpo/margin_mean": 19.195751190185547,
"margin_dpo/margin_std": 23.792537689208984,
"step": 91
},
{
"KL/chosen_KL_mean": -8.991003036499023,
"KL/mean": -18.25010108947754,
"KL/rejected_KL_mean": -27.509204864501953,
"KL/std": 19.42957878112793,
"epoch": 0.13509544787077826,
"fcm_dpo/beta": 0.046396687626838684,
"fcm_dpo/delta": -0.06554846465587616,
"fcm_dpo/margin": 18.51820182800293,
"fcm_dpo/q_t": 0.3381293714046478,
"grad_norm": 36.30815887451172,
"learning_rate": 4.984074589033043e-07,
"logits/chosen": -0.7363908886909485,
"logits/rejected": -0.7118117809295654,
"logps/chosen": -60.631771087646484,
"logps/ref_chosen": -51.640769958496094,
"logps/ref_rejected": -77.88117980957031,
"logps/rejected": -105.39038848876953,
"loss": 0.9257,
"margin_dpo/margin_mean": 18.51820182800293,
"margin_dpo/margin_std": 23.363544464111328,
"step": 92
},
{
"KL/chosen_KL_mean": -9.855321884155273,
"KL/mean": -19.377309799194336,
"KL/rejected_KL_mean": -28.89929962158203,
"KL/std": 18.807472229003906,
"epoch": 0.13656387665198239,
"fcm_dpo/beta": 0.0453658290207386,
"fcm_dpo/delta": -0.07005032896995544,
"fcm_dpo/margin": 19.043975830078125,
"fcm_dpo/q_t": 0.3281649351119995,
"grad_norm": 33.93098068237305,
"learning_rate": 4.982595640958425e-07,
"logits/chosen": -0.7472556233406067,
"logits/rejected": -0.6917161345481873,
"logps/chosen": -62.384559631347656,
"logps/ref_chosen": -52.529239654541016,
"logps/ref_rejected": -77.16075134277344,
"logps/rejected": -106.06004333496094,
"loss": 0.8607,
"margin_dpo/margin_mean": 19.043975830078125,
"margin_dpo/margin_std": 21.03558349609375,
"step": 93
},
{
"KL/chosen_KL_mean": -10.773126602172852,
"KL/mean": -21.24502944946289,
"KL/rejected_KL_mean": -31.716930389404297,
"KL/std": 20.481773376464844,
"epoch": 0.13803230543318648,
"fcm_dpo/beta": 0.04396844655275345,
"fcm_dpo/delta": -0.13568861782550812,
"fcm_dpo/margin": 20.943805694580078,
"fcm_dpo/q_t": 0.317252516746521,
"grad_norm": 34.56655502319336,
"learning_rate": 4.98105127417984e-07,
"logits/chosen": -0.6687697768211365,
"logits/rejected": -0.6449911594390869,
"logps/chosen": -71.99574279785156,
"logps/ref_chosen": -61.22261047363281,
"logps/ref_rejected": -99.59902954101562,
"logps/rejected": -131.3159637451172,
"loss": 0.8318,
"margin_dpo/margin_mean": 20.943805694580078,
"margin_dpo/margin_std": 21.08070182800293,
"step": 94
},
{
"KL/chosen_KL_mean": -9.53125,
"KL/mean": -18.02033233642578,
"KL/rejected_KL_mean": -26.509416580200195,
"KL/std": 18.12246322631836,
"epoch": 0.1395007342143906,
"fcm_dpo/beta": 0.044233426451683044,
"fcm_dpo/delta": 0.05294659361243248,
"fcm_dpo/margin": 16.978164672851562,
"fcm_dpo/q_t": 0.3434492349624634,
"grad_norm": 33.531463623046875,
"learning_rate": 4.979441529392784e-07,
"logits/chosen": -0.7102452516555786,
"logits/rejected": -0.6761167049407959,
"logps/chosen": -62.054893493652344,
"logps/ref_chosen": -52.523643493652344,
"logps/ref_rejected": -75.8803482055664,
"logps/rejected": -102.3897705078125,
"loss": 0.914,
"margin_dpo/margin_mean": 16.978164672851562,
"margin_dpo/margin_std": 18.86602783203125,
"step": 95
},
{
"KL/chosen_KL_mean": -8.65886116027832,
"KL/mean": -20.042604446411133,
"KL/rejected_KL_mean": -31.426353454589844,
"KL/std": 22.305217742919922,
"epoch": 0.14096916299559473,
"fcm_dpo/beta": 0.042825281620025635,
"fcm_dpo/delta": -0.19741219282150269,
"fcm_dpo/margin": 22.767492294311523,
"fcm_dpo/q_t": 0.3105483055114746,
"grad_norm": 33.64408493041992,
"learning_rate": 4.977766449015534e-07,
"logits/chosen": -0.7110648155212402,
"logits/rejected": -0.6743212938308716,
"logps/chosen": -70.81583404541016,
"logps/ref_chosen": -62.15697479248047,
"logps/ref_rejected": -96.59601593017578,
"logps/rejected": -128.02236938476562,
"loss": 0.814,
"margin_dpo/margin_mean": 22.767492294311523,
"margin_dpo/margin_std": 24.09496307373047,
"step": 96
},
{
"KL/chosen_KL_mean": -9.822381973266602,
"KL/mean": -18.966472625732422,
"KL/rejected_KL_mean": -28.110565185546875,
"KL/std": 18.120513916015625,
"epoch": 0.14243759177679882,
"fcm_dpo/beta": 0.04335338622331619,
"fcm_dpo/delta": 0.00421653687953949,
"fcm_dpo/margin": 18.288185119628906,
"fcm_dpo/q_t": 0.3337208330631256,
"grad_norm": 36.38238525390625,
"learning_rate": 4.976026077188012e-07,
"logits/chosen": -0.6568082571029663,
"logits/rejected": -0.6052370667457581,
"logps/chosen": -64.46875,
"logps/ref_chosen": -54.646366119384766,
"logps/ref_rejected": -76.96475219726562,
"logps/rejected": -105.0753173828125,
"loss": 0.88,
"margin_dpo/margin_mean": 18.288185119628906,
"margin_dpo/margin_std": 17.953163146972656,
"step": 97
},
{
"KL/chosen_KL_mean": -11.879871368408203,
"KL/mean": -22.035350799560547,
"KL/rejected_KL_mean": -32.19082260131836,
"KL/std": 20.222049713134766,
"epoch": 0.14390602055800295,
"fcm_dpo/beta": 0.04232582449913025,
"fcm_dpo/delta": -0.06543941795825958,
"fcm_dpo/margin": 20.310956954956055,
"fcm_dpo/q_t": 0.3265250325202942,
"grad_norm": 37.31782150268555,
"learning_rate": 4.974220459770639e-07,
"logits/chosen": -0.7118183374404907,
"logits/rejected": -0.691297709941864,
"logps/chosen": -77.13850402832031,
"logps/ref_chosen": -65.25862884521484,
"logps/ref_rejected": -96.5274887084961,
"logps/rejected": -128.7183074951172,
"loss": 0.903,
"margin_dpo/margin_mean": 20.310956954956055,
"margin_dpo/margin_std": 23.548891067504883,
"step": 98
},
{
"KL/chosen_KL_mean": -9.833019256591797,
"KL/mean": -21.514665603637695,
"KL/rejected_KL_mean": -33.19630432128906,
"KL/std": 21.859981536865234,
"epoch": 0.14537444933920704,
"fcm_dpo/beta": 0.04106743261218071,
"fcm_dpo/delta": -0.17689181864261627,
"fcm_dpo/margin": 23.363290786743164,
"fcm_dpo/q_t": 0.3184036612510681,
"grad_norm": 31.978910446166992,
"learning_rate": 4.972349644343108e-07,
"logits/chosen": -0.6774739027023315,
"logits/rejected": -0.6715878248214722,
"logps/chosen": -55.47150421142578,
"logps/ref_chosen": -45.638484954833984,
"logps/ref_rejected": -86.43793487548828,
"logps/rejected": -119.63423919677734,
"loss": 0.8333,
"margin_dpo/margin_mean": 23.363290786743164,
"margin_dpo/margin_std": 26.378826141357422,
"step": 99
},
{
"KL/chosen_KL_mean": -10.707719802856445,
"KL/mean": -18.45950698852539,
"KL/rejected_KL_mean": -26.211288452148438,
"KL/std": 17.909923553466797,
"epoch": 0.14684287812041116,
"fcm_dpo/beta": 0.041958265006542206,
"fcm_dpo/delta": 0.15857800841331482,
"fcm_dpo/margin": 15.503573417663574,
"fcm_dpo/q_t": 0.36995577812194824,
"grad_norm": 38.36394119262695,
"learning_rate": 4.970413680203148e-07,
"logits/chosen": -0.6655231714248657,
"logits/rejected": -0.6192318201065063,
"logps/chosen": -68.30169677734375,
"logps/ref_chosen": -57.59397888183594,
"logps/ref_rejected": -74.06021118164062,
"logps/rejected": -100.27149963378906,
"loss": 1.031,
"margin_dpo/margin_mean": 15.503572463989258,
"margin_dpo/margin_std": 22.743520736694336,
"step": 100
},
{
"KL/chosen_KL_mean": -12.964332580566406,
"KL/mean": -21.973020553588867,
"KL/rejected_KL_mean": -30.981712341308594,
"KL/std": 20.87362289428711,
"epoch": 0.14831130690161526,
"fcm_dpo/beta": 0.04202631860971451,
"fcm_dpo/delta": 0.045243848115205765,
"fcm_dpo/margin": 18.01737403869629,
"fcm_dpo/q_t": 0.35849660634994507,
"grad_norm": 39.82667541503906,
"learning_rate": 4.968412618365215e-07,
"logits/chosen": -0.6872289180755615,
"logits/rejected": -0.6496012210845947,
"logps/chosen": -74.61318969726562,
"logps/ref_chosen": -61.64885330200195,
"logps/ref_rejected": -83.18968200683594,
"logps/rejected": -114.17140197753906,
"loss": 0.9806,
"margin_dpo/margin_mean": 18.01737403869629,
"margin_dpo/margin_std": 25.271141052246094,
"step": 101
},
{
"KL/chosen_KL_mean": -14.69502067565918,
"KL/mean": -21.546857833862305,
"KL/rejected_KL_mean": -28.398696899414062,
"KL/std": 20.012174606323242,
"epoch": 0.14977973568281938,
"fcm_dpo/beta": 0.04423201084136963,
"fcm_dpo/delta": 0.2019134908914566,
"fcm_dpo/margin": 13.70367431640625,
"fcm_dpo/q_t": 0.3829655945301056,
"grad_norm": 49.598167419433594,
"learning_rate": 4.966346511559149e-07,
"logits/chosen": -0.7153527736663818,
"logits/rejected": -0.6670923233032227,
"logps/chosen": -78.77389526367188,
"logps/ref_chosen": -64.0788803100586,
"logps/ref_rejected": -68.18707275390625,
"logps/rejected": -96.58576965332031,
"loss": 1.0845,
"margin_dpo/margin_mean": 13.703676223754883,
"margin_dpo/margin_std": 22.846996307373047,
"step": 102
},
{
"KL/chosen_KL_mean": -10.015007019042969,
"KL/mean": -22.716508865356445,
"KL/rejected_KL_mean": -35.41801071166992,
"KL/std": 21.94977569580078,
"epoch": 0.1512481644640235,
"fcm_dpo/beta": 0.042147666215896606,
"fcm_dpo/delta": -0.3043977618217468,
"fcm_dpo/margin": 25.402999877929688,
"fcm_dpo/q_t": 0.29378920793533325,
"grad_norm": 33.92921829223633,
"learning_rate": 4.964215414228785e-07,
"logits/chosen": -0.7019765377044678,
"logits/rejected": -0.6657599210739136,
"logps/chosen": -71.31428527832031,
"logps/ref_chosen": -61.299278259277344,
"logps/ref_rejected": -93.57270812988281,
"logps/rejected": -128.99072265625,
"loss": 0.7667,
"margin_dpo/margin_mean": 25.402996063232422,
"margin_dpo/margin_std": 24.849088668823242,
"step": 103
},
{
"KL/chosen_KL_mean": -11.889623641967773,
"KL/mean": -23.839998245239258,
"KL/rejected_KL_mean": -35.790382385253906,
"KL/std": 24.284305572509766,
"epoch": 0.1527165932452276,
"fcm_dpo/beta": 0.040641963481903076,
"fcm_dpo/delta": -0.18938633799552917,
"fcm_dpo/margin": 23.9007568359375,
"fcm_dpo/q_t": 0.3229272961616516,
"grad_norm": 34.873538970947266,
"learning_rate": 4.96201938253052e-07,
"logits/chosen": -0.7309329509735107,
"logits/rejected": -0.6916954517364502,
"logps/chosen": -66.26239776611328,
"logps/ref_chosen": -54.372772216796875,
"logps/ref_rejected": -89.5647201538086,
"logps/rejected": -125.3551025390625,
"loss": 0.8711,
"margin_dpo/margin_mean": 23.9007568359375,
"margin_dpo/margin_std": 28.508708953857422,
"step": 104
},
{
"KL/chosen_KL_mean": -10.295255661010742,
"KL/mean": -26.896554946899414,
"KL/rejected_KL_mean": -43.49786376953125,
"KL/std": 24.715675354003906,
"epoch": 0.15418502202643172,
"fcm_dpo/beta": 0.0372607484459877,
"fcm_dpo/delta": -0.4987417459487915,
"fcm_dpo/margin": 33.20261001586914,
"fcm_dpo/q_t": 0.2553773522377014,
"grad_norm": 30.176359176635742,
"learning_rate": 4.959758474331832e-07,
"logits/chosen": -0.7114518880844116,
"logits/rejected": -0.6793392896652222,
"logps/chosen": -64.9342041015625,
"logps/ref_chosen": -54.638946533203125,
"logps/ref_rejected": -97.97351837158203,
"logps/rejected": -141.47137451171875,
"loss": 0.6601,
"margin_dpo/margin_mean": 33.20261001586914,
"margin_dpo/margin_std": 26.298891067504883,
"step": 105
},
{
"KL/chosen_KL_mean": -12.043193817138672,
"KL/mean": -22.559654235839844,
"KL/rejected_KL_mean": -33.07612228393555,
"KL/std": 20.727237701416016,
"epoch": 0.15565345080763582,
"fcm_dpo/beta": 0.0364394336938858,
"fcm_dpo/delta": 0.03637177497148514,
"fcm_dpo/margin": 21.03292465209961,
"fcm_dpo/q_t": 0.3410166800022125,
"grad_norm": 31.546979904174805,
"learning_rate": 4.957432749209755e-07,
"logits/chosen": -0.6615180969238281,
"logits/rejected": -0.6151139140129089,
"logps/chosen": -66.87609100341797,
"logps/ref_chosen": -54.83289337158203,
"logps/ref_rejected": -85.22461700439453,
"logps/rejected": -118.30073547363281,
"loss": 0.8977,
"margin_dpo/margin_mean": 21.03292465209961,
"margin_dpo/margin_std": 22.555233001708984,
"step": 106
},
{
"KL/chosen_KL_mean": -14.550010681152344,
"KL/mean": -26.484832763671875,
"KL/rejected_KL_mean": -38.419654846191406,
"KL/std": 24.61465072631836,
"epoch": 0.15712187958883994,
"fcm_dpo/beta": 0.0360017865896225,
"fcm_dpo/delta": -0.06571710109710693,
"fcm_dpo/margin": 23.869640350341797,
"fcm_dpo/q_t": 0.3263270854949951,
"grad_norm": 30.792583465576172,
"learning_rate": 4.955042268449307e-07,
"logits/chosen": -0.7245312929153442,
"logits/rejected": -0.6730589866638184,
"logps/chosen": -84.25782775878906,
"logps/ref_chosen": -69.70780944824219,
"logps/ref_rejected": -94.73950958251953,
"logps/rejected": -133.15916442871094,
"loss": 0.8737,
"margin_dpo/margin_mean": 23.869640350341797,
"margin_dpo/margin_std": 25.510066986083984,
"step": 107
},
{
"KL/chosen_KL_mean": -13.597505569458008,
"KL/mean": -26.603561401367188,
"KL/rejected_KL_mean": -39.60961151123047,
"KL/std": 26.066404342651367,
"epoch": 0.15859030837004406,
"fcm_dpo/beta": 0.03532955050468445,
"fcm_dpo/delta": -0.1311628818511963,
"fcm_dpo/margin": 26.012113571166992,
"fcm_dpo/q_t": 0.33340299129486084,
"grad_norm": 34.07270050048828,
"learning_rate": 4.952587095041881e-07,
"logits/chosen": -0.6948635578155518,
"logits/rejected": -0.6569874882698059,
"logps/chosen": -69.60739135742188,
"logps/ref_chosen": -56.0098876953125,
"logps/ref_rejected": -95.79601287841797,
"logps/rejected": -135.40562438964844,
"loss": 0.9154,
"margin_dpo/margin_mean": 26.012113571166992,
"margin_dpo/margin_std": 33.049766540527344,
"step": 108
},
{
"KL/chosen_KL_mean": -12.223468780517578,
"KL/mean": -26.6085205078125,
"KL/rejected_KL_mean": -40.99357604980469,
"KL/std": 25.966686248779297,
"epoch": 0.16005873715124816,
"fcm_dpo/beta": 0.03398079425096512,
"fcm_dpo/delta": -0.19790343940258026,
"fcm_dpo/margin": 28.770111083984375,
"fcm_dpo/q_t": 0.3054792881011963,
"grad_norm": 30.806764602661133,
"learning_rate": 4.95006729368358e-07,
"logits/chosen": -0.6443037986755371,
"logits/rejected": -0.618833065032959,
"logps/chosen": -75.10895538330078,
"logps/ref_chosen": -62.88549041748047,
"logps/ref_rejected": -98.68573760986328,
"logps/rejected": -139.6793212890625,
"loss": 0.8036,
"margin_dpo/margin_mean": 28.770111083984375,
"margin_dpo/margin_std": 27.74138641357422,
"step": 109
},
{
"KL/chosen_KL_mean": -11.736026763916016,
"KL/mean": -24.941783905029297,
"KL/rejected_KL_mean": -38.14754104614258,
"KL/std": 25.426937103271484,
"epoch": 0.16152716593245228,
"fcm_dpo/beta": 0.0328492596745491,
"fcm_dpo/delta": -0.08019863814115524,
"fcm_dpo/margin": 26.411510467529297,
"fcm_dpo/q_t": 0.32906657457351685,
"grad_norm": 29.275253295898438,
"learning_rate": 4.947482930773511e-07,
"logits/chosen": -0.6451650857925415,
"logits/rejected": -0.5977135896682739,
"logps/chosen": -70.48971557617188,
"logps/ref_chosen": -58.753684997558594,
"logps/ref_rejected": -79.75001525878906,
"logps/rejected": -117.89755249023438,
"loss": 0.893,
"margin_dpo/margin_mean": 26.411510467529297,
"margin_dpo/margin_std": 29.890933990478516,
"step": 110
},
{
"KL/chosen_KL_mean": -14.619787216186523,
"KL/mean": -28.84766387939453,
"KL/rejected_KL_mean": -43.075538635253906,
"KL/std": 28.174575805664062,
"epoch": 0.16299559471365638,
"fcm_dpo/beta": 0.032323651015758514,
"fcm_dpo/delta": -0.13409429788589478,
"fcm_dpo/margin": 28.45575523376465,
"fcm_dpo/q_t": 0.32236313819885254,
"grad_norm": 30.668977737426758,
"learning_rate": 4.944834074412042e-07,
"logits/chosen": -0.7059623003005981,
"logits/rejected": -0.6807644367218018,
"logps/chosen": -83.243896484375,
"logps/ref_chosen": -68.62410736083984,
"logps/ref_rejected": -98.42886352539062,
"logps/rejected": -141.50439453125,
"loss": 0.8786,
"margin_dpo/margin_mean": 28.45575523376465,
"margin_dpo/margin_std": 32.44450378417969,
"step": 111
},
{
"KL/chosen_KL_mean": -13.895004272460938,
"KL/mean": -23.563627243041992,
"KL/rejected_KL_mean": -33.23225402832031,
"KL/std": 21.184219360351562,
"epoch": 0.1644640234948605,
"fcm_dpo/beta": 0.032971762120723724,
"fcm_dpo/delta": 0.1737350970506668,
"fcm_dpo/margin": 19.33725357055664,
"fcm_dpo/q_t": 0.3689352869987488,
"grad_norm": 31.53493309020996,
"learning_rate": 4.942120794399002e-07,
"logits/chosen": -0.6969665884971619,
"logits/rejected": -0.6505686640739441,
"logps/chosen": -64.14464569091797,
"logps/ref_chosen": -50.24964141845703,
"logps/ref_rejected": -64.77442932128906,
"logps/rejected": -98.00668334960938,
"loss": 0.9915,
"margin_dpo/margin_mean": 19.33725357055664,
"margin_dpo/margin_std": 24.64777183532715,
"step": 112
},
{
"KL/chosen_KL_mean": -15.778499603271484,
"KL/mean": -26.65062713623047,
"KL/rejected_KL_mean": -37.522762298583984,
"KL/std": 20.39154815673828,
"epoch": 0.16593245227606462,
"fcm_dpo/beta": 0.03382117301225662,
"fcm_dpo/delta": 0.06972439587116241,
"fcm_dpo/margin": 21.744258880615234,
"fcm_dpo/q_t": 0.34526845812797546,
"grad_norm": 30.338754653930664,
"learning_rate": 4.939343162231841e-07,
"logits/chosen": -0.6443773508071899,
"logits/rejected": -0.5929930210113525,
"logps/chosen": -82.491455078125,
"logps/ref_chosen": -66.71295166015625,
"logps/ref_rejected": -77.96870422363281,
"logps/rejected": -115.49147033691406,
"loss": 0.9041,
"margin_dpo/margin_mean": 21.744258880615234,
"margin_dpo/margin_std": 22.552532196044922,
"step": 113
},
{
"KL/chosen_KL_mean": -14.374372482299805,
"KL/mean": -30.3890438079834,
"KL/rejected_KL_mean": -46.403717041015625,
"KL/std": 31.068870544433594,
"epoch": 0.16740088105726872,
"fcm_dpo/beta": 0.03228652477264404,
"fcm_dpo/delta": -0.2666097581386566,
"fcm_dpo/margin": 32.02934265136719,
"fcm_dpo/q_t": 0.30826088786125183,
"grad_norm": 28.729209899902344,
"learning_rate": 4.936501251103751e-07,
"logits/chosen": -0.6775549650192261,
"logits/rejected": -0.6369887590408325,
"logps/chosen": -72.15945434570312,
"logps/ref_chosen": -57.78507995605469,
"logps/ref_rejected": -87.10966491699219,
"logps/rejected": -133.5133819580078,
"loss": 0.8171,
"margin_dpo/margin_mean": 32.02934265136719,
"margin_dpo/margin_std": 36.730525970458984,
"step": 114
},
{
"KL/chosen_KL_mean": -19.756134033203125,
"KL/mean": -31.28628921508789,
"KL/rejected_KL_mean": -42.816444396972656,
"KL/std": 28.551877975463867,
"epoch": 0.16886930983847284,
"fcm_dpo/beta": 0.03259563446044922,
"fcm_dpo/delta": 0.0514962375164032,
"fcm_dpo/margin": 23.0603084564209,
"fcm_dpo/q_t": 0.3605424761772156,
"grad_norm": 44.56121826171875,
"learning_rate": 4.933595135901732e-07,
"logits/chosen": -0.7127076387405396,
"logits/rejected": -0.6788430213928223,
"logps/chosen": -85.33877563476562,
"logps/ref_chosen": -65.5826416015625,
"logps/ref_rejected": -98.56552124023438,
"logps/rejected": -141.3819580078125,
"loss": 1.0188,
"margin_dpo/margin_mean": 23.0603084564209,
"margin_dpo/margin_std": 35.82677459716797,
"step": 115
},
{
"KL/chosen_KL_mean": -14.837322235107422,
"KL/mean": -26.82990837097168,
"KL/rejected_KL_mean": -38.82250213623047,
"KL/std": 24.2340145111084,
"epoch": 0.17033773861967694,
"fcm_dpo/beta": 0.03275762125849724,
"fcm_dpo/delta": 0.014920920133590698,
"fcm_dpo/margin": 23.98516845703125,
"fcm_dpo/q_t": 0.33930882811546326,
"grad_norm": 31.535541534423828,
"learning_rate": 4.930624893204624e-07,
"logits/chosen": -0.6988054513931274,
"logits/rejected": -0.6746259331703186,
"logps/chosen": -66.23763275146484,
"logps/ref_chosen": -51.40031433105469,
"logps/ref_rejected": -80.5218505859375,
"logps/rejected": -119.34435272216797,
"loss": 0.8796,
"margin_dpo/margin_mean": 23.985170364379883,
"margin_dpo/margin_std": 25.571834564208984,
"step": 116
},
{
"KL/chosen_KL_mean": -17.652626037597656,
"KL/mean": -28.647598266601562,
"KL/rejected_KL_mean": -39.642574310302734,
"KL/std": 25.565113067626953,
"epoch": 0.17180616740088106,
"fcm_dpo/beta": 0.03300020843744278,
"fcm_dpo/delta": 0.080161452293396,
"fcm_dpo/margin": 21.989948272705078,
"fcm_dpo/q_t": 0.3538960814476013,
"grad_norm": 39.028663635253906,
"learning_rate": 4.927590601281083e-07,
"logits/chosen": -0.6682844161987305,
"logits/rejected": -0.628246545791626,
"logps/chosen": -86.95103454589844,
"logps/ref_chosen": -69.29840850830078,
"logps/ref_rejected": -66.583984375,
"logps/rejected": -106.2265625,
"loss": 0.9556,
"margin_dpo/margin_mean": 21.989948272705078,
"margin_dpo/margin_std": 28.185462951660156,
"step": 117
},
{
"KL/chosen_KL_mean": -15.85186767578125,
"KL/mean": -27.46802520751953,
"KL/rejected_KL_mean": -39.08417892456055,
"KL/std": 23.371967315673828,
"epoch": 0.17327459618208516,
"fcm_dpo/beta": 0.03330120071768761,
"fcm_dpo/delta": 0.028526771813631058,
"fcm_dpo/margin": 23.232316970825195,
"fcm_dpo/q_t": 0.3429677188396454,
"grad_norm": 48.863182067871094,
"learning_rate": 4.924492340087524e-07,
"logits/chosen": -0.6977423429489136,
"logits/rejected": -0.6685633659362793,
"logps/chosen": -71.49284362792969,
"logps/ref_chosen": -55.6409797668457,
"logps/ref_rejected": -75.66905975341797,
"logps/rejected": -114.75323486328125,
"loss": 0.8972,
"margin_dpo/margin_mean": 23.232318878173828,
"margin_dpo/margin_std": 25.548391342163086,
"step": 118
},
{
"KL/chosen_KL_mean": -19.729721069335938,
"KL/mean": -31.524303436279297,
"KL/rejected_KL_mean": -43.31888961791992,
"KL/std": 25.345006942749023,
"epoch": 0.17474302496328928,
"fcm_dpo/beta": 0.03315918147563934,
"fcm_dpo/delta": 0.018118254840373993,
"fcm_dpo/margin": 23.58917236328125,
"fcm_dpo/q_t": 0.3480648994445801,
"grad_norm": 34.203346252441406,
"learning_rate": 4.92133019126601e-07,
"logits/chosen": -0.6777770519256592,
"logits/rejected": -0.6563413739204407,
"logps/chosen": -93.23991394042969,
"logps/ref_chosen": -73.51019287109375,
"logps/ref_rejected": -102.977294921875,
"logps/rejected": -146.29617309570312,
"loss": 0.9387,
"margin_dpo/margin_mean": 23.58917236328125,
"margin_dpo/margin_std": 29.872432708740234,
"step": 119
},
{
"KL/chosen_KL_mean": -21.17425537109375,
"KL/mean": -36.770111083984375,
"KL/rejected_KL_mean": -52.365962982177734,
"KL/std": 31.41073226928711,
"epoch": 0.1762114537444934,
"fcm_dpo/beta": 0.03230596333742142,
"fcm_dpo/delta": -0.23119711875915527,
"fcm_dpo/margin": 31.19170379638672,
"fcm_dpo/q_t": 0.30633899569511414,
"grad_norm": 32.835208892822266,
"learning_rate": 4.918104238142103e-07,
"logits/chosen": -0.6844779253005981,
"logits/rejected": -0.6430919766426086,
"logps/chosen": -97.95509338378906,
"logps/ref_chosen": -76.78083801269531,
"logps/ref_rejected": -108.02374267578125,
"logps/rejected": -160.38970947265625,
"loss": 0.8181,
"margin_dpo/margin_mean": 31.19170379638672,
"margin_dpo/margin_std": 32.8134765625,
"step": 120
},
{
"KL/chosen_KL_mean": -20.149927139282227,
"KL/mean": -37.27117156982422,
"KL/rejected_KL_mean": -54.39240646362305,
"KL/std": 31.102367401123047,
"epoch": 0.1776798825256975,
"fcm_dpo/beta": 0.03047577664256096,
"fcm_dpo/delta": -0.27470406889915466,
"fcm_dpo/margin": 34.24247360229492,
"fcm_dpo/q_t": 0.30704832077026367,
"grad_norm": 32.69995880126953,
"learning_rate": 4.91481456572267e-07,
"logits/chosen": -0.6667851209640503,
"logits/rejected": -0.6555989384651184,
"logps/chosen": -81.9398193359375,
"logps/ref_chosen": -61.789894104003906,
"logps/ref_rejected": -109.99456787109375,
"logps/rejected": -164.38697814941406,
"loss": 0.8259,
"margin_dpo/margin_mean": 34.24247360229492,
"margin_dpo/margin_std": 37.63690948486328,
"step": 121
},
{
"KL/chosen_KL_mean": -18.689184188842773,
"KL/mean": -39.89842224121094,
"KL/rejected_KL_mean": -61.107669830322266,
"KL/std": 36.409278869628906,
"epoch": 0.17914831130690162,
"fcm_dpo/beta": 0.02825929969549179,
"fcm_dpo/delta": -0.4536348581314087,
"fcm_dpo/margin": 42.418479919433594,
"fcm_dpo/q_t": 0.26600882411003113,
"grad_norm": 31.267995834350586,
"learning_rate": 4.911461260693638e-07,
"logits/chosen": -0.6460617780685425,
"logits/rejected": -0.6521382927894592,
"logps/chosen": -65.59139251708984,
"logps/ref_chosen": -46.9022102355957,
"logps/ref_rejected": -106.71418762207031,
"logps/rejected": -167.82186889648438,
"loss": 0.6902,
"margin_dpo/margin_mean": 42.41847610473633,
"margin_dpo/margin_std": 35.58713150024414,
"step": 122
},
{
"KL/chosen_KL_mean": -19.802492141723633,
"KL/mean": -33.98395919799805,
"KL/rejected_KL_mean": -48.16543197631836,
"KL/std": 30.98874282836914,
"epoch": 0.18061674008810572,
"fcm_dpo/beta": 0.027563009411096573,
"fcm_dpo/delta": 0.01940111070871353,
"fcm_dpo/margin": 28.362932205200195,
"fcm_dpo/q_t": 0.35159093141555786,
"grad_norm": 32.10316467285156,
"learning_rate": 4.908044411417711e-07,
"logits/chosen": -0.6464723348617554,
"logits/rejected": -0.6184071898460388,
"logps/chosen": -81.14112854003906,
"logps/ref_chosen": -61.33863830566406,
"logps/ref_rejected": -87.775390625,
"logps/rejected": -135.94082641601562,
"loss": 0.984,
"margin_dpo/margin_mean": 28.362932205200195,
"margin_dpo/margin_std": 40.174957275390625,
"step": 123
},
{
"KL/chosen_KL_mean": -21.405563354492188,
"KL/mean": -40.972625732421875,
"KL/rejected_KL_mean": -60.53968811035156,
"KL/std": 40.26324462890625,
"epoch": 0.18208516886930984,
"fcm_dpo/beta": 0.0265456922352314,
"fcm_dpo/delta": -0.26791417598724365,
"fcm_dpo/margin": 39.134124755859375,
"fcm_dpo/q_t": 0.3195335566997528,
"grad_norm": 32.24935531616211,
"learning_rate": 4.904564107932048e-07,
"logits/chosen": -0.664286732673645,
"logits/rejected": -0.6608208417892456,
"logps/chosen": -92.85389709472656,
"logps/ref_chosen": -71.44833374023438,
"logps/ref_rejected": -117.58056640625,
"logps/rejected": -178.12025451660156,
"loss": 0.8779,
"margin_dpo/margin_mean": 39.134124755859375,
"margin_dpo/margin_std": 50.061702728271484,
"step": 124
},
{
"KL/chosen_KL_mean": -17.320648193359375,
"KL/mean": -35.29172134399414,
"KL/rejected_KL_mean": -53.2628059387207,
"KL/std": 32.852684020996094,
"epoch": 0.18355359765051396,
"fcm_dpo/beta": 0.02581089735031128,
"fcm_dpo/delta": -0.14052706956863403,
"fcm_dpo/margin": 35.94215393066406,
"fcm_dpo/q_t": 0.31883522868156433,
"grad_norm": 27.765161514282227,
"learning_rate": 4.90102044194588e-07,
"logits/chosen": -0.6267136335372925,
"logits/rejected": -0.615674614906311,
"logps/chosen": -67.45758819580078,
"logps/ref_chosen": -50.136940002441406,
"logps/ref_rejected": -83.98861694335938,
"logps/rejected": -137.2514190673828,
"loss": 0.8366,
"margin_dpo/margin_mean": 35.94215393066406,
"margin_dpo/margin_std": 37.57044982910156,
"step": 125
},
{
"KL/chosen_KL_mean": -19.33326530456543,
"KL/mean": -36.09067153930664,
"KL/rejected_KL_mean": -52.84807586669922,
"KL/std": 30.532608032226562,
"epoch": 0.18502202643171806,
"fcm_dpo/beta": 0.02531757578253746,
"fcm_dpo/delta": -0.05310361459851265,
"fcm_dpo/margin": 33.51481628417969,
"fcm_dpo/q_t": 0.330096036195755,
"grad_norm": 30.099266052246094,
"learning_rate": 4.897413506838102e-07,
"logits/chosen": -0.6573269367218018,
"logits/rejected": -0.6356841921806335,
"logps/chosen": -75.00033569335938,
"logps/ref_chosen": -55.66706848144531,
"logps/ref_rejected": -98.1297607421875,
"logps/rejected": -150.97784423828125,
"loss": 0.8715,
"margin_dpo/margin_mean": 33.514808654785156,
"margin_dpo/margin_std": 36.43012237548828,
"step": 126
},
{
"KL/chosen_KL_mean": -17.67395782470703,
"KL/mean": -31.366748809814453,
"KL/rejected_KL_mean": -45.05954360961914,
"KL/std": 26.2576904296875,
"epoch": 0.18649045521292218,
"fcm_dpo/beta": 0.025727272033691406,
"fcm_dpo/delta": 0.1022951602935791,
"fcm_dpo/margin": 27.38558006286621,
"fcm_dpo/q_t": 0.35243093967437744,
"grad_norm": 29.7404727935791,
"learning_rate": 4.89374339765481e-07,
"logits/chosen": -0.6145305633544922,
"logits/rejected": -0.5852265357971191,
"logps/chosen": -74.2286376953125,
"logps/ref_chosen": -56.55467987060547,
"logps/ref_rejected": -76.7957763671875,
"logps/rejected": -121.8553237915039,
"loss": 0.9369,
"margin_dpo/margin_mean": 27.38558006286621,
"margin_dpo/margin_std": 30.984207153320312,
"step": 127
},
{
"KL/chosen_KL_mean": -19.56759262084961,
"KL/mean": -33.658756256103516,
"KL/rejected_KL_mean": -47.749916076660156,
"KL/std": 32.29600524902344,
"epoch": 0.18795888399412627,
"fcm_dpo/beta": 0.026202715933322906,
"fcm_dpo/delta": 0.06536929309368134,
"fcm_dpo/margin": 28.182331085205078,
"fcm_dpo/q_t": 0.355803906917572,
"grad_norm": 37.05172348022461,
"learning_rate": 4.890010211106795e-07,
"logits/chosen": -0.6373168230056763,
"logits/rejected": -0.598369836807251,
"logps/chosen": -77.68855285644531,
"logps/ref_chosen": -58.12095642089844,
"logps/ref_rejected": -76.43896484375,
"logps/rejected": -124.18888092041016,
"loss": 0.9771,
"margin_dpo/margin_mean": 28.182331085205078,
"margin_dpo/margin_std": 38.03752136230469,
"step": 128
},
{
"KL/chosen_KL_mean": -25.679073333740234,
"KL/mean": -40.25211715698242,
"KL/rejected_KL_mean": -54.825164794921875,
"KL/std": 33.20759201049805,
"epoch": 0.1894273127753304,
"fcm_dpo/beta": 0.02634388394653797,
"fcm_dpo/delta": 0.03461331129074097,
"fcm_dpo/margin": 29.14608383178711,
"fcm_dpo/q_t": 0.3556867241859436,
"grad_norm": 34.6231575012207,
"learning_rate": 4.88621404556699e-07,
"logits/chosen": -0.6257126331329346,
"logits/rejected": -0.6025162935256958,
"logps/chosen": -92.59544372558594,
"logps/ref_chosen": -66.91637420654297,
"logps/ref_rejected": -96.6422119140625,
"logps/rejected": -151.46737670898438,
"loss": 0.9911,
"margin_dpo/margin_mean": 29.14608383178711,
"margin_dpo/margin_std": 41.93766784667969,
"step": 129
},
{
"KL/chosen_KL_mean": -19.17430305480957,
"KL/mean": -40.231685638427734,
"KL/rejected_KL_mean": -61.28907012939453,
"KL/std": 37.78228759765625,
"epoch": 0.19089574155653452,
"fcm_dpo/beta": 0.02557522803544998,
"fcm_dpo/delta": -0.3111729323863983,
"fcm_dpo/margin": 42.11476135253906,
"fcm_dpo/q_t": 0.2972610890865326,
"grad_norm": 27.576147079467773,
"learning_rate": 4.882355001067891e-07,
"logits/chosen": -0.6214011907577515,
"logits/rejected": -0.6146156191825867,
"logps/chosen": -63.841156005859375,
"logps/ref_chosen": -44.66685104370117,
"logps/ref_rejected": -82.78165435791016,
"logps/rejected": -144.0707244873047,
"loss": 0.8184,
"margin_dpo/margin_mean": 42.11476135253906,
"margin_dpo/margin_std": 42.95219802856445,
"step": 130
},
{
"KL/chosen_KL_mean": -15.329935073852539,
"KL/mean": -35.1269416809082,
"KL/rejected_KL_mean": -54.923946380615234,
"KL/std": 34.3599967956543,
"epoch": 0.19236417033773862,
"fcm_dpo/beta": 0.02407342568039894,
"fcm_dpo/delta": -0.169725239276886,
"fcm_dpo/margin": 39.59401321411133,
"fcm_dpo/q_t": 0.3058611750602722,
"grad_norm": 28.302913665771484,
"learning_rate": 4.878433179298909e-07,
"logits/chosen": -0.6235455274581909,
"logits/rejected": -0.6231019496917725,
"logps/chosen": -60.25452423095703,
"logps/ref_chosen": -44.924591064453125,
"logps/ref_rejected": -88.44401550292969,
"logps/rejected": -143.36795043945312,
"loss": 0.8077,
"margin_dpo/margin_mean": 39.59400939941406,
"margin_dpo/margin_std": 37.83281326293945,
"step": 131
},
{
"KL/chosen_KL_mean": -21.70701026916504,
"KL/mean": -39.37970733642578,
"KL/rejected_KL_mean": -57.05239486694336,
"KL/std": 35.41930389404297,
"epoch": 0.19383259911894274,
"fcm_dpo/beta": 0.023686932399868965,
"fcm_dpo/delta": -0.04097011685371399,
"fcm_dpo/margin": 35.34538269042969,
"fcm_dpo/q_t": 0.33768439292907715,
"grad_norm": 29.296648025512695,
"learning_rate": 4.874448683603694e-07,
"logits/chosen": -0.6430600881576538,
"logits/rejected": -0.6259501576423645,
"logps/chosen": -80.70809936523438,
"logps/ref_chosen": -59.00108337402344,
"logps/ref_rejected": -87.89215087890625,
"logps/rejected": -144.94454956054688,
"loss": 0.9004,
"margin_dpo/margin_mean": 35.34538269042969,
"margin_dpo/margin_std": 43.01177978515625,
"step": 132
},
{
"KL/chosen_KL_mean": -25.513275146484375,
"KL/mean": -41.42220687866211,
"KL/rejected_KL_mean": -57.33114242553711,
"KL/std": 34.27680206298828,
"epoch": 0.19530102790014683,
"fcm_dpo/beta": 0.023830700665712357,
"fcm_dpo/delta": 0.04520042985677719,
"fcm_dpo/margin": 31.81786346435547,
"fcm_dpo/q_t": 0.35364824533462524,
"grad_norm": 29.429222106933594,
"learning_rate": 4.870401618977415e-07,
"logits/chosen": -0.6595304012298584,
"logits/rejected": -0.6431485414505005,
"logps/chosen": -92.1177749633789,
"logps/ref_chosen": -66.60449981689453,
"logps/ref_rejected": -96.33355712890625,
"logps/rejected": -153.66470336914062,
"loss": 0.9495,
"margin_dpo/margin_mean": 31.81786346435547,
"margin_dpo/margin_std": 41.558082580566406,
"step": 133
},
{
"KL/chosen_KL_mean": -19.793434143066406,
"KL/mean": -37.411041259765625,
"KL/rejected_KL_mean": -55.02864456176758,
"KL/std": 33.86731719970703,
"epoch": 0.19676945668135096,
"fcm_dpo/beta": 0.023871179670095444,
"fcm_dpo/delta": -0.045072250068187714,
"fcm_dpo/margin": 35.2352180480957,
"fcm_dpo/q_t": 0.3298512101173401,
"grad_norm": 27.502216339111328,
"learning_rate": 4.866292092063986e-07,
"logits/chosen": -0.6473065614700317,
"logits/rejected": -0.6194508075714111,
"logps/chosen": -71.86268615722656,
"logps/ref_chosen": -52.06925582885742,
"logps/ref_rejected": -87.6545181274414,
"logps/rejected": -142.68316650390625,
"loss": 0.8507,
"margin_dpo/margin_mean": 35.23521423339844,
"margin_dpo/margin_std": 36.012550354003906,
"step": 134
},
{
"KL/chosen_KL_mean": -24.061479568481445,
"KL/mean": -45.499755859375,
"KL/rejected_KL_mean": -66.93803405761719,
"KL/std": 40.95748519897461,
"epoch": 0.19823788546255505,
"fcm_dpo/beta": 0.023015718907117844,
"fcm_dpo/delta": -0.20716674625873566,
"fcm_dpo/margin": 42.87656021118164,
"fcm_dpo/q_t": 0.3147345781326294,
"grad_norm": 29.010801315307617,
"learning_rate": 4.862120211153265e-07,
"logits/chosen": -0.6196516752243042,
"logits/rejected": -0.6404733657836914,
"logps/chosen": -74.41534423828125,
"logps/ref_chosen": -50.353858947753906,
"logps/ref_rejected": -115.97975158691406,
"logps/rejected": -182.91778564453125,
"loss": 0.8439,
"margin_dpo/margin_mean": 42.876556396484375,
"margin_dpo/margin_std": 47.684814453125,
"step": 135
},
{
"KL/chosen_KL_mean": -27.37831687927246,
"KL/mean": -42.851158142089844,
"KL/rejected_KL_mean": -58.32400131225586,
"KL/std": 36.29102325439453,
"epoch": 0.19970631424375918,
"fcm_dpo/beta": 0.022867830470204353,
"fcm_dpo/delta": 0.09842132776975632,
"fcm_dpo/margin": 30.945680618286133,
"fcm_dpo/q_t": 0.36907869577407837,
"grad_norm": 30.862062454223633,
"learning_rate": 4.857886086178193e-07,
"logits/chosen": -0.6548190116882324,
"logits/rejected": -0.6352522373199463,
"logps/chosen": -92.4508285522461,
"logps/ref_chosen": -65.072509765625,
"logps/ref_rejected": -96.32122802734375,
"logps/rejected": -154.64523315429688,
"loss": 0.9884,
"margin_dpo/margin_mean": 30.945680618286133,
"margin_dpo/margin_std": 44.39238357543945,
"step": 136
},
{
"KL/chosen_KL_mean": -28.5081844329834,
"KL/mean": -51.93169403076172,
"KL/rejected_KL_mean": -75.35520935058594,
"KL/std": 50.29423141479492,
"epoch": 0.2011747430249633,
"fcm_dpo/beta": 0.022213459014892578,
"fcm_dpo/delta": -0.26994192600250244,
"fcm_dpo/margin": 46.847015380859375,
"fcm_dpo/q_t": 0.3159807026386261,
"grad_norm": 29.10369110107422,
"learning_rate": 4.853589828711902e-07,
"logits/chosen": -0.5867665410041809,
"logits/rejected": -0.6017188429832458,
"logps/chosen": -77.26730346679688,
"logps/ref_chosen": -48.759117126464844,
"logps/ref_rejected": -113.86376953125,
"logps/rejected": -189.21897888183594,
"loss": 0.8694,
"margin_dpo/margin_mean": 46.847015380859375,
"margin_dpo/margin_std": 57.939029693603516,
"step": 137
},
{
"KL/chosen_KL_mean": -29.796592712402344,
"KL/mean": -47.36405563354492,
"KL/rejected_KL_mean": -64.9315185546875,
"KL/std": 38.10211944580078,
"epoch": 0.2026431718061674,
"fcm_dpo/beta": 0.022151703014969826,
"fcm_dpo/delta": 0.023063896223902702,
"fcm_dpo/margin": 35.134925842285156,
"fcm_dpo/q_t": 0.340358704328537,
"grad_norm": 30.460988998413086,
"learning_rate": 4.849231551964771e-07,
"logits/chosen": -0.5743345022201538,
"logits/rejected": -0.5551059246063232,
"logps/chosen": -90.31623840332031,
"logps/ref_chosen": -60.519649505615234,
"logps/ref_rejected": -93.19694519042969,
"logps/rejected": -158.1284637451172,
"loss": 0.9103,
"margin_dpo/margin_mean": 35.134925842285156,
"margin_dpo/margin_std": 39.568973541259766,
"step": 138
},
{
"KL/chosen_KL_mean": -23.029796600341797,
"KL/mean": -45.01148986816406,
"KL/rejected_KL_mean": -66.9931869506836,
"KL/std": 37.149497985839844,
"epoch": 0.20411160058737152,
"fcm_dpo/beta": 0.021606380119919777,
"fcm_dpo/delta": -0.16534382104873657,
"fcm_dpo/margin": 43.96338653564453,
"fcm_dpo/q_t": 0.31225913763046265,
"grad_norm": 27.354082107543945,
"learning_rate": 4.844811370781446e-07,
"logits/chosen": -0.6141539812088013,
"logits/rejected": -0.5956014394760132,
"logps/chosen": -69.92118072509766,
"logps/ref_chosen": -46.89138412475586,
"logps/ref_rejected": -79.72798156738281,
"logps/rejected": -146.72116088867188,
"loss": 0.8173,
"margin_dpo/margin_mean": 43.96338653564453,
"margin_dpo/margin_std": 44.648704528808594,
"step": 139
},
{
"KL/chosen_KL_mean": -27.68996238708496,
"KL/mean": -48.107666015625,
"KL/rejected_KL_mean": -68.52537536621094,
"KL/std": 42.47513961791992,
"epoch": 0.2055800293685756,
"fcm_dpo/beta": 0.02111215889453888,
"fcm_dpo/delta": -0.06818899512290955,
"fcm_dpo/margin": 40.83540344238281,
"fcm_dpo/q_t": 0.33521249890327454,
"grad_norm": 28.752126693725586,
"learning_rate": 4.840329401637809e-07,
"logits/chosen": -0.614875316619873,
"logits/rejected": -0.5954533815383911,
"logps/chosen": -86.6646728515625,
"logps/ref_chosen": -58.97471618652344,
"logps/ref_rejected": -83.28410339355469,
"logps/rejected": -151.80947875976562,
"loss": 0.9081,
"margin_dpo/margin_mean": 40.83540344238281,
"margin_dpo/margin_std": 49.618003845214844,
"step": 140
},
{
"KL/chosen_KL_mean": -27.771591186523438,
"KL/mean": -47.71379470825195,
"KL/rejected_KL_mean": -67.65599822998047,
"KL/std": 41.21592712402344,
"epoch": 0.20704845814977973,
"fcm_dpo/beta": 0.0209644865244627,
"fcm_dpo/delta": -0.03944108262658119,
"fcm_dpo/margin": 39.88440704345703,
"fcm_dpo/q_t": 0.33007389307022095,
"grad_norm": 30.820947647094727,
"learning_rate": 4.83578576263792e-07,
"logits/chosen": -0.6197242736816406,
"logits/rejected": -0.5986499190330505,
"logps/chosen": -102.84725952148438,
"logps/ref_chosen": -75.07566833496094,
"logps/ref_rejected": -98.1922607421875,
"logps/rejected": -165.8482666015625,
"loss": 0.9013,
"margin_dpo/margin_mean": 39.88440704345703,
"margin_dpo/margin_std": 45.809532165527344,
"step": 141
},
{
"KL/chosen_KL_mean": -32.35279846191406,
"KL/mean": -55.03770446777344,
"KL/rejected_KL_mean": -77.72261047363281,
"KL/std": 48.86778259277344,
"epoch": 0.20851688693098386,
"fcm_dpo/beta": 0.0206030011177063,
"fcm_dpo/delta": -0.14863386750221252,
"fcm_dpo/margin": 45.36981964111328,
"fcm_dpo/q_t": 0.32887327671051025,
"grad_norm": 30.7178955078125,
"learning_rate": 4.83118057351089e-07,
"logits/chosen": -0.5786978006362915,
"logits/rejected": -0.5709136128425598,
"logps/chosen": -90.38072204589844,
"logps/ref_chosen": -58.027931213378906,
"logps/ref_rejected": -94.58222961425781,
"logps/rejected": -172.30484008789062,
"loss": 0.9218,
"margin_dpo/margin_mean": 45.36981964111328,
"margin_dpo/margin_std": 57.22596740722656,
"step": 142
},
{
"KL/chosen_KL_mean": -35.13017654418945,
"KL/mean": -49.68499755859375,
"KL/rejected_KL_mean": -64.23982238769531,
"KL/std": 42.61761474609375,
"epoch": 0.20998531571218795,
"fcm_dpo/beta": 0.0208355113863945,
"fcm_dpo/delta": 0.20649181306362152,
"fcm_dpo/margin": 29.10964584350586,
"fcm_dpo/q_t": 0.38302189111709595,
"grad_norm": 36.91963195800781,
"learning_rate": 4.826513955607734e-07,
"logits/chosen": -0.56812983751297,
"logits/rejected": -0.543799102306366,
"logps/chosen": -92.72662353515625,
"logps/ref_chosen": -57.59645080566406,
"logps/ref_rejected": -78.99957275390625,
"logps/rejected": -143.23939514160156,
"loss": 1.0707,
"margin_dpo/margin_mean": 29.10964584350586,
"margin_dpo/margin_std": 47.77528762817383,
"step": 143
},
{
"KL/chosen_KL_mean": -28.555076599121094,
"KL/mean": -47.98714065551758,
"KL/rejected_KL_mean": -67.41920471191406,
"KL/std": 36.25341033935547,
"epoch": 0.21145374449339208,
"fcm_dpo/beta": 0.021065259352326393,
"fcm_dpo/delta": -0.02040477842092514,
"fcm_dpo/margin": 38.8641242980957,
"fcm_dpo/q_t": 0.33234626054763794,
"grad_norm": 28.399396896362305,
"learning_rate": 4.821786031898176e-07,
"logits/chosen": -0.5759509801864624,
"logits/rejected": -0.5413784980773926,
"logps/chosen": -88.46144104003906,
"logps/ref_chosen": -59.90636444091797,
"logps/ref_rejected": -82.00025939941406,
"logps/rejected": -149.41946411132812,
"loss": 0.8764,
"margin_dpo/margin_mean": 38.8641242980957,
"margin_dpo/margin_std": 41.08622741699219,
"step": 144
},
{
"KL/chosen_KL_mean": -31.771686553955078,
"KL/mean": -52.151512145996094,
"KL/rejected_KL_mean": -72.53134155273438,
"KL/std": 41.140968322753906,
"epoch": 0.21292217327459617,
"fcm_dpo/beta": 0.020855430513620377,
"fcm_dpo/delta": -0.05491182580590248,
"fcm_dpo/margin": 40.75965881347656,
"fcm_dpo/q_t": 0.3329458236694336,
"grad_norm": 33.59305953979492,
"learning_rate": 4.816996926967401e-07,
"logits/chosen": -0.5763427019119263,
"logits/rejected": -0.544802725315094,
"logps/chosen": -88.37235260009766,
"logps/ref_chosen": -56.60066604614258,
"logps/ref_rejected": -77.86631774902344,
"logps/rejected": -150.3976593017578,
"loss": 0.8897,
"margin_dpo/margin_mean": 40.75965881347656,
"margin_dpo/margin_std": 46.583221435546875,
"step": 145
},
{
"KL/chosen_KL_mean": -44.07976150512695,
"KL/mean": -60.510459899902344,
"KL/rejected_KL_mean": -76.94116973876953,
"KL/std": 42.426719665527344,
"epoch": 0.2143906020558003,
"fcm_dpo/beta": 0.021105386316776276,
"fcm_dpo/delta": 0.1144891306757927,
"fcm_dpo/margin": 32.861412048339844,
"fcm_dpo/q_t": 0.3631611764431,
"grad_norm": 44.34247970581055,
"learning_rate": 4.812146767012779e-07,
"logits/chosen": -0.5526940822601318,
"logits/rejected": -0.5075163841247559,
"logps/chosen": -110.08021545410156,
"logps/ref_chosen": -66.00045013427734,
"logps/ref_rejected": -81.70278930664062,
"logps/rejected": -158.64395141601562,
"loss": 1.0376,
"margin_dpo/margin_mean": 32.861412048339844,
"margin_dpo/margin_std": 49.71653747558594,
"step": 146
},
{
"KL/chosen_KL_mean": -33.446563720703125,
"KL/mean": -55.87425231933594,
"KL/rejected_KL_mean": -78.30194091796875,
"KL/std": 45.601898193359375,
"epoch": 0.21585903083700442,
"fcm_dpo/beta": 0.020772868767380714,
"fcm_dpo/delta": -0.14638394117355347,
"fcm_dpo/margin": 44.855384826660156,
"fcm_dpo/q_t": 0.3264108896255493,
"grad_norm": 38.08588409423828,
"learning_rate": 4.807235679840536e-07,
"logits/chosen": -0.5531934499740601,
"logits/rejected": -0.5197881460189819,
"logps/chosen": -86.85205078125,
"logps/ref_chosen": -53.405487060546875,
"logps/ref_rejected": -71.39060974121094,
"logps/rejected": -149.6925506591797,
"loss": 0.8932,
"margin_dpo/margin_mean": 44.85538101196289,
"margin_dpo/margin_std": 54.91014862060547,
"step": 147
},
{
"KL/chosen_KL_mean": -32.03932571411133,
"KL/mean": -49.7523307800293,
"KL/rejected_KL_mean": -67.46534729003906,
"KL/std": 41.54022216796875,
"epoch": 0.2173274596182085,
"fcm_dpo/beta": 0.021175362169742584,
"fcm_dpo/delta": 0.048354372382164,
"fcm_dpo/margin": 35.426021575927734,
"fcm_dpo/q_t": 0.355111300945282,
"grad_norm": 30.767854690551758,
"learning_rate": 4.802263794862384e-07,
"logits/chosen": -0.5951809287071228,
"logits/rejected": -0.5793430805206299,
"logps/chosen": -96.97640991210938,
"logps/ref_chosen": -64.93708038330078,
"logps/ref_rejected": -103.09384155273438,
"logps/rejected": -170.55918884277344,
"loss": 0.9545,
"margin_dpo/margin_mean": 35.426021575927734,
"margin_dpo/margin_std": 43.80073547363281,
"step": 148
},
{
"KL/chosen_KL_mean": -31.97378158569336,
"KL/mean": -54.779823303222656,
"KL/rejected_KL_mean": -77.58586120605469,
"KL/std": 41.12773132324219,
"epoch": 0.21879588839941264,
"fcm_dpo/beta": 0.02021482028067112,
"fcm_dpo/delta": -0.13983891904354095,
"fcm_dpo/margin": 45.612083435058594,
"fcm_dpo/q_t": 0.3158126473426819,
"grad_norm": 31.028966903686523,
"learning_rate": 4.797231243092118e-07,
"logits/chosen": -0.6176996231079102,
"logits/rejected": -0.596400797367096,
"logps/chosen": -90.44754028320312,
"logps/ref_chosen": -58.47376251220703,
"logps/ref_rejected": -99.31474304199219,
"logps/rejected": -176.90060424804688,
"loss": 0.8414,
"margin_dpo/margin_mean": 45.612083435058594,
"margin_dpo/margin_std": 47.153900146484375,
"step": 149
},
{
"KL/chosen_KL_mean": -29.563812255859375,
"KL/mean": -51.22605895996094,
"KL/rejected_KL_mean": -72.8883056640625,
"KL/std": 48.249725341796875,
"epoch": 0.22026431718061673,
"fcm_dpo/beta": 0.019835665822029114,
"fcm_dpo/delta": -0.07024933397769928,
"fcm_dpo/margin": 43.324493408203125,
"fcm_dpo/q_t": 0.3418238162994385,
"grad_norm": 30.609365463256836,
"learning_rate": 4.792138157142157e-07,
"logits/chosen": -0.5815879106521606,
"logits/rejected": -0.5770955681800842,
"logps/chosen": -75.26962280273438,
"logps/ref_chosen": -45.705810546875,
"logps/ref_rejected": -83.34759521484375,
"logps/rejected": -156.23590087890625,
"loss": 0.9152,
"margin_dpo/margin_mean": 43.324493408203125,
"margin_dpo/margin_std": 56.31333923339844,
"step": 150
},
{
"KL/chosen_KL_mean": -33.465301513671875,
"KL/mean": -56.45378112792969,
"KL/rejected_KL_mean": -79.4422607421875,
"KL/std": 45.18247985839844,
"epoch": 0.22173274596182085,
"fcm_dpo/beta": 0.019716549664735794,
"fcm_dpo/delta": -0.1170579120516777,
"fcm_dpo/margin": 45.976966857910156,
"fcm_dpo/q_t": 0.31761401891708374,
"grad_norm": 29.541719436645508,
"learning_rate": 4.786984671220053e-07,
"logits/chosen": -0.6623918414115906,
"logits/rejected": -0.6282558441162109,
"logps/chosen": -104.0361328125,
"logps/ref_chosen": -70.57083129882812,
"logps/ref_rejected": -100.46382141113281,
"logps/rejected": -179.90606689453125,
"loss": 0.8425,
"margin_dpo/margin_mean": 45.976966857910156,
"margin_dpo/margin_std": 47.510108947753906,
"step": 151
},
{
"KL/chosen_KL_mean": -28.917739868164062,
"KL/mean": -54.12328338623047,
"KL/rejected_KL_mean": -79.32882690429688,
"KL/std": 45.731834411621094,
"epoch": 0.22320117474302498,
"fcm_dpo/beta": 0.019173946231603622,
"fcm_dpo/delta": -0.18405033648014069,
"fcm_dpo/margin": 50.41108703613281,
"fcm_dpo/q_t": 0.31479883193969727,
"grad_norm": 27.932998657226562,
"learning_rate": 4.78177092112495e-07,
"logits/chosen": -0.6130504608154297,
"logits/rejected": -0.6028048992156982,
"logps/chosen": -89.08212280273438,
"logps/ref_chosen": -60.16438674926758,
"logps/ref_rejected": -106.14045715332031,
"logps/rejected": -185.4692840576172,
"loss": 0.8384,
"margin_dpo/margin_mean": 50.41108703613281,
"margin_dpo/margin_std": 53.3874397277832,
"step": 152
},
{
"KL/chosen_KL_mean": -28.78639030456543,
"KL/mean": -50.361366271972656,
"KL/rejected_KL_mean": -71.93634033203125,
"KL/std": 47.18241500854492,
"epoch": 0.22466960352422907,
"fcm_dpo/beta": 0.018890701234340668,
"fcm_dpo/delta": -0.016644855961203575,
"fcm_dpo/margin": 43.14995193481445,
"fcm_dpo/q_t": 0.3440234959125519,
"grad_norm": 25.5745792388916,
"learning_rate": 4.776497044244016e-07,
"logits/chosen": -0.616934061050415,
"logits/rejected": -0.6062427163124084,
"logps/chosen": -85.10166931152344,
"logps/ref_chosen": -56.315277099609375,
"logps/ref_rejected": -85.65583801269531,
"logps/rejected": -157.59217834472656,
"loss": 0.9313,
"margin_dpo/margin_mean": 43.14995193481445,
"margin_dpo/margin_std": 55.69709014892578,
"step": 153
},
{
"KL/chosen_KL_mean": -33.958927154541016,
"KL/mean": -55.02075958251953,
"KL/rejected_KL_mean": -76.08258056640625,
"KL/std": 47.15727615356445,
"epoch": 0.2261380323054332,
"fcm_dpo/beta": 0.018977440893650055,
"fcm_dpo/delta": -0.0009660199284553528,
"fcm_dpo/margin": 42.1236572265625,
"fcm_dpo/q_t": 0.3485991954803467,
"grad_norm": 31.434709548950195,
"learning_rate": 4.771163179548808e-07,
"logits/chosen": -0.6276504993438721,
"logits/rejected": -0.6189817190170288,
"logps/chosen": -96.70149230957031,
"logps/ref_chosen": -62.74256896972656,
"logps/ref_rejected": -104.24420166015625,
"logps/rejected": -180.3267822265625,
"loss": 0.9699,
"margin_dpo/margin_mean": 42.123653411865234,
"margin_dpo/margin_std": 56.15239715576172,
"step": 154
},
{
"KL/chosen_KL_mean": -28.95609474182129,
"KL/mean": -51.265785217285156,
"KL/rejected_KL_mean": -73.57547760009766,
"KL/std": 45.23421859741211,
"epoch": 0.2276064610866373,
"fcm_dpo/beta": 0.01880238763988018,
"fcm_dpo/delta": -0.04300792142748833,
"fcm_dpo/margin": 44.61937713623047,
"fcm_dpo/q_t": 0.33316513895988464,
"grad_norm": 27.293209075927734,
"learning_rate": 4.7657694675916247e-07,
"logits/chosen": -0.6241185665130615,
"logits/rejected": -0.5972343683242798,
"logps/chosen": -89.6092758178711,
"logps/ref_chosen": -60.65318298339844,
"logps/ref_rejected": -77.49220275878906,
"logps/rejected": -151.06768798828125,
"loss": 0.896,
"margin_dpo/margin_mean": 44.61937713623047,
"margin_dpo/margin_std": 52.05685043334961,
"step": 155
},
{
"KL/chosen_KL_mean": -41.33610534667969,
"KL/mean": -55.682823181152344,
"KL/rejected_KL_mean": -70.02952575683594,
"KL/std": 44.41982650756836,
"epoch": 0.2290748898678414,
"fcm_dpo/beta": 0.019381992518901825,
"fcm_dpo/delta": 0.25817906856536865,
"fcm_dpo/margin": 28.693424224853516,
"fcm_dpo/q_t": 0.39033377170562744,
"grad_norm": 43.62647247314453,
"learning_rate": 4.7603160505017893e-07,
"logits/chosen": -0.561880350112915,
"logits/rejected": -0.5312899947166443,
"logps/chosen": -110.82798767089844,
"logps/ref_chosen": -69.49188232421875,
"logps/ref_rejected": -77.16929626464844,
"logps/rejected": -147.19882202148438,
"loss": 1.176,
"margin_dpo/margin_mean": 28.693424224853516,
"margin_dpo/margin_std": 59.12420654296875,
"step": 156
},
{
"KL/chosen_KL_mean": -38.10362243652344,
"KL/mean": -64.24569702148438,
"KL/rejected_KL_mean": -90.38777923583984,
"KL/std": 48.962730407714844,
"epoch": 0.2305433186490455,
"fcm_dpo/beta": 0.018893180415034294,
"fcm_dpo/delta": -0.21114027500152588,
"fcm_dpo/margin": 52.28416061401367,
"fcm_dpo/q_t": 0.30563968420028687,
"grad_norm": 30.84547233581543,
"learning_rate": 4.7548030719819154e-07,
"logits/chosen": -0.608643651008606,
"logits/rejected": -0.6031553745269775,
"logps/chosen": -99.47206115722656,
"logps/ref_chosen": -61.368438720703125,
"logps/ref_rejected": -107.64636993408203,
"logps/rejected": -198.03414916992188,
"loss": 0.8289,
"margin_dpo/margin_mean": 52.28416061401367,
"margin_dpo/margin_std": 53.84667205810547,
"step": 157
},
{
"KL/chosen_KL_mean": -34.95962142944336,
"KL/mean": -60.97590255737305,
"KL/rejected_KL_mean": -86.9921875,
"KL/std": 58.75188446044922,
"epoch": 0.23201174743024963,
"fcm_dpo/beta": 0.018339555710554123,
"fcm_dpo/delta": -0.1707034558057785,
"fcm_dpo/margin": 52.03257751464844,
"fcm_dpo/q_t": 0.3332700729370117,
"grad_norm": 25.7038631439209,
"learning_rate": 4.7492306773041136e-07,
"logits/chosen": -0.5895273685455322,
"logits/rejected": -0.5938005447387695,
"logps/chosen": -92.57254028320312,
"logps/ref_chosen": -57.612918853759766,
"logps/ref_rejected": -113.6946792602539,
"logps/rejected": -200.68685913085938,
"loss": 0.9016,
"margin_dpo/margin_mean": 52.03257751464844,
"margin_dpo/margin_std": 68.59899139404297,
"step": 158
},
{
"KL/chosen_KL_mean": -39.52997970581055,
"KL/mean": -60.189544677734375,
"KL/rejected_KL_mean": -80.84910583496094,
"KL/std": 51.77936553955078,
"epoch": 0.23348017621145375,
"fcm_dpo/beta": 0.018413972109556198,
"fcm_dpo/delta": 0.0413820706307888,
"fcm_dpo/margin": 41.31913375854492,
"fcm_dpo/q_t": 0.35460609197616577,
"grad_norm": 34.12101745605469,
"learning_rate": 4.743599013306165e-07,
"logits/chosen": -0.5919934511184692,
"logits/rejected": -0.5467137694358826,
"logps/chosen": -121.09032440185547,
"logps/ref_chosen": -81.56034851074219,
"logps/ref_rejected": -88.89871215820312,
"logps/rejected": -169.74781799316406,
"loss": 0.9757,
"margin_dpo/margin_mean": 41.31913757324219,
"margin_dpo/margin_std": 57.08066940307617,
"step": 159
},
{
"KL/chosen_KL_mean": -39.19923400878906,
"KL/mean": -63.417083740234375,
"KL/rejected_KL_mean": -87.63493347167969,
"KL/std": 52.864295959472656,
"epoch": 0.23494860499265785,
"fcm_dpo/beta": 0.01803383231163025,
"fcm_dpo/delta": -0.081442691385746,
"fcm_dpo/margin": 48.43570327758789,
"fcm_dpo/q_t": 0.34352871775627136,
"grad_norm": 30.687528610229492,
"learning_rate": 4.737908228387656e-07,
"logits/chosen": -0.5659890174865723,
"logits/rejected": -0.5416566729545593,
"logps/chosen": -104.93011474609375,
"logps/ref_chosen": -65.73088073730469,
"logps/ref_rejected": -97.21781921386719,
"logps/rejected": -184.85275268554688,
"loss": 0.9425,
"margin_dpo/margin_mean": 48.43570327758789,
"margin_dpo/margin_std": 65.908935546875,
"step": 160
},
{
"KL/chosen_KL_mean": -35.278465270996094,
"KL/mean": -57.27626037597656,
"KL/rejected_KL_mean": -79.2740478515625,
"KL/std": 46.100494384765625,
"epoch": 0.23641703377386197,
"fcm_dpo/beta": 0.0180988647043705,
"fcm_dpo/delta": 0.0039350297302007675,
"fcm_dpo/margin": 43.995582580566406,
"fcm_dpo/q_t": 0.34731143712997437,
"grad_norm": 30.22264862060547,
"learning_rate": 4.7321584725060594e-07,
"logits/chosen": -0.5966364741325378,
"logits/rejected": -0.5794234275817871,
"logps/chosen": -87.71493530273438,
"logps/ref_chosen": -52.43647003173828,
"logps/ref_rejected": -83.43095397949219,
"logps/rejected": -162.7050018310547,
"loss": 0.9439,
"margin_dpo/margin_mean": 43.995582580566406,
"margin_dpo/margin_std": 56.44627380371094,
"step": 161
},
{
"KL/chosen_KL_mean": -33.17871856689453,
"KL/mean": -56.08073425292969,
"KL/rejected_KL_mean": -78.98275756835938,
"KL/std": 51.76576232910156,
"epoch": 0.23788546255506607,
"fcm_dpo/beta": 0.017755115404725075,
"fcm_dpo/delta": -0.01901736855506897,
"fcm_dpo/margin": 45.804046630859375,
"fcm_dpo/q_t": 0.3460782766342163,
"grad_norm": 25.410987854003906,
"learning_rate": 4.7263498971727905e-07,
"logits/chosen": -0.6189196109771729,
"logits/rejected": -0.5900764465332031,
"logps/chosen": -95.789306640625,
"logps/ref_chosen": -62.6105842590332,
"logps/ref_rejected": -89.39057922363281,
"logps/rejected": -168.37335205078125,
"loss": 0.9361,
"margin_dpo/margin_mean": 45.804046630859375,
"margin_dpo/margin_std": 59.16350173950195,
"step": 162
},
{
"KL/chosen_KL_mean": -34.73664474487305,
"KL/mean": -56.72509002685547,
"KL/rejected_KL_mean": -78.71353149414062,
"KL/std": 45.937137603759766,
"epoch": 0.2393538913362702,
"fcm_dpo/beta": 0.0180361308157444,
"fcm_dpo/delta": 0.007369140163064003,
"fcm_dpo/margin": 43.976890563964844,
"fcm_dpo/q_t": 0.34393471479415894,
"grad_norm": 26.971351623535156,
"learning_rate": 4.720482655449212e-07,
"logits/chosen": -0.5807976126670837,
"logits/rejected": -0.5515669584274292,
"logps/chosen": -89.75827026367188,
"logps/ref_chosen": -55.021629333496094,
"logps/ref_rejected": -75.418212890625,
"logps/rejected": -154.13174438476562,
"loss": 0.9156,
"margin_dpo/margin_mean": 43.976890563964844,
"margin_dpo/margin_std": 52.59187316894531,
"step": 163
},
{
"KL/chosen_KL_mean": -32.905269622802734,
"KL/mean": -58.444881439208984,
"KL/rejected_KL_mean": -83.98448181152344,
"KL/std": 46.80143737792969,
"epoch": 0.24082232011747431,
"fcm_dpo/beta": 0.017528720200061798,
"fcm_dpo/delta": -0.10878677666187286,
"fcm_dpo/margin": 51.079219818115234,
"fcm_dpo/q_t": 0.3195659816265106,
"grad_norm": 27.89398765563965,
"learning_rate": 4.714556901942599e-07,
"logits/chosen": -0.5946141481399536,
"logits/rejected": -0.5662475228309631,
"logps/chosen": -88.54593658447266,
"logps/ref_chosen": -55.64066696166992,
"logps/ref_rejected": -79.66463470458984,
"logps/rejected": -163.64910888671875,
"loss": 0.8391,
"margin_dpo/margin_mean": 51.07921600341797,
"margin_dpo/margin_std": 52.178611755371094,
"step": 164
},
{
"KL/chosen_KL_mean": -36.89280700683594,
"KL/mean": -54.11333084106445,
"KL/rejected_KL_mean": -71.3338623046875,
"KL/std": 39.27880859375,
"epoch": 0.2422907488986784,
"fcm_dpo/beta": 0.018116144463419914,
"fcm_dpo/delta": 0.18791693449020386,
"fcm_dpo/margin": 34.4410514831543,
"fcm_dpo/q_t": 0.37251168489456177,
"grad_norm": 35.580360412597656,
"learning_rate": 4.708572792802069e-07,
"logits/chosen": -0.6063022613525391,
"logits/rejected": -0.5636542439460754,
"logps/chosen": -98.2034912109375,
"logps/ref_chosen": -61.310691833496094,
"logps/ref_rejected": -73.67060852050781,
"logps/rejected": -145.0044708251953,
"loss": 1.0123,
"margin_dpo/margin_mean": 34.4410514831543,
"margin_dpo/margin_std": 47.09235382080078,
"step": 165
},
{
"KL/chosen_KL_mean": -34.758277893066406,
"KL/mean": -64.99706268310547,
"KL/rejected_KL_mean": -95.23585510253906,
"KL/std": 59.283321380615234,
"epoch": 0.24375917767988253,
"fcm_dpo/beta": 0.017409585416316986,
"fcm_dpo/delta": -0.2863610088825226,
"fcm_dpo/margin": 60.477569580078125,
"fcm_dpo/q_t": 0.3133409023284912,
"grad_norm": 25.431591033935547,
"learning_rate": 4.702530485714461e-07,
"logits/chosen": -0.5450478792190552,
"logits/rejected": -0.5523861050605774,
"logps/chosen": -85.74188232421875,
"logps/ref_chosen": -50.98360061645508,
"logps/ref_rejected": -98.09512329101562,
"logps/rejected": -193.3309783935547,
"loss": 0.8277,
"margin_dpo/margin_mean": 60.477569580078125,
"margin_dpo/margin_std": 71.7076187133789,
"step": 166
},
{
"KL/chosen_KL_mean": -35.36009979248047,
"KL/mean": -67.31937408447266,
"KL/rejected_KL_mean": -99.27864837646484,
"KL/std": 54.32081604003906,
"epoch": 0.24522760646108663,
"fcm_dpo/beta": 0.01653553918004036,
"fcm_dpo/delta": -0.28801417350769043,
"fcm_dpo/margin": 63.918548583984375,
"fcm_dpo/q_t": 0.29091453552246094,
"grad_norm": 25.4301815032959,
"learning_rate": 4.6964301399001877e-07,
"logits/chosen": -0.49997496604919434,
"logits/rejected": -0.4990348219871521,
"logps/chosen": -85.78419494628906,
"logps/ref_chosen": -50.424095153808594,
"logps/ref_rejected": -96.03042602539062,
"logps/rejected": -195.30908203125,
"loss": 0.7685,
"margin_dpo/margin_mean": 63.918548583984375,
"margin_dpo/margin_std": 59.32880783081055,
"step": 167
},
{
"KL/chosen_KL_mean": -39.87940979003906,
"KL/mean": -64.94029235839844,
"KL/rejected_KL_mean": -90.00116729736328,
"KL/std": 51.81836700439453,
"epoch": 0.24669603524229075,
"fcm_dpo/beta": 0.0162336602807045,
"fcm_dpo/delta": -0.015112994238734245,
"fcm_dpo/margin": 50.12175750732422,
"fcm_dpo/q_t": 0.33712950348854065,
"grad_norm": 28.887313842773438,
"learning_rate": 4.690271916109034e-07,
"logits/chosen": -0.5341615676879883,
"logits/rejected": -0.5141159296035767,
"logps/chosen": -89.34223937988281,
"logps/ref_chosen": -49.462825775146484,
"logps/ref_rejected": -75.30855560302734,
"logps/rejected": -165.30972290039062,
"loss": 0.8878,
"margin_dpo/margin_mean": 50.12175750732422,
"margin_dpo/margin_std": 55.842750549316406,
"step": 168
},
{
"KL/chosen_KL_mean": -38.55489730834961,
"KL/mean": -60.85240936279297,
"KL/rejected_KL_mean": -83.14991760253906,
"KL/std": 52.6884765625,
"epoch": 0.24816446402349487,
"fcm_dpo/beta": 0.016293514519929886,
"fcm_dpo/delta": 0.07776357233524323,
"fcm_dpo/margin": 44.59502029418945,
"fcm_dpo/q_t": 0.3672984838485718,
"grad_norm": 31.865880966186523,
"learning_rate": 4.6840559766159235e-07,
"logits/chosen": -0.5834333896636963,
"logits/rejected": -0.5621183514595032,
"logps/chosen": -98.35833740234375,
"logps/ref_chosen": -59.803443908691406,
"logps/ref_rejected": -83.34574890136719,
"logps/rejected": -166.49566650390625,
"loss": 1.0159,
"margin_dpo/margin_mean": 44.59502410888672,
"margin_dpo/margin_std": 69.36247253417969,
"step": 169
},
{
"KL/chosen_KL_mean": -35.542457580566406,
"KL/mean": -63.328102111816406,
"KL/rejected_KL_mean": -91.11375427246094,
"KL/std": 50.676841735839844,
"epoch": 0.24963289280469897,
"fcm_dpo/beta": 0.016118060797452927,
"fcm_dpo/delta": -0.10781846940517426,
"fcm_dpo/margin": 55.5712890625,
"fcm_dpo/q_t": 0.3201714754104614,
"grad_norm": 27.23699188232422,
"learning_rate": 4.6777824852166437e-07,
"logits/chosen": -0.504392147064209,
"logits/rejected": -0.48632174730300903,
"logps/chosen": -85.01423645019531,
"logps/ref_chosen": -49.471771240234375,
"logps/ref_rejected": -75.91734313964844,
"logps/rejected": -167.03109741210938,
"loss": 0.8564,
"margin_dpo/margin_mean": 55.571292877197266,
"margin_dpo/margin_std": 57.82054138183594,
"step": 170
},
{
"KL/chosen_KL_mean": -52.33512878417969,
"KL/mean": -75.137939453125,
"KL/rejected_KL_mean": -97.94075775146484,
"KL/std": 56.281578063964844,
"epoch": 0.2511013215859031,
"fcm_dpo/beta": 0.01631205342710018,
"fcm_dpo/delta": 0.060608040541410446,
"fcm_dpo/margin": 45.605628967285156,
"fcm_dpo/q_t": 0.3616752624511719,
"grad_norm": 37.65127182006836,
"learning_rate": 4.6714516072235273e-07,
"logits/chosen": -0.5643775463104248,
"logits/rejected": -0.5289890170097351,
"logps/chosen": -136.83444213867188,
"logps/ref_chosen": -84.49931335449219,
"logps/ref_rejected": -109.38209533691406,
"logps/rejected": -207.32284545898438,
"loss": 1.0027,
"margin_dpo/margin_mean": 45.605628967285156,
"margin_dpo/margin_std": 68.96087646484375,
"step": 171
},
{
"KL/chosen_KL_mean": -51.69752502441406,
"KL/mean": -75.31649780273438,
"KL/rejected_KL_mean": -98.93547058105469,
"KL/std": 57.76668930053711,
"epoch": 0.2525697503671072,
"fcm_dpo/beta": 0.016372080892324448,
"fcm_dpo/delta": 0.028607267886400223,
"fcm_dpo/margin": 47.237937927246094,
"fcm_dpo/q_t": 0.3496333956718445,
"grad_norm": 35.594093322753906,
"learning_rate": 4.6650635094610966e-07,
"logits/chosen": -0.5606852769851685,
"logits/rejected": -0.5259276032447815,
"logps/chosen": -120.3514404296875,
"logps/ref_chosen": -68.65391540527344,
"logps/ref_rejected": -85.43667602539062,
"logps/rejected": -184.37213134765625,
"loss": 0.9619,
"margin_dpo/margin_mean": 47.237937927246094,
"margin_dpo/margin_std": 63.03688049316406,
"step": 172
},
{
"KL/chosen_KL_mean": -45.648590087890625,
"KL/mean": -69.3135757446289,
"KL/rejected_KL_mean": -92.97855377197266,
"KL/std": 52.03070831298828,
"epoch": 0.2540381791483113,
"fcm_dpo/beta": 0.016575288027524948,
"fcm_dpo/delta": 0.016358211636543274,
"fcm_dpo/margin": 47.3299560546875,
"fcm_dpo/q_t": 0.3418443202972412,
"grad_norm": 34.87374496459961,
"learning_rate": 4.6586183602616687e-07,
"logits/chosen": -0.5738804340362549,
"logits/rejected": -0.5336560606956482,
"logps/chosen": -108.69947052001953,
"logps/ref_chosen": -63.050880432128906,
"logps/ref_rejected": -78.68392181396484,
"logps/rejected": -171.6624755859375,
"loss": 0.9043,
"margin_dpo/margin_mean": 47.329952239990234,
"margin_dpo/margin_std": 54.01713943481445,
"step": 173
},
{
"KL/chosen_KL_mean": -46.467247009277344,
"KL/mean": -72.35551452636719,
"KL/rejected_KL_mean": -98.24378967285156,
"KL/std": 57.77963638305664,
"epoch": 0.2555066079295154,
"fcm_dpo/beta": 0.01650991104543209,
"fcm_dpo/delta": -0.06092230975627899,
"fcm_dpo/margin": 51.776527404785156,
"fcm_dpo/q_t": 0.33887800574302673,
"grad_norm": 39.11122131347656,
"learning_rate": 4.652116329460919e-07,
"logits/chosen": -0.49907761812210083,
"logits/rejected": -0.5138338804244995,
"logps/chosen": -99.83021545410156,
"logps/ref_chosen": -53.36296844482422,
"logps/ref_rejected": -101.91120910644531,
"logps/rejected": -200.15499877929688,
"loss": 0.918,
"margin_dpo/margin_mean": 51.776527404785156,
"margin_dpo/margin_std": 64.13847351074219,
"step": 174
},
{
"KL/chosen_KL_mean": -44.84705352783203,
"KL/mean": -81.27606201171875,
"KL/rejected_KL_mean": -117.70506286621094,
"KL/std": 65.36940002441406,
"epoch": 0.25697503671071953,
"fcm_dpo/beta": 0.01541909109801054,
"fcm_dpo/delta": -0.36564406752586365,
"fcm_dpo/margin": 72.85799407958984,
"fcm_dpo/q_t": 0.28254061937332153,
"grad_norm": 37.300682067871094,
"learning_rate": 4.645557588393406e-07,
"logits/chosen": -0.4535369873046875,
"logits/rejected": -0.4410516023635864,
"logps/chosen": -90.26481628417969,
"logps/ref_chosen": -45.417762756347656,
"logps/ref_rejected": -89.50579833984375,
"logps/rejected": -207.2108612060547,
"loss": 0.7328,
"margin_dpo/margin_mean": 72.85800170898438,
"margin_dpo/margin_std": 66.75201416015625,
"step": 175
},
{
"KL/chosen_KL_mean": -46.950782775878906,
"KL/mean": -76.24058532714844,
"KL/rejected_KL_mean": -105.53038024902344,
"KL/std": 59.06817626953125,
"epoch": 0.25844346549192365,
"fcm_dpo/beta": 0.015022006817162037,
"fcm_dpo/delta": -0.08763821423053741,
"fcm_dpo/margin": 58.57960510253906,
"fcm_dpo/q_t": 0.3318549394607544,
"grad_norm": 40.32584762573242,
"learning_rate": 4.638942309888058e-07,
"logits/chosen": -0.42935019731521606,
"logits/rejected": -0.44252365827560425,
"logps/chosen": -97.40362548828125,
"logps/ref_chosen": -50.452842712402344,
"logps/ref_rejected": -95.5589599609375,
"logps/rejected": -201.08935546875,
"loss": 0.8749,
"margin_dpo/margin_mean": 58.57960510253906,
"margin_dpo/margin_std": 67.5830307006836,
"step": 176
},
{
"KL/chosen_KL_mean": -52.787109375,
"KL/mean": -83.38577270507812,
"KL/rejected_KL_mean": -113.98443603515625,
"KL/std": 65.90585327148438,
"epoch": 0.2599118942731278,
"fcm_dpo/beta": 0.014719298109412193,
"fcm_dpo/delta": -0.11064187437295914,
"fcm_dpo/margin": 61.197330474853516,
"fcm_dpo/q_t": 0.32784807682037354,
"grad_norm": 42.42573928833008,
"learning_rate": 4.6322706682636137e-07,
"logits/chosen": -0.5027165412902832,
"logits/rejected": -0.48637324571609497,
"logps/chosen": -114.00357055664062,
"logps/ref_chosen": -61.216468811035156,
"logps/ref_rejected": -95.89378356933594,
"logps/rejected": -209.8782196044922,
"loss": 0.8677,
"margin_dpo/margin_mean": 61.19733428955078,
"margin_dpo/margin_std": 69.97938537597656,
"step": 177
},
{
"KL/chosen_KL_mean": -60.033023834228516,
"KL/mean": -97.84568786621094,
"KL/rejected_KL_mean": -135.65835571289062,
"KL/std": 80.01490020751953,
"epoch": 0.26138032305433184,
"fcm_dpo/beta": 0.013894779607653618,
"fcm_dpo/delta": -0.282906174659729,
"fcm_dpo/margin": 75.62533569335938,
"fcm_dpo/q_t": 0.30688485503196716,
"grad_norm": 39.59940719604492,
"learning_rate": 4.6255428393240354e-07,
"logits/chosen": -0.4129112660884857,
"logits/rejected": -0.4165341258049011,
"logps/chosen": -118.29781341552734,
"logps/ref_chosen": -58.26478958129883,
"logps/ref_rejected": -105.3653335571289,
"logps/rejected": -241.02369689941406,
"loss": 0.8254,
"margin_dpo/margin_mean": 75.62533569335938,
"margin_dpo/margin_std": 86.36256408691406,
"step": 178
},
{
"KL/chosen_KL_mean": -57.656192779541016,
"KL/mean": -87.85360717773438,
"KL/rejected_KL_mean": -118.05103302001953,
"KL/std": 66.25624084472656,
"epoch": 0.26284875183553597,
"fcm_dpo/beta": 0.01362140104174614,
"fcm_dpo/delta": -0.02604658529162407,
"fcm_dpo/margin": 60.394832611083984,
"fcm_dpo/q_t": 0.3425036668777466,
"grad_norm": 57.43539047241211,
"learning_rate": 4.6187590003538724e-07,
"logits/chosen": -0.4739261269569397,
"logits/rejected": -0.47013968229293823,
"logps/chosen": -118.71451568603516,
"logps/ref_chosen": -61.05832290649414,
"logps/ref_rejected": -90.52782440185547,
"logps/rejected": -208.578857421875,
"loss": 0.9654,
"margin_dpo/margin_mean": 60.394832611083984,
"margin_dpo/margin_std": 80.08963775634766,
"step": 179
},
{
"KL/chosen_KL_mean": -48.66742706298828,
"KL/mean": -82.73980712890625,
"KL/rejected_KL_mean": -116.81217193603516,
"KL/std": 60.23838424682617,
"epoch": 0.2643171806167401,
"fcm_dpo/beta": 0.013405999168753624,
"fcm_dpo/delta": -0.12565259635448456,
"fcm_dpo/margin": 68.14473724365234,
"fcm_dpo/q_t": 0.3189559280872345,
"grad_norm": 52.571556091308594,
"learning_rate": 4.611919330113591e-07,
"logits/chosen": -0.47179630398750305,
"logits/rejected": -0.45903003215789795,
"logps/chosen": -103.01014709472656,
"logps/ref_chosen": -54.34272003173828,
"logps/ref_rejected": -98.21183776855469,
"logps/rejected": -215.02401733398438,
"loss": 0.8536,
"margin_dpo/margin_mean": 68.14473724365234,
"margin_dpo/margin_std": 72.47191619873047,
"step": 180
},
{
"KL/chosen_KL_mean": -39.98610305786133,
"KL/mean": -62.656532287597656,
"KL/rejected_KL_mean": -85.32696533203125,
"KL/std": 53.778289794921875,
"epoch": 0.2657856093979442,
"fcm_dpo/beta": 0.013776125386357307,
"fcm_dpo/delta": 0.18641813099384308,
"fcm_dpo/margin": 45.34085464477539,
"fcm_dpo/q_t": 0.37443986535072327,
"grad_norm": 26.93648338317871,
"learning_rate": 4.605024008834863e-07,
"logits/chosen": -0.5331726670265198,
"logits/rejected": -0.5003119707107544,
"logps/chosen": -94.98655700683594,
"logps/ref_chosen": -55.000457763671875,
"logps/ref_rejected": -61.656166076660156,
"logps/rejected": -146.98312377929688,
"loss": 1.0225,
"margin_dpo/margin_mean": 45.34085464477539,
"margin_dpo/margin_std": 65.32242584228516,
"step": 181
},
{
"KL/chosen_KL_mean": -38.98612594604492,
"KL/mean": -74.91706848144531,
"KL/rejected_KL_mean": -110.84800720214844,
"KL/std": 66.33081817626953,
"epoch": 0.26725403817914833,
"fcm_dpo/beta": 0.013408947736024857,
"fcm_dpo/delta": -0.18331019580364227,
"fcm_dpo/margin": 71.86188507080078,
"fcm_dpo/q_t": 0.3110619783401489,
"grad_norm": 27.608600616455078,
"learning_rate": 4.598073218215817e-07,
"logits/chosen": -0.4613451063632965,
"logits/rejected": -0.4604541063308716,
"logps/chosen": -80.09397888183594,
"logps/ref_chosen": -41.107852935791016,
"logps/ref_rejected": -89.5215835571289,
"logps/rejected": -200.36959838867188,
"loss": 0.8519,
"margin_dpo/margin_mean": 71.86187744140625,
"margin_dpo/margin_std": 77.41173553466797,
"step": 182
},
{
"KL/chosen_KL_mean": -55.72860336303711,
"KL/mean": -78.53468322753906,
"KL/rejected_KL_mean": -101.34075927734375,
"KL/std": 50.84642791748047,
"epoch": 0.2687224669603524,
"fcm_dpo/beta": 0.013560278341174126,
"fcm_dpo/delta": 0.19175204634666443,
"fcm_dpo/margin": 45.612159729003906,
"fcm_dpo/q_t": 0.3727971911430359,
"grad_norm": 30.786500930786133,
"learning_rate": 4.5910671414162484e-07,
"logits/chosen": -0.5146129131317139,
"logits/rejected": -0.4842214584350586,
"logps/chosen": -113.25316619873047,
"logps/ref_chosen": -57.52456283569336,
"logps/ref_rejected": -75.97572326660156,
"logps/rejected": -177.3164825439453,
"loss": 1.0039,
"margin_dpo/margin_mean": 45.612159729003906,
"margin_dpo/margin_std": 55.62657928466797,
"step": 183
},
{
"KL/chosen_KL_mean": -42.92633056640625,
"KL/mean": -64.14622497558594,
"KL/rejected_KL_mean": -85.3661117553711,
"KL/std": 48.59588623046875,
"epoch": 0.2701908957415565,
"fcm_dpo/beta": 0.014429578557610512,
"fcm_dpo/delta": 0.19666652381420135,
"fcm_dpo/margin": 42.43977737426758,
"fcm_dpo/q_t": 0.37740087509155273,
"grad_norm": 24.578208923339844,
"learning_rate": 4.5840059630527985e-07,
"logits/chosen": -0.5529508590698242,
"logits/rejected": -0.5374755263328552,
"logps/chosen": -101.47128295898438,
"logps/ref_chosen": -58.544952392578125,
"logps/ref_rejected": -76.63406372070312,
"logps/rejected": -162.00018310546875,
"loss": 1.0119,
"margin_dpo/margin_mean": 42.43977737426758,
"margin_dpo/margin_std": 58.118682861328125,
"step": 184
},
{
"KL/chosen_KL_mean": -47.32001495361328,
"KL/mean": -64.94092559814453,
"KL/rejected_KL_mean": -82.56183624267578,
"KL/std": 51.73418045043945,
"epoch": 0.27165932452276065,
"fcm_dpo/beta": 0.01505757961422205,
"fcm_dpo/delta": 0.2848103940486908,
"fcm_dpo/margin": 35.24181365966797,
"fcm_dpo/q_t": 0.3963721990585327,
"grad_norm": 37.11123275756836,
"learning_rate": 4.5768898691940836e-07,
"logits/chosen": -0.549095094203949,
"logits/rejected": -0.5053994655609131,
"logps/chosen": -109.34586334228516,
"logps/ref_chosen": -62.025848388671875,
"logps/ref_rejected": -73.7625961303711,
"logps/rejected": -156.32443237304688,
"loss": 1.0906,
"margin_dpo/margin_mean": 35.24181365966797,
"margin_dpo/margin_std": 59.88246154785156,
"step": 185
},
{
"KL/chosen_KL_mean": -44.749820709228516,
"KL/mean": -75.26815795898438,
"KL/rejected_KL_mean": -105.7864990234375,
"KL/std": 57.48197937011719,
"epoch": 0.27312775330396477,
"fcm_dpo/beta": 0.014994775876402855,
"fcm_dpo/delta": -0.12706023454666138,
"fcm_dpo/margin": 61.03667449951172,
"fcm_dpo/q_t": 0.3158210217952728,
"grad_norm": 40.3712043762207,
"learning_rate": 4.5697190473557947e-07,
"logits/chosen": -0.5640110373497009,
"logits/rejected": -0.5185353755950928,
"logps/chosen": -114.10328674316406,
"logps/ref_chosen": -69.35346984863281,
"logps/ref_rejected": -88.07244873046875,
"logps/rejected": -193.85894775390625,
"loss": 0.8233,
"margin_dpo/margin_mean": 61.03667068481445,
"margin_dpo/margin_std": 60.81195831298828,
"step": 186
},
{
"KL/chosen_KL_mean": -43.97365951538086,
"KL/mean": -68.62959289550781,
"KL/rejected_KL_mean": -93.28553009033203,
"KL/std": 53.63896179199219,
"epoch": 0.2745961820851689,
"fcm_dpo/beta": 0.015185382217168808,
"fcm_dpo/delta": 0.05268273502588272,
"fcm_dpo/margin": 49.31187438964844,
"fcm_dpo/q_t": 0.35000455379486084,
"grad_norm": 30.36644172668457,
"learning_rate": 4.5624936864957555e-07,
"logits/chosen": -0.5303990840911865,
"logits/rejected": -0.5252684354782104,
"logps/chosen": -96.73011779785156,
"logps/ref_chosen": -52.7564582824707,
"logps/ref_rejected": -81.96910095214844,
"logps/rejected": -175.254638671875,
"loss": 0.9437,
"margin_dpo/margin_mean": 49.3118782043457,
"margin_dpo/margin_std": 60.085113525390625,
"step": 187
},
{
"KL/chosen_KL_mean": -39.8115234375,
"KL/mean": -70.60044860839844,
"KL/rejected_KL_mean": -101.38937377929688,
"KL/std": 61.41886520385742,
"epoch": 0.27606461086637296,
"fcm_dpo/beta": 0.01480124332010746,
"fcm_dpo/delta": -0.12262607365846634,
"fcm_dpo/margin": 61.577857971191406,
"fcm_dpo/q_t": 0.3225354552268982,
"grad_norm": 36.456275939941406,
"learning_rate": 4.5552139770089454e-07,
"logits/chosen": -0.5304605960845947,
"logits/rejected": -0.525503396987915,
"logps/chosen": -89.22700500488281,
"logps/ref_chosen": -49.415489196777344,
"logps/ref_rejected": -89.54043579101562,
"logps/rejected": -190.9298095703125,
"loss": 0.8553,
"margin_dpo/margin_mean": 61.577857971191406,
"margin_dpo/margin_std": 67.12760925292969,
"step": 188
},
{
"KL/chosen_KL_mean": -46.008487701416016,
"KL/mean": -73.29472351074219,
"KL/rejected_KL_mean": -100.58096313476562,
"KL/std": 59.5657844543457,
"epoch": 0.2775330396475771,
"fcm_dpo/beta": 0.014734582975506783,
"fcm_dpo/delta": -0.004572561010718346,
"fcm_dpo/margin": 54.572471618652344,
"fcm_dpo/q_t": 0.3462943434715271,
"grad_norm": 32.416141510009766,
"learning_rate": 4.5478801107224794e-07,
"logits/chosen": -0.5101655721664429,
"logits/rejected": -0.47071373462677,
"logps/chosen": -98.4074478149414,
"logps/ref_chosen": -52.39896011352539,
"logps/ref_rejected": -72.16735076904297,
"logps/rejected": -172.74832153320312,
"loss": 0.9359,
"margin_dpo/margin_mean": 54.572471618652344,
"margin_dpo/margin_std": 71.166015625,
"step": 189
},
{
"KL/chosen_KL_mean": -49.97734069824219,
"KL/mean": -79.54035949707031,
"KL/rejected_KL_mean": -109.10337829589844,
"KL/std": 62.99628829956055,
"epoch": 0.2790014684287812,
"fcm_dpo/beta": 0.014730885624885559,
"fcm_dpo/delta": -0.08081679046154022,
"fcm_dpo/margin": 59.12603759765625,
"fcm_dpo/q_t": 0.336655855178833,
"grad_norm": 32.33114242553711,
"learning_rate": 4.5404922808905543e-07,
"logits/chosen": -0.5516223907470703,
"logits/rejected": -0.525041937828064,
"logps/chosen": -114.660400390625,
"logps/ref_chosen": -64.68305969238281,
"logps/ref_rejected": -102.55052185058594,
"logps/rejected": -211.65390014648438,
"loss": 0.9259,
"margin_dpo/margin_mean": 59.12604522705078,
"margin_dpo/margin_std": 73.30282592773438,
"step": 190
},
{
"KL/chosen_KL_mean": -43.832725524902344,
"KL/mean": -86.67939758300781,
"KL/rejected_KL_mean": -129.52606201171875,
"KL/std": 77.8781967163086,
"epoch": 0.28046989720998533,
"fcm_dpo/beta": 0.013516737148165703,
"fcm_dpo/delta": -0.40941235423088074,
"fcm_dpo/margin": 85.6933364868164,
"fcm_dpo/q_t": 0.2823325991630554,
"grad_norm": 29.902969360351562,
"learning_rate": 4.5330506821893565e-07,
"logits/chosen": -0.5110219717025757,
"logits/rejected": -0.49608632922172546,
"logps/chosen": -112.49159240722656,
"logps/ref_chosen": -68.65887451171875,
"logps/ref_rejected": -110.1396713256836,
"logps/rejected": -239.66574096679688,
"loss": 0.7371,
"margin_dpo/margin_mean": 85.6933364868164,
"margin_dpo/margin_std": 83.78256225585938,
"step": 191
},
{
"KL/chosen_KL_mean": -60.607242584228516,
"KL/mean": -91.75163269042969,
"KL/rejected_KL_mean": -122.89604187011719,
"KL/std": 67.90768432617188,
"epoch": 0.28193832599118945,
"fcm_dpo/beta": 0.01330653764307499,
"fcm_dpo/delta": -0.031592316925525665,
"fcm_dpo/margin": 62.288795471191406,
"fcm_dpo/q_t": 0.3421240448951721,
"grad_norm": 32.041038513183594,
"learning_rate": 4.5255555107119336e-07,
"logits/chosen": -0.5030392408370972,
"logits/rejected": -0.4946763217449188,
"logps/chosen": -130.3341522216797,
"logps/ref_chosen": -69.72691345214844,
"logps/ref_rejected": -103.32135009765625,
"logps/rejected": -226.21737670898438,
"loss": 0.9197,
"margin_dpo/margin_mean": 62.28879928588867,
"margin_dpo/margin_std": 77.69465637207031,
"step": 192
},
{
"KL/chosen_KL_mean": -63.92502212524414,
"KL/mean": -82.90367889404297,
"KL/rejected_KL_mean": -101.88233947753906,
"KL/std": 57.34717559814453,
"epoch": 0.2834067547723935,
"fcm_dpo/beta": 0.013927249237895012,
"fcm_dpo/delta": 0.2825108766555786,
"fcm_dpo/margin": 37.95730972290039,
"fcm_dpo/q_t": 0.3886550962924957,
"grad_norm": 43.13148498535156,
"learning_rate": 4.5180069639630236e-07,
"logits/chosen": -0.5010656118392944,
"logits/rejected": -0.4694574475288391,
"logps/chosen": -124.11552429199219,
"logps/ref_chosen": -60.19049835205078,
"logps/ref_rejected": -76.40755462646484,
"logps/rejected": -178.28988647460938,
"loss": 1.1345,
"margin_dpo/margin_mean": 37.95730972290039,
"margin_dpo/margin_std": 70.45993041992188,
"step": 193
},
{
"KL/chosen_KL_mean": -39.9882698059082,
"KL/mean": -70.1005859375,
"KL/rejected_KL_mean": -100.21292114257812,
"KL/std": 56.194984436035156,
"epoch": 0.28487518355359764,
"fcm_dpo/beta": 0.013896675780415535,
"fcm_dpo/delta": -0.040464356541633606,
"fcm_dpo/margin": 60.22463607788086,
"fcm_dpo/q_t": 0.3275321424007416,
"grad_norm": 26.552078247070312,
"learning_rate": 4.510405240853854e-07,
"logits/chosen": -0.4238455891609192,
"logits/rejected": -0.4025278389453888,
"logps/chosen": -77.82864379882812,
"logps/ref_chosen": -37.84037399291992,
"logps/ref_rejected": -60.684783935546875,
"logps/rejected": -160.897705078125,
"loss": 0.8503,
"margin_dpo/margin_mean": 60.224639892578125,
"margin_dpo/margin_std": 58.84598922729492,
"step": 194
},
{
"KL/chosen_KL_mean": -66.05638122558594,
"KL/mean": -98.05033874511719,
"KL/rejected_KL_mean": -130.04428100585938,
"KL/std": 64.62409210205078,
"epoch": 0.28634361233480177,
"fcm_dpo/beta": 0.013699124567210674,
"fcm_dpo/delta": -0.0841585025191307,
"fcm_dpo/margin": 63.98790740966797,
"fcm_dpo/q_t": 0.32316848635673523,
"grad_norm": 37.01463317871094,
"learning_rate": 4.5027505416968985e-07,
"logits/chosen": -0.4037872552871704,
"logits/rejected": -0.4184109568595886,
"logps/chosen": -120.94795227050781,
"logps/ref_chosen": -54.891571044921875,
"logps/ref_rejected": -96.77095794677734,
"logps/rejected": -226.81524658203125,
"loss": 0.8456,
"margin_dpo/margin_mean": 63.98790740966797,
"margin_dpo/margin_std": 66.32554626464844,
"step": 195
},
{
"KL/chosen_KL_mean": -52.29330825805664,
"KL/mean": -89.10833740234375,
"KL/rejected_KL_mean": -125.9233627319336,
"KL/std": 72.76502990722656,
"epoch": 0.2878120411160059,
"fcm_dpo/beta": 0.013190150260925293,
"fcm_dpo/delta": -0.19196948409080505,
"fcm_dpo/margin": 73.63005828857422,
"fcm_dpo/q_t": 0.3175709843635559,
"grad_norm": 33.40938949584961,
"learning_rate": 4.495043068200599e-07,
"logits/chosen": -0.44675952196121216,
"logits/rejected": -0.4224822521209717,
"logps/chosen": -105.5385513305664,
"logps/ref_chosen": -53.245243072509766,
"logps/ref_rejected": -76.05294799804688,
"logps/rejected": -201.976318359375,
"loss": 0.8571,
"margin_dpo/margin_mean": 73.63005065917969,
"margin_dpo/margin_std": 82.35310363769531,
"step": 196
},
{
"KL/chosen_KL_mean": -53.17420959472656,
"KL/mean": -82.42511749267578,
"KL/rejected_KL_mean": -111.676025390625,
"KL/std": 61.11585235595703,
"epoch": 0.28928046989721,
"fcm_dpo/beta": 0.013252440840005875,
"fcm_dpo/delta": 0.02636050246655941,
"fcm_dpo/margin": 58.50181198120117,
"fcm_dpo/q_t": 0.34405672550201416,
"grad_norm": 43.14238739013672,
"learning_rate": 4.4872830234640493e-07,
"logits/chosen": -0.3930720090866089,
"logits/rejected": -0.38080155849456787,
"logps/chosen": -113.59454345703125,
"logps/ref_chosen": -60.42033386230469,
"logps/ref_rejected": -77.20890808105469,
"logps/rejected": -188.8849334716797,
"loss": 0.9062,
"margin_dpo/margin_mean": 58.50181198120117,
"margin_dpo/margin_std": 66.8076400756836,
"step": 197
},
{
"KL/chosen_KL_mean": -57.16075134277344,
"KL/mean": -91.8175277709961,
"KL/rejected_KL_mean": -126.47431945800781,
"KL/std": 73.49171447753906,
"epoch": 0.2907488986784141,
"fcm_dpo/beta": 0.013048326596617699,
"fcm_dpo/delta": -0.1146821528673172,
"fcm_dpo/margin": 69.31356048583984,
"fcm_dpo/q_t": 0.3285406827926636,
"grad_norm": 33.66745376586914,
"learning_rate": 4.479470611971645e-07,
"logits/chosen": -0.4547293186187744,
"logits/rejected": -0.4567200243473053,
"logps/chosen": -112.19693756103516,
"logps/ref_chosen": -55.03618621826172,
"logps/ref_rejected": -97.24325561523438,
"logps/rejected": -223.7175750732422,
"loss": 0.8665,
"margin_dpo/margin_mean": 69.31356048583984,
"margin_dpo/margin_std": 82.79634094238281,
"step": 198
},
{
"KL/chosen_KL_mean": -57.0550537109375,
"KL/mean": -89.62132263183594,
"KL/rejected_KL_mean": -122.18759155273438,
"KL/std": 66.67041015625,
"epoch": 0.2922173274596182,
"fcm_dpo/beta": 0.012743506580591202,
"fcm_dpo/delta": -0.03456920385360718,
"fcm_dpo/margin": 65.13253021240234,
"fcm_dpo/q_t": 0.33725377917289734,
"grad_norm": 36.01947021484375,
"learning_rate": 4.471606039587695e-07,
"logits/chosen": -0.4379258155822754,
"logits/rejected": -0.4226016104221344,
"logps/chosen": -113.88388061523438,
"logps/ref_chosen": -56.828826904296875,
"logps/ref_rejected": -84.64820861816406,
"logps/rejected": -206.83580017089844,
"loss": 0.9154,
"margin_dpo/margin_mean": 65.13253021240234,
"margin_dpo/margin_std": 77.29293060302734,
"step": 199
},
{
"KL/chosen_KL_mean": -54.75391387939453,
"KL/mean": -90.25743103027344,
"KL/rejected_KL_mean": -125.76095581054688,
"KL/std": 74.021484375,
"epoch": 0.2936857562408223,
"fcm_dpo/beta": 0.012570657767355442,
"fcm_dpo/delta": -0.10279311239719391,
"fcm_dpo/margin": 71.00704956054688,
"fcm_dpo/q_t": 0.33011093735694885,
"grad_norm": 34.09638977050781,
"learning_rate": 4.4636895135509966e-07,
"logits/chosen": -0.398794949054718,
"logits/rejected": -0.3711370825767517,
"logps/chosen": -107.82096862792969,
"logps/ref_chosen": -53.06706237792969,
"logps/ref_rejected": -80.60843658447266,
"logps/rejected": -206.369384765625,
"loss": 0.8994,
"margin_dpo/margin_mean": 71.00704956054688,
"margin_dpo/margin_std": 86.69010925292969,
"step": 200
},
{
"KL/chosen_KL_mean": -54.42976379394531,
"KL/mean": -89.63890075683594,
"KL/rejected_KL_mean": -124.8480224609375,
"KL/std": 72.89269256591797,
"epoch": 0.29515418502202645,
"fcm_dpo/beta": 0.012411890551447868,
"fcm_dpo/delta": -0.08116737008094788,
"fcm_dpo/margin": 70.41826629638672,
"fcm_dpo/q_t": 0.32952481508255005,
"grad_norm": 35.047454833984375,
"learning_rate": 4.455721242469372e-07,
"logits/chosen": -0.4590557813644409,
"logits/rejected": -0.4471399188041687,
"logps/chosen": -129.8319854736328,
"logps/ref_chosen": -75.4022216796875,
"logps/ref_rejected": -114.80821990966797,
"logps/rejected": -239.65625,
"loss": 0.8873,
"margin_dpo/margin_mean": 70.41826629638672,
"margin_dpo/margin_std": 82.54559326171875,
"step": 201
},
{
"KL/chosen_KL_mean": -58.886390686035156,
"KL/mean": -83.84607696533203,
"KL/rejected_KL_mean": -108.8057632446289,
"KL/std": 64.16035461425781,
"epoch": 0.2966226138032305,
"fcm_dpo/beta": 0.012727733701467514,
"fcm_dpo/delta": 0.17515869438648224,
"fcm_dpo/margin": 49.91937255859375,
"fcm_dpo/q_t": 0.3769991397857666,
"grad_norm": 33.56517791748047,
"learning_rate": 4.4477014363141755e-07,
"logits/chosen": -0.43319663405418396,
"logits/rejected": -0.435769647359848,
"logps/chosen": -108.98770904541016,
"logps/ref_chosen": -50.101318359375,
"logps/ref_rejected": -86.98503112792969,
"logps/rejected": -195.79080200195312,
"loss": 1.053,
"margin_dpo/margin_mean": 49.919376373291016,
"margin_dpo/margin_std": 78.88993835449219,
"step": 202
},
{
"KL/chosen_KL_mean": -55.39906311035156,
"KL/mean": -85.28140258789062,
"KL/rejected_KL_mean": -115.16374206542969,
"KL/std": 64.33355712890625,
"epoch": 0.29809104258443464,
"fcm_dpo/beta": 0.012859424576163292,
"fcm_dpo/delta": 0.03409314155578613,
"fcm_dpo/margin": 59.764678955078125,
"fcm_dpo/q_t": 0.34479662775993347,
"grad_norm": 36.9451904296875,
"learning_rate": 4.439630306414758e-07,
"logits/chosen": -0.4340917766094208,
"logits/rejected": -0.4139591157436371,
"logps/chosen": -116.00875854492188,
"logps/ref_chosen": -60.60969543457031,
"logps/ref_rejected": -85.89596557617188,
"logps/rejected": -201.05970764160156,
"loss": 0.9075,
"margin_dpo/margin_mean": 59.764678955078125,
"margin_dpo/margin_std": 68.13716125488281,
"step": 203
},
{
"KL/chosen_KL_mean": -66.02056121826172,
"KL/mean": -94.26062774658203,
"KL/rejected_KL_mean": -122.50070190429688,
"KL/std": 72.59966278076172,
"epoch": 0.29955947136563876,
"fcm_dpo/beta": 0.013025782071053982,
"fcm_dpo/delta": 0.06942086666822433,
"fcm_dpo/margin": 56.480140686035156,
"fcm_dpo/q_t": 0.3637349605560303,
"grad_norm": 41.804168701171875,
"learning_rate": 4.431508065452897e-07,
"logits/chosen": -0.4650980234146118,
"logits/rejected": -0.42012155055999756,
"logps/chosen": -146.18551635742188,
"logps/ref_chosen": -80.16496276855469,
"logps/ref_rejected": -87.69590759277344,
"logps/rejected": -210.1966094970703,
"loss": 0.9857,
"margin_dpo/margin_mean": 56.480140686035156,
"margin_dpo/margin_std": 81.36882019042969,
"step": 204
},
{
"KL/chosen_KL_mean": -67.76717376708984,
"KL/mean": -103.71720886230469,
"KL/rejected_KL_mean": -139.667236328125,
"KL/std": 76.89108276367188,
"epoch": 0.3010279001468429,
"fcm_dpo/beta": 0.012646548449993134,
"fcm_dpo/delta": -0.1265978217124939,
"fcm_dpo/margin": 71.9000473022461,
"fcm_dpo/q_t": 0.32160481810569763,
"grad_norm": 31.984642028808594,
"learning_rate": 4.4233349274571974e-07,
"logits/chosen": -0.40963441133499146,
"logits/rejected": -0.3742339015007019,
"logps/chosen": -127.15190887451172,
"logps/ref_chosen": -59.384735107421875,
"logps/ref_rejected": -85.12505340576172,
"logps/rejected": -224.7922821044922,
"loss": 0.898,
"margin_dpo/margin_mean": 71.9000473022461,
"margin_dpo/margin_std": 81.00496673583984,
"step": 205
},
{
"KL/chosen_KL_mean": -65.09518432617188,
"KL/mean": -104.39747619628906,
"KL/rejected_KL_mean": -143.6997833251953,
"KL/std": 76.27721405029297,
"epoch": 0.302496328928047,
"fcm_dpo/beta": 0.01231662929058075,
"fcm_dpo/delta": -0.18836334347724915,
"fcm_dpo/margin": 78.60458374023438,
"fcm_dpo/q_t": 0.3099524676799774,
"grad_norm": 42.828025817871094,
"learning_rate": 4.415111107797445e-07,
"logits/chosen": -0.39149826765060425,
"logits/rejected": -0.3944673538208008,
"logps/chosen": -112.0596923828125,
"logps/ref_chosen": -46.964500427246094,
"logps/ref_rejected": -98.9534912109375,
"logps/rejected": -242.6532745361328,
"loss": 0.8418,
"margin_dpo/margin_mean": 78.60458374023438,
"margin_dpo/margin_std": 83.66462707519531,
"step": 206
},
{
"KL/chosen_KL_mean": -51.632415771484375,
"KL/mean": -96.7965087890625,
"KL/rejected_KL_mean": -141.96058654785156,
"KL/std": 84.35016632080078,
"epoch": 0.3039647577092511,
"fcm_dpo/beta": 0.011775311082601547,
"fcm_dpo/delta": -0.2946253716945648,
"fcm_dpo/margin": 90.32818603515625,
"fcm_dpo/q_t": 0.30258244276046753,
"grad_norm": 29.97208595275879,
"learning_rate": 4.4068368231789365e-07,
"logits/chosen": -0.4526169002056122,
"logits/rejected": -0.42235511541366577,
"logps/chosen": -107.68867492675781,
"logps/ref_chosen": -56.05625915527344,
"logps/ref_rejected": -84.44779968261719,
"logps/rejected": -226.40838623046875,
"loss": 0.7953,
"margin_dpo/margin_mean": 90.32817840576172,
"margin_dpo/margin_std": 96.11925506591797,
"step": 207
},
{
"KL/chosen_KL_mean": -75.16487121582031,
"KL/mean": -111.59002685546875,
"KL/rejected_KL_mean": -148.01516723632812,
"KL/std": 76.63873291015625,
"epoch": 0.3054331864904552,
"fcm_dpo/beta": 0.011480957269668579,
"fcm_dpo/delta": -0.03990346938371658,
"fcm_dpo/margin": 72.85030364990234,
"fcm_dpo/q_t": 0.33449286222457886,
"grad_norm": 32.494258880615234,
"learning_rate": 4.398512291636768e-07,
"logits/chosen": -0.4866720736026764,
"logits/rejected": -0.466644287109375,
"logps/chosen": -142.23248291015625,
"logps/ref_chosen": -67.06761169433594,
"logps/ref_rejected": -94.28689575195312,
"logps/rejected": -242.30206298828125,
"loss": 0.9049,
"margin_dpo/margin_mean": 72.85029602050781,
"margin_dpo/margin_std": 86.24763488769531,
"step": 208
},
{
"KL/chosen_KL_mean": -63.27362060546875,
"KL/mean": -94.08541107177734,
"KL/rejected_KL_mean": -124.89720916748047,
"KL/std": 64.80196380615234,
"epoch": 0.3069016152716593,
"fcm_dpo/beta": 0.011625498533248901,
"fcm_dpo/delta": 0.09011469781398773,
"fcm_dpo/margin": 61.62358093261719,
"fcm_dpo/q_t": 0.35670265555381775,
"grad_norm": 35.18954086303711,
"learning_rate": 4.3901377325300857e-07,
"logits/chosen": -0.4050047993659973,
"logits/rejected": -0.390215665102005,
"logps/chosen": -119.45531463623047,
"logps/ref_chosen": -56.18169403076172,
"logps/ref_rejected": -80.94152069091797,
"logps/rejected": -205.83872985839844,
"loss": 0.9732,
"margin_dpo/margin_mean": 61.62357711791992,
"margin_dpo/margin_std": 79.23416137695312,
"step": 209
},
{
"KL/chosen_KL_mean": -55.82368469238281,
"KL/mean": -89.87747955322266,
"KL/rejected_KL_mean": -123.9312744140625,
"KL/std": 65.2939682006836,
"epoch": 0.30837004405286345,
"fcm_dpo/beta": 0.011724742129445076,
"fcm_dpo/delta": 0.0014328230172395706,
"fcm_dpo/margin": 68.10758972167969,
"fcm_dpo/q_t": 0.3393843472003937,
"grad_norm": 25.020299911499023,
"learning_rate": 4.381713366536311e-07,
"logits/chosen": -0.4426725208759308,
"logits/rejected": -0.42541852593421936,
"logps/chosen": -102.19551086425781,
"logps/ref_chosen": -46.371822357177734,
"logps/ref_rejected": -76.68162536621094,
"logps/rejected": -200.61288452148438,
"loss": 0.8903,
"margin_dpo/margin_mean": 68.10758972167969,
"margin_dpo/margin_std": 74.40350341796875,
"step": 210
},
{
"KL/chosen_KL_mean": -80.90629577636719,
"KL/mean": -109.43853759765625,
"KL/rejected_KL_mean": -137.9707794189453,
"KL/std": 67.03673553466797,
"epoch": 0.30983847283406757,
"fcm_dpo/beta": 0.011907186359167099,
"fcm_dpo/delta": 0.1294347047805786,
"fcm_dpo/margin": 57.064483642578125,
"fcm_dpo/q_t": 0.36575454473495483,
"grad_norm": 36.52604675292969,
"learning_rate": 4.373239415645323e-07,
"logits/chosen": -0.48629775643348694,
"logits/rejected": -0.43736910820007324,
"logps/chosen": -159.83865356445312,
"logps/ref_chosen": -78.93235778808594,
"logps/ref_rejected": -86.82098388671875,
"logps/rejected": -224.79176330566406,
"loss": 0.9892,
"margin_dpo/margin_mean": 57.064483642578125,
"margin_dpo/margin_std": 76.28172302246094,
"step": 211
},
{
"KL/chosen_KL_mean": -65.99049377441406,
"KL/mean": -107.37837982177734,
"KL/rejected_KL_mean": -148.76626586914062,
"KL/std": 81.93545532226562,
"epoch": 0.31130690161527164,
"fcm_dpo/beta": 0.011535340920090675,
"fcm_dpo/delta": -0.17738127708435059,
"fcm_dpo/margin": 82.77577209472656,
"fcm_dpo/q_t": 0.31630581617355347,
"grad_norm": 26.731220245361328,
"learning_rate": 4.3647161031536086e-07,
"logits/chosen": -0.49336063861846924,
"logits/rejected": -0.48159685730934143,
"logps/chosen": -124.18750762939453,
"logps/ref_chosen": -58.19701385498047,
"logps/ref_rejected": -103.05785369873047,
"logps/rejected": -251.82412719726562,
"loss": 0.8567,
"margin_dpo/margin_mean": 82.77577209472656,
"margin_dpo/margin_std": 87.4062728881836,
"step": 212
},
{
"KL/chosen_KL_mean": -59.183773040771484,
"KL/mean": -100.06889343261719,
"KL/rejected_KL_mean": -140.95401000976562,
"KL/std": 71.30158996582031,
"epoch": 0.31277533039647576,
"fcm_dpo/beta": 0.011260983534157276,
"fcm_dpo/delta": -0.13558340072631836,
"fcm_dpo/margin": 81.77023315429688,
"fcm_dpo/q_t": 0.316591739654541,
"grad_norm": 29.752309799194336,
"learning_rate": 4.3561436536583774e-07,
"logits/chosen": -0.4815593957901001,
"logits/rejected": -0.443857342004776,
"logps/chosen": -126.69648742675781,
"logps/ref_chosen": -67.51271057128906,
"logps/ref_rejected": -93.91471862792969,
"logps/rejected": -234.8687286376953,
"loss": 0.843,
"margin_dpo/margin_mean": 81.77023315429688,
"margin_dpo/margin_std": 84.07875061035156,
"step": 213
},
{
"KL/chosen_KL_mean": -57.820125579833984,
"KL/mean": -92.26338195800781,
"KL/rejected_KL_mean": -126.70663452148438,
"KL/std": 73.20215606689453,
"epoch": 0.3142437591776799,
"fcm_dpo/beta": 0.011330001056194305,
"fcm_dpo/delta": 0.02105110138654709,
"fcm_dpo/margin": 68.88651275634766,
"fcm_dpo/q_t": 0.3464926779270172,
"grad_norm": 35.64681625366211,
"learning_rate": 4.3475222930516473e-07,
"logits/chosen": -0.43472471833229065,
"logits/rejected": -0.4402396082878113,
"logps/chosen": -99.42501831054688,
"logps/ref_chosen": -41.604888916015625,
"logps/ref_rejected": -77.51741027832031,
"logps/rejected": -204.22406005859375,
"loss": 0.9323,
"margin_dpo/margin_mean": 68.88650512695312,
"margin_dpo/margin_std": 85.51469421386719,
"step": 214
},
{
"KL/chosen_KL_mean": -63.354827880859375,
"KL/mean": -100.20165252685547,
"KL/rejected_KL_mean": -137.04847717285156,
"KL/std": 71.63882446289062,
"epoch": 0.315712187958884,
"fcm_dpo/beta": 0.011217828840017319,
"fcm_dpo/delta": -0.02969864010810852,
"fcm_dpo/margin": 73.69364929199219,
"fcm_dpo/q_t": 0.33209890127182007,
"grad_norm": 32.47001647949219,
"learning_rate": 4.3388522485142885e-07,
"logits/chosen": -0.4393133521080017,
"logits/rejected": -0.42265331745147705,
"logps/chosen": -116.63409423828125,
"logps/ref_chosen": -53.279266357421875,
"logps/ref_rejected": -89.96464538574219,
"logps/rejected": -227.01312255859375,
"loss": 0.8721,
"margin_dpo/margin_mean": 73.69364929199219,
"margin_dpo/margin_std": 77.87715911865234,
"step": 215
},
{
"KL/chosen_KL_mean": -65.12640380859375,
"KL/mean": -104.15386199951172,
"KL/rejected_KL_mean": -143.18133544921875,
"KL/std": 75.26974487304688,
"epoch": 0.31718061674008813,
"fcm_dpo/beta": 0.011194197461009026,
"fcm_dpo/delta": -0.08120834827423096,
"fcm_dpo/margin": 78.054931640625,
"fcm_dpo/q_t": 0.3296348750591278,
"grad_norm": 33.180416107177734,
"learning_rate": 4.330133748510036e-07,
"logits/chosen": -0.46606436371803284,
"logits/rejected": -0.44526204466819763,
"logps/chosen": -114.01419067382812,
"logps/ref_chosen": -48.887794494628906,
"logps/ref_rejected": -77.19892883300781,
"logps/rejected": -220.3802490234375,
"loss": 0.8957,
"margin_dpo/margin_mean": 78.054931640625,
"margin_dpo/margin_std": 89.83686828613281,
"step": 216
},
{
"KL/chosen_KL_mean": -61.40077209472656,
"KL/mean": -104.93330383300781,
"KL/rejected_KL_mean": -148.4658203125,
"KL/std": 74.56744384765625,
"epoch": 0.3186490455212922,
"fcm_dpo/beta": 0.010798503644764423,
"fcm_dpo/delta": -0.15515443682670593,
"fcm_dpo/margin": 87.06504821777344,
"fcm_dpo/q_t": 0.3086259365081787,
"grad_norm": 29.548845291137695,
"learning_rate": 4.3213670227794757e-07,
"logits/chosen": -0.461231529712677,
"logits/rejected": -0.4476476311683655,
"logps/chosen": -111.24607849121094,
"logps/ref_chosen": -49.845306396484375,
"logps/ref_rejected": -100.07832336425781,
"logps/rejected": -248.54415893554688,
"loss": 0.7976,
"margin_dpo/margin_mean": 87.06504821777344,
"margin_dpo/margin_std": 80.49182891845703,
"step": 217
},
{
"KL/chosen_KL_mean": -65.54306030273438,
"KL/mean": -100.61727142333984,
"KL/rejected_KL_mean": -135.6914825439453,
"KL/std": 72.58134460449219,
"epoch": 0.3201174743024963,
"fcm_dpo/beta": 0.010796687565743923,
"fcm_dpo/delta": 0.046147942543029785,
"fcm_dpo/margin": 70.14842224121094,
"fcm_dpo/q_t": 0.3473072052001953,
"grad_norm": 28.300230026245117,
"learning_rate": 4.3125523023339815e-07,
"logits/chosen": -0.4414158761501312,
"logits/rejected": -0.42348459362983704,
"logps/chosen": -124.11973571777344,
"logps/ref_chosen": -58.576683044433594,
"logps/ref_rejected": -87.84639739990234,
"logps/rejected": -223.53787231445312,
"loss": 0.9229,
"margin_dpo/margin_mean": 70.14842224121094,
"margin_dpo/margin_std": 81.61410522460938,
"step": 218
},
{
"KL/chosen_KL_mean": -69.83723449707031,
"KL/mean": -99.55513000488281,
"KL/rejected_KL_mean": -129.27301025390625,
"KL/std": 70.54728698730469,
"epoch": 0.32158590308370044,
"fcm_dpo/beta": 0.011129364371299744,
"fcm_dpo/delta": 0.14702974259853363,
"fcm_dpo/margin": 59.4357795715332,
"fcm_dpo/q_t": 0.3667854070663452,
"grad_norm": 33.41205596923828,
"learning_rate": 4.303689819449636e-07,
"logits/chosen": -0.470586359500885,
"logits/rejected": -0.45085620880126953,
"logps/chosen": -130.9210968017578,
"logps/ref_chosen": -61.083858489990234,
"logps/ref_rejected": -85.83042907714844,
"logps/rejected": -215.1034393310547,
"loss": 0.9974,
"margin_dpo/margin_mean": 59.43577575683594,
"margin_dpo/margin_std": 81.409912109375,
"step": 219
},
{
"KL/chosen_KL_mean": -76.43949890136719,
"KL/mean": -103.16355895996094,
"KL/rejected_KL_mean": -129.88760375976562,
"KL/std": 62.09259796142578,
"epoch": 0.32305433186490456,
"fcm_dpo/beta": 0.011432506144046783,
"fcm_dpo/delta": 0.20173068344593048,
"fcm_dpo/margin": 53.44810485839844,
"fcm_dpo/q_t": 0.3675389885902405,
"grad_norm": 29.649065017700195,
"learning_rate": 4.2947798076611047e-07,
"logits/chosen": -0.46881765127182007,
"logits/rejected": -0.4399240016937256,
"logps/chosen": -146.47079467773438,
"logps/ref_chosen": -70.03128051757812,
"logps/ref_rejected": -87.68551635742188,
"logps/rejected": -217.5731201171875,
"loss": 0.9669,
"margin_dpo/margin_mean": 53.4481086730957,
"margin_dpo/margin_std": 57.649662017822266,
"step": 220
},
{
"KL/chosen_KL_mean": -77.00071716308594,
"KL/mean": -127.14787292480469,
"KL/rejected_KL_mean": -177.29502868652344,
"KL/std": 87.18041229248047,
"epoch": 0.3245227606461087,
"fcm_dpo/beta": 0.011109141632914543,
"fcm_dpo/delta": -0.3521580696105957,
"fcm_dpo/margin": 100.2943115234375,
"fcm_dpo/q_t": 0.2772984504699707,
"grad_norm": 28.87704086303711,
"learning_rate": 4.285822501755485e-07,
"logits/chosen": -0.4105208218097687,
"logits/rejected": -0.416123628616333,
"logps/chosen": -129.15542602539062,
"logps/ref_chosen": -52.15470886230469,
"logps/ref_rejected": -106.46768188476562,
"logps/rejected": -283.7626953125,
"loss": 0.7239,
"margin_dpo/margin_mean": 100.2943115234375,
"margin_dpo/margin_std": 82.78035736083984,
"step": 221
},
{
"KL/chosen_KL_mean": -80.87545013427734,
"KL/mean": -119.49418640136719,
"KL/rejected_KL_mean": -158.1129150390625,
"KL/std": 77.89344024658203,
"epoch": 0.32599118942731276,
"fcm_dpo/beta": 0.010816777125000954,
"fcm_dpo/delta": -0.0389985591173172,
"fcm_dpo/margin": 77.23746490478516,
"fcm_dpo/q_t": 0.33166736364364624,
"grad_norm": 46.94334411621094,
"learning_rate": 4.276818137766118e-07,
"logits/chosen": -0.4480839669704437,
"logits/rejected": -0.4343772232532501,
"logps/chosen": -141.8465576171875,
"logps/ref_chosen": -60.971099853515625,
"logps/ref_rejected": -100.00115203857422,
"logps/rejected": -258.11407470703125,
"loss": 0.8845,
"margin_dpo/margin_mean": 77.23746490478516,
"margin_dpo/margin_std": 84.92472839355469,
"step": 222
},
{
"KL/chosen_KL_mean": -88.08724212646484,
"KL/mean": -121.95101928710938,
"KL/rejected_KL_mean": -155.81478881835938,
"KL/std": 75.61756896972656,
"epoch": 0.3274596182085169,
"fcm_dpo/beta": 0.010851439088582993,
"fcm_dpo/delta": 0.07023735344409943,
"fcm_dpo/margin": 67.72756958007812,
"fcm_dpo/q_t": 0.3549031615257263,
"grad_norm": 38.04423522949219,
"learning_rate": 4.2677669529663686e-07,
"logits/chosen": -0.382046639919281,
"logits/rejected": -0.36027002334594727,
"logps/chosen": -140.72781372070312,
"logps/ref_chosen": -52.64057540893555,
"logps/ref_rejected": -82.82502746582031,
"logps/rejected": -238.63983154296875,
"loss": 1.002,
"margin_dpo/margin_mean": 67.72756958007812,
"margin_dpo/margin_std": 95.45079040527344,
"step": 223
},
{
"KL/chosen_KL_mean": -76.32827758789062,
"KL/mean": -117.27737426757812,
"KL/rejected_KL_mean": -158.2264862060547,
"KL/std": 86.974365234375,
"epoch": 0.328928046989721,
"fcm_dpo/beta": 0.010628387331962585,
"fcm_dpo/delta": -0.0815625712275505,
"fcm_dpo/margin": 81.89820861816406,
"fcm_dpo/q_t": 0.3337979316711426,
"grad_norm": 36.384342193603516,
"learning_rate": 4.2586691858633747e-07,
"logits/chosen": -0.3687441349029541,
"logits/rejected": -0.3405214548110962,
"logps/chosen": -124.9236831665039,
"logps/ref_chosen": -48.59541320800781,
"logps/ref_rejected": -77.11648559570312,
"logps/rejected": -235.3429718017578,
"loss": 0.9033,
"margin_dpo/margin_mean": 81.89820861816406,
"margin_dpo/margin_std": 97.37171936035156,
"step": 224
},
{
"KL/chosen_KL_mean": -89.83382415771484,
"KL/mean": -139.490966796875,
"KL/rejected_KL_mean": -189.14810180664062,
"KL/std": 89.47193908691406,
"epoch": 0.3303964757709251,
"fcm_dpo/beta": 0.010304208844900131,
"fcm_dpo/delta": -0.2496224194765091,
"fcm_dpo/margin": 99.31427001953125,
"fcm_dpo/q_t": 0.3028189539909363,
"grad_norm": 30.707063674926758,
"learning_rate": 4.249525076191759e-07,
"logits/chosen": -0.4086596965789795,
"logits/rejected": -0.3914273977279663,
"logps/chosen": -147.83428955078125,
"logps/ref_chosen": -58.000465393066406,
"logps/ref_rejected": -99.90291595458984,
"logps/rejected": -289.051025390625,
"loss": 0.8152,
"margin_dpo/margin_mean": 99.31427001953125,
"margin_dpo/margin_std": 103.35362243652344,
"step": 225
},
{
"KL/chosen_KL_mean": -76.8957290649414,
"KL/mean": -115.76048278808594,
"KL/rejected_KL_mean": -154.625244140625,
"KL/std": 85.1742935180664,
"epoch": 0.33186490455212925,
"fcm_dpo/beta": 0.010075679048895836,
"fcm_dpo/delta": 0.014316320419311523,
"fcm_dpo/margin": 77.7295150756836,
"fcm_dpo/q_t": 0.34836602210998535,
"grad_norm": 35.826454162597656,
"learning_rate": 4.2403348649073167e-07,
"logits/chosen": -0.3905969262123108,
"logits/rejected": -0.3390669822692871,
"logps/chosen": -135.79452514648438,
"logps/ref_chosen": -58.898799896240234,
"logps/ref_rejected": -78.68775939941406,
"logps/rejected": -233.31300354003906,
"loss": 0.9375,
"margin_dpo/margin_mean": 77.7295150756836,
"margin_dpo/margin_std": 94.68783569335938,
"step": 226
},
{
"KL/chosen_KL_mean": -88.30928802490234,
"KL/mean": -137.89309692382812,
"KL/rejected_KL_mean": -187.4768829345703,
"KL/std": 100.16815948486328,
"epoch": 0.3333333333333333,
"fcm_dpo/beta": 0.009845739230513573,
"fcm_dpo/delta": -0.19868168234825134,
"fcm_dpo/margin": 99.1676025390625,
"fcm_dpo/q_t": 0.3060121238231659,
"grad_norm": 42.023475646972656,
"learning_rate": 4.2310987941806615e-07,
"logits/chosen": -0.41084548830986023,
"logits/rejected": -0.3941611647605896,
"logps/chosen": -147.3814697265625,
"logps/ref_chosen": -59.072181701660156,
"logps/ref_rejected": -99.41236877441406,
"logps/rejected": -286.8892517089844,
"loss": 0.8302,
"margin_dpo/margin_mean": 99.1676025390625,
"margin_dpo/margin_std": 103.44757080078125,
"step": 227
},
{
"KL/chosen_KL_mean": -86.19977569580078,
"KL/mean": -121.89463806152344,
"KL/rejected_KL_mean": -157.5894775390625,
"KL/std": 78.76786804199219,
"epoch": 0.33480176211453744,
"fcm_dpo/beta": 0.009987346827983856,
"fcm_dpo/delta": 0.09334658831357956,
"fcm_dpo/margin": 71.38971710205078,
"fcm_dpo/q_t": 0.35477495193481445,
"grad_norm": 29.949947357177734,
"learning_rate": 4.2218171073908463e-07,
"logits/chosen": -0.39673376083374023,
"logits/rejected": -0.37368690967559814,
"logps/chosen": -152.091064453125,
"logps/ref_chosen": -65.89128875732422,
"logps/ref_rejected": -91.04875183105469,
"logps/rejected": -248.6382293701172,
"loss": 0.961,
"margin_dpo/margin_mean": 71.38971710205078,
"margin_dpo/margin_std": 89.57862091064453,
"step": 228
},
{
"KL/chosen_KL_mean": -86.62499237060547,
"KL/mean": -127.21802520751953,
"KL/rejected_KL_mean": -167.81106567382812,
"KL/std": 92.57134246826172,
"epoch": 0.33627019089574156,
"fcm_dpo/beta": 0.010008785873651505,
"fcm_dpo/delta": -0.013723690994083881,
"fcm_dpo/margin": 81.18606567382812,
"fcm_dpo/q_t": 0.33619070053100586,
"grad_norm": 38.73611068725586,
"learning_rate": 4.212490049118951e-07,
"logits/chosen": -0.49296534061431885,
"logits/rejected": -0.46168243885040283,
"logps/chosen": -157.33135986328125,
"logps/ref_chosen": -70.70637512207031,
"logps/ref_rejected": -84.52741241455078,
"logps/rejected": -252.33847045898438,
"loss": 0.9191,
"margin_dpo/margin_mean": 81.18606567382812,
"margin_dpo/margin_std": 97.13127899169922,
"step": 229
},
{
"KL/chosen_KL_mean": -73.65254211425781,
"KL/mean": -126.47062683105469,
"KL/rejected_KL_mean": -179.28871154785156,
"KL/std": 91.7388916015625,
"epoch": 0.3377386196769457,
"fcm_dpo/beta": 0.009593733586370945,
"fcm_dpo/delta": -0.23911604285240173,
"fcm_dpo/margin": 105.63615417480469,
"fcm_dpo/q_t": 0.29790061712265015,
"grad_norm": 28.22924041748047,
"learning_rate": 4.203117865141635e-07,
"logits/chosen": -0.3197930157184601,
"logits/rejected": -0.3206769824028015,
"logps/chosen": -112.93455505371094,
"logps/ref_chosen": -39.282005310058594,
"logps/ref_rejected": -85.62191009521484,
"logps/rejected": -264.9106140136719,
"loss": 0.793,
"margin_dpo/margin_mean": 105.63615417480469,
"margin_dpo/margin_std": 102.19107055664062,
"step": 230
},
{
"KL/chosen_KL_mean": -75.81376647949219,
"KL/mean": -114.97604370117188,
"KL/rejected_KL_mean": -154.13833618164062,
"KL/std": 78.87530517578125,
"epoch": 0.3392070484581498,
"fcm_dpo/beta": 0.009577736258506775,
"fcm_dpo/delta": 0.05389944836497307,
"fcm_dpo/margin": 78.32456970214844,
"fcm_dpo/q_t": 0.34839433431625366,
"grad_norm": 44.196414947509766,
"learning_rate": 4.1937008024246625e-07,
"logits/chosen": -0.41479164361953735,
"logits/rejected": -0.38350310921669006,
"logps/chosen": -139.0902099609375,
"logps/ref_chosen": -63.27644348144531,
"logps/ref_rejected": -74.1239013671875,
"logps/rejected": -228.26223754882812,
"loss": 0.9068,
"margin_dpo/margin_mean": 78.32456970214844,
"margin_dpo/margin_std": 87.84222412109375,
"step": 231
},
{
"KL/chosen_KL_mean": -92.11940002441406,
"KL/mean": -123.58616638183594,
"KL/rejected_KL_mean": -155.05294799804688,
"KL/std": 88.39076232910156,
"epoch": 0.3406754772393539,
"fcm_dpo/beta": 0.009844740852713585,
"fcm_dpo/delta": 0.19277611374855042,
"fcm_dpo/margin": 62.93353271484375,
"fcm_dpo/q_t": 0.3812215328216553,
"grad_norm": 31.937786102294922,
"learning_rate": 4.1842391091163933e-07,
"logits/chosen": -0.4062514305114746,
"logits/rejected": -0.37542420625686646,
"logps/chosen": -162.8681640625,
"logps/ref_chosen": -70.74876403808594,
"logps/ref_rejected": -83.97706604003906,
"logps/rejected": -239.03001403808594,
"loss": 1.0336,
"margin_dpo/margin_mean": 62.93353271484375,
"margin_dpo/margin_std": 95.02970886230469,
"step": 232
},
{
"KL/chosen_KL_mean": -80.99494171142578,
"KL/mean": -128.9691162109375,
"KL/rejected_KL_mean": -176.94326782226562,
"KL/std": 97.7330322265625,
"epoch": 0.342143906020558,
"fcm_dpo/beta": 0.009784163907170296,
"fcm_dpo/delta": -0.15304714441299438,
"fcm_dpo/margin": 95.94833374023438,
"fcm_dpo/q_t": 0.33123689889907837,
"grad_norm": 36.1855354309082,
"learning_rate": 4.174733034541245e-07,
"logits/chosen": -0.4026934802532196,
"logits/rejected": -0.39757007360458374,
"logps/chosen": -135.87786865234375,
"logps/ref_chosen": -54.8829345703125,
"logps/ref_rejected": -107.4800796508789,
"logps/rejected": -284.42333984375,
"loss": 0.9175,
"margin_dpo/margin_mean": 95.94833374023438,
"margin_dpo/margin_std": 123.36692810058594,
"step": 233
},
{
"KL/chosen_KL_mean": -76.04293823242188,
"KL/mean": -124.41976928710938,
"KL/rejected_KL_mean": -172.7965850830078,
"KL/std": 83.86376953125,
"epoch": 0.3436123348017621,
"fcm_dpo/beta": 0.009408731944859028,
"fcm_dpo/delta": -0.12575244903564453,
"fcm_dpo/margin": 96.75364685058594,
"fcm_dpo/q_t": 0.3136642873287201,
"grad_norm": 33.3326530456543,
"learning_rate": 4.165182829193126e-07,
"logits/chosen": -0.36011114716529846,
"logits/rejected": -0.3862539529800415,
"logps/chosen": -120.13746643066406,
"logps/ref_chosen": -44.094520568847656,
"logps/ref_rejected": -100.00663757324219,
"logps/rejected": -272.80322265625,
"loss": 0.8233,
"margin_dpo/margin_mean": 96.75364685058594,
"margin_dpo/margin_std": 91.30296325683594,
"step": 234
},
{
"KL/chosen_KL_mean": -94.54927062988281,
"KL/mean": -127.44790649414062,
"KL/rejected_KL_mean": -160.34654235839844,
"KL/std": 81.41526794433594,
"epoch": 0.34508076358296624,
"fcm_dpo/beta": 0.009737150743603706,
"fcm_dpo/delta": 0.16920268535614014,
"fcm_dpo/margin": 65.79728698730469,
"fcm_dpo/q_t": 0.3712586462497711,
"grad_norm": 35.418949127197266,
"learning_rate": 4.1555887447288255e-07,
"logits/chosen": -0.4128161072731018,
"logits/rejected": -0.38685333728790283,
"logps/chosen": -156.7871856689453,
"logps/ref_chosen": -62.237911224365234,
"logps/ref_rejected": -90.39506530761719,
"logps/rejected": -250.74160766601562,
"loss": 1.0157,
"margin_dpo/margin_mean": 65.79728698730469,
"margin_dpo/margin_std": 92.96914672851562,
"step": 235
},
{
"KL/chosen_KL_mean": -71.0826187133789,
"KL/mean": -118.72409057617188,
"KL/rejected_KL_mean": -166.3655548095703,
"KL/std": 86.84884643554688,
"epoch": 0.3465491923641703,
"fcm_dpo/beta": 0.009598923847079277,
"fcm_dpo/delta": -0.12620067596435547,
"fcm_dpo/margin": 95.28294372558594,
"fcm_dpo/q_t": 0.3114420175552368,
"grad_norm": 42.24114990234375,
"learning_rate": 4.1459510339613946e-07,
"logits/chosen": -0.34095823764801025,
"logits/rejected": -0.34158775210380554,
"logps/chosen": -120.42398071289062,
"logps/ref_chosen": -49.34136199951172,
"logps/ref_rejected": -103.51162719726562,
"logps/rejected": -269.877197265625,
"loss": 0.8009,
"margin_dpo/margin_mean": 95.2829360961914,
"margin_dpo/margin_std": 85.3992691040039,
"step": 236
},
{
"KL/chosen_KL_mean": -89.83419799804688,
"KL/mean": -129.98583984375,
"KL/rejected_KL_mean": -170.13746643066406,
"KL/std": 84.88386535644531,
"epoch": 0.34801762114537443,
"fcm_dpo/beta": 0.009580841287970543,
"fcm_dpo/delta": 0.0331917479634285,
"fcm_dpo/margin": 80.30326843261719,
"fcm_dpo/q_t": 0.3448735475540161,
"grad_norm": 36.753971099853516,
"learning_rate": 4.136269950853473e-07,
"logits/chosen": -0.3638402223587036,
"logits/rejected": -0.3566875457763672,
"logps/chosen": -144.0023193359375,
"logps/ref_chosen": -54.168121337890625,
"logps/ref_rejected": -94.78036499023438,
"logps/rejected": -264.9178466796875,
"loss": 0.9228,
"margin_dpo/margin_mean": 80.30326843261719,
"margin_dpo/margin_std": 94.76294708251953,
"step": 237
},
{
"KL/chosen_KL_mean": -87.62262725830078,
"KL/mean": -125.47653198242188,
"KL/rejected_KL_mean": -163.3304443359375,
"KL/std": 84.19970703125,
"epoch": 0.34948604992657856,
"fcm_dpo/beta": 0.009638959541916847,
"fcm_dpo/delta": 0.0752006322145462,
"fcm_dpo/margin": 75.70780944824219,
"fcm_dpo/q_t": 0.35555845499038696,
"grad_norm": 36.07196044921875,
"learning_rate": 4.126545750510605e-07,
"logits/chosen": -0.33357322216033936,
"logits/rejected": -0.3504067659378052,
"logps/chosen": -141.5957489013672,
"logps/ref_chosen": -53.973121643066406,
"logps/ref_rejected": -89.41795349121094,
"logps/rejected": -252.74838256835938,
"loss": 0.94,
"margin_dpo/margin_mean": 75.70780944824219,
"margin_dpo/margin_std": 92.58589172363281,
"step": 238
},
{
"KL/chosen_KL_mean": -94.18663024902344,
"KL/mean": -137.20260620117188,
"KL/rejected_KL_mean": -180.21859741210938,
"KL/std": 83.78192138671875,
"epoch": 0.3509544787077827,
"fcm_dpo/beta": 0.009586915373802185,
"fcm_dpo/delta": -0.03049781173467636,
"fcm_dpo/margin": 86.03196716308594,
"fcm_dpo/q_t": 0.3317285478115082,
"grad_norm": 34.0923957824707,
"learning_rate": 4.116778689174514e-07,
"logits/chosen": -0.3576841354370117,
"logits/rejected": -0.33470335602760315,
"logps/chosen": -152.28445434570312,
"logps/ref_chosen": -58.09782409667969,
"logps/ref_rejected": -93.59294128417969,
"logps/rejected": -273.8115234375,
"loss": 0.8762,
"margin_dpo/margin_mean": 86.03196716308594,
"margin_dpo/margin_std": 86.97271728515625,
"step": 239
},
{
"KL/chosen_KL_mean": -98.12992858886719,
"KL/mean": -137.9344482421875,
"KL/rejected_KL_mean": -177.7389678955078,
"KL/std": 91.66488647460938,
"epoch": 0.3524229074889868,
"fcm_dpo/beta": 0.009678095579147339,
"fcm_dpo/delta": 0.031485963612794876,
"fcm_dpo/margin": 79.60904693603516,
"fcm_dpo/q_t": 0.3433542251586914,
"grad_norm": 35.59693908691406,
"learning_rate": 4.106969024216348e-07,
"logits/chosen": -0.34670311212539673,
"logits/rejected": -0.32258307933807373,
"logps/chosen": -158.74441528320312,
"logps/ref_chosen": -60.6144905090332,
"logps/ref_rejected": -74.1185302734375,
"logps/rejected": -251.8574981689453,
"loss": 0.9304,
"margin_dpo/margin_mean": 79.60903930664062,
"margin_dpo/margin_std": 93.16104125976562,
"step": 240
},
{
"KL/chosen_KL_mean": -81.70965576171875,
"KL/mean": -138.0463409423828,
"KL/rejected_KL_mean": -194.38299560546875,
"KL/std": 101.58145141601562,
"epoch": 0.35389133627019087,
"fcm_dpo/beta": 0.009372754022479057,
"fcm_dpo/delta": -0.28571805357933044,
"fcm_dpo/margin": 112.67334747314453,
"fcm_dpo/q_t": 0.29759377241134644,
"grad_norm": 34.78485870361328,
"learning_rate": 4.097117014129903e-07,
"logits/chosen": -0.3599684238433838,
"logits/rejected": -0.3252626061439514,
"logps/chosen": -147.80072021484375,
"logps/ref_chosen": -66.091064453125,
"logps/ref_rejected": -88.06088256835938,
"logps/rejected": -282.44390869140625,
"loss": 0.7835,
"margin_dpo/margin_mean": 112.67334747314453,
"margin_dpo/margin_std": 113.32879638671875,
"step": 241
},
{
"KL/chosen_KL_mean": -99.06041717529297,
"KL/mean": -137.75704956054688,
"KL/rejected_KL_mean": -176.4536590576172,
"KL/std": 85.93316650390625,
"epoch": 0.355359765051395,
"fcm_dpo/beta": 0.009386187419295311,
"fcm_dpo/delta": 0.07753384858369827,
"fcm_dpo/margin": 77.39323425292969,
"fcm_dpo/q_t": 0.3551555573940277,
"grad_norm": 39.032955169677734,
"learning_rate": 4.087222918524807e-07,
"logits/chosen": -0.29589563608169556,
"logits/rejected": -0.26947832107543945,
"logps/chosen": -166.92434692382812,
"logps/ref_chosen": -67.86392974853516,
"logps/ref_rejected": -83.36033630371094,
"logps/rejected": -259.8139953613281,
"loss": 0.9389,
"margin_dpo/margin_mean": 77.39323425292969,
"margin_dpo/margin_std": 94.50575256347656,
"step": 242
},
{
"KL/chosen_KL_mean": -101.89714050292969,
"KL/mean": -153.20147705078125,
"KL/rejected_KL_mean": -204.5058135986328,
"KL/std": 97.73469543457031,
"epoch": 0.3568281938325991,
"fcm_dpo/beta": 0.009123127907514572,
"fcm_dpo/delta": -0.15070104598999023,
"fcm_dpo/margin": 102.60867309570312,
"fcm_dpo/q_t": 0.3097137212753296,
"grad_norm": 33.98747634887695,
"learning_rate": 4.07728699811968e-07,
"logits/chosen": -0.34125879406929016,
"logits/rejected": -0.2980634570121765,
"logps/chosen": -164.98138427734375,
"logps/ref_chosen": -63.0842399597168,
"logps/ref_rejected": -76.33563232421875,
"logps/rejected": -280.8414306640625,
"loss": 0.8068,
"margin_dpo/margin_mean": 102.60867309570312,
"margin_dpo/margin_std": 97.82844543457031,
"step": 243
},
{
"KL/chosen_KL_mean": -88.12919616699219,
"KL/mean": -137.82073974609375,
"KL/rejected_KL_mean": -187.51231384277344,
"KL/std": 93.26637268066406,
"epoch": 0.35829662261380324,
"fcm_dpo/beta": 0.008928896859288216,
"fcm_dpo/delta": -0.09616594016551971,
"fcm_dpo/margin": 99.38310241699219,
"fcm_dpo/q_t": 0.31590431928634644,
"grad_norm": 32.82498550415039,
"learning_rate": 4.067309514735267e-07,
"logits/chosen": -0.4070456027984619,
"logits/rejected": -0.3957531154155731,
"logps/chosen": -149.26988220214844,
"logps/ref_chosen": -61.140689849853516,
"logps/ref_rejected": -94.89193725585938,
"logps/rejected": -282.40423583984375,
"loss": 0.8176,
"margin_dpo/margin_mean": 99.38311004638672,
"margin_dpo/margin_std": 90.44486999511719,
"step": 244
},
{
"KL/chosen_KL_mean": -102.30755615234375,
"KL/mean": -147.13352966308594,
"KL/rejected_KL_mean": -191.95950317382812,
"KL/std": 100.39794921875,
"epoch": 0.35976505139500736,
"fcm_dpo/beta": 0.00877309963107109,
"fcm_dpo/delta": 0.009379252791404724,
"fcm_dpo/margin": 89.65194702148438,
"fcm_dpo/q_t": 0.34968093037605286,
"grad_norm": 49.55756759643555,
"learning_rate": 4.057290731287531e-07,
"logits/chosen": -0.40635842084884644,
"logits/rejected": -0.3686367869377136,
"logps/chosen": -169.56985473632812,
"logps/ref_chosen": -67.26228332519531,
"logps/ref_rejected": -87.64010620117188,
"logps/rejected": -279.599609375,
"loss": 0.9526,
"margin_dpo/margin_mean": 89.65194702148438,
"margin_dpo/margin_std": 113.21443176269531,
"step": 245
},
{
"KL/chosen_KL_mean": -94.72513580322266,
"KL/mean": -136.26837158203125,
"KL/rejected_KL_mean": -177.8115997314453,
"KL/std": 99.95783996582031,
"epoch": 0.36123348017621143,
"fcm_dpo/beta": 0.008978039026260376,
"fcm_dpo/delta": 0.05843641608953476,
"fcm_dpo/margin": 83.08646392822266,
"fcm_dpo/q_t": 0.35539257526397705,
"grad_norm": 33.92315673828125,
"learning_rate": 4.047230911780736e-07,
"logits/chosen": -0.45548391342163086,
"logits/rejected": -0.41600751876831055,
"logps/chosen": -161.42210388183594,
"logps/ref_chosen": -66.69696807861328,
"logps/ref_rejected": -84.34634399414062,
"logps/rejected": -262.157958984375,
"loss": 0.9558,
"margin_dpo/margin_mean": 83.08645629882812,
"margin_dpo/margin_std": 109.47613525390625,
"step": 246
},
{
"KL/chosen_KL_mean": -109.76311492919922,
"KL/mean": -170.4364013671875,
"KL/rejected_KL_mean": -231.10968017578125,
"KL/std": 111.96722412109375,
"epoch": 0.36270190895741555,
"fcm_dpo/beta": 0.008609181270003319,
"fcm_dpo/delta": -0.27586841583251953,
"fcm_dpo/margin": 121.34656524658203,
"fcm_dpo/q_t": 0.2987045645713806,
"grad_norm": 29.220176696777344,
"learning_rate": 4.0371303213004814e-07,
"logits/chosen": -0.38043373823165894,
"logits/rejected": -0.37580886483192444,
"logps/chosen": -166.36846923828125,
"logps/ref_chosen": -56.6053466796875,
"logps/ref_rejected": -106.29326629638672,
"logps/rejected": -337.4029541015625,
"loss": 0.8024,
"margin_dpo/margin_mean": 121.34656524658203,
"margin_dpo/margin_std": 122.87495422363281,
"step": 247
},
{
"KL/chosen_KL_mean": -87.84927368164062,
"KL/mean": -134.8021697998047,
"KL/rejected_KL_mean": -181.75506591796875,
"KL/std": 83.55899047851562,
"epoch": 0.3641703377386197,
"fcm_dpo/beta": 0.008513124659657478,
"fcm_dpo/delta": 0.0004185140132904053,
"fcm_dpo/margin": 93.90579223632812,
"fcm_dpo/q_t": 0.3283466696739197,
"grad_norm": 48.29165267944336,
"learning_rate": 4.0269892260067197e-07,
"logits/chosen": -0.37384480237960815,
"logits/rejected": -0.3868759274482727,
"logps/chosen": -131.89248657226562,
"logps/ref_chosen": -44.043216705322266,
"logps/ref_rejected": -91.85687255859375,
"logps/rejected": -273.6119384765625,
"loss": 0.8374,
"margin_dpo/margin_mean": 93.90579223632812,
"margin_dpo/margin_std": 80.23796081542969,
"step": 248
},
{
"KL/chosen_KL_mean": -112.0308609008789,
"KL/mean": -145.31228637695312,
"KL/rejected_KL_mean": -178.59371948242188,
"KL/std": 91.14482116699219,
"epoch": 0.3656387665198238,
"fcm_dpo/beta": 0.008806029334664345,
"fcm_dpo/delta": 0.22767800092697144,
"fcm_dpo/margin": 66.5628662109375,
"fcm_dpo/q_t": 0.38379842042922974,
"grad_norm": 38.947914123535156,
"learning_rate": 4.0168078931267426e-07,
"logits/chosen": -0.4205751419067383,
"logits/rejected": -0.394936203956604,
"logps/chosen": -174.47320556640625,
"logps/ref_chosen": -62.442352294921875,
"logps/ref_rejected": -80.46806335449219,
"logps/rejected": -259.061767578125,
"loss": 1.0696,
"margin_dpo/margin_mean": 66.5628662109375,
"margin_dpo/margin_std": 105.92036437988281,
"step": 249
},
{
"KL/chosen_KL_mean": -95.54054260253906,
"KL/mean": -145.7799072265625,
"KL/rejected_KL_mean": -196.019287109375,
"KL/std": 85.64958190917969,
"epoch": 0.3671071953010279,
"fcm_dpo/beta": 0.008858118206262589,
"fcm_dpo/delta": -0.09895782917737961,
"fcm_dpo/margin": 100.47872924804688,
"fcm_dpo/q_t": 0.3090527057647705,
"grad_norm": 46.5678596496582,
"learning_rate": 4.006586590948141e-07,
"logits/chosen": -0.3768117129802704,
"logits/rejected": -0.31270933151245117,
"logps/chosen": -161.17723083496094,
"logps/ref_chosen": -65.63668823242188,
"logps/ref_rejected": -73.87184143066406,
"logps/rejected": -269.89111328125,
"loss": 0.7945,
"margin_dpo/margin_mean": 100.4787368774414,
"margin_dpo/margin_std": 79.92070007324219,
"step": 250
},
{
"KL/chosen_KL_mean": -116.66812896728516,
"KL/mean": -158.00106811523438,
"KL/rejected_KL_mean": -199.3340301513672,
"KL/std": 99.83149719238281,
"epoch": 0.368575624082232,
"fcm_dpo/beta": 0.008802777156233788,
"fcm_dpo/delta": 0.07752127945423126,
"fcm_dpo/margin": 82.6658935546875,
"fcm_dpo/q_t": 0.35495853424072266,
"grad_norm": 48.53387451171875,
"learning_rate": 3.9963255888117325e-07,
"logits/chosen": -0.3401832580566406,
"logits/rejected": -0.3037635087966919,
"logps/chosen": -173.85084533691406,
"logps/ref_chosen": -57.182716369628906,
"logps/ref_rejected": -77.66343688964844,
"logps/rejected": -276.9974670410156,
"loss": 0.968,
"margin_dpo/margin_mean": 82.6658935546875,
"margin_dpo/margin_std": 105.33753967285156,
"step": 251
},
{
"KL/chosen_KL_mean": -115.15412902832031,
"KL/mean": -166.94317626953125,
"KL/rejected_KL_mean": -218.7322540283203,
"KL/std": 89.63955688476562,
"epoch": 0.3700440528634361,
"fcm_dpo/beta": 0.008697323501110077,
"fcm_dpo/delta": -0.11240165680646896,
"fcm_dpo/margin": 103.57811737060547,
"fcm_dpo/q_t": 0.30444052815437317,
"grad_norm": 34.029197692871094,
"learning_rate": 3.9860251571044666e-07,
"logits/chosen": -0.39935481548309326,
"logits/rejected": -0.35331428050994873,
"logps/chosen": -186.8397674560547,
"logps/ref_chosen": -71.68563842773438,
"logps/ref_rejected": -84.75799560546875,
"logps/rejected": -303.490234375,
"loss": 0.7994,
"margin_dpo/margin_mean": 103.57811737060547,
"margin_dpo/margin_std": 86.66522216796875,
"step": 252
},
{
"KL/chosen_KL_mean": -109.57987976074219,
"KL/mean": -156.1011962890625,
"KL/rejected_KL_mean": -202.62249755859375,
"KL/std": 100.52079010009766,
"epoch": 0.37151248164464024,
"fcm_dpo/beta": 0.008769432082772255,
"fcm_dpo/delta": -0.019483350217342377,
"fcm_dpo/margin": 93.04261779785156,
"fcm_dpo/q_t": 0.3355570137500763,
"grad_norm": 38.78324508666992,
"learning_rate": 3.9756855672522986e-07,
"logits/chosen": -0.4148038625717163,
"logits/rejected": -0.4067913293838501,
"logps/chosen": -178.71380615234375,
"logps/ref_chosen": -69.1339340209961,
"logps/ref_rejected": -98.70252990722656,
"logps/rejected": -301.32501220703125,
"loss": 0.9266,
"margin_dpo/margin_mean": 93.04261779785156,
"margin_dpo/margin_std": 110.63420867919922,
"step": 253
},
{
"KL/chosen_KL_mean": -120.14738464355469,
"KL/mean": -163.5091552734375,
"KL/rejected_KL_mean": -206.8709716796875,
"KL/std": 104.5792007446289,
"epoch": 0.37298091042584436,
"fcm_dpo/beta": 0.00873111467808485,
"fcm_dpo/delta": 0.046288929879665375,
"fcm_dpo/margin": 86.72355651855469,
"fcm_dpo/q_t": 0.36139702796936035,
"grad_norm": 52.724220275878906,
"learning_rate": 3.965307091713037e-07,
"logits/chosen": -0.33431658148765564,
"logits/rejected": -0.30974212288856506,
"logps/chosen": -174.30238342285156,
"logps/ref_chosen": -54.154998779296875,
"logps/ref_rejected": -90.30764770507812,
"logps/rejected": -297.1785888671875,
"loss": 0.9987,
"margin_dpo/margin_mean": 86.72355651855469,
"margin_dpo/margin_std": 128.11468505859375,
"step": 254
},
{
"KL/chosen_KL_mean": -123.59725952148438,
"KL/mean": -169.68804931640625,
"KL/rejected_KL_mean": -215.7788543701172,
"KL/std": 99.052001953125,
"epoch": 0.3744493392070485,
"fcm_dpo/beta": 0.008680582977831364,
"fcm_dpo/delta": -0.0014177486300468445,
"fcm_dpo/margin": 92.18161010742188,
"fcm_dpo/q_t": 0.34611931443214417,
"grad_norm": 48.63689041137695,
"learning_rate": 3.954890003969163e-07,
"logits/chosen": -0.30374979972839355,
"logits/rejected": -0.2946004271507263,
"logps/chosen": -180.73892211914062,
"logps/ref_chosen": -57.14167022705078,
"logps/ref_rejected": -90.2085952758789,
"logps/rejected": -305.9874572753906,
"loss": 0.9727,
"margin_dpo/margin_mean": 92.18161010742188,
"margin_dpo/margin_std": 123.77351379394531,
"step": 255
},
{
"KL/chosen_KL_mean": -105.18153381347656,
"KL/mean": -155.66441345214844,
"KL/rejected_KL_mean": -206.14730834960938,
"KL/std": 106.35296630859375,
"epoch": 0.37591776798825255,
"fcm_dpo/beta": 0.008626077324151993,
"fcm_dpo/delta": -0.0779896229505539,
"fcm_dpo/margin": 100.96575927734375,
"fcm_dpo/q_t": 0.3262209892272949,
"grad_norm": 29.191234588623047,
"learning_rate": 3.944434578520628e-07,
"logits/chosen": -0.30876320600509644,
"logits/rejected": -0.30598020553588867,
"logps/chosen": -160.34503173828125,
"logps/ref_chosen": -55.163490295410156,
"logps/ref_rejected": -92.56291961669922,
"logps/rejected": -298.710205078125,
"loss": 0.8687,
"margin_dpo/margin_mean": 100.96575927734375,
"margin_dpo/margin_std": 109.50328063964844,
"step": 256
},
{
"KL/chosen_KL_mean": -97.52660369873047,
"KL/mean": -149.33360290527344,
"KL/rejected_KL_mean": -201.14059448242188,
"KL/std": 103.94354248046875,
"epoch": 0.37738619676945667,
"fcm_dpo/beta": 0.00838499329984188,
"fcm_dpo/delta": -0.08100643008947372,
"fcm_dpo/margin": 103.61398315429688,
"fcm_dpo/q_t": 0.3288852870464325,
"grad_norm": 28.96324348449707,
"learning_rate": 3.933941090877615e-07,
"logits/chosen": -0.3473857343196869,
"logits/rejected": -0.33152860403060913,
"logps/chosen": -146.95030212402344,
"logps/ref_chosen": -49.42369842529297,
"logps/ref_rejected": -79.53791809082031,
"logps/rejected": -280.67852783203125,
"loss": 0.8754,
"margin_dpo/margin_mean": 103.61398315429688,
"margin_dpo/margin_std": 109.51292419433594,
"step": 257
},
{
"KL/chosen_KL_mean": -112.60980224609375,
"KL/mean": -157.80865478515625,
"KL/rejected_KL_mean": -203.00750732421875,
"KL/std": 97.22439575195312,
"epoch": 0.3788546255506608,
"fcm_dpo/beta": 0.008527854457497597,
"fcm_dpo/delta": 0.03139530122280121,
"fcm_dpo/margin": 90.397705078125,
"fcm_dpo/q_t": 0.34141361713409424,
"grad_norm": 37.659915924072266,
"learning_rate": 3.923409817553284e-07,
"logits/chosen": -0.32153844833374023,
"logits/rejected": -0.31202760338783264,
"logps/chosen": -171.99392700195312,
"logps/ref_chosen": -59.384124755859375,
"logps/ref_rejected": -95.99010467529297,
"logps/rejected": -298.99761962890625,
"loss": 0.9525,
"margin_dpo/margin_mean": 90.39769744873047,
"margin_dpo/margin_std": 114.62164306640625,
"step": 258
},
{
"KL/chosen_KL_mean": -101.266845703125,
"KL/mean": -141.80453491210938,
"KL/rejected_KL_mean": -182.3422088623047,
"KL/std": 91.08666229248047,
"epoch": 0.3803230543318649,
"fcm_dpo/beta": 0.008614392951130867,
"fcm_dpo/delta": 0.10913976281881332,
"fcm_dpo/margin": 81.07535552978516,
"fcm_dpo/q_t": 0.3593909740447998,
"grad_norm": 40.97324752807617,
"learning_rate": 3.9128410360564793e-07,
"logits/chosen": -0.39695829153060913,
"logits/rejected": -0.3891292214393616,
"logps/chosen": -154.09518432617188,
"logps/ref_chosen": -52.828346252441406,
"logps/ref_rejected": -89.191650390625,
"logps/rejected": -271.53387451171875,
"loss": 0.9726,
"margin_dpo/margin_mean": 81.07535552978516,
"margin_dpo/margin_std": 102.6622314453125,
"step": 259
},
{
"KL/chosen_KL_mean": -96.43135833740234,
"KL/mean": -142.50563049316406,
"KL/rejected_KL_mean": -188.57989501953125,
"KL/std": 89.96287536621094,
"epoch": 0.38179148311306904,
"fcm_dpo/beta": 0.008739551529288292,
"fcm_dpo/delta": -0.006093751639127731,
"fcm_dpo/margin": 92.14852905273438,
"fcm_dpo/q_t": 0.33704131841659546,
"grad_norm": 45.9984016418457,
"learning_rate": 3.9022350248844246e-07,
"logits/chosen": -0.3932284116744995,
"logits/rejected": -0.4087497591972351,
"logps/chosen": -143.84902954101562,
"logps/ref_chosen": -47.41767501831055,
"logps/ref_rejected": -95.08978271484375,
"logps/rejected": -283.669677734375,
"loss": 0.8789,
"margin_dpo/margin_mean": 92.14852905273438,
"margin_dpo/margin_std": 98.96159362792969,
"step": 260
},
{
"KL/chosen_KL_mean": -85.78534698486328,
"KL/mean": -136.07266235351562,
"KL/rejected_KL_mean": -186.35995483398438,
"KL/std": 94.51484680175781,
"epoch": 0.3832599118942731,
"fcm_dpo/beta": 0.008594926446676254,
"fcm_dpo/delta": -0.0707588791847229,
"fcm_dpo/margin": 100.57462310791016,
"fcm_dpo/q_t": 0.32291507720947266,
"grad_norm": 32.12379837036133,
"learning_rate": 3.891592063515376e-07,
"logits/chosen": -0.34815192222595215,
"logits/rejected": -0.3409876227378845,
"logps/chosen": -138.8167266845703,
"logps/ref_chosen": -53.03137969970703,
"logps/ref_rejected": -88.51494598388672,
"logps/rejected": -274.8749084472656,
"loss": 0.8429,
"margin_dpo/margin_mean": 100.57461547851562,
"margin_dpo/margin_std": 100.58946990966797,
"step": 261
},
{
"KL/chosen_KL_mean": -99.97845458984375,
"KL/mean": -139.77328491210938,
"KL/rejected_KL_mean": -179.568115234375,
"KL/std": 84.39540100097656,
"epoch": 0.38472834067547723,
"fcm_dpo/beta": 0.008759372867643833,
"fcm_dpo/delta": 0.10967819392681122,
"fcm_dpo/margin": 79.58967590332031,
"fcm_dpo/q_t": 0.3497657775878906,
"grad_norm": 31.033384323120117,
"learning_rate": 3.880912432401264e-07,
"logits/chosen": -0.3228227496147156,
"logits/rejected": -0.2938918173313141,
"logps/chosen": -159.59860229492188,
"logps/ref_chosen": -59.620140075683594,
"logps/ref_rejected": -86.41853332519531,
"logps/rejected": -265.98663330078125,
"loss": 0.9248,
"margin_dpo/margin_mean": 79.58967590332031,
"margin_dpo/margin_std": 84.16622924804688,
"step": 262
},
{
"KL/chosen_KL_mean": -91.52005767822266,
"KL/mean": -147.3134002685547,
"KL/rejected_KL_mean": -203.10671997070312,
"KL/std": 100.20832061767578,
"epoch": 0.38619676945668135,
"fcm_dpo/beta": 0.008518587797880173,
"fcm_dpo/delta": -0.1675512194633484,
"fcm_dpo/margin": 111.58668518066406,
"fcm_dpo/q_t": 0.3041931092739105,
"grad_norm": 32.451473236083984,
"learning_rate": 3.870196412960302e-07,
"logits/chosen": -0.3325992822647095,
"logits/rejected": -0.29685109853744507,
"logps/chosen": -150.94100952148438,
"logps/ref_chosen": -59.42094421386719,
"logps/ref_rejected": -96.85720825195312,
"logps/rejected": -299.96392822265625,
"loss": 0.7942,
"margin_dpo/margin_mean": 111.58668518066406,
"margin_dpo/margin_std": 99.71820068359375,
"step": 263
},
{
"KL/chosen_KL_mean": -96.28630828857422,
"KL/mean": -141.96990966796875,
"KL/rejected_KL_mean": -187.65350341796875,
"KL/std": 88.48318481445312,
"epoch": 0.3876651982378855,
"fcm_dpo/beta": 0.00846080295741558,
"fcm_dpo/delta": 0.028465010225772858,
"fcm_dpo/margin": 91.36720275878906,
"fcm_dpo/q_t": 0.33899131417274475,
"grad_norm": 36.09725570678711,
"learning_rate": 3.8594442875695665e-07,
"logits/chosen": -0.33131206035614014,
"logits/rejected": -0.3153602182865143,
"logps/chosen": -159.00839233398438,
"logps/ref_chosen": -62.722084045410156,
"logps/ref_rejected": -93.85620880126953,
"logps/rejected": -281.50970458984375,
"loss": 0.8846,
"margin_dpo/margin_mean": 91.36720275878906,
"margin_dpo/margin_std": 92.86323547363281,
"step": 264
},
{
"KL/chosen_KL_mean": -110.69151306152344,
"KL/mean": -156.87734985351562,
"KL/rejected_KL_mean": -203.06320190429688,
"KL/std": 102.73265075683594,
"epoch": 0.3891336270190896,
"fcm_dpo/beta": 0.008556404151022434,
"fcm_dpo/delta": 0.010393224656581879,
"fcm_dpo/margin": 92.3717041015625,
"fcm_dpo/q_t": 0.3452988862991333,
"grad_norm": 44.408111572265625,
"learning_rate": 3.848656339557562e-07,
"logits/chosen": -0.2442556917667389,
"logits/rejected": -0.22311729192733765,
"logps/chosen": -172.66299438476562,
"logps/ref_chosen": -61.971466064453125,
"logps/ref_rejected": -88.02059936523438,
"logps/rejected": -291.08380126953125,
"loss": 0.9235,
"margin_dpo/margin_mean": 92.3717041015625,
"margin_dpo/margin_std": 112.91886138916016,
"step": 265
},
{
"KL/chosen_KL_mean": -107.76649475097656,
"KL/mean": -154.02108764648438,
"KL/rejected_KL_mean": -200.27566528320312,
"KL/std": 99.5924072265625,
"epoch": 0.39060205580029367,
"fcm_dpo/beta": 0.008531241677701473,
"fcm_dpo/delta": 0.01146605983376503,
"fcm_dpo/margin": 92.50917053222656,
"fcm_dpo/q_t": 0.33881711959838867,
"grad_norm": 54.370849609375,
"learning_rate": 3.8378328531967507e-07,
"logits/chosen": -0.3046592175960541,
"logits/rejected": -0.24882930517196655,
"logps/chosen": -174.8661651611328,
"logps/ref_chosen": -67.09967041015625,
"logps/ref_rejected": -67.97122192382812,
"logps/rejected": -268.24688720703125,
"loss": 0.8963,
"margin_dpo/margin_mean": 92.50917053222656,
"margin_dpo/margin_std": 100.46192932128906,
"step": 266
},
{
"KL/chosen_KL_mean": -111.7256088256836,
"KL/mean": -161.23422241210938,
"KL/rejected_KL_mean": -210.74285888671875,
"KL/std": 98.82826232910156,
"epoch": 0.3920704845814978,
"fcm_dpo/beta": 0.008480764925479889,
"fcm_dpo/delta": -0.04383730888366699,
"fcm_dpo/margin": 99.01722717285156,
"fcm_dpo/q_t": 0.3242519497871399,
"grad_norm": 41.02730941772461,
"learning_rate": 3.8269741136960646e-07,
"logits/chosen": -0.26957884430885315,
"logits/rejected": -0.2300885021686554,
"logps/chosen": -180.6963653564453,
"logps/ref_chosen": -68.97075653076172,
"logps/ref_rejected": -90.16844940185547,
"logps/rejected": -300.91131591796875,
"loss": 0.8434,
"margin_dpo/margin_mean": 99.01722717285156,
"margin_dpo/margin_std": 95.28477478027344,
"step": 267
},
{
"KL/chosen_KL_mean": -126.69245910644531,
"KL/mean": -174.75869750976562,
"KL/rejected_KL_mean": -222.824951171875,
"KL/std": 94.38973236083984,
"epoch": 0.3935389133627019,
"fcm_dpo/beta": 0.008443417027592659,
"fcm_dpo/delta": -0.013090118765830994,
"fcm_dpo/margin": 96.13249206542969,
"fcm_dpo/q_t": 0.3327893316745758,
"grad_norm": 47.08164978027344,
"learning_rate": 3.8160804071933894e-07,
"logits/chosen": -0.20129185914993286,
"logits/rejected": -0.20935595035552979,
"logps/chosen": -182.5927734375,
"logps/ref_chosen": -55.90031051635742,
"logps/ref_rejected": -101.64763641357422,
"logps/rejected": -324.47259521484375,
"loss": 0.8821,
"margin_dpo/margin_mean": 96.13249206542969,
"margin_dpo/margin_std": 103.66873931884766,
"step": 268
},
{
"KL/chosen_KL_mean": -144.3126220703125,
"KL/mean": -207.30047607421875,
"KL/rejected_KL_mean": -270.288330078125,
"KL/std": 112.39837646484375,
"epoch": 0.39500734214390604,
"fcm_dpo/beta": 0.008144214749336243,
"fcm_dpo/delta": -0.25245746970176697,
"fcm_dpo/margin": 125.9757080078125,
"fcm_dpo/q_t": 0.30247214436531067,
"grad_norm": 60.25404357910156,
"learning_rate": 3.8051520207480204e-07,
"logits/chosen": -0.2495516538619995,
"logits/rejected": -0.2241310477256775,
"logps/chosen": -214.35218811035156,
"logps/ref_chosen": -70.03955841064453,
"logps/ref_rejected": -107.34937286376953,
"logps/rejected": -377.6376953125,
"loss": 0.8291,
"margin_dpo/margin_mean": 125.9757080078125,
"margin_dpo/margin_std": 133.42056274414062,
"step": 269
},
{
"KL/chosen_KL_mean": -136.29586791992188,
"KL/mean": -183.19334411621094,
"KL/rejected_KL_mean": -230.0908203125,
"KL/std": 105.35012817382812,
"epoch": 0.3964757709251101,
"fcm_dpo/beta": 0.008130217902362347,
"fcm_dpo/delta": 0.04011562466621399,
"fcm_dpo/margin": 93.7949447631836,
"fcm_dpo/q_t": 0.35046982765197754,
"grad_norm": 50.62470245361328,
"learning_rate": 3.794189242333106e-07,
"logits/chosen": -0.29408591985702515,
"logits/rejected": -0.2865193486213684,
"logps/chosen": -205.829345703125,
"logps/ref_chosen": -69.53347778320312,
"logps/ref_rejected": -109.92864990234375,
"logps/rejected": -340.01947021484375,
"loss": 0.9617,
"margin_dpo/margin_mean": 93.79493713378906,
"margin_dpo/margin_std": 123.04978942871094,
"step": 270
},
{
"KL/chosen_KL_mean": -136.893310546875,
"KL/mean": -195.67169189453125,
"KL/rejected_KL_mean": -254.4500274658203,
"KL/std": 104.8451156616211,
"epoch": 0.39794419970631423,
"fcm_dpo/beta": 0.007922438904643059,
"fcm_dpo/delta": -0.1454475224018097,
"fcm_dpo/margin": 117.55671691894531,
"fcm_dpo/q_t": 0.3143436908721924,
"grad_norm": 33.90323257446289,
"learning_rate": 3.7831923608280514e-07,
"logits/chosen": -0.20189276337623596,
"logits/rejected": -0.17152735590934753,
"logps/chosen": -193.65789794921875,
"logps/ref_chosen": -56.76456832885742,
"logps/ref_rejected": -92.51383972167969,
"logps/rejected": -346.9638671875,
"loss": 0.8179,
"margin_dpo/margin_mean": 117.55671691894531,
"margin_dpo/margin_std": 115.97308349609375,
"step": 271
},
{
"KL/chosen_KL_mean": -128.50242614746094,
"KL/mean": -197.3144073486328,
"KL/rejected_KL_mean": -266.1263732910156,
"KL/std": 122.61846160888672,
"epoch": 0.39941262848751835,
"fcm_dpo/beta": 0.00761681143194437,
"fcm_dpo/delta": -0.27643194794654846,
"fcm_dpo/margin": 137.62393188476562,
"fcm_dpo/q_t": 0.2913571000099182,
"grad_norm": 40.26355743408203,
"learning_rate": 3.772161666010912e-07,
"logits/chosen": -0.16119909286499023,
"logits/rejected": -0.17272311449050903,
"logps/chosen": -177.9995880126953,
"logps/ref_chosen": -49.497154235839844,
"logps/ref_rejected": -105.54279327392578,
"logps/rejected": -371.669189453125,
"loss": 0.7877,
"margin_dpo/margin_mean": 137.62393188476562,
"margin_dpo/margin_std": 130.5201416015625,
"step": 272
},
{
"KL/chosen_KL_mean": -142.26593017578125,
"KL/mean": -211.93959045410156,
"KL/rejected_KL_mean": -281.61322021484375,
"KL/std": 124.14240264892578,
"epoch": 0.4008810572687225,
"fcm_dpo/beta": 0.007157725282013416,
"fcm_dpo/delta": -0.22283779084682465,
"fcm_dpo/margin": 139.3472900390625,
"fcm_dpo/q_t": 0.3012547492980957,
"grad_norm": 32.3958854675293,
"learning_rate": 3.761097448550755e-07,
"logits/chosen": -0.19352608919143677,
"logits/rejected": -0.16837653517723083,
"logps/chosen": -205.2413330078125,
"logps/ref_chosen": -62.97539520263672,
"logps/ref_rejected": -92.49858093261719,
"logps/rejected": -374.11181640625,
"loss": 0.7903,
"margin_dpo/margin_mean": 139.34730529785156,
"margin_dpo/margin_std": 130.78785705566406,
"step": 273
},
{
"KL/chosen_KL_mean": -173.3973388671875,
"KL/mean": -229.00474548339844,
"KL/rejected_KL_mean": -284.6121520996094,
"KL/std": 112.57540893554688,
"epoch": 0.4023494860499266,
"fcm_dpo/beta": 0.007162545807659626,
"fcm_dpo/delta": 0.0034975409507751465,
"fcm_dpo/margin": 111.2148208618164,
"fcm_dpo/q_t": 0.33956801891326904,
"grad_norm": 45.80356979370117,
"learning_rate": 3.75e-07,
"logits/chosen": -0.11668691784143448,
"logits/rejected": -0.08984885364770889,
"logps/chosen": -229.06503295898438,
"logps/ref_chosen": -55.66770935058594,
"logps/ref_rejected": -77.33308410644531,
"logps/rejected": -361.94525146484375,
"loss": 0.9127,
"margin_dpo/margin_mean": 111.21481323242188,
"margin_dpo/margin_std": 127.79832458496094,
"step": 274
},
{
"KL/chosen_KL_mean": -135.02244567871094,
"KL/mean": -191.32406616210938,
"KL/rejected_KL_mean": -247.62567138671875,
"KL/std": 117.45345306396484,
"epoch": 0.40381791483113066,
"fcm_dpo/beta": 0.007176370359957218,
"fcm_dpo/delta": -0.009557224810123444,
"fcm_dpo/margin": 112.60321044921875,
"fcm_dpo/q_t": 0.339751273393631,
"grad_norm": 38.28922653198242,
"learning_rate": 3.738869612786737e-07,
"logits/chosen": -0.20961615443229675,
"logits/rejected": -0.21241699159145355,
"logps/chosen": -183.61715698242188,
"logps/ref_chosen": -48.594703674316406,
"logps/ref_rejected": -93.30369567871094,
"logps/rejected": -340.92938232421875,
"loss": 0.9185,
"margin_dpo/margin_mean": 112.60321044921875,
"margin_dpo/margin_std": 132.3553466796875,
"step": 275
},
{
"KL/chosen_KL_mean": -146.90843200683594,
"KL/mean": -208.81777954101562,
"KL/rejected_KL_mean": -270.72711181640625,
"KL/std": 123.26631164550781,
"epoch": 0.4052863436123348,
"fcm_dpo/beta": 0.007015553303062916,
"fcm_dpo/delta": -0.07598960399627686,
"fcm_dpo/margin": 123.81869506835938,
"fcm_dpo/q_t": 0.3311005234718323,
"grad_norm": 38.56481170654297,
"learning_rate": 3.7277065802070204e-07,
"logits/chosen": -0.19943909347057343,
"logits/rejected": -0.16966615617275238,
"logps/chosen": -203.48583984375,
"logps/ref_chosen": -56.57740783691406,
"logps/ref_rejected": -70.36566925048828,
"logps/rejected": -341.0927734375,
"loss": 0.8881,
"margin_dpo/margin_mean": 123.81867980957031,
"margin_dpo/margin_std": 144.08128356933594,
"step": 276
},
{
"KL/chosen_KL_mean": -158.66197204589844,
"KL/mean": -219.00457763671875,
"KL/rejected_KL_mean": -279.34716796875,
"KL/std": 118.71607208251953,
"epoch": 0.4067547723935389,
"fcm_dpo/beta": 0.007000372279435396,
"fcm_dpo/delta": -0.04916992783546448,
"fcm_dpo/margin": 120.68519592285156,
"fcm_dpo/q_t": 0.3313535451889038,
"grad_norm": 34.503780364990234,
"learning_rate": 3.71651119641714e-07,
"logits/chosen": -0.1559976041316986,
"logits/rejected": -0.1391219049692154,
"logps/chosen": -214.93353271484375,
"logps/ref_chosen": -56.27156066894531,
"logps/ref_rejected": -92.88127136230469,
"logps/rejected": -372.22845458984375,
"loss": 0.8744,
"margin_dpo/margin_mean": 120.68519592285156,
"margin_dpo/margin_std": 130.1613311767578,
"step": 277
},
{
"KL/chosen_KL_mean": -151.7635955810547,
"KL/mean": -215.63674926757812,
"KL/rejected_KL_mean": -279.5098876953125,
"KL/std": 126.30606079101562,
"epoch": 0.40822320117474303,
"fcm_dpo/beta": 0.006845751777291298,
"fcm_dpo/delta": -0.08213748037815094,
"fcm_dpo/margin": 127.74630737304688,
"fcm_dpo/q_t": 0.3263602554798126,
"grad_norm": 38.64687728881836,
"learning_rate": 3.705283756425872e-07,
"logits/chosen": -0.17192326486110687,
"logits/rejected": -0.17586654424667358,
"logps/chosen": -204.70553588867188,
"logps/ref_chosen": -52.94194030761719,
"logps/ref_rejected": -91.25357818603516,
"logps/rejected": -370.7634582519531,
"loss": 0.8594,
"margin_dpo/margin_mean": 127.74630737304688,
"margin_dpo/margin_std": 135.8467559814453,
"step": 278
},
{
"KL/chosen_KL_mean": -168.48736572265625,
"KL/mean": -237.06497192382812,
"KL/rejected_KL_mean": -305.642578125,
"KL/std": 130.68185424804688,
"epoch": 0.40969162995594716,
"fcm_dpo/beta": 0.006637511774897575,
"fcm_dpo/delta": -0.12550733983516693,
"fcm_dpo/margin": 137.15524291992188,
"fcm_dpo/q_t": 0.3223535418510437,
"grad_norm": 44.90956497192383,
"learning_rate": 3.6940245560867e-07,
"logits/chosen": -0.06602071225643158,
"logits/rejected": -0.05742315948009491,
"logps/chosen": -217.12867736816406,
"logps/ref_chosen": -48.641319274902344,
"logps/ref_rejected": -87.8514404296875,
"logps/rejected": -393.4940185546875,
"loss": 0.8748,
"margin_dpo/margin_mean": 137.15524291992188,
"margin_dpo/margin_std": 151.80455017089844,
"step": 279
},
{
"KL/chosen_KL_mean": -147.5689697265625,
"KL/mean": -216.75357055664062,
"KL/rejected_KL_mean": -285.93817138671875,
"KL/std": 113.64622497558594,
"epoch": 0.4111600587371512,
"fcm_dpo/beta": 0.006537692155689001,
"fcm_dpo/delta": -0.1155073344707489,
"fcm_dpo/margin": 138.36923217773438,
"fcm_dpo/q_t": 0.3063077926635742,
"grad_norm": 55.07219314575195,
"learning_rate": 3.6827338920900253e-07,
"logits/chosen": -0.15549999475479126,
"logits/rejected": -0.15061771869659424,
"logps/chosen": -206.3660888671875,
"logps/ref_chosen": -58.797122955322266,
"logps/ref_rejected": -98.61885070800781,
"logps/rejected": -384.5570373535156,
"loss": 0.7877,
"margin_dpo/margin_mean": 138.3692169189453,
"margin_dpo/margin_std": 111.95368957519531,
"step": 280
},
{
"KL/chosen_KL_mean": -141.1031494140625,
"KL/mean": -203.22886657714844,
"KL/rejected_KL_mean": -265.35455322265625,
"KL/std": 114.74964904785156,
"epoch": 0.41262848751835535,
"fcm_dpo/beta": 0.006536037661135197,
"fcm_dpo/delta": -0.013637367635965347,
"fcm_dpo/margin": 124.25141143798828,
"fcm_dpo/q_t": 0.32296425104141235,
"grad_norm": 49.556819915771484,
"learning_rate": 3.6714120619553435e-07,
"logits/chosen": -0.2126597762107849,
"logits/rejected": -0.17675194144248962,
"logps/chosen": -196.5916748046875,
"logps/ref_chosen": -55.488521575927734,
"logps/ref_rejected": -80.88258361816406,
"logps/rejected": -346.2371520996094,
"loss": 0.8527,
"margin_dpo/margin_mean": 124.25141143798828,
"margin_dpo/margin_std": 114.7934799194336,
"step": 281
},
{
"KL/chosen_KL_mean": -156.95260620117188,
"KL/mean": -207.12942504882812,
"KL/rejected_KL_mean": -257.3062438964844,
"KL/std": 108.96510314941406,
"epoch": 0.41409691629955947,
"fcm_dpo/beta": 0.006672097835689783,
"fcm_dpo/delta": 0.13827921450138092,
"fcm_dpo/margin": 100.3536376953125,
"fcm_dpo/q_t": 0.36175787448883057,
"grad_norm": 41.35542297363281,
"learning_rate": 3.660059364023408e-07,
"logits/chosen": -0.2611793577671051,
"logits/rejected": -0.22383667528629303,
"logps/chosen": -230.02276611328125,
"logps/ref_chosen": -73.07014465332031,
"logps/ref_rejected": -95.35098266601562,
"logps/rejected": -352.6572265625,
"loss": 0.9472,
"margin_dpo/margin_mean": 100.3536376953125,
"margin_dpo/margin_std": 119.709716796875,
"step": 282
},
{
"KL/chosen_KL_mean": -165.55203247070312,
"KL/mean": -235.3925018310547,
"KL/rejected_KL_mean": -305.2330017089844,
"KL/std": 139.40737915039062,
"epoch": 0.4155653450807636,
"fcm_dpo/beta": 0.0065710763446986675,
"fcm_dpo/delta": -0.1295718550682068,
"fcm_dpo/margin": 139.68096923828125,
"fcm_dpo/q_t": 0.30924761295318604,
"grad_norm": 42.45329666137695,
"learning_rate": 3.6486760974483685e-07,
"logits/chosen": -0.184697687625885,
"logits/rejected": -0.17743512988090515,
"logps/chosen": -227.45046997070312,
"logps/ref_chosen": -61.89844512939453,
"logps/ref_rejected": -96.98655700683594,
"logps/rejected": -402.21954345703125,
"loss": 0.8136,
"margin_dpo/margin_mean": 139.68096923828125,
"margin_dpo/margin_std": 129.1880340576172,
"step": 283
},
{
"KL/chosen_KL_mean": -171.84423828125,
"KL/mean": -239.68804931640625,
"KL/rejected_KL_mean": -307.5318603515625,
"KL/std": 133.614990234375,
"epoch": 0.4170337738619677,
"fcm_dpo/beta": 0.006425424944609404,
"fcm_dpo/delta": -0.07886916399002075,
"fcm_dpo/margin": 135.68765258789062,
"fcm_dpo/q_t": 0.3206017017364502,
"grad_norm": 45.275691986083984,
"learning_rate": 3.6372625621898863e-07,
"logits/chosen": -0.16477805376052856,
"logits/rejected": -0.14584243297576904,
"logps/chosen": -230.27975463867188,
"logps/ref_chosen": -58.4355354309082,
"logps/ref_rejected": -93.46926879882812,
"logps/rejected": -401.0011291503906,
"loss": 0.8227,
"margin_dpo/margin_mean": 135.68765258789062,
"margin_dpo/margin_std": 128.515380859375,
"step": 284
},
{
"KL/chosen_KL_mean": -185.8503875732422,
"KL/mean": -248.33926391601562,
"KL/rejected_KL_mean": -310.8281555175781,
"KL/std": 114.6676025390625,
"epoch": 0.4185022026431718,
"fcm_dpo/beta": 0.00645300280302763,
"fcm_dpo/delta": -0.008379101753234863,
"fcm_dpo/margin": 124.9777603149414,
"fcm_dpo/q_t": 0.3266201615333557,
"grad_norm": 48.88724899291992,
"learning_rate": 3.625819059005228e-07,
"logits/chosen": -0.20939572155475616,
"logits/rejected": -0.18859480321407318,
"logps/chosen": -252.08258056640625,
"logps/ref_chosen": -66.23219299316406,
"logps/ref_rejected": -99.1268310546875,
"logps/rejected": -409.9549865722656,
"loss": 0.85,
"margin_dpo/margin_mean": 124.9777603149414,
"margin_dpo/margin_std": 111.27662658691406,
"step": 285
},
{
"KL/chosen_KL_mean": -186.20968627929688,
"KL/mean": -254.66603088378906,
"KL/rejected_KL_mean": -323.12237548828125,
"KL/std": 133.13705444335938,
"epoch": 0.4199706314243759,
"fcm_dpo/beta": 0.0063394177705049515,
"fcm_dpo/delta": -0.07433885335922241,
"fcm_dpo/margin": 136.91268920898438,
"fcm_dpo/q_t": 0.32065117359161377,
"grad_norm": 33.741024017333984,
"learning_rate": 3.614345889441346e-07,
"logits/chosen": -0.19529663026332855,
"logits/rejected": -0.17008443176746368,
"logps/chosen": -259.16070556640625,
"logps/ref_chosen": -72.95100402832031,
"logps/ref_rejected": -88.58845520019531,
"logps/rejected": -411.7108154296875,
"loss": 0.8456,
"margin_dpo/margin_mean": 136.91268920898438,
"margin_dpo/margin_std": 134.44070434570312,
"step": 286
},
{
"KL/chosen_KL_mean": -194.35592651367188,
"KL/mean": -251.70867919921875,
"KL/rejected_KL_mean": -309.0614013671875,
"KL/std": 127.58320617675781,
"epoch": 0.42143906020558003,
"fcm_dpo/beta": 0.006387564353644848,
"fcm_dpo/delta": 0.07230030000209808,
"fcm_dpo/margin": 114.70552825927734,
"fcm_dpo/q_t": 0.3514924943447113,
"grad_norm": 35.43113708496094,
"learning_rate": 3.6028433558269275e-07,
"logits/chosen": -0.09007483720779419,
"logits/rejected": -0.06799876689910889,
"logps/chosen": -255.89706420898438,
"logps/ref_chosen": -61.54115295410156,
"logps/ref_rejected": -77.69607543945312,
"logps/rejected": -386.75750732421875,
"loss": 0.9391,
"margin_dpo/margin_mean": 114.70552825927734,
"margin_dpo/margin_std": 139.09039306640625,
"step": 287
},
{
"KL/chosen_KL_mean": -190.921142578125,
"KL/mean": -257.931396484375,
"KL/rejected_KL_mean": -324.941650390625,
"KL/std": 118.37663269042969,
"epoch": 0.42290748898678415,
"fcm_dpo/beta": 0.0062710437923669815,
"fcm_dpo/delta": -0.04802323132753372,
"fcm_dpo/margin": 134.02052307128906,
"fcm_dpo/q_t": 0.3206808567047119,
"grad_norm": 33.899898529052734,
"learning_rate": 3.5913117612644327e-07,
"logits/chosen": -0.14238840341567993,
"logits/rejected": -0.12358824163675308,
"logps/chosen": -247.58236694335938,
"logps/ref_chosen": -56.661224365234375,
"logps/ref_rejected": -87.33570098876953,
"logps/rejected": -412.27734375,
"loss": 0.8339,
"margin_dpo/margin_mean": 134.02052307128906,
"margin_dpo/margin_std": 115.77912139892578,
"step": 288
},
{
"KL/chosen_KL_mean": -187.45516967773438,
"KL/mean": -263.76019287109375,
"KL/rejected_KL_mean": -340.065185546875,
"KL/std": 135.79379272460938,
"epoch": 0.4243759177679883,
"fcm_dpo/beta": 0.006219586357474327,
"fcm_dpo/delta": -0.1645835041999817,
"fcm_dpo/margin": 152.6099853515625,
"fcm_dpo/q_t": 0.31164634227752686,
"grad_norm": 37.334537506103516,
"learning_rate": 3.5797514096221024e-07,
"logits/chosen": -0.028572116047143936,
"logits/rejected": -0.022187216207385063,
"logps/chosen": -232.68557739257812,
"logps/ref_chosen": -45.23039245605469,
"logps/ref_rejected": -87.64266967773438,
"logps/rejected": -427.70782470703125,
"loss": 0.8196,
"margin_dpo/margin_mean": 152.6099853515625,
"margin_dpo/margin_std": 149.71493530273438,
"step": 289
},
{
"KL/chosen_KL_mean": -209.78424072265625,
"KL/mean": -285.5294189453125,
"KL/rejected_KL_mean": -361.274658203125,
"KL/std": 147.62713623046875,
"epoch": 0.42584434654919234,
"fcm_dpo/beta": 0.006003453396260738,
"fcm_dpo/delta": -0.12091632187366486,
"fcm_dpo/margin": 151.49041748046875,
"fcm_dpo/q_t": 0.32481837272644043,
"grad_norm": 39.83347702026367,
"learning_rate": 3.568162605525952e-07,
"logits/chosen": -0.036675482988357544,
"logits/rejected": -0.05918949097394943,
"logps/chosen": -265.2557373046875,
"logps/ref_chosen": -55.47149658203125,
"logps/ref_rejected": -116.70857238769531,
"logps/rejected": -477.98321533203125,
"loss": 0.8687,
"margin_dpo/margin_mean": 151.4904022216797,
"margin_dpo/margin_std": 172.48365783691406,
"step": 290
},
{
"KL/chosen_KL_mean": -181.04437255859375,
"KL/mean": -251.22125244140625,
"KL/rejected_KL_mean": -321.39813232421875,
"KL/std": 126.02388000488281,
"epoch": 0.42731277533039647,
"fcm_dpo/beta": 0.005997511558234692,
"fcm_dpo/delta": -0.04703492671251297,
"fcm_dpo/margin": 140.35374450683594,
"fcm_dpo/q_t": 0.32573235034942627,
"grad_norm": 40.698150634765625,
"learning_rate": 3.5565456543517485e-07,
"logits/chosen": -0.08782187104225159,
"logits/rejected": -0.07490293681621552,
"logps/chosen": -244.30474853515625,
"logps/ref_chosen": -63.26036834716797,
"logps/ref_rejected": -89.29708862304688,
"logps/rejected": -410.6952209472656,
"loss": 0.8633,
"margin_dpo/margin_mean": 140.35374450683594,
"margin_dpo/margin_std": 138.77685546875,
"step": 291
},
{
"KL/chosen_KL_mean": -203.10101318359375,
"KL/mean": -273.4295654296875,
"KL/rejected_KL_mean": -343.75811767578125,
"KL/std": 140.24241638183594,
"epoch": 0.4287812041116006,
"fcm_dpo/beta": 0.005930889863520861,
"fcm_dpo/delta": -0.03794461488723755,
"fcm_dpo/margin": 140.6571044921875,
"fcm_dpo/q_t": 0.32827457785606384,
"grad_norm": 52.17742156982422,
"learning_rate": 3.5449008622169583e-07,
"logits/chosen": -0.009431811049580574,
"logits/rejected": 0.00803426280617714,
"logps/chosen": -257.01953125,
"logps/ref_chosen": -53.91852951049805,
"logps/ref_rejected": -89.96138000488281,
"logps/rejected": -433.7195129394531,
"loss": 0.9046,
"margin_dpo/margin_mean": 140.6571044921875,
"margin_dpo/margin_std": 160.97979736328125,
"step": 292
},
{
"KL/chosen_KL_mean": -186.6754913330078,
"KL/mean": -246.67132568359375,
"KL/rejected_KL_mean": -306.66717529296875,
"KL/std": 132.06301879882812,
"epoch": 0.4302496328928047,
"fcm_dpo/beta": 0.005984225310385227,
"fcm_dpo/delta": 0.08749930560588837,
"fcm_dpo/margin": 119.99168395996094,
"fcm_dpo/q_t": 0.3568994402885437,
"grad_norm": 45.794559478759766,
"learning_rate": 3.5332285359726846e-07,
"logits/chosen": -0.056711144745349884,
"logits/rejected": -0.039980124682188034,
"logps/chosen": -247.051513671875,
"logps/ref_chosen": -60.376033782958984,
"logps/ref_rejected": -77.85244750976562,
"logps/rejected": -384.5196228027344,
"loss": 0.9677,
"margin_dpo/margin_mean": 119.99169158935547,
"margin_dpo/margin_std": 153.38482666015625,
"step": 293
},
{
"KL/chosen_KL_mean": -164.24774169921875,
"KL/mean": -227.4092559814453,
"KL/rejected_KL_mean": -290.5707702636719,
"KL/std": 132.69287109375,
"epoch": 0.43171806167400884,
"fcm_dpo/beta": 0.0059990473091602325,
"fcm_dpo/delta": 0.04543343558907509,
"fcm_dpo/margin": 126.32302856445312,
"fcm_dpo/q_t": 0.35002630949020386,
"grad_norm": 38.24577713012695,
"learning_rate": 3.5215289831955786e-07,
"logits/chosen": -0.07339977473020554,
"logits/rejected": -0.07351949065923691,
"logps/chosen": -212.3352813720703,
"logps/ref_chosen": -48.0875358581543,
"logps/ref_rejected": -81.89698791503906,
"logps/rejected": -372.4677734375,
"loss": 0.9284,
"margin_dpo/margin_mean": 126.32302856445312,
"margin_dpo/margin_std": 151.93936157226562,
"step": 294
},
{
"KL/chosen_KL_mean": -187.2119598388672,
"KL/mean": -249.87669372558594,
"KL/rejected_KL_mean": -312.54144287109375,
"KL/std": 127.01498413085938,
"epoch": 0.4331864904552129,
"fcm_dpo/beta": 0.0060641467571258545,
"fcm_dpo/delta": 0.04325701668858528,
"fcm_dpo/margin": 125.32946014404297,
"fcm_dpo/q_t": 0.3435119688510895,
"grad_norm": 37.575748443603516,
"learning_rate": 3.509802512179737e-07,
"logits/chosen": -0.06643851101398468,
"logits/rejected": -0.06460548937320709,
"logps/chosen": -237.13662719726562,
"logps/ref_chosen": -49.92467498779297,
"logps/ref_rejected": -87.45632934570312,
"logps/rejected": -399.99774169921875,
"loss": 0.9108,
"margin_dpo/margin_mean": 125.3294677734375,
"margin_dpo/margin_std": 138.71270751953125,
"step": 295
},
{
"KL/chosen_KL_mean": -210.7562255859375,
"KL/mean": -261.470947265625,
"KL/rejected_KL_mean": -312.18572998046875,
"KL/std": 124.9061508178711,
"epoch": 0.434654919236417,
"fcm_dpo/beta": 0.0062162261456251144,
"fcm_dpo/delta": 0.1811649054288864,
"fcm_dpo/margin": 101.42951965332031,
"fcm_dpo/q_t": 0.37338194251060486,
"grad_norm": 66.42903137207031,
"learning_rate": 3.498049431928577e-07,
"logits/chosen": -0.0994357317686081,
"logits/rejected": -0.07531915605068207,
"logps/chosen": -276.2474365234375,
"logps/ref_chosen": -65.49124145507812,
"logps/ref_rejected": -93.08908081054688,
"logps/rejected": -405.27484130859375,
"loss": 1.0423,
"margin_dpo/margin_mean": 101.42951965332031,
"margin_dpo/margin_std": 150.9395294189453,
"step": 296
},
{
"KL/chosen_KL_mean": -186.30535888671875,
"KL/mean": -245.25868225097656,
"KL/rejected_KL_mean": -304.2120056152344,
"KL/std": 122.74223327636719,
"epoch": 0.43612334801762115,
"fcm_dpo/beta": 0.006364773027598858,
"fcm_dpo/delta": 0.05358999967575073,
"fcm_dpo/margin": 117.90662384033203,
"fcm_dpo/q_t": 0.34062981605529785,
"grad_norm": 36.62229919433594,
"learning_rate": 3.486270052146694e-07,
"logits/chosen": -0.14602035284042358,
"logits/rejected": -0.14422789216041565,
"logps/chosen": -242.78231811523438,
"logps/ref_chosen": -56.476951599121094,
"logps/ref_rejected": -95.1385498046875,
"logps/rejected": -399.3505554199219,
"loss": 0.8876,
"margin_dpo/margin_mean": 117.90663146972656,
"margin_dpo/margin_std": 117.52006530761719,
"step": 297
},
{
"KL/chosen_KL_mean": -183.15670776367188,
"KL/mean": -253.70401000976562,
"KL/rejected_KL_mean": -324.25128173828125,
"KL/std": 148.4191436767578,
"epoch": 0.43759177679882527,
"fcm_dpo/beta": 0.006309167016297579,
"fcm_dpo/delta": -0.09888706356287003,
"fcm_dpo/margin": 141.0946044921875,
"fcm_dpo/q_t": 0.3272933065891266,
"grad_norm": 44.55033493041992,
"learning_rate": 3.474464683231698e-07,
"logits/chosen": -0.1122806966304779,
"logits/rejected": -0.13088667392730713,
"logps/chosen": -250.48187255859375,
"logps/ref_chosen": -67.32516479492188,
"logps/ref_rejected": -116.66217041015625,
"logps/rejected": -440.9134521484375,
"loss": 0.8618,
"margin_dpo/margin_mean": 141.0946044921875,
"margin_dpo/margin_std": 156.77532958984375,
"step": 298
},
{
"KL/chosen_KL_mean": -160.49649047851562,
"KL/mean": -227.01962280273438,
"KL/rejected_KL_mean": -293.54278564453125,
"KL/std": 127.14181518554688,
"epoch": 0.4390602055800294,
"fcm_dpo/beta": 0.0062877098098397255,
"fcm_dpo/delta": -0.0417584553360939,
"fcm_dpo/margin": 133.04627990722656,
"fcm_dpo/q_t": 0.33548539876937866,
"grad_norm": 44.61274337768555,
"learning_rate": 3.462633636266041e-07,
"logits/chosen": -0.17389070987701416,
"logits/rejected": -0.16740745306015015,
"logps/chosen": -209.45858764648438,
"logps/ref_chosen": -48.96209716796875,
"logps/ref_rejected": -84.32823944091797,
"logps/rejected": -377.87103271484375,
"loss": 0.9029,
"margin_dpo/margin_mean": 133.04627990722656,
"margin_dpo/margin_std": 153.0653076171875,
"step": 299
},
{
"KL/chosen_KL_mean": -201.64463806152344,
"KL/mean": -279.50714111328125,
"KL/rejected_KL_mean": -357.3695983886719,
"KL/std": 150.42608642578125,
"epoch": 0.44052863436123346,
"fcm_dpo/beta": 0.00608530780300498,
"fcm_dpo/delta": -0.16277390718460083,
"fcm_dpo/margin": 155.72494506835938,
"fcm_dpo/q_t": 0.3149811625480652,
"grad_norm": 62.3836784362793,
"learning_rate": 3.4507772230088147e-07,
"logits/chosen": -0.12772464752197266,
"logits/rejected": -0.11891786009073257,
"logps/chosen": -260.7183532714844,
"logps/ref_chosen": -59.07371139526367,
"logps/ref_rejected": -95.9664535522461,
"logps/rejected": -453.3360595703125,
"loss": 0.8776,
"margin_dpo/margin_mean": 155.72494506835938,
"margin_dpo/margin_std": 176.29461669921875,
"step": 300
},
{
"KL/chosen_KL_mean": -177.3877716064453,
"KL/mean": -250.5698699951172,
"KL/rejected_KL_mean": -323.751953125,
"KL/std": 135.59161376953125,
"epoch": 0.4419970631424376,
"fcm_dpo/beta": 0.005912788212299347,
"fcm_dpo/delta": -0.0726335421204567,
"fcm_dpo/margin": 146.36419677734375,
"fcm_dpo/q_t": 0.3244306743144989,
"grad_norm": 47.50827407836914,
"learning_rate": 3.4388957558875316e-07,
"logits/chosen": -0.10594043135643005,
"logits/rejected": -0.09627512097358704,
"logps/chosen": -234.63714599609375,
"logps/ref_chosen": -57.249366760253906,
"logps/ref_rejected": -92.35354614257812,
"logps/rejected": -416.10552978515625,
"loss": 0.8447,
"margin_dpo/margin_mean": 146.36419677734375,
"margin_dpo/margin_std": 145.9784393310547,
"step": 301
},
{
"KL/chosen_KL_mean": -150.9648895263672,
"KL/mean": -216.1134033203125,
"KL/rejected_KL_mean": -281.26190185546875,
"KL/std": 123.28852844238281,
"epoch": 0.4434654919236417,
"fcm_dpo/beta": 0.005917476490139961,
"fcm_dpo/delta": 0.03093942627310753,
"fcm_dpo/margin": 130.2970428466797,
"fcm_dpo/q_t": 0.3413656949996948,
"grad_norm": 45.89029312133789,
"learning_rate": 3.426989547989902e-07,
"logits/chosen": -0.13110940158367157,
"logits/rejected": -0.12976717948913574,
"logps/chosen": -202.16287231445312,
"logps/ref_chosen": -51.197994232177734,
"logps/ref_rejected": -97.22636413574219,
"logps/rejected": -378.48828125,
"loss": 0.9054,
"margin_dpo/margin_mean": 130.2970428466797,
"margin_dpo/margin_std": 143.6829071044922,
"step": 302
},
{
"KL/chosen_KL_mean": -161.89434814453125,
"KL/mean": -225.97076416015625,
"KL/rejected_KL_mean": -290.04718017578125,
"KL/std": 136.81471252441406,
"epoch": 0.44493392070484583,
"fcm_dpo/beta": 0.005963338539004326,
"fcm_dpo/delta": 0.03840842843055725,
"fcm_dpo/margin": 128.15284729003906,
"fcm_dpo/q_t": 0.34974241256713867,
"grad_norm": 39.980674743652344,
"learning_rate": 3.4150589130555773e-07,
"logits/chosen": -0.12430644035339355,
"logits/rejected": -0.09778697788715363,
"logps/chosen": -228.60829162597656,
"logps/ref_chosen": -66.71394348144531,
"logps/ref_rejected": -86.94542694091797,
"logps/rejected": -376.99261474609375,
"loss": 0.9415,
"margin_dpo/margin_mean": 128.15284729003906,
"margin_dpo/margin_std": 157.72259521484375,
"step": 303
},
{
"KL/chosen_KL_mean": -149.53477478027344,
"KL/mean": -216.70220947265625,
"KL/rejected_KL_mean": -283.86962890625,
"KL/std": 127.11853790283203,
"epoch": 0.44640234948604995,
"fcm_dpo/beta": 0.006036032922565937,
"fcm_dpo/delta": -0.01273266226053238,
"fcm_dpo/margin": 134.3348388671875,
"fcm_dpo/q_t": 0.3229079246520996,
"grad_norm": 56.761173248291016,
"learning_rate": 3.403104165467883e-07,
"logits/chosen": -0.13418614864349365,
"logits/rejected": -0.11346621066331863,
"logps/chosen": -221.4854736328125,
"logps/ref_chosen": -71.95069885253906,
"logps/ref_rejected": -90.47203063964844,
"logps/rejected": -374.3416442871094,
"loss": 0.829,
"margin_dpo/margin_mean": 134.3348388671875,
"margin_dpo/margin_std": 107.37701416015625,
"step": 304
},
{
"KL/chosen_KL_mean": -170.97412109375,
"KL/mean": -234.58453369140625,
"KL/rejected_KL_mean": -298.1949462890625,
"KL/std": 147.8951416015625,
"epoch": 0.447870778267254,
"fcm_dpo/beta": 0.005968155339360237,
"fcm_dpo/delta": 0.04241461306810379,
"fcm_dpo/margin": 127.22084045410156,
"fcm_dpo/q_t": 0.34988775849342346,
"grad_norm": 41.87591552734375,
"learning_rate": 3.391125620245535e-07,
"logits/chosen": -0.14380377531051636,
"logits/rejected": -0.11721345782279968,
"logps/chosen": -237.76934814453125,
"logps/ref_chosen": -66.79523468017578,
"logps/ref_rejected": -92.75459289550781,
"logps/rejected": -390.94952392578125,
"loss": 0.9292,
"margin_dpo/margin_mean": 127.22084045410156,
"margin_dpo/margin_std": 151.81414794921875,
"step": 305
},
{
"KL/chosen_KL_mean": -176.7681427001953,
"KL/mean": -238.14883422851562,
"KL/rejected_KL_mean": -299.529541015625,
"KL/std": 127.993408203125,
"epoch": 0.44933920704845814,
"fcm_dpo/beta": 0.006088206544518471,
"fcm_dpo/delta": 0.05688028782606125,
"fcm_dpo/margin": 122.76139831542969,
"fcm_dpo/q_t": 0.3437831401824951,
"grad_norm": 42.77898025512695,
"learning_rate": 3.3791235930343417e-07,
"logits/chosen": -0.08690765500068665,
"logits/rejected": -0.056158192455768585,
"logps/chosen": -246.45204162597656,
"logps/ref_chosen": -69.68389892578125,
"logps/ref_rejected": -85.15919494628906,
"logps/rejected": -384.688720703125,
"loss": 0.9036,
"margin_dpo/margin_mean": 122.76140594482422,
"margin_dpo/margin_std": 131.52952575683594,
"step": 306
},
{
"KL/chosen_KL_mean": -173.75515747070312,
"KL/mean": -236.67062377929688,
"KL/rejected_KL_mean": -299.5860595703125,
"KL/std": 121.59797668457031,
"epoch": 0.45080763582966227,
"fcm_dpo/beta": 0.0061085280030965805,
"fcm_dpo/delta": 0.03357131406664848,
"fcm_dpo/margin": 125.83091735839844,
"fcm_dpo/q_t": 0.3417189121246338,
"grad_norm": 44.65004348754883,
"learning_rate": 3.367098400098881e-07,
"logits/chosen": -0.06562909483909607,
"logits/rejected": -0.04333928972482681,
"logps/chosen": -243.92059326171875,
"logps/ref_chosen": -70.16542053222656,
"logps/ref_rejected": -86.97230529785156,
"logps/rejected": -386.5583801269531,
"loss": 0.9153,
"margin_dpo/margin_mean": 125.83091735839844,
"margin_dpo/margin_std": 141.49557495117188,
"step": 307
},
{
"KL/chosen_KL_mean": -178.3945770263672,
"KL/mean": -248.44857788085938,
"KL/rejected_KL_mean": -318.5025634765625,
"KL/std": 136.55337524414062,
"epoch": 0.4522760646108664,
"fcm_dpo/beta": 0.00610099732875824,
"fcm_dpo/delta": -0.059890978038311005,
"fcm_dpo/margin": 140.10801696777344,
"fcm_dpo/q_t": 0.3226816654205322,
"grad_norm": 44.11930847167969,
"learning_rate": 3.355050358314172e-07,
"logits/chosen": -0.03536057472229004,
"logits/rejected": -0.012509562075138092,
"logps/chosen": -233.6395721435547,
"logps/ref_chosen": -55.2449951171875,
"logps/ref_rejected": -79.37226104736328,
"logps/rejected": -397.8748474121094,
"loss": 0.8489,
"margin_dpo/margin_mean": 140.10801696777344,
"margin_dpo/margin_std": 136.1858367919922,
"step": 308
},
{
"KL/chosen_KL_mean": -182.00643920898438,
"KL/mean": -252.7855224609375,
"KL/rejected_KL_mean": -323.56463623046875,
"KL/std": 137.23944091796875,
"epoch": 0.45374449339207046,
"fcm_dpo/beta": 0.006070786155760288,
"fcm_dpo/delta": -0.06589590013027191,
"fcm_dpo/margin": 141.55816650390625,
"fcm_dpo/q_t": 0.3239052891731262,
"grad_norm": 40.19969177246094,
"learning_rate": 3.3429797851573183e-07,
"logits/chosen": -0.010355365462601185,
"logits/rejected": 0.010411388240754604,
"logps/chosen": -230.9655303955078,
"logps/ref_chosen": -48.959083557128906,
"logps/ref_rejected": -82.34072875976562,
"logps/rejected": -405.90533447265625,
"loss": 0.8536,
"margin_dpo/margin_mean": 141.5581512451172,
"margin_dpo/margin_std": 140.39210510253906,
"step": 309
},
{
"KL/chosen_KL_mean": -226.34396362304688,
"KL/mean": -292.3310546875,
"KL/rejected_KL_mean": -358.31805419921875,
"KL/std": 120.46112060546875,
"epoch": 0.4552129221732746,
"fcm_dpo/beta": 0.006043789908289909,
"fcm_dpo/delta": 0.001462891697883606,
"fcm_dpo/margin": 131.97410583496094,
"fcm_dpo/q_t": 0.3287044167518616,
"grad_norm": 69.3414535522461,
"learning_rate": 3.3308869986991487e-07,
"logits/chosen": 0.0008979048579931259,
"logits/rejected": 0.024873077869415283,
"logps/chosen": -289.08575439453125,
"logps/ref_chosen": -62.74177932739258,
"logps/ref_rejected": -79.9302978515625,
"logps/rejected": -438.2483825683594,
"loss": 0.8421,
"margin_dpo/margin_mean": 131.97410583496094,
"margin_dpo/margin_std": 113.02054595947266,
"step": 310
},
{
"KL/chosen_KL_mean": -230.95571899414062,
"KL/mean": -309.91021728515625,
"KL/rejected_KL_mean": -388.86474609375,
"KL/std": 167.3114013671875,
"epoch": 0.4566813509544787,
"fcm_dpo/beta": 0.005860151723027229,
"fcm_dpo/delta": -0.1387987583875656,
"fcm_dpo/margin": 157.90902709960938,
"fcm_dpo/q_t": 0.3198055028915405,
"grad_norm": 36.636531829833984,
"learning_rate": 3.3187723175958346e-07,
"logits/chosen": -0.007311457768082619,
"logits/rejected": 0.02352697029709816,
"logps/chosen": -283.98370361328125,
"logps/ref_chosen": -53.02798080444336,
"logps/ref_rejected": -77.43820190429688,
"logps/rejected": -466.3029479980469,
"loss": 0.85,
"margin_dpo/margin_mean": 157.90902709960938,
"margin_dpo/margin_std": 170.51254272460938,
"step": 311
},
{
"KL/chosen_KL_mean": -220.95730590820312,
"KL/mean": -290.677001953125,
"KL/rejected_KL_mean": -360.3967590332031,
"KL/std": 133.24356079101562,
"epoch": 0.4581497797356828,
"fcm_dpo/beta": 0.005845385603606701,
"fcm_dpo/delta": -0.01669427752494812,
"fcm_dpo/margin": 139.439453125,
"fcm_dpo/q_t": 0.33251118659973145,
"grad_norm": 41.69878005981445,
"learning_rate": 3.306636061080487e-07,
"logits/chosen": 0.03852830454707146,
"logits/rejected": 0.05525093898177147,
"logps/chosen": -270.3495178222656,
"logps/ref_chosen": -49.39221954345703,
"logps/ref_rejected": -75.79280853271484,
"logps/rejected": -436.1895751953125,
"loss": 0.8841,
"margin_dpo/margin_mean": 139.439453125,
"margin_dpo/margin_std": 151.3997039794922,
"step": 312
},
{
"KL/chosen_KL_mean": -219.05848693847656,
"KL/mean": -286.2684020996094,
"KL/rejected_KL_mean": -353.47833251953125,
"KL/std": 144.6976318359375,
"epoch": 0.45961820851688695,
"fcm_dpo/beta": 0.005889068823307753,
"fcm_dpo/delta": 0.006327494978904724,
"fcm_dpo/margin": 134.4198455810547,
"fcm_dpo/q_t": 0.3411150276660919,
"grad_norm": 43.50709533691406,
"learning_rate": 3.2944785489547537e-07,
"logits/chosen": 0.026514366269111633,
"logits/rejected": 0.03467664122581482,
"logps/chosen": -269.21124267578125,
"logps/ref_chosen": -50.152740478515625,
"logps/ref_rejected": -86.40620422363281,
"logps/rejected": -439.884521484375,
"loss": 0.932,
"margin_dpo/margin_mean": 134.41983032226562,
"margin_dpo/margin_std": 159.38235473632812,
"step": 313
},
{
"KL/chosen_KL_mean": -235.50747680664062,
"KL/mean": -299.62200927734375,
"KL/rejected_KL_mean": -363.73651123046875,
"KL/std": 141.76861572265625,
"epoch": 0.461086637298091,
"fcm_dpo/beta": 0.005829677917063236,
"fcm_dpo/delta": 0.05599457398056984,
"fcm_dpo/margin": 128.22901916503906,
"fcm_dpo/q_t": 0.35217374563217163,
"grad_norm": 33.33452224731445,
"learning_rate": 3.2823001015803857e-07,
"logits/chosen": -0.06579890102148056,
"logits/rejected": -0.06030385196208954,
"logps/chosen": -292.74505615234375,
"logps/ref_chosen": -57.237579345703125,
"logps/ref_rejected": -97.5965347290039,
"logps/rejected": -461.3330383300781,
"loss": 0.9653,
"margin_dpo/margin_mean": 128.22901916503906,
"margin_dpo/margin_std": 166.1632080078125,
"step": 314
},
{
"KL/chosen_KL_mean": -230.164306640625,
"KL/mean": -291.27130126953125,
"KL/rejected_KL_mean": -352.3782958984375,
"KL/std": 119.44525146484375,
"epoch": 0.46255506607929514,
"fcm_dpo/beta": 0.005951759405434132,
"fcm_dpo/delta": 0.07836839556694031,
"fcm_dpo/margin": 122.21400451660156,
"fcm_dpo/q_t": 0.3487578332424164,
"grad_norm": 51.986087799072266,
"learning_rate": 3.270101039870797e-07,
"logits/chosen": 0.05683774873614311,
"logits/rejected": 0.0601893775165081,
"logps/chosen": -279.23388671875,
"logps/ref_chosen": -49.06958770751953,
"logps/ref_rejected": -85.68087768554688,
"logps/rejected": -438.0592041015625,
"loss": 0.9161,
"margin_dpo/margin_mean": 122.21400451660156,
"margin_dpo/margin_std": 134.08251953125,
"step": 315
},
{
"KL/chosen_KL_mean": -211.80429077148438,
"KL/mean": -290.564208984375,
"KL/rejected_KL_mean": -369.32415771484375,
"KL/std": 143.4363250732422,
"epoch": 0.46402349486049926,
"fcm_dpo/beta": 0.005822393111884594,
"fcm_dpo/delta": -0.13124880194664001,
"fcm_dpo/margin": 157.5198974609375,
"fcm_dpo/q_t": 0.31461644172668457,
"grad_norm": 42.63252258300781,
"learning_rate": 3.2578816852826086e-07,
"logits/chosen": -0.016533054411411285,
"logits/rejected": -0.012745167128741741,
"logps/chosen": -266.0650329589844,
"logps/ref_chosen": -54.26074981689453,
"logps/ref_rejected": -101.2814712524414,
"logps/rejected": -470.60565185546875,
"loss": 0.8139,
"margin_dpo/margin_mean": 157.5198974609375,
"margin_dpo/margin_std": 150.1714324951172,
"step": 316
},
{
"KL/chosen_KL_mean": -211.88148498535156,
"KL/mean": -298.3778991699219,
"KL/rejected_KL_mean": -384.8742980957031,
"KL/std": 135.2079620361328,
"epoch": 0.4654919236417034,
"fcm_dpo/beta": 0.005675586871802807,
"fcm_dpo/delta": -0.20120689272880554,
"fcm_dpo/margin": 172.99285888671875,
"fcm_dpo/q_t": 0.29535406827926636,
"grad_norm": 43.179039001464844,
"learning_rate": 3.2456423598071783e-07,
"logits/chosen": -0.02477482333779335,
"logits/rejected": -0.006942405365407467,
"logps/chosen": -267.9757080078125,
"logps/ref_chosen": -56.094207763671875,
"logps/ref_rejected": -100.69905090332031,
"logps/rejected": -485.5733642578125,
"loss": 0.7599,
"margin_dpo/margin_mean": 172.9928436279297,
"margin_dpo/margin_std": 140.1666259765625,
"step": 317
},
{
"KL/chosen_KL_mean": -212.18310546875,
"KL/mean": -289.941162109375,
"KL/rejected_KL_mean": -367.69927978515625,
"KL/std": 141.094970703125,
"epoch": 0.4669603524229075,
"fcm_dpo/beta": 0.005568217486143112,
"fcm_dpo/delta": -0.07241801917552948,
"fcm_dpo/margin": 155.5161590576172,
"fcm_dpo/q_t": 0.31813758611679077,
"grad_norm": 38.82643508911133,
"learning_rate": 3.233383385962115e-07,
"logits/chosen": -0.08865199983119965,
"logits/rejected": -0.057291001081466675,
"logps/chosen": -276.82879638671875,
"logps/ref_chosen": -64.64569854736328,
"logps/ref_rejected": -82.76425170898438,
"logps/rejected": -450.4635009765625,
"loss": 0.8259,
"margin_dpo/margin_mean": 155.51617431640625,
"margin_dpo/margin_std": 140.63192749023438,
"step": 318
},
{
"KL/chosen_KL_mean": -218.0725860595703,
"KL/mean": -295.23101806640625,
"KL/rejected_KL_mean": -372.389404296875,
"KL/std": 148.6660919189453,
"epoch": 0.4684287812041116,
"fcm_dpo/beta": 0.005480584688484669,
"fcm_dpo/delta": -0.04994754120707512,
"fcm_dpo/margin": 154.31683349609375,
"fcm_dpo/q_t": 0.32287895679473877,
"grad_norm": 31.670095443725586,
"learning_rate": 3.2211050867827805e-07,
"logits/chosen": -0.042262256145477295,
"logits/rejected": -0.05314238369464874,
"logps/chosen": -267.45635986328125,
"logps/ref_chosen": -49.383758544921875,
"logps/ref_rejected": -113.90650939941406,
"logps/rejected": -486.2959289550781,
"loss": 0.8427,
"margin_dpo/margin_mean": 154.31683349609375,
"margin_dpo/margin_std": 146.704833984375,
"step": 319
},
{
"KL/chosen_KL_mean": -237.90200805664062,
"KL/mean": -325.74908447265625,
"KL/rejected_KL_mean": -413.5962219238281,
"KL/std": 172.5784149169922,
"epoch": 0.4698972099853157,
"fcm_dpo/beta": 0.005322999320924282,
"fcm_dpo/delta": -0.15001486241817474,
"fcm_dpo/margin": 175.69418334960938,
"fcm_dpo/q_t": 0.3112744092941284,
"grad_norm": 39.102046966552734,
"learning_rate": 3.208807785813777e-07,
"logits/chosen": -0.01164646353572607,
"logits/rejected": -0.012258168309926987,
"logps/chosen": -297.4068908691406,
"logps/ref_chosen": -59.50489044189453,
"logps/ref_rejected": -97.66717529296875,
"logps/rejected": -511.26336669921875,
"loss": 0.8104,
"margin_dpo/margin_mean": 175.69418334960938,
"margin_dpo/margin_std": 170.92335510253906,
"step": 320
},
{
"KL/chosen_KL_mean": -300.34954833984375,
"KL/mean": -372.80084228515625,
"KL/rejected_KL_mean": -445.2521667480469,
"KL/std": 164.09548950195312,
"epoch": 0.4713656387665198,
"fcm_dpo/beta": 0.005263539031147957,
"fcm_dpo/delta": 0.03748725354671478,
"fcm_dpo/margin": 144.90261840820312,
"fcm_dpo/q_t": 0.3461604416370392,
"grad_norm": 42.18912124633789,
"learning_rate": 3.1964918071004217e-07,
"logits/chosen": 0.18075895309448242,
"logits/rejected": 0.189798966050148,
"logps/chosen": -361.898193359375,
"logps/ref_chosen": -61.548683166503906,
"logps/ref_rejected": -91.64103698730469,
"logps/rejected": -536.8931884765625,
"loss": 0.9492,
"margin_dpo/margin_mean": 144.90261840820312,
"margin_dpo/margin_std": 174.24356079101562,
"step": 321
},
{
"KL/chosen_KL_mean": -268.3616638183594,
"KL/mean": -350.6944580078125,
"KL/rejected_KL_mean": -433.0272216796875,
"KL/std": 155.32608032226562,
"epoch": 0.47283406754772395,
"fcm_dpo/beta": 0.005247790366411209,
"fcm_dpo/delta": -0.0717114806175232,
"fcm_dpo/margin": 164.66555786132812,
"fcm_dpo/q_t": 0.32040244340896606,
"grad_norm": 47.565155029296875,
"learning_rate": 3.184157475180207e-07,
"logits/chosen": 0.15934163331985474,
"logits/rejected": 0.16657748818397522,
"logps/chosen": -325.6517028808594,
"logps/ref_chosen": -57.29003143310547,
"logps/ref_rejected": -95.74992370605469,
"logps/rejected": -528.7771606445312,
"loss": 0.8352,
"margin_dpo/margin_mean": 164.66555786132812,
"margin_dpo/margin_std": 153.64788818359375,
"step": 322
},
{
"KL/chosen_KL_mean": -287.80499267578125,
"KL/mean": -361.19696044921875,
"KL/rejected_KL_mean": -434.5888671875,
"KL/std": 157.22926330566406,
"epoch": 0.47430249632892807,
"fcm_dpo/beta": 0.005266541615128517,
"fcm_dpo/delta": 0.029003996402025223,
"fcm_dpo/margin": 146.7838897705078,
"fcm_dpo/q_t": 0.33969342708587646,
"grad_norm": 94.16871643066406,
"learning_rate": 3.171805115074251e-07,
"logits/chosen": 0.12828153371810913,
"logits/rejected": 0.13148348033428192,
"logps/chosen": -339.0389404296875,
"logps/ref_chosen": -51.23395919799805,
"logps/ref_rejected": -75.06192016601562,
"logps/rejected": -509.6507873535156,
"loss": 0.9039,
"margin_dpo/margin_mean": 146.7838897705078,
"margin_dpo/margin_std": 155.2939910888672,
"step": 323
},
{
"KL/chosen_KL_mean": -308.6185302734375,
"KL/mean": -386.80194091796875,
"KL/rejected_KL_mean": -464.9853820800781,
"KL/std": 166.91270446777344,
"epoch": 0.47577092511013214,
"fcm_dpo/beta": 0.005319544579833746,
"fcm_dpo/delta": -0.036364443600177765,
"fcm_dpo/margin": 156.36685180664062,
"fcm_dpo/q_t": 0.33428698778152466,
"grad_norm": 102.16787719726562,
"learning_rate": 3.1594350522787295e-07,
"logits/chosen": 0.04309145733714104,
"logits/rejected": 0.06093376502394676,
"logps/chosen": -373.75372314453125,
"logps/ref_chosen": -65.13516998291016,
"logps/ref_rejected": -86.47750854492188,
"logps/rejected": -551.462890625,
"loss": 0.9097,
"margin_dpo/margin_mean": 156.36685180664062,
"margin_dpo/margin_std": 177.79598999023438,
"step": 324
},
{
"KL/chosen_KL_mean": -250.1693878173828,
"KL/mean": -315.89727783203125,
"KL/rejected_KL_mean": -381.625244140625,
"KL/std": 165.94528198242188,
"epoch": 0.47723935389133626,
"fcm_dpo/beta": 0.005339835770428181,
"fcm_dpo/delta": 0.10544876754283905,
"fcm_dpo/margin": 131.45582580566406,
"fcm_dpo/q_t": 0.35164737701416016,
"grad_norm": 72.66465759277344,
"learning_rate": 3.147047612756302e-07,
"logits/chosen": -0.03497472405433655,
"logits/rejected": -0.006054544821381569,
"logps/chosen": -306.3849792480469,
"logps/ref_chosen": -56.215599060058594,
"logps/ref_rejected": -70.08592987060547,
"logps/rejected": -451.7111511230469,
"loss": 0.9295,
"margin_dpo/margin_mean": 131.45584106445312,
"margin_dpo/margin_std": 139.95791625976562,
"step": 325
},
{
"KL/chosen_KL_mean": -265.7798767089844,
"KL/mean": -331.59429931640625,
"KL/rejected_KL_mean": -397.40869140625,
"KL/std": 143.18637084960938,
"epoch": 0.4787077826725404,
"fcm_dpo/beta": 0.005434621125459671,
"fcm_dpo/delta": 0.09125562757253647,
"fcm_dpo/margin": 131.62879943847656,
"fcm_dpo/q_t": 0.34775280952453613,
"grad_norm": 52.132789611816406,
"learning_rate": 3.134643122927519e-07,
"logits/chosen": -0.028336074203252792,
"logits/rejected": 0.004011590965092182,
"logps/chosen": -338.5048522949219,
"logps/ref_chosen": -72.72496032714844,
"logps/ref_rejected": -79.8467788696289,
"logps/rejected": -477.2554931640625,
"loss": 0.8987,
"margin_dpo/margin_mean": 131.62879943847656,
"margin_dpo/margin_std": 130.20550537109375,
"step": 326
},
{
"KL/chosen_KL_mean": -220.91165161132812,
"KL/mean": -315.80706787109375,
"KL/rejected_KL_mean": -410.7025146484375,
"KL/std": 157.95884704589844,
"epoch": 0.4801762114537445,
"fcm_dpo/beta": 0.005260833539068699,
"fcm_dpo/delta": -0.2224833071231842,
"fcm_dpo/margin": 189.79087829589844,
"fcm_dpo/q_t": 0.29437848925590515,
"grad_norm": 56.85773468017578,
"learning_rate": 3.1222219096622264e-07,
"logits/chosen": -0.03550358861684799,
"logits/rejected": -0.011621898040175438,
"logps/chosen": -290.04608154296875,
"logps/ref_chosen": -69.13441467285156,
"logps/ref_rejected": -111.93377685546875,
"logps/rejected": -522.6362915039062,
"loss": 0.7735,
"margin_dpo/margin_mean": 189.7908935546875,
"margin_dpo/margin_std": 165.75531005859375,
"step": 327
},
{
"KL/chosen_KL_mean": -234.7518768310547,
"KL/mean": -312.504638671875,
"KL/rejected_KL_mean": -390.25738525390625,
"KL/std": 155.8070068359375,
"epoch": 0.48164464023494863,
"fcm_dpo/beta": 0.005238644778728485,
"fcm_dpo/delta": -0.01621408388018608,
"fcm_dpo/margin": 155.5054931640625,
"fcm_dpo/q_t": 0.3311880826950073,
"grad_norm": 46.42213439941406,
"learning_rate": 3.1097843002709427e-07,
"logits/chosen": -0.05805660039186478,
"logits/rejected": -0.05620625987648964,
"logps/chosen": -294.4390869140625,
"logps/ref_chosen": -59.68719482421875,
"logps/ref_rejected": -90.85499572753906,
"logps/rejected": -481.11236572265625,
"loss": 0.8629,
"margin_dpo/margin_mean": 155.5054931640625,
"margin_dpo/margin_std": 155.49652099609375,
"step": 328
},
{
"KL/chosen_KL_mean": -249.91339111328125,
"KL/mean": -333.9342956542969,
"KL/rejected_KL_mean": -417.9552001953125,
"KL/std": 158.51559448242188,
"epoch": 0.4831130690161527,
"fcm_dpo/beta": 0.005131312645971775,
"fcm_dpo/delta": -0.0691857784986496,
"fcm_dpo/margin": 168.04180908203125,
"fcm_dpo/q_t": 0.32326847314834595,
"grad_norm": 43.19540023803711,
"learning_rate": 3.0973306224962437e-07,
"logits/chosen": 0.032189346849918365,
"logits/rejected": 0.0457240529358387,
"logps/chosen": -315.1595764160156,
"logps/ref_chosen": -65.2461929321289,
"logps/ref_rejected": -100.69770812988281,
"logps/rejected": -518.6528930664062,
"loss": 0.8604,
"margin_dpo/margin_mean": 168.04180908203125,
"margin_dpo/margin_std": 168.69622802734375,
"step": 329
},
{
"KL/chosen_KL_mean": -229.13954162597656,
"KL/mean": -311.7605285644531,
"KL/rejected_KL_mean": -394.3815002441406,
"KL/std": 157.78042602539062,
"epoch": 0.4845814977973568,
"fcm_dpo/beta": 0.005100243724882603,
"fcm_dpo/delta": -0.04678843915462494,
"fcm_dpo/margin": 165.24192810058594,
"fcm_dpo/q_t": 0.3230311870574951,
"grad_norm": 47.458736419677734,
"learning_rate": 3.084861204504122e-07,
"logits/chosen": 0.039801113307476044,
"logits/rejected": 0.03778820112347603,
"logps/chosen": -276.13787841796875,
"logps/ref_chosen": -46.998348236083984,
"logps/ref_rejected": -86.87684631347656,
"logps/rejected": -481.25836181640625,
"loss": 0.8448,
"margin_dpo/margin_mean": 165.241943359375,
"margin_dpo/margin_std": 157.53878784179688,
"step": 330
},
{
"KL/chosen_KL_mean": -226.46022033691406,
"KL/mean": -313.51763916015625,
"KL/rejected_KL_mean": -400.5750732421875,
"KL/std": 132.8838348388672,
"epoch": 0.48604992657856094,
"fcm_dpo/beta": 0.005020303651690483,
"fcm_dpo/delta": -0.08152840286493301,
"fcm_dpo/margin": 174.1148681640625,
"fcm_dpo/q_t": 0.3074917197227478,
"grad_norm": 47.758262634277344,
"learning_rate": 3.072376374875335e-07,
"logits/chosen": 0.02427823096513748,
"logits/rejected": 0.03714999929070473,
"logps/chosen": -276.9844665527344,
"logps/ref_chosen": -50.52424621582031,
"logps/ref_rejected": -89.01544189453125,
"logps/rejected": -489.59051513671875,
"loss": 0.7647,
"margin_dpo/margin_mean": 174.1148681640625,
"margin_dpo/margin_std": 117.05330657958984,
"step": 331
},
{
"KL/chosen_KL_mean": -238.39886474609375,
"KL/mean": -301.0345458984375,
"KL/rejected_KL_mean": -363.67022705078125,
"KL/std": 137.32064819335938,
"epoch": 0.48751835535976507,
"fcm_dpo/beta": 0.005146116018295288,
"fcm_dpo/delta": 0.16562792658805847,
"fcm_dpo/margin": 125.2713623046875,
"fcm_dpo/q_t": 0.362199991941452,
"grad_norm": 37.70057678222656,
"learning_rate": 3.059876462596758e-07,
"logits/chosen": 0.062703438103199,
"logits/rejected": 0.07360726594924927,
"logps/chosen": -287.57916259765625,
"logps/ref_chosen": -49.18028259277344,
"logps/ref_rejected": -76.48515319824219,
"logps/rejected": -440.1553955078125,
"loss": 0.949,
"margin_dpo/margin_mean": 125.2713623046875,
"margin_dpo/margin_std": 136.968017578125,
"step": 332
},
{
"KL/chosen_KL_mean": -251.75677490234375,
"KL/mean": -335.0605773925781,
"KL/rejected_KL_mean": -418.3643798828125,
"KL/std": 162.35433959960938,
"epoch": 0.4889867841409692,
"fcm_dpo/beta": 0.005091669969260693,
"fcm_dpo/delta": -0.05500401183962822,
"fcm_dpo/margin": 166.6075897216797,
"fcm_dpo/q_t": 0.3314821720123291,
"grad_norm": 37.29023742675781,
"learning_rate": 3.0473617970527015e-07,
"logits/chosen": 0.00028313975781202316,
"logits/rejected": 0.0058424267917871475,
"logps/chosen": -315.51251220703125,
"logps/ref_chosen": -63.75574493408203,
"logps/ref_rejected": -95.04411315917969,
"logps/rejected": -513.408447265625,
"loss": 0.8865,
"margin_dpo/margin_mean": 166.6075897216797,
"margin_dpo/margin_std": 180.63882446289062,
"step": 333
},
{
"KL/chosen_KL_mean": -250.1717529296875,
"KL/mean": -329.30615234375,
"KL/rejected_KL_mean": -408.4405822753906,
"KL/std": 176.424072265625,
"epoch": 0.49045521292217326,
"fcm_dpo/beta": 0.005118607543408871,
"fcm_dpo/delta": -0.011011124588549137,
"fcm_dpo/margin": 158.2688446044922,
"fcm_dpo/q_t": 0.33261972665786743,
"grad_norm": 39.07373809814453,
"learning_rate": 3.034832708016243e-07,
"logits/chosen": -0.023045614361763,
"logits/rejected": -0.018983149901032448,
"logps/chosen": -317.1514892578125,
"logps/ref_chosen": -66.97975158691406,
"logps/ref_rejected": -95.31692504882812,
"logps/rejected": -503.75750732421875,
"loss": 0.8932,
"margin_dpo/margin_mean": 158.26882934570312,
"margin_dpo/margin_std": 170.21026611328125,
"step": 334
},
{
"KL/chosen_KL_mean": -293.2091979980469,
"KL/mean": -356.44830322265625,
"KL/rejected_KL_mean": -419.68743896484375,
"KL/std": 160.33944702148438,
"epoch": 0.4919236417033774,
"fcm_dpo/beta": 0.005251425784081221,
"fcm_dpo/delta": 0.1440257877111435,
"fcm_dpo/margin": 126.47823333740234,
"fcm_dpo/q_t": 0.36713701486587524,
"grad_norm": 71.73628234863281,
"learning_rate": 3.022289525640531e-07,
"logits/chosen": 0.0950508713722229,
"logits/rejected": 0.11580017954111099,
"logps/chosen": -355.7516784667969,
"logps/ref_chosen": -62.54248046875,
"logps/ref_rejected": -87.61770629882812,
"logps/rejected": -507.30511474609375,
"loss": 1.0083,
"margin_dpo/margin_mean": 126.47822570800781,
"margin_dpo/margin_std": 176.1946258544922,
"step": 335
},
{
"KL/chosen_KL_mean": -285.6962585449219,
"KL/mean": -374.312744140625,
"KL/rejected_KL_mean": -462.92919921875,
"KL/std": 179.97970581054688,
"epoch": 0.4933920704845815,
"fcm_dpo/beta": 0.005166906863451004,
"fcm_dpo/delta": -0.1273384988307953,
"fcm_dpo/margin": 177.23297119140625,
"fcm_dpo/q_t": 0.31727689504623413,
"grad_norm": 44.789981842041016,
"learning_rate": 3.009732580450086e-07,
"logits/chosen": 0.06262758374214172,
"logits/rejected": 0.06866885721683502,
"logps/chosen": -340.2274169921875,
"logps/ref_chosen": -54.53115463256836,
"logps/ref_rejected": -104.40424346923828,
"logps/rejected": -567.33349609375,
"loss": 0.8731,
"margin_dpo/margin_mean": 177.23297119140625,
"margin_dpo/margin_std": 196.13185119628906,
"step": 336
},
{
"KL/chosen_KL_mean": -257.1330871582031,
"KL/mean": -352.3122253417969,
"KL/rejected_KL_mean": -447.4913635253906,
"KL/std": 158.125244140625,
"epoch": 0.4948604992657856,
"fcm_dpo/beta": 0.004982330836355686,
"fcm_dpo/delta": -0.165202796459198,
"fcm_dpo/margin": 190.35829162597656,
"fcm_dpo/q_t": 0.30310046672821045,
"grad_norm": 44.3165283203125,
"learning_rate": 2.9971622033320914e-07,
"logits/chosen": 0.0833229273557663,
"logits/rejected": 0.098611980676651,
"logps/chosen": -322.26177978515625,
"logps/ref_chosen": -65.12869262695312,
"logps/ref_rejected": -101.72701263427734,
"logps/rejected": -549.2183837890625,
"loss": 0.7904,
"margin_dpo/margin_mean": 190.35829162597656,
"margin_dpo/margin_std": 169.05369567871094,
"step": 337
},
{
"KL/chosen_KL_mean": -253.24179077148438,
"KL/mean": -342.19232177734375,
"KL/rejected_KL_mean": -431.1428527832031,
"KL/std": 160.1546630859375,
"epoch": 0.49632892804698975,
"fcm_dpo/beta": 0.004904153756797314,
"fcm_dpo/delta": -0.07938512414693832,
"fcm_dpo/margin": 177.90106201171875,
"fcm_dpo/q_t": 0.3190838396549225,
"grad_norm": 40.79475784301758,
"learning_rate": 2.984578725527675e-07,
"logits/chosen": 0.05606719106435776,
"logits/rejected": 0.0772060826420784,
"logps/chosen": -311.66448974609375,
"logps/ref_chosen": -58.422706604003906,
"logps/ref_rejected": -89.06854248046875,
"logps/rejected": -520.2113647460938,
"loss": 0.8318,
"margin_dpo/margin_mean": 177.90106201171875,
"margin_dpo/margin_std": 167.9755096435547,
"step": 338
},
{
"KL/chosen_KL_mean": -238.2997283935547,
"KL/mean": -323.3148193359375,
"KL/rejected_KL_mean": -408.3299560546875,
"KL/std": 160.2054443359375,
"epoch": 0.4977973568281938,
"fcm_dpo/beta": 0.0049000889994204044,
"fcm_dpo/delta": -0.03744722157716751,
"fcm_dpo/margin": 170.03021240234375,
"fcm_dpo/q_t": 0.32219401001930237,
"grad_norm": 38.09584045410156,
"learning_rate": 2.9719824786231796e-07,
"logits/chosen": 0.05355466157197952,
"logits/rejected": 0.06788984686136246,
"logps/chosen": -298.2950439453125,
"logps/ref_chosen": -59.99531555175781,
"logps/ref_rejected": -103.9109115600586,
"logps/rejected": -512.2408447265625,
"loss": 0.8261,
"margin_dpo/margin_mean": 170.03021240234375,
"margin_dpo/margin_std": 143.59585571289062,
"step": 339
},
{
"KL/chosen_KL_mean": -240.2142333984375,
"KL/mean": -317.741455078125,
"KL/rejected_KL_mean": -395.26861572265625,
"KL/std": 146.14395141601562,
"epoch": 0.49926578560939794,
"fcm_dpo/beta": 0.0048608845099806786,
"fcm_dpo/delta": 0.04984866455197334,
"fcm_dpo/margin": 155.05438232421875,
"fcm_dpo/q_t": 0.3420332968235016,
"grad_norm": 35.83970260620117,
"learning_rate": 2.959373794541426e-07,
"logits/chosen": 0.038565389811992645,
"logits/rejected": 0.06811020523309708,
"logps/chosen": -293.0444641113281,
"logps/ref_chosen": -52.83022689819336,
"logps/ref_rejected": -73.10723114013672,
"logps/rejected": -468.3758544921875,
"loss": 0.8975,
"margin_dpo/margin_mean": 155.0543975830078,
"margin_dpo/margin_std": 162.0891876220703,
"step": 340
},
{
"KL/chosen_KL_mean": -218.0098876953125,
"KL/mean": -304.40362548828125,
"KL/rejected_KL_mean": -390.79736328125,
"KL/std": 153.83116149902344,
"epoch": 0.5007342143906021,
"fcm_dpo/beta": 0.004849275108426809,
"fcm_dpo/delta": -0.04175635054707527,
"fcm_dpo/margin": 172.7874755859375,
"fcm_dpo/q_t": 0.32173144817352295,
"grad_norm": 34.084495544433594,
"learning_rate": 2.946753005532965e-07,
"logits/chosen": -0.0034986878745257854,
"logits/rejected": -0.00451930146664381,
"logps/chosen": -265.90966796875,
"logps/ref_chosen": -47.899803161621094,
"logps/ref_rejected": -101.80987548828125,
"logps/rejected": -492.60723876953125,
"loss": 0.8354,
"margin_dpo/margin_mean": 172.7874755859375,
"margin_dpo/margin_std": 155.39715576171875,
"step": 341
},
{
"KL/chosen_KL_mean": -207.13504028320312,
"KL/mean": -284.558349609375,
"KL/rejected_KL_mean": -361.981689453125,
"KL/std": 152.72943115234375,
"epoch": 0.5022026431718062,
"fcm_dpo/beta": 0.004914829973131418,
"fcm_dpo/delta": 0.041156478226184845,
"fcm_dpo/margin": 154.84661865234375,
"fcm_dpo/q_t": 0.3427678644657135,
"grad_norm": 34.09379959106445,
"learning_rate": 2.934120444167326e-07,
"logits/chosen": -0.09954602271318436,
"logits/rejected": -0.07372716814279556,
"logps/chosen": -279.1316833496094,
"logps/ref_chosen": -71.99664306640625,
"logps/ref_rejected": -92.58959197998047,
"logps/rejected": -454.5712585449219,
"loss": 0.908,
"margin_dpo/margin_mean": 154.84661865234375,
"margin_dpo/margin_std": 171.0548095703125,
"step": 342
},
{
"KL/chosen_KL_mean": -189.46636962890625,
"KL/mean": -279.3402099609375,
"KL/rejected_KL_mean": -369.21405029296875,
"KL/std": 155.5145263671875,
"epoch": 0.5036710719530103,
"fcm_dpo/beta": 0.004830049350857735,
"fcm_dpo/delta": -0.07497724145650864,
"fcm_dpo/margin": 179.74769592285156,
"fcm_dpo/q_t": 0.31141358613967896,
"grad_norm": 55.63039016723633,
"learning_rate": 2.9214764433242476e-07,
"logits/chosen": -0.04048309847712517,
"logits/rejected": -0.02954668179154396,
"logps/chosen": -243.87197875976562,
"logps/ref_chosen": -54.405616760253906,
"logps/ref_rejected": -111.04142761230469,
"logps/rejected": -480.2554931640625,
"loss": 0.7885,
"margin_dpo/margin_mean": 179.74771118164062,
"margin_dpo/margin_std": 134.98263549804688,
"step": 343
},
{
"KL/chosen_KL_mean": -204.9948272705078,
"KL/mean": -283.34881591796875,
"KL/rejected_KL_mean": -361.70281982421875,
"KL/std": 170.766357421875,
"epoch": 0.5051395007342144,
"fcm_dpo/beta": 0.004901527427136898,
"fcm_dpo/delta": 0.03143307566642761,
"fcm_dpo/margin": 156.7080078125,
"fcm_dpo/q_t": 0.34064382314682007,
"grad_norm": 54.35795974731445,
"learning_rate": 2.9088213361849126e-07,
"logits/chosen": -0.04297008737921715,
"logits/rejected": -0.03432103246450424,
"logps/chosen": -258.95947265625,
"logps/ref_chosen": -53.96466827392578,
"logps/ref_rejected": -90.62336730957031,
"logps/rejected": -452.3262023925781,
"loss": 0.8886,
"margin_dpo/margin_mean": 156.7080078125,
"margin_dpo/margin_std": 158.40911865234375,
"step": 344
},
{
"KL/chosen_KL_mean": -249.43026733398438,
"KL/mean": -335.73760986328125,
"KL/rejected_KL_mean": -422.04498291015625,
"KL/std": 163.56846618652344,
"epoch": 0.5066079295154186,
"fcm_dpo/beta": 0.004851485602557659,
"fcm_dpo/delta": -0.04130681976675987,
"fcm_dpo/margin": 172.61471557617188,
"fcm_dpo/q_t": 0.32575783133506775,
"grad_norm": 45.86104965209961,
"learning_rate": 2.896155456223163e-07,
"logits/chosen": -0.04765704274177551,
"logits/rejected": -0.031171437352895737,
"logps/chosen": -311.115966796875,
"logps/ref_chosen": -61.685699462890625,
"logps/ref_rejected": -99.49041748046875,
"logps/rejected": -521.535400390625,
"loss": 0.8542,
"margin_dpo/margin_mean": 172.61473083496094,
"margin_dpo/margin_std": 169.39071655273438,
"step": 345
},
{
"KL/chosen_KL_mean": -275.68389892578125,
"KL/mean": -357.99700927734375,
"KL/rejected_KL_mean": -440.31011962890625,
"KL/std": 157.96783447265625,
"epoch": 0.5080763582966226,
"fcm_dpo/beta": 0.004806653130799532,
"fcm_dpo/delta": 0.009312797337770462,
"fcm_dpo/margin": 164.626220703125,
"fcm_dpo/q_t": 0.33165764808654785,
"grad_norm": 36.5350341796875,
"learning_rate": 2.883479137196714e-07,
"logits/chosen": 0.11625039577484131,
"logits/rejected": 0.13555414974689484,
"logps/chosen": -330.9401550292969,
"logps/ref_chosen": -55.256263732910156,
"logps/ref_rejected": -77.41532135009766,
"logps/rejected": -517.7254638671875,
"loss": 0.877,
"margin_dpo/margin_mean": 164.626220703125,
"margin_dpo/margin_std": 163.69869995117188,
"step": 346
},
{
"KL/chosen_KL_mean": -289.11468505859375,
"KL/mean": -371.96826171875,
"KL/rejected_KL_mean": -454.8218078613281,
"KL/std": 172.33523559570312,
"epoch": 0.5095447870778267,
"fcm_dpo/beta": 0.004811136983335018,
"fcm_dpo/delta": 0.002851152792572975,
"fcm_dpo/margin": 165.7071075439453,
"fcm_dpo/q_t": 0.3361477851867676,
"grad_norm": 40.908966064453125,
"learning_rate": 2.8707927131383614e-07,
"logits/chosen": 0.10305792093276978,
"logits/rejected": 0.11407352983951569,
"logps/chosen": -346.6809387207031,
"logps/ref_chosen": -57.56623840332031,
"logps/ref_rejected": -92.35509490966797,
"logps/rejected": -547.1768798828125,
"loss": 0.9029,
"margin_dpo/margin_mean": 165.7071075439453,
"margin_dpo/margin_std": 183.31068420410156,
"step": 347
},
{
"KL/chosen_KL_mean": -262.79388427734375,
"KL/mean": -337.61395263671875,
"KL/rejected_KL_mean": -412.4340515136719,
"KL/std": 159.94393920898438,
"epoch": 0.5110132158590308,
"fcm_dpo/beta": 0.004831107333302498,
"fcm_dpo/delta": 0.08148808032274246,
"fcm_dpo/margin": 149.6401824951172,
"fcm_dpo/q_t": 0.3513612449169159,
"grad_norm": 45.900245666503906,
"learning_rate": 2.858096518347179e-07,
"logits/chosen": 0.0368988960981369,
"logits/rejected": 0.042798325419425964,
"logps/chosen": -319.111572265625,
"logps/ref_chosen": -56.31770324707031,
"logps/ref_rejected": -89.13836669921875,
"logps/rejected": -501.5724182128906,
"loss": 0.9404,
"margin_dpo/margin_mean": 149.6401824951172,
"margin_dpo/margin_std": 172.48953247070312,
"step": 348
},
{
"KL/chosen_KL_mean": -276.16656494140625,
"KL/mean": -357.14825439453125,
"KL/rejected_KL_mean": -438.1300048828125,
"KL/std": 166.79330444335938,
"epoch": 0.5124816446402349,
"fcm_dpo/beta": 0.004948951303958893,
"fcm_dpo/delta": -0.0035505443811416626,
"fcm_dpo/margin": 161.96343994140625,
"fcm_dpo/q_t": 0.34265533089637756,
"grad_norm": 38.599334716796875,
"learning_rate": 2.845390887379706e-07,
"logits/chosen": 0.12962684035301208,
"logits/rejected": 0.1368744671344757,
"logps/chosen": -334.19207763671875,
"logps/ref_chosen": -58.025516510009766,
"logps/ref_rejected": -97.50515747070312,
"logps/rejected": -535.6351318359375,
"loss": 0.9288,
"margin_dpo/margin_mean": 161.96343994140625,
"margin_dpo/margin_std": 197.3702392578125,
"step": 349
},
{
"KL/chosen_KL_mean": -290.5421142578125,
"KL/mean": -370.68487548828125,
"KL/rejected_KL_mean": -450.82763671875,
"KL/std": 159.0634765625,
"epoch": 0.5139500734214391,
"fcm_dpo/beta": 0.004874521866440773,
"fcm_dpo/delta": 0.019010931253433228,
"fcm_dpo/margin": 160.2855224609375,
"fcm_dpo/q_t": 0.33771812915802,
"grad_norm": 52.47856521606445,
"learning_rate": 2.8326761550411346e-07,
"logits/chosen": 0.047298818826675415,
"logits/rejected": 0.0552336648106575,
"logps/chosen": -354.87261962890625,
"logps/ref_chosen": -64.33049011230469,
"logps/ref_rejected": -89.87164306640625,
"logps/rejected": -540.6992797851562,
"loss": 0.9145,
"margin_dpo/margin_mean": 160.2855224609375,
"margin_dpo/margin_std": 178.19192504882812,
"step": 350
},
{
"KL/chosen_KL_mean": -277.630859375,
"KL/mean": -372.445556640625,
"KL/rejected_KL_mean": -467.26031494140625,
"KL/std": 192.2869415283203,
"epoch": 0.5154185022026432,
"fcm_dpo/beta": 0.004822996445000172,
"fcm_dpo/delta": -0.12629525363445282,
"fcm_dpo/margin": 189.62950134277344,
"fcm_dpo/q_t": 0.31934210658073425,
"grad_norm": 39.9083251953125,
"learning_rate": 2.819952656376487e-07,
"logits/chosen": 0.024956397712230682,
"logits/rejected": 0.0348396897315979,
"logps/chosen": -338.302978515625,
"logps/ref_chosen": -60.6721305847168,
"logps/ref_rejected": -101.5654296875,
"logps/rejected": -568.8257446289062,
"loss": 0.8645,
"margin_dpo/margin_mean": 189.62950134277344,
"margin_dpo/margin_std": 208.29437255859375,
"step": 351
},
{
"KL/chosen_KL_mean": -329.90045166015625,
"KL/mean": -390.7127685546875,
"KL/rejected_KL_mean": -451.52508544921875,
"KL/std": 176.01547241210938,
"epoch": 0.5168869309838473,
"fcm_dpo/beta": 0.004911536350846291,
"fcm_dpo/delta": 0.21583901345729828,
"fcm_dpo/margin": 121.62461853027344,
"fcm_dpo/q_t": 0.3810230791568756,
"grad_norm": 78.79871368408203,
"learning_rate": 2.8072207266617854e-07,
"logits/chosen": 0.135191410779953,
"logits/rejected": 0.1668703854084015,
"logps/chosen": -400.8439025878906,
"logps/ref_chosen": -70.9434585571289,
"logps/ref_rejected": -76.6419677734375,
"logps/rejected": -528.1670532226562,
"loss": 1.0591,
"margin_dpo/margin_mean": 121.62461853027344,
"margin_dpo/margin_std": 185.77769470214844,
"step": 352
},
{
"KL/chosen_KL_mean": -304.80230712890625,
"KL/mean": -384.9066162109375,
"KL/rejected_KL_mean": -465.01092529296875,
"KL/std": 172.11077880859375,
"epoch": 0.5183553597650514,
"fcm_dpo/beta": 0.004962503910064697,
"fcm_dpo/delta": 0.003439009189605713,
"fcm_dpo/margin": 160.20860290527344,
"fcm_dpo/q_t": 0.3434601128101349,
"grad_norm": 39.81656265258789,
"learning_rate": 2.794480701395219e-07,
"logits/chosen": 0.17260809242725372,
"logits/rejected": 0.19139912724494934,
"logps/chosen": -363.1976623535156,
"logps/ref_chosen": -58.39533996582031,
"logps/ref_rejected": -80.33553314208984,
"logps/rejected": -545.346435546875,
"loss": 0.9295,
"margin_dpo/margin_mean": 160.20860290527344,
"margin_dpo/margin_std": 190.71505737304688,
"step": 353
},
{
"KL/chosen_KL_mean": -259.389404296875,
"KL/mean": -348.7931823730469,
"KL/rejected_KL_mean": -438.19696044921875,
"KL/std": 158.2752685546875,
"epoch": 0.5198237885462555,
"fcm_dpo/beta": 0.004931221716105938,
"fcm_dpo/delta": -0.09008358418941498,
"fcm_dpo/margin": 178.80755615234375,
"fcm_dpo/q_t": 0.31218570470809937,
"grad_norm": 39.17101287841797,
"learning_rate": 2.781732916288303e-07,
"logits/chosen": 0.1611638367176056,
"logits/rejected": 0.18379822373390198,
"logps/chosen": -319.1923828125,
"logps/ref_chosen": -59.80299377441406,
"logps/ref_rejected": -88.75750732421875,
"logps/rejected": -526.9544677734375,
"loss": 0.7976,
"margin_dpo/margin_mean": 178.80755615234375,
"margin_dpo/margin_std": 147.66680908203125,
"step": 354
},
{
"KL/chosen_KL_mean": -251.05117797851562,
"KL/mean": -341.536865234375,
"KL/rejected_KL_mean": -432.0225524902344,
"KL/std": 175.2047119140625,
"epoch": 0.5212922173274597,
"fcm_dpo/beta": 0.004824481904506683,
"fcm_dpo/delta": -0.08206754922866821,
"fcm_dpo/margin": 180.97137451171875,
"fcm_dpo/q_t": 0.31630951166152954,
"grad_norm": 51.323665618896484,
"learning_rate": 2.7689777072570284e-07,
"logits/chosen": 0.11667881906032562,
"logits/rejected": 0.14499685168266296,
"logps/chosen": -305.1796875,
"logps/ref_chosen": -54.12849807739258,
"logps/ref_rejected": -82.40606689453125,
"logps/rejected": -514.4285888671875,
"loss": 0.8141,
"margin_dpo/margin_mean": 180.97137451171875,
"margin_dpo/margin_std": 157.3625030517578,
"step": 355
},
{
"KL/chosen_KL_mean": -290.5279541015625,
"KL/mean": -346.39935302734375,
"KL/rejected_KL_mean": -402.2707214355469,
"KL/std": 172.05081176757812,
"epoch": 0.5227606461086637,
"fcm_dpo/beta": 0.005076043773442507,
"fcm_dpo/delta": 0.24142667651176453,
"fcm_dpo/margin": 111.74278259277344,
"fcm_dpo/q_t": 0.39116936922073364,
"grad_norm": 69.66039276123047,
"learning_rate": 2.7562154104130176e-07,
"logits/chosen": 0.11355781555175781,
"logits/rejected": 0.12858667969703674,
"logps/chosen": -355.2017517089844,
"logps/ref_chosen": -64.6738052368164,
"logps/ref_rejected": -75.89926147460938,
"logps/rejected": -478.16998291015625,
"loss": 1.1223,
"margin_dpo/margin_mean": 111.74279022216797,
"margin_dpo/margin_std": 199.85433959960938,
"step": 356
},
{
"KL/chosen_KL_mean": -250.232177734375,
"KL/mean": -336.46234130859375,
"KL/rejected_KL_mean": -422.6925048828125,
"KL/std": 173.98196411132812,
"epoch": 0.5242290748898678,
"fcm_dpo/beta": 0.004994173534214497,
"fcm_dpo/delta": -0.06866991519927979,
"fcm_dpo/margin": 172.4603271484375,
"fcm_dpo/q_t": 0.31907975673675537,
"grad_norm": 39.19725799560547,
"learning_rate": 2.7434463620546594e-07,
"logits/chosen": 0.11515648663043976,
"logits/rejected": 0.13530930876731873,
"logps/chosen": -302.9579772949219,
"logps/ref_chosen": -52.725799560546875,
"logps/ref_rejected": -86.84115600585938,
"logps/rejected": -509.5336608886719,
"loss": 0.8274,
"margin_dpo/margin_mean": 172.46034240722656,
"margin_dpo/margin_std": 156.9672393798828,
"step": 357
},
{
"KL/chosen_KL_mean": -236.85372924804688,
"KL/mean": -309.67779541015625,
"KL/rejected_KL_mean": -382.50189208984375,
"KL/std": 164.68772888183594,
"epoch": 0.5256975036710719,
"fcm_dpo/beta": 0.005067657679319382,
"fcm_dpo/delta": 0.06674303859472275,
"fcm_dpo/margin": 145.64813232421875,
"fcm_dpo/q_t": 0.35047268867492676,
"grad_norm": 39.29417419433594,
"learning_rate": 2.730670898658255e-07,
"logits/chosen": -0.0026182327419519424,
"logits/rejected": 0.017924560233950615,
"logps/chosen": -300.0591735839844,
"logps/ref_chosen": -63.20543670654297,
"logps/ref_rejected": -88.373291015625,
"logps/rejected": -470.87518310546875,
"loss": 0.9235,
"margin_dpo/margin_mean": 145.64813232421875,
"margin_dpo/margin_std": 169.32843017578125,
"step": 358
},
{
"KL/chosen_KL_mean": -261.9931640625,
"KL/mean": -344.5310974121094,
"KL/rejected_KL_mean": -427.0690612792969,
"KL/std": 152.8817138671875,
"epoch": 0.527165932452276,
"fcm_dpo/beta": 0.005035270005464554,
"fcm_dpo/delta": -0.03444623947143555,
"fcm_dpo/margin": 165.0758819580078,
"fcm_dpo/q_t": 0.33515065908432007,
"grad_norm": 47.493412017822266,
"learning_rate": 2.717889356869146e-07,
"logits/chosen": 0.1439620852470398,
"logits/rejected": 0.16959136724472046,
"logps/chosen": -318.3633728027344,
"logps/ref_chosen": -56.370216369628906,
"logps/ref_rejected": -82.17375183105469,
"logps/rejected": -509.2427978515625,
"loss": 0.8822,
"margin_dpo/margin_mean": 165.07586669921875,
"margin_dpo/margin_std": 180.14077758789062,
"step": 359
},
{
"KL/chosen_KL_mean": -242.22293090820312,
"KL/mean": -319.6527099609375,
"KL/rejected_KL_mean": -397.08245849609375,
"KL/std": 135.70486450195312,
"epoch": 0.5286343612334802,
"fcm_dpo/beta": 0.005076201166957617,
"fcm_dpo/delta": 0.014695055782794952,
"fcm_dpo/margin": 154.85951232910156,
"fcm_dpo/q_t": 0.3311746418476105,
"grad_norm": 39.480186462402344,
"learning_rate": 2.7051020734928443e-07,
"logits/chosen": 0.1422397643327713,
"logits/rejected": 0.15580809116363525,
"logps/chosen": -293.6833190917969,
"logps/ref_chosen": -51.460384368896484,
"logps/ref_rejected": -69.83892059326172,
"logps/rejected": -466.92138671875,
"loss": 0.85,
"margin_dpo/margin_mean": 154.85951232910156,
"margin_dpo/margin_std": 136.16104125976562,
"step": 360
},
{
"KL/chosen_KL_mean": -244.15972900390625,
"KL/mean": -311.66357421875,
"KL/rejected_KL_mean": -379.16741943359375,
"KL/std": 147.64564514160156,
"epoch": 0.5301027900146843,
"fcm_dpo/beta": 0.005172476172447205,
"fcm_dpo/delta": 0.10789966583251953,
"fcm_dpo/margin": 135.00767517089844,
"fcm_dpo/q_t": 0.35620319843292236,
"grad_norm": 40.17778015136719,
"learning_rate": 2.6923093854861593e-07,
"logits/chosen": 0.06714671850204468,
"logits/rejected": 0.0685434490442276,
"logps/chosen": -298.02923583984375,
"logps/ref_chosen": -53.86951446533203,
"logps/ref_rejected": -90.7692642211914,
"logps/rejected": -469.9366760253906,
"loss": 0.9569,
"margin_dpo/margin_mean": 135.0076904296875,
"margin_dpo/margin_std": 163.23028564453125,
"step": 361
},
{
"KL/chosen_KL_mean": -209.6553955078125,
"KL/mean": -306.86016845703125,
"KL/rejected_KL_mean": -404.0649108886719,
"KL/std": 155.64691162109375,
"epoch": 0.5315712187958884,
"fcm_dpo/beta": 0.0050034294836223125,
"fcm_dpo/delta": -0.19257816672325134,
"fcm_dpo/margin": 194.40948486328125,
"fcm_dpo/q_t": 0.2972559332847595,
"grad_norm": 34.24903869628906,
"learning_rate": 2.679511629948319e-07,
"logits/chosen": 0.06750906258821487,
"logits/rejected": 0.06738758087158203,
"logps/chosen": -268.2944641113281,
"logps/ref_chosen": -58.639060974121094,
"logps/ref_rejected": -105.58195495605469,
"logps/rejected": -509.6468505859375,
"loss": 0.7571,
"margin_dpo/margin_mean": 194.40948486328125,
"margin_dpo/margin_std": 160.616455078125,
"step": 362
},
{
"KL/chosen_KL_mean": -185.4617919921875,
"KL/mean": -288.15216064453125,
"KL/rejected_KL_mean": -390.8425598144531,
"KL/std": 156.06210327148438,
"epoch": 0.5330396475770925,
"fcm_dpo/beta": 0.004827477503567934,
"fcm_dpo/delta": -0.21232551336288452,
"fcm_dpo/margin": 205.38076782226562,
"fcm_dpo/q_t": 0.2914503514766693,
"grad_norm": 42.787349700927734,
"learning_rate": 2.6667091441120816e-07,
"logits/chosen": 0.10203859210014343,
"logits/rejected": 0.12859748303890228,
"logps/chosen": -230.02017211914062,
"logps/ref_chosen": -44.558380126953125,
"logps/ref_rejected": -74.69496154785156,
"logps/rejected": -465.53753662109375,
"loss": 0.7421,
"margin_dpo/margin_mean": 205.38076782226562,
"margin_dpo/margin_std": 159.50564575195312,
"step": 363
},
{
"KL/chosen_KL_mean": -205.2596435546875,
"KL/mean": -279.29486083984375,
"KL/rejected_KL_mean": -353.3301086425781,
"KL/std": 146.0043182373047,
"epoch": 0.5345080763582967,
"fcm_dpo/beta": 0.004854832775890827,
"fcm_dpo/delta": 0.08639908581972122,
"fcm_dpo/margin": 148.07046508789062,
"fcm_dpo/q_t": 0.3479136824607849,
"grad_norm": 30.45250129699707,
"learning_rate": 2.6539022653348575e-07,
"logits/chosen": -0.006259392946958542,
"logits/rejected": -0.008601278066635132,
"logps/chosen": -254.15426635742188,
"logps/ref_chosen": -48.894622802734375,
"logps/ref_rejected": -91.395751953125,
"logps/rejected": -444.725830078125,
"loss": 0.9176,
"margin_dpo/margin_mean": 148.07046508789062,
"margin_dpo/margin_std": 158.57998657226562,
"step": 364
},
{
"KL/chosen_KL_mean": -199.53936767578125,
"KL/mean": -275.67108154296875,
"KL/rejected_KL_mean": -351.80279541015625,
"KL/std": 160.76339721679688,
"epoch": 0.5359765051395007,
"fcm_dpo/beta": 0.004905831068754196,
"fcm_dpo/delta": 0.05698072165250778,
"fcm_dpo/margin": 152.26344299316406,
"fcm_dpo/q_t": 0.346111923456192,
"grad_norm": 37.139583587646484,
"learning_rate": 2.641091331089811e-07,
"logits/chosen": 0.01942162960767746,
"logits/rejected": 0.012668520212173462,
"logps/chosen": -251.0321044921875,
"logps/ref_chosen": -51.49274444580078,
"logps/ref_rejected": -92.70166778564453,
"logps/rejected": -444.50445556640625,
"loss": 0.897,
"margin_dpo/margin_mean": 152.263427734375,
"margin_dpo/margin_std": 162.59872436523438,
"step": 365
},
{
"KL/chosen_KL_mean": -197.44915771484375,
"KL/mean": -276.533203125,
"KL/rejected_KL_mean": -355.6172790527344,
"KL/std": 154.4979248046875,
"epoch": 0.5374449339207048,
"fcm_dpo/beta": 0.004907799884676933,
"fcm_dpo/delta": 0.025566942989826202,
"fcm_dpo/margin": 158.16812133789062,
"fcm_dpo/q_t": 0.3392502963542938,
"grad_norm": 35.77729034423828,
"learning_rate": 2.6282766789569736e-07,
"logits/chosen": 0.06100982427597046,
"logits/rejected": 0.05832307040691376,
"logps/chosen": -242.16970825195312,
"logps/ref_chosen": -44.7205696105957,
"logps/ref_rejected": -83.31040954589844,
"logps/rejected": -438.92767333984375,
"loss": 0.89,
"margin_dpo/margin_mean": 158.16812133789062,
"margin_dpo/margin_std": 166.03973388671875,
"step": 366
},
{
"KL/chosen_KL_mean": -210.3382568359375,
"KL/mean": -274.84576416015625,
"KL/rejected_KL_mean": -339.353271484375,
"KL/std": 141.46987915039062,
"epoch": 0.5389133627019089,
"fcm_dpo/beta": 0.005055001471191645,
"fcm_dpo/delta": 0.15791787207126617,
"fcm_dpo/margin": 129.01502990722656,
"fcm_dpo/q_t": 0.3598521947860718,
"grad_norm": 33.63018798828125,
"learning_rate": 2.615458646614349e-07,
"logits/chosen": 0.01299678348004818,
"logits/rejected": 0.038110073655843735,
"logps/chosen": -268.7436828613281,
"logps/ref_chosen": -58.405418395996094,
"logps/ref_rejected": -76.75132751464844,
"logps/rejected": -416.1046142578125,
"loss": 0.9426,
"margin_dpo/margin_mean": 129.01502990722656,
"margin_dpo/margin_std": 137.5300750732422,
"step": 367
},
{
"KL/chosen_KL_mean": -205.1249237060547,
"KL/mean": -294.59051513671875,
"KL/rejected_KL_mean": -384.05609130859375,
"KL/std": 149.47842407226562,
"epoch": 0.540381791483113,
"fcm_dpo/beta": 0.005028966814279556,
"fcm_dpo/delta": -0.10961154103279114,
"fcm_dpo/margin": 178.93115234375,
"fcm_dpo/q_t": 0.3045145273208618,
"grad_norm": 58.015506744384766,
"learning_rate": 2.6026375718290083e-07,
"logits/chosen": 0.03757400065660477,
"logits/rejected": 0.0448252409696579,
"logps/chosen": -249.5774383544922,
"logps/ref_chosen": -44.452518463134766,
"logps/ref_rejected": -98.55526733398438,
"logps/rejected": -482.6113586425781,
"loss": 0.765,
"margin_dpo/margin_mean": 178.93115234375,
"margin_dpo/margin_std": 128.32945251464844,
"step": 368
},
{
"KL/chosen_KL_mean": -243.73876953125,
"KL/mean": -307.91265869140625,
"KL/rejected_KL_mean": -372.08648681640625,
"KL/std": 149.72775268554688,
"epoch": 0.5418502202643172,
"fcm_dpo/beta": 0.0051218681037425995,
"fcm_dpo/delta": 0.15110597014427185,
"fcm_dpo/margin": 128.3477020263672,
"fcm_dpo/q_t": 0.3617558777332306,
"grad_norm": 54.27680969238281,
"learning_rate": 2.589813792448196e-07,
"logits/chosen": 0.04510130733251572,
"logits/rejected": 0.063881054520607,
"logps/chosen": -315.12030029296875,
"logps/ref_chosen": -71.38150024414062,
"logps/ref_rejected": -91.29582214355469,
"logps/rejected": -463.38232421875,
"loss": 0.9757,
"margin_dpo/margin_mean": 128.3477020263672,
"margin_dpo/margin_std": 156.42678833007812,
"step": 369
},
{
"KL/chosen_KL_mean": -277.29473876953125,
"KL/mean": -331.53424072265625,
"KL/rejected_KL_mean": -385.7738037109375,
"KL/std": 153.50787353515625,
"epoch": 0.5433186490455213,
"fcm_dpo/beta": 0.005329503677785397,
"fcm_dpo/delta": 0.2346639633178711,
"fcm_dpo/margin": 108.47908020019531,
"fcm_dpo/q_t": 0.3781193196773529,
"grad_norm": 43.87196731567383,
"learning_rate": 2.5769876463904263e-07,
"logits/chosen": 0.07844299077987671,
"logits/rejected": 0.08098983019590378,
"logps/chosen": -348.9022216796875,
"logps/ref_chosen": -71.60749816894531,
"logps/ref_rejected": -97.25978088378906,
"logps/rejected": -483.0335693359375,
"loss": 1.0206,
"margin_dpo/margin_mean": 108.47908020019531,
"margin_dpo/margin_std": 144.95086669921875,
"step": 370
},
{
"KL/chosen_KL_mean": -262.21136474609375,
"KL/mean": -335.576171875,
"KL/rejected_KL_mean": -408.94091796875,
"KL/std": 151.41275024414062,
"epoch": 0.5447870778267254,
"fcm_dpo/beta": 0.005401424132287502,
"fcm_dpo/delta": 0.007885420694947243,
"fcm_dpo/margin": 146.72958374023438,
"fcm_dpo/q_t": 0.33563292026519775,
"grad_norm": 42.30845642089844,
"learning_rate": 2.5641594716365744e-07,
"logits/chosen": 0.06122782081365585,
"logits/rejected": 0.07856868207454681,
"logps/chosen": -331.6258544921875,
"logps/ref_chosen": -69.41448974609375,
"logps/ref_rejected": -99.17217254638672,
"logps/rejected": -508.11309814453125,
"loss": 0.8961,
"margin_dpo/margin_mean": 146.72958374023438,
"margin_dpo/margin_std": 158.2304229736328,
"step": 371
},
{
"KL/chosen_KL_mean": -268.47515869140625,
"KL/mean": -363.11712646484375,
"KL/rejected_KL_mean": -457.75909423828125,
"KL/std": 186.968017578125,
"epoch": 0.5462555066079295,
"fcm_dpo/beta": 0.005196971818804741,
"fcm_dpo/delta": -0.20561370253562927,
"fcm_dpo/margin": 189.283935546875,
"fcm_dpo/q_t": 0.307757705450058,
"grad_norm": 38.48178482055664,
"learning_rate": 2.551329606220976e-07,
"logits/chosen": 0.14845509827136993,
"logits/rejected": 0.18487697839736938,
"logps/chosen": -330.29315185546875,
"logps/ref_chosen": -61.8179931640625,
"logps/ref_rejected": -78.53948974609375,
"logps/rejected": -536.298583984375,
"loss": 0.8196,
"margin_dpo/margin_mean": 189.283935546875,
"margin_dpo/margin_std": 192.58099365234375,
"step": 372
},
{
"KL/chosen_KL_mean": -282.3094177246094,
"KL/mean": -373.96246337890625,
"KL/rejected_KL_mean": -465.6155090332031,
"KL/std": 172.3424530029297,
"epoch": 0.5477239353891337,
"fcm_dpo/beta": 0.005096299573779106,
"fcm_dpo/delta": -0.1480431854724884,
"fcm_dpo/margin": 183.30609130859375,
"fcm_dpo/q_t": 0.30489110946655273,
"grad_norm": 46.1933479309082,
"learning_rate": 2.538498388222517e-07,
"logits/chosen": 0.16775904595851898,
"logits/rejected": 0.2070429027080536,
"logps/chosen": -346.52655029296875,
"logps/ref_chosen": -64.21713256835938,
"logps/ref_rejected": -85.95960998535156,
"logps/rejected": -551.5751342773438,
"loss": 0.7973,
"margin_dpo/margin_mean": 183.30609130859375,
"margin_dpo/margin_std": 159.746826171875,
"step": 373
},
{
"KL/chosen_KL_mean": -259.88995361328125,
"KL/mean": -342.8182373046875,
"KL/rejected_KL_mean": -425.74652099609375,
"KL/std": 191.01841735839844,
"epoch": 0.5491923641703378,
"fcm_dpo/beta": 0.00495956651866436,
"fcm_dpo/delta": -0.02621932327747345,
"fcm_dpo/margin": 165.8565673828125,
"fcm_dpo/q_t": 0.33612874150276184,
"grad_norm": 43.68080139160156,
"learning_rate": 2.525666155755725e-07,
"logits/chosen": 0.014575677923858166,
"logits/rejected": 0.04540920630097389,
"logps/chosen": -330.5401306152344,
"logps/ref_chosen": -70.65018463134766,
"logps/ref_rejected": -93.64016723632812,
"logps/rejected": -519.38671875,
"loss": 0.8808,
"margin_dpo/margin_mean": 165.85658264160156,
"margin_dpo/margin_std": 178.73219299316406,
"step": 374
},
{
"KL/chosen_KL_mean": -302.646484375,
"KL/mean": -383.7525634765625,
"KL/rejected_KL_mean": -464.8586730957031,
"KL/std": 173.11106872558594,
"epoch": 0.5506607929515418,
"fcm_dpo/beta": 0.004924945533275604,
"fcm_dpo/delta": -0.0026347339153289795,
"fcm_dpo/margin": 162.21218872070312,
"fcm_dpo/q_t": 0.33666491508483887,
"grad_norm": 42.200557708740234,
"learning_rate": 2.512833246961859e-07,
"logits/chosen": 0.13259188830852509,
"logits/rejected": 0.14011941850185394,
"logps/chosen": -362.7267150878906,
"logps/ref_chosen": -60.080223083496094,
"logps/ref_rejected": -88.93830871582031,
"logps/rejected": -553.7969970703125,
"loss": 0.9159,
"margin_dpo/margin_mean": 162.21218872070312,
"margin_dpo/margin_std": 178.72955322265625,
"step": 375
},
{
"KL/chosen_KL_mean": -284.9696350097656,
"KL/mean": -380.9700927734375,
"KL/rejected_KL_mean": -476.97052001953125,
"KL/std": 175.26528930664062,
"epoch": 0.5521292217327459,
"fcm_dpo/beta": 0.0048746224492788315,
"fcm_dpo/delta": -0.15022152662277222,
"fcm_dpo/margin": 192.00088500976562,
"fcm_dpo/q_t": 0.31024953722953796,
"grad_norm": 39.80452346801758,
"learning_rate": 2.5e-07,
"logits/chosen": 0.10005451738834381,
"logits/rejected": 0.11893795430660248,
"logps/chosen": -347.62994384765625,
"logps/ref_chosen": -62.660308837890625,
"logps/ref_rejected": -105.52660369873047,
"logps/rejected": -582.4970703125,
"loss": 0.8218,
"margin_dpo/margin_mean": 192.00088500976562,
"margin_dpo/margin_std": 187.91757202148438,
"step": 376
},
{
"KL/chosen_KL_mean": -312.7281188964844,
"KL/mean": -402.2392883300781,
"KL/rejected_KL_mean": -491.75048828125,
"KL/std": 178.24826049804688,
"epoch": 0.55359765051395,
"fcm_dpo/beta": 0.004800399765372276,
"fcm_dpo/delta": -0.06492528319358826,
"fcm_dpo/margin": 179.02236938476562,
"fcm_dpo/q_t": 0.32147014141082764,
"grad_norm": 41.688804626464844,
"learning_rate": 2.487166753038141e-07,
"logits/chosen": 0.17259091138839722,
"logits/rejected": 0.1837162971496582,
"logps/chosen": -367.20684814453125,
"logps/ref_chosen": -54.478736877441406,
"logps/ref_rejected": -98.70335388183594,
"logps/rejected": -590.453857421875,
"loss": 0.8519,
"margin_dpo/margin_mean": 179.02236938476562,
"margin_dpo/margin_std": 176.6744384765625,
"step": 377
},
{
"KL/chosen_KL_mean": -329.71490478515625,
"KL/mean": -428.11505126953125,
"KL/rejected_KL_mean": -526.5152587890625,
"KL/std": 173.01528930664062,
"epoch": 0.5550660792951542,
"fcm_dpo/beta": 0.0046610780991613865,
"fcm_dpo/delta": -0.1307007521390915,
"fcm_dpo/margin": 196.8003692626953,
"fcm_dpo/q_t": 0.30623000860214233,
"grad_norm": 49.080684661865234,
"learning_rate": 2.4743338442442754e-07,
"logits/chosen": 0.33253681659698486,
"logits/rejected": 0.3434640169143677,
"logps/chosen": -374.73541259765625,
"logps/ref_chosen": -45.02053451538086,
"logps/ref_rejected": -88.0469741821289,
"logps/rejected": -614.562255859375,
"loss": 0.7959,
"margin_dpo/margin_mean": 196.8003692626953,
"margin_dpo/margin_std": 168.24053955078125,
"step": 378
},
{
"KL/chosen_KL_mean": -347.2498779296875,
"KL/mean": -441.2623291015625,
"KL/rejected_KL_mean": -535.2748413085938,
"KL/std": 171.8399658203125,
"epoch": 0.5565345080763583,
"fcm_dpo/beta": 0.004580418113619089,
"fcm_dpo/delta": -0.0683475211262703,
"fcm_dpo/margin": 188.02496337890625,
"fcm_dpo/q_t": 0.3261038661003113,
"grad_norm": 56.207244873046875,
"learning_rate": 2.461501611777483e-07,
"logits/chosen": 0.23632130026817322,
"logits/rejected": 0.22753140330314636,
"logps/chosen": -400.4319763183594,
"logps/ref_chosen": -53.182098388671875,
"logps/ref_rejected": -114.3001708984375,
"logps/rejected": -649.574951171875,
"loss": 0.8721,
"margin_dpo/margin_mean": 188.0249481201172,
"margin_dpo/margin_std": 201.0380859375,
"step": 379
},
{
"KL/chosen_KL_mean": -346.2129211425781,
"KL/mean": -453.26580810546875,
"KL/rejected_KL_mean": -560.3186645507812,
"KL/std": 215.87939453125,
"epoch": 0.5580029368575624,
"fcm_dpo/beta": 0.004501840099692345,
"fcm_dpo/delta": -0.18104371428489685,
"fcm_dpo/margin": 214.10574340820312,
"fcm_dpo/q_t": 0.3031790256500244,
"grad_norm": 38.03917694091797,
"learning_rate": 2.4486703937790243e-07,
"logits/chosen": 0.21140098571777344,
"logits/rejected": 0.19853919744491577,
"logps/chosen": -397.5659484863281,
"logps/ref_chosen": -51.3530387878418,
"logps/ref_rejected": -104.19169616699219,
"logps/rejected": -664.5103759765625,
"loss": 0.8193,
"margin_dpo/margin_mean": 214.10574340820312,
"margin_dpo/margin_std": 205.07131958007812,
"step": 380
},
{
"KL/chosen_KL_mean": -359.42694091796875,
"KL/mean": -447.6293029785156,
"KL/rejected_KL_mean": -535.8316650390625,
"KL/std": 172.031494140625,
"epoch": 0.5594713656387665,
"fcm_dpo/beta": 0.004441162571310997,
"fcm_dpo/delta": 0.01796901971101761,
"fcm_dpo/margin": 176.40469360351562,
"fcm_dpo/q_t": 0.34440547227859497,
"grad_norm": 51.2555046081543,
"learning_rate": 2.435840528363426e-07,
"logits/chosen": 0.264553427696228,
"logits/rejected": 0.2853265404701233,
"logps/chosen": -417.2300109863281,
"logps/ref_chosen": -57.80306625366211,
"logps/ref_rejected": -79.21940612792969,
"logps/rejected": -615.051025390625,
"loss": 0.9444,
"margin_dpo/margin_mean": 176.40469360351562,
"margin_dpo/margin_std": 220.600341796875,
"step": 381
},
{
"KL/chosen_KL_mean": -359.14471435546875,
"KL/mean": -449.2369689941406,
"KL/rejected_KL_mean": -539.3292236328125,
"KL/std": 180.70993041992188,
"epoch": 0.5609397944199707,
"fcm_dpo/beta": 0.004444487858563662,
"fcm_dpo/delta": -0.0009026522748172283,
"fcm_dpo/margin": 180.18447875976562,
"fcm_dpo/q_t": 0.3322370648384094,
"grad_norm": 40.537044525146484,
"learning_rate": 2.4230123536095745e-07,
"logits/chosen": 0.19075685739517212,
"logits/rejected": 0.19895179569721222,
"logps/chosen": -425.1650390625,
"logps/ref_chosen": -66.02030181884766,
"logps/ref_rejected": -110.71016693115234,
"logps/rejected": -650.0394287109375,
"loss": 0.868,
"margin_dpo/margin_mean": 180.18447875976562,
"margin_dpo/margin_std": 177.90121459960938,
"step": 382
},
{
"KL/chosen_KL_mean": -351.47509765625,
"KL/mean": -444.9332275390625,
"KL/rejected_KL_mean": -538.391357421875,
"KL/std": 187.61868286132812,
"epoch": 0.5624082232011748,
"fcm_dpo/beta": 0.004414629191160202,
"fcm_dpo/delta": -0.027692466974258423,
"fcm_dpo/margin": 186.916259765625,
"fcm_dpo/q_t": 0.3323889970779419,
"grad_norm": 50.61336898803711,
"learning_rate": 2.4101862075518037e-07,
"logits/chosen": 0.2367815375328064,
"logits/rejected": 0.23397362232208252,
"logps/chosen": -401.8665771484375,
"logps/ref_chosen": -50.39148712158203,
"logps/ref_rejected": -93.71589660644531,
"logps/rejected": -632.1072998046875,
"loss": 0.9123,
"margin_dpo/margin_mean": 186.916259765625,
"margin_dpo/margin_std": 220.93585205078125,
"step": 383
},
{
"KL/chosen_KL_mean": -350.86578369140625,
"KL/mean": -428.3222351074219,
"KL/rejected_KL_mean": -505.7787170410156,
"KL/std": 191.98023986816406,
"epoch": 0.5638766519823789,
"fcm_dpo/beta": 0.004525776021182537,
"fcm_dpo/delta": 0.10422958433628082,
"fcm_dpo/margin": 154.9129638671875,
"fcm_dpo/q_t": 0.3500513434410095,
"grad_norm": 42.488250732421875,
"learning_rate": 2.397362428170992e-07,
"logits/chosen": 0.13534685969352722,
"logits/rejected": 0.14616771042346954,
"logps/chosen": -402.911865234375,
"logps/ref_chosen": -52.046104431152344,
"logps/ref_rejected": -85.76089477539062,
"logps/rejected": -591.5396118164062,
"loss": 0.9165,
"margin_dpo/margin_mean": 154.91297912597656,
"margin_dpo/margin_std": 157.85365295410156,
"step": 384
},
{
"KL/chosen_KL_mean": -349.88470458984375,
"KL/mean": -442.55535888671875,
"KL/rejected_KL_mean": -535.2260131835938,
"KL/std": 163.54031372070312,
"epoch": 0.5653450807635829,
"fcm_dpo/beta": 0.004492057487368584,
"fcm_dpo/delta": -0.035512764006853104,
"fcm_dpo/margin": 185.34132385253906,
"fcm_dpo/q_t": 0.32157087326049805,
"grad_norm": 45.575958251953125,
"learning_rate": 2.3845413533856514e-07,
"logits/chosen": 0.24671843647956848,
"logits/rejected": 0.2860070466995239,
"logps/chosen": -415.4368591308594,
"logps/ref_chosen": -65.55215454101562,
"logps/ref_rejected": -77.82792663574219,
"logps/rejected": -613.053955078125,
"loss": 0.8289,
"margin_dpo/margin_mean": 185.34132385253906,
"margin_dpo/margin_std": 160.22723388671875,
"step": 385
},
{
"KL/chosen_KL_mean": -359.40350341796875,
"KL/mean": -456.8614501953125,
"KL/rejected_KL_mean": -554.3193359375,
"KL/std": 196.9163360595703,
"epoch": 0.566813509544787,
"fcm_dpo/beta": 0.004448466002941132,
"fcm_dpo/delta": -0.07342489063739777,
"fcm_dpo/margin": 194.91583251953125,
"fcm_dpo/q_t": 0.324515163898468,
"grad_norm": 42.45697784423828,
"learning_rate": 2.3717233210430254e-07,
"logits/chosen": 0.16371941566467285,
"logits/rejected": 0.1747831106185913,
"logps/chosen": -417.6253662109375,
"logps/ref_chosen": -58.22185516357422,
"logps/ref_rejected": -92.32742309570312,
"logps/rejected": -646.646728515625,
"loss": 0.8613,
"margin_dpo/margin_mean": 194.91583251953125,
"margin_dpo/margin_std": 205.5290069580078,
"step": 386
},
{
"KL/chosen_KL_mean": -372.31109619140625,
"KL/mean": -450.6243591308594,
"KL/rejected_KL_mean": -528.9375610351562,
"KL/std": 182.24484252929688,
"epoch": 0.5682819383259912,
"fcm_dpo/beta": 0.004475068300962448,
"fcm_dpo/delta": 0.1066901832818985,
"fcm_dpo/margin": 156.62646484375,
"fcm_dpo/q_t": 0.35136184096336365,
"grad_norm": 49.93186569213867,
"learning_rate": 2.3589086689101889e-07,
"logits/chosen": 0.1222163736820221,
"logits/rejected": 0.1534290611743927,
"logps/chosen": -438.73052978515625,
"logps/ref_chosen": -66.41944885253906,
"logps/ref_rejected": -92.16915893554688,
"logps/rejected": -621.1067504882812,
"loss": 0.9202,
"margin_dpo/margin_mean": 156.62648010253906,
"margin_dpo/margin_std": 165.41067504882812,
"step": 387
},
{
"KL/chosen_KL_mean": -345.98480224609375,
"KL/mean": -450.4599914550781,
"KL/rejected_KL_mean": -554.9351806640625,
"KL/std": 201.11993408203125,
"epoch": 0.5697503671071953,
"fcm_dpo/beta": 0.00440019927918911,
"fcm_dpo/delta": -0.13290224969387054,
"fcm_dpo/margin": 208.95037841796875,
"fcm_dpo/q_t": 0.31572598218917847,
"grad_norm": 45.941741943359375,
"learning_rate": 2.3460977346651428e-07,
"logits/chosen": 0.2326255887746811,
"logits/rejected": 0.22134818136692047,
"logps/chosen": -396.1142578125,
"logps/ref_chosen": -50.129459381103516,
"logps/ref_rejected": -104.43305969238281,
"logps/rejected": -659.3682250976562,
"loss": 0.8373,
"margin_dpo/margin_mean": 208.95037841796875,
"margin_dpo/margin_std": 217.96234130859375,
"step": 388
},
{
"KL/chosen_KL_mean": -377.5467834472656,
"KL/mean": -470.57232666015625,
"KL/rejected_KL_mean": -563.5978393554688,
"KL/std": 187.90357971191406,
"epoch": 0.5712187958883994,
"fcm_dpo/beta": 0.004369369708001614,
"fcm_dpo/delta": -0.014375818893313408,
"fcm_dpo/margin": 186.05105590820312,
"fcm_dpo/q_t": 0.33482223749160767,
"grad_norm": 44.91924285888672,
"learning_rate": 2.3332908558879177e-07,
"logits/chosen": 0.22911961376667023,
"logits/rejected": 0.2511199712753296,
"logps/chosen": -435.453369140625,
"logps/ref_chosen": -57.906593322753906,
"logps/ref_rejected": -77.91454315185547,
"logps/rejected": -641.5123901367188,
"loss": 0.8948,
"margin_dpo/margin_mean": 186.05105590820312,
"margin_dpo/margin_std": 203.96209716796875,
"step": 389
},
{
"KL/chosen_KL_mean": -358.6050720214844,
"KL/mean": -444.9243469238281,
"KL/rejected_KL_mean": -531.2435913085938,
"KL/std": 184.04891967773438,
"epoch": 0.5726872246696035,
"fcm_dpo/beta": 0.00439393799751997,
"fcm_dpo/delta": 0.044710010290145874,
"fcm_dpo/margin": 172.63853454589844,
"fcm_dpo/q_t": 0.3482525050640106,
"grad_norm": 39.236610412597656,
"learning_rate": 2.320488370051681e-07,
"logits/chosen": 0.1646582931280136,
"logits/rejected": 0.18338599801063538,
"logps/chosen": -407.83099365234375,
"logps/ref_chosen": -49.22591781616211,
"logps/ref_rejected": -85.5281982421875,
"logps/rejected": -616.7717895507812,
"loss": 0.9706,
"margin_dpo/margin_mean": 172.63853454589844,
"margin_dpo/margin_std": 231.70809936523438,
"step": 390
},
{
"KL/chosen_KL_mean": -329.9169006347656,
"KL/mean": -392.60296630859375,
"KL/rejected_KL_mean": -455.28900146484375,
"KL/std": 190.03704833984375,
"epoch": 0.5741556534508077,
"fcm_dpo/beta": 0.004583893343806267,
"fcm_dpo/delta": 0.2388683259487152,
"fcm_dpo/margin": 125.37210845947266,
"fcm_dpo/q_t": 0.3850371241569519,
"grad_norm": 55.410194396972656,
"learning_rate": 2.3076906145138405e-07,
"logits/chosen": 0.14501428604125977,
"logits/rejected": 0.15393584966659546,
"logps/chosen": -394.2465515136719,
"logps/ref_chosen": -64.32965087890625,
"logps/ref_rejected": -86.73820495605469,
"logps/rejected": -542.0272216796875,
"loss": 1.0517,
"margin_dpo/margin_mean": 125.37210845947266,
"margin_dpo/margin_std": 190.40362548828125,
"step": 391
},
{
"KL/chosen_KL_mean": -277.9818115234375,
"KL/mean": -376.48553466796875,
"KL/rejected_KL_mean": -474.9892883300781,
"KL/std": 181.52938842773438,
"epoch": 0.5756240822320118,
"fcm_dpo/beta": 0.004586499184370041,
"fcm_dpo/delta": -0.1139056608080864,
"fcm_dpo/margin": 197.00746154785156,
"fcm_dpo/q_t": 0.311518132686615,
"grad_norm": 51.724281311035156,
"learning_rate": 2.294897926507156e-07,
"logits/chosen": 0.06240957975387573,
"logits/rejected": 0.07568895816802979,
"logps/chosen": -331.48577880859375,
"logps/ref_chosen": -53.50397872924805,
"logps/ref_rejected": -102.34584045410156,
"logps/rejected": -577.3350830078125,
"loss": 0.8014,
"margin_dpo/margin_mean": 197.00747680664062,
"margin_dpo/margin_std": 170.6578369140625,
"step": 392
},
{
"KL/chosen_KL_mean": -279.2279052734375,
"KL/mean": -355.4307861328125,
"KL/rejected_KL_mean": -431.6336669921875,
"KL/std": 180.89584350585938,
"epoch": 0.5770925110132159,
"fcm_dpo/beta": 0.004597535356879234,
"fcm_dpo/delta": 0.10691172629594803,
"fcm_dpo/margin": 152.40574645996094,
"fcm_dpo/q_t": 0.36370301246643066,
"grad_norm": 33.592952728271484,
"learning_rate": 2.2821106431308543e-07,
"logits/chosen": 0.13287240266799927,
"logits/rejected": 0.14408522844314575,
"logps/chosen": -325.7018127441406,
"logps/ref_chosen": -46.473915100097656,
"logps/ref_rejected": -71.96885681152344,
"logps/rejected": -503.6025390625,
"loss": 0.9837,
"margin_dpo/margin_mean": 152.40574645996094,
"margin_dpo/margin_std": 210.06790161132812,
"step": 393
},
{
"KL/chosen_KL_mean": -281.6102294921875,
"KL/mean": -368.0687255859375,
"KL/rejected_KL_mean": -454.5271911621094,
"KL/std": 181.987060546875,
"epoch": 0.57856093979442,
"fcm_dpo/beta": 0.004621568135917187,
"fcm_dpo/delta": 0.0008383970707654953,
"fcm_dpo/margin": 172.91696166992188,
"fcm_dpo/q_t": 0.3378085494041443,
"grad_norm": 48.079124450683594,
"learning_rate": 2.2693291013417452e-07,
"logits/chosen": 0.05104764923453331,
"logits/rejected": 0.05909823626279831,
"logps/chosen": -334.52178955078125,
"logps/ref_chosen": -52.91154861450195,
"logps/ref_rejected": -90.8226318359375,
"logps/rejected": -545.349853515625,
"loss": 0.8954,
"margin_dpo/margin_mean": 172.91696166992188,
"margin_dpo/margin_std": 190.8291778564453,
"step": 394
},
{
"KL/chosen_KL_mean": -271.431640625,
"KL/mean": -362.2996520996094,
"KL/rejected_KL_mean": -453.16766357421875,
"KL/std": 188.79595947265625,
"epoch": 0.580029368575624,
"fcm_dpo/beta": 0.004584716632962227,
"fcm_dpo/delta": -0.03685643523931503,
"fcm_dpo/margin": 181.7360076904297,
"fcm_dpo/q_t": 0.3331453800201416,
"grad_norm": 40.45199966430664,
"learning_rate": 2.2565536379453404e-07,
"logits/chosen": 0.03748972713947296,
"logits/rejected": 0.048588261008262634,
"logps/chosen": -333.977783203125,
"logps/ref_chosen": -62.546112060546875,
"logps/ref_rejected": -83.78262329101562,
"logps/rejected": -536.9503173828125,
"loss": 0.8995,
"margin_dpo/margin_mean": 181.73599243164062,
"margin_dpo/margin_std": 206.99716186523438,
"step": 395
},
{
"KL/chosen_KL_mean": -254.0207977294922,
"KL/mean": -335.33380126953125,
"KL/rejected_KL_mean": -416.64678955078125,
"KL/std": 170.42239379882812,
"epoch": 0.5814977973568282,
"fcm_dpo/beta": 0.004617318511009216,
"fcm_dpo/delta": 0.05301779881119728,
"fcm_dpo/margin": 162.62600708007812,
"fcm_dpo/q_t": 0.3398272395133972,
"grad_norm": 42.922828674316406,
"learning_rate": 2.2437845895869825e-07,
"logits/chosen": -0.03782034292817116,
"logits/rejected": -0.010618546046316624,
"logps/chosen": -323.0167236328125,
"logps/ref_chosen": -68.99594116210938,
"logps/ref_rejected": -88.64665985107422,
"logps/rejected": -505.29345703125,
"loss": 0.8781,
"margin_dpo/margin_mean": 162.62600708007812,
"margin_dpo/margin_std": 156.22705078125,
"step": 396
},
{
"KL/chosen_KL_mean": -230.32138061523438,
"KL/mean": -323.5391845703125,
"KL/rejected_KL_mean": -416.7569580078125,
"KL/std": 165.25631713867188,
"epoch": 0.5829662261380323,
"fcm_dpo/beta": 0.004583822563290596,
"fcm_dpo/delta": -0.060415178537368774,
"fcm_dpo/margin": 186.43557739257812,
"fcm_dpo/q_t": 0.31888556480407715,
"grad_norm": 55.33768081665039,
"learning_rate": 2.2310222927429716e-07,
"logits/chosen": 0.06463508307933807,
"logits/rejected": 0.08042390644550323,
"logps/chosen": -291.59857177734375,
"logps/ref_chosen": -61.27716827392578,
"logps/ref_rejected": -103.11612701416016,
"logps/rejected": -519.873046875,
"loss": 0.8176,
"margin_dpo/margin_mean": 186.43557739257812,
"margin_dpo/margin_std": 161.7078857421875,
"step": 397
},
{
"KL/chosen_KL_mean": -252.4274139404297,
"KL/mean": -340.54632568359375,
"KL/rejected_KL_mean": -428.6652526855469,
"KL/std": 179.56756591796875,
"epoch": 0.5844346549192364,
"fcm_dpo/beta": 0.004576831124722958,
"fcm_dpo/delta": -0.0072813937440514565,
"fcm_dpo/margin": 176.23779296875,
"fcm_dpo/q_t": 0.33511707186698914,
"grad_norm": 33.04292297363281,
"learning_rate": 2.2182670837116972e-07,
"logits/chosen": -0.08657974749803543,
"logits/rejected": -0.07474084943532944,
"logps/chosen": -320.5789794921875,
"logps/ref_chosen": -68.15155029296875,
"logps/ref_rejected": -108.52360534667969,
"logps/rejected": -537.1888427734375,
"loss": 0.884,
"margin_dpo/margin_mean": 176.23780822753906,
"margin_dpo/margin_std": 189.9404296875,
"step": 398
},
{
"KL/chosen_KL_mean": -215.05654907226562,
"KL/mean": -306.60455322265625,
"KL/rejected_KL_mean": -398.15252685546875,
"KL/std": 165.11349487304688,
"epoch": 0.5859030837004405,
"fcm_dpo/beta": 0.004536399617791176,
"fcm_dpo/delta": -0.03415355086326599,
"fcm_dpo/margin": 183.09597778320312,
"fcm_dpo/q_t": 0.3255794048309326,
"grad_norm": 30.613500595092773,
"learning_rate": 2.2055192986047804e-07,
"logits/chosen": 0.02934134751558304,
"logits/rejected": 0.0851244181394577,
"logps/chosen": -275.94635009765625,
"logps/ref_chosen": -60.889801025390625,
"logps/ref_rejected": -77.965576171875,
"logps/rejected": -476.11810302734375,
"loss": 0.8583,
"margin_dpo/margin_mean": 183.09597778320312,
"margin_dpo/margin_std": 178.49737548828125,
"step": 399
},
{
"KL/chosen_KL_mean": -231.09521484375,
"KL/mean": -350.0788879394531,
"KL/rejected_KL_mean": -469.06256103515625,
"KL/std": 186.72280883789062,
"epoch": 0.5873715124816447,
"fcm_dpo/beta": 0.004352675750851631,
"fcm_dpo/delta": -0.26532918214797974,
"fcm_dpo/margin": 237.9673309326172,
"fcm_dpo/q_t": 0.2851880192756653,
"grad_norm": 44.4686164855957,
"learning_rate": 2.192779273338215e-07,
"logits/chosen": 0.05125574767589569,
"logits/rejected": 0.06881796568632126,
"logps/chosen": -294.73883056640625,
"logps/ref_chosen": -63.64359664916992,
"logps/ref_rejected": -105.252685546875,
"logps/rejected": -574.3152465820312,
"loss": 0.7417,
"margin_dpo/margin_mean": 237.9673614501953,
"margin_dpo/margin_std": 195.74899291992188,
"step": 400
},
{
"KL/chosen_KL_mean": -268.1168212890625,
"KL/mean": -336.1388854980469,
"KL/rejected_KL_mean": -404.1609191894531,
"KL/std": 176.03399658203125,
"epoch": 0.5888399412628488,
"fcm_dpo/beta": 0.004442146047949791,
"fcm_dpo/delta": 0.2086574286222458,
"fcm_dpo/margin": 136.04408264160156,
"fcm_dpo/q_t": 0.3828345537185669,
"grad_norm": 39.05641555786133,
"learning_rate": 2.1800473436235136e-07,
"logits/chosen": 0.07737800478935242,
"logits/rejected": 0.08538491278886795,
"logps/chosen": -325.27984619140625,
"logps/ref_chosen": -57.16303253173828,
"logps/ref_rejected": -83.79249572753906,
"logps/rejected": -487.95343017578125,
"loss": 1.0859,
"margin_dpo/margin_mean": 136.0440673828125,
"margin_dpo/margin_std": 228.36392211914062,
"step": 401
},
{
"KL/chosen_KL_mean": -213.17237854003906,
"KL/mean": -337.1200256347656,
"KL/rejected_KL_mean": -461.06768798828125,
"KL/std": 202.32061767578125,
"epoch": 0.5903083700440529,
"fcm_dpo/beta": 0.004299672320485115,
"fcm_dpo/delta": -0.29827845096588135,
"fcm_dpo/margin": 247.89532470703125,
"fcm_dpo/q_t": 0.2777937650680542,
"grad_norm": 44.950050354003906,
"learning_rate": 2.1673238449588665e-07,
"logits/chosen": 0.08306563645601273,
"logits/rejected": 0.10771772265434265,
"logps/chosen": -263.9127502441406,
"logps/ref_chosen": -50.74037170410156,
"logps/ref_rejected": -81.0460433959961,
"logps/rejected": -542.11376953125,
"loss": 0.7104,
"margin_dpo/margin_mean": 247.89532470703125,
"margin_dpo/margin_std": 189.4843292236328,
"step": 402
},
{
"KL/chosen_KL_mean": -266.988037109375,
"KL/mean": -361.72894287109375,
"KL/rejected_KL_mean": -456.4698181152344,
"KL/std": 186.5218505859375,
"epoch": 0.591776798825257,
"fcm_dpo/beta": 0.00422732625156641,
"fcm_dpo/delta": -0.0012762416154146194,
"fcm_dpo/margin": 189.48178100585938,
"fcm_dpo/q_t": 0.33159884810447693,
"grad_norm": 44.431949615478516,
"learning_rate": 2.154609112620295e-07,
"logits/chosen": 0.18813556432724,
"logits/rejected": 0.19890309870243073,
"logps/chosen": -314.1353454589844,
"logps/ref_chosen": -47.14731216430664,
"logps/ref_rejected": -77.2666015625,
"logps/rejected": -533.7364501953125,
"loss": 0.8712,
"margin_dpo/margin_mean": 189.4817657470703,
"margin_dpo/margin_std": 184.798583984375,
"step": 403
},
{
"KL/chosen_KL_mean": -298.219482421875,
"KL/mean": -398.1234130859375,
"KL/rejected_KL_mean": -498.02740478515625,
"KL/std": 176.61068725585938,
"epoch": 0.593245227606461,
"fcm_dpo/beta": 0.004205549135804176,
"fcm_dpo/delta": -0.044117238372564316,
"fcm_dpo/margin": 199.80795288085938,
"fcm_dpo/q_t": 0.3256443738937378,
"grad_norm": 42.222572326660156,
"learning_rate": 2.1419034816528218e-07,
"logits/chosen": 0.2749345898628235,
"logits/rejected": 0.29835087060928345,
"logps/chosen": -346.0947265625,
"logps/ref_chosen": -47.875274658203125,
"logps/ref_rejected": -77.15499877929688,
"logps/rejected": -575.182373046875,
"loss": 0.8728,
"margin_dpo/margin_mean": 199.80795288085938,
"margin_dpo/margin_std": 206.64486694335938,
"step": 404
},
{
"KL/chosen_KL_mean": -318.4598083496094,
"KL/mean": -398.934326171875,
"KL/rejected_KL_mean": -479.40887451171875,
"KL/std": 186.36871337890625,
"epoch": 0.5947136563876652,
"fcm_dpo/beta": 0.004319686442613602,
"fcm_dpo/delta": 0.10862886905670166,
"fcm_dpo/margin": 160.9490203857422,
"fcm_dpo/q_t": 0.3593287467956543,
"grad_norm": 67.92670440673828,
"learning_rate": 2.129207286861638e-07,
"logits/chosen": 0.15380798280239105,
"logits/rejected": 0.1734359711408615,
"logps/chosen": -383.62274169921875,
"logps/ref_chosen": -65.16290283203125,
"logps/ref_rejected": -87.18678283691406,
"logps/rejected": -566.5956420898438,
"loss": 0.9771,
"margin_dpo/margin_mean": 160.94903564453125,
"margin_dpo/margin_std": 202.40145874023438,
"step": 405
},
{
"KL/chosen_KL_mean": -313.0897216796875,
"KL/mean": -410.04827880859375,
"KL/rejected_KL_mean": -507.0068359375,
"KL/std": 204.11318969726562,
"epoch": 0.5961820851688693,
"fcm_dpo/beta": 0.004310654476284981,
"fcm_dpo/delta": -0.0407475009560585,
"fcm_dpo/margin": 193.91712951660156,
"fcm_dpo/q_t": 0.33301207423210144,
"grad_norm": 39.647132873535156,
"learning_rate": 2.1165208628032861e-07,
"logits/chosen": 0.2193293273448944,
"logits/rejected": 0.24135665595531464,
"logps/chosen": -362.83050537109375,
"logps/ref_chosen": -49.740814208984375,
"logps/ref_rejected": -92.07862854003906,
"logps/rejected": -599.08544921875,
"loss": 0.8882,
"margin_dpo/margin_mean": 193.91714477539062,
"margin_dpo/margin_std": 212.67950439453125,
"step": 406
},
{
"KL/chosen_KL_mean": -339.6982421875,
"KL/mean": -417.26922607421875,
"KL/rejected_KL_mean": -494.8402099609375,
"KL/std": 171.96035766601562,
"epoch": 0.5976505139500734,
"fcm_dpo/beta": 0.004385569132864475,
"fcm_dpo/delta": 0.12469163537025452,
"fcm_dpo/margin": 155.1419677734375,
"fcm_dpo/q_t": 0.35749387741088867,
"grad_norm": 62.01276779174805,
"learning_rate": 2.1038445437768375e-07,
"logits/chosen": 0.3068956434726715,
"logits/rejected": 0.3434358835220337,
"logps/chosen": -396.0289306640625,
"logps/ref_chosen": -56.33069610595703,
"logps/ref_rejected": -77.51209259033203,
"logps/rejected": -572.352294921875,
"loss": 0.9859,
"margin_dpo/margin_mean": 155.1419677734375,
"margin_dpo/margin_std": 195.72064208984375,
"step": 407
},
{
"KL/chosen_KL_mean": -355.1640625,
"KL/mean": -427.469482421875,
"KL/rejected_KL_mean": -499.77490234375,
"KL/std": 156.67254638671875,
"epoch": 0.5991189427312775,
"fcm_dpo/beta": 0.00446933601051569,
"fcm_dpo/delta": 0.16449061036109924,
"fcm_dpo/margin": 144.61085510253906,
"fcm_dpo/q_t": 0.36376452445983887,
"grad_norm": 48.290897369384766,
"learning_rate": 2.0911786638150872e-07,
"logits/chosen": 0.24663367867469788,
"logits/rejected": 0.28335240483283997,
"logps/chosen": -424.953369140625,
"logps/ref_chosen": -69.789306640625,
"logps/ref_rejected": -90.09693908691406,
"logps/rejected": -589.871826171875,
"loss": 0.9679,
"margin_dpo/margin_mean": 144.61085510253906,
"margin_dpo/margin_std": 171.00955200195312,
"step": 408
},
{
"KL/chosen_KL_mean": -332.8118896484375,
"KL/mean": -415.9732360839844,
"KL/rejected_KL_mean": -499.1345520019531,
"KL/std": 192.5561981201172,
"epoch": 0.6005873715124816,
"fcm_dpo/beta": 0.0045572854578495026,
"fcm_dpo/delta": 0.045145727694034576,
"fcm_dpo/margin": 166.3226776123047,
"fcm_dpo/q_t": 0.34245866537094116,
"grad_norm": 54.05333709716797,
"learning_rate": 2.0785235566757517e-07,
"logits/chosen": 0.17667712271213531,
"logits/rejected": 0.1955818235874176,
"logps/chosen": -400.12933349609375,
"logps/ref_chosen": -67.31744384765625,
"logps/ref_rejected": -84.904296875,
"logps/rejected": -584.038818359375,
"loss": 0.8977,
"margin_dpo/margin_mean": 166.3226776123047,
"margin_dpo/margin_std": 175.45404052734375,
"step": 409
},
{
"KL/chosen_KL_mean": -328.254150390625,
"KL/mean": -412.47406005859375,
"KL/rejected_KL_mean": -496.6939392089844,
"KL/std": 183.95452880859375,
"epoch": 0.6020558002936858,
"fcm_dpo/beta": 0.004573145881295204,
"fcm_dpo/delta": 0.032191164791584015,
"fcm_dpo/margin": 168.43978881835938,
"fcm_dpo/q_t": 0.33852922916412354,
"grad_norm": 47.99551010131836,
"learning_rate": 2.065879555832674e-07,
"logits/chosen": 0.1765487790107727,
"logits/rejected": 0.1874597668647766,
"logps/chosen": -379.7195129394531,
"logps/ref_chosen": -51.465354919433594,
"logps/ref_rejected": -83.198974609375,
"logps/rejected": -579.8929443359375,
"loss": 0.9054,
"margin_dpo/margin_mean": 168.43978881835938,
"margin_dpo/margin_std": 182.5813751220703,
"step": 410
},
{
"KL/chosen_KL_mean": -345.7894287109375,
"KL/mean": -429.1628112792969,
"KL/rejected_KL_mean": -512.5361938476562,
"KL/std": 178.01425170898438,
"epoch": 0.6035242290748899,
"fcm_dpo/beta": 0.004561232402920723,
"fcm_dpo/delta": 0.040590718388557434,
"fcm_dpo/margin": 166.74673461914062,
"fcm_dpo/q_t": 0.3462197184562683,
"grad_norm": 67.33753967285156,
"learning_rate": 2.0532469944670343e-07,
"logits/chosen": 0.30428582429885864,
"logits/rejected": 0.32055914402008057,
"logps/chosen": -398.0967102050781,
"logps/ref_chosen": -52.30727005004883,
"logps/ref_rejected": -80.69495391845703,
"logps/rejected": -593.2311401367188,
"loss": 0.9128,
"margin_dpo/margin_mean": 166.7467498779297,
"margin_dpo/margin_std": 185.79019165039062,
"step": 411
},
{
"KL/chosen_KL_mean": -347.623046875,
"KL/mean": -430.669921875,
"KL/rejected_KL_mean": -513.716796875,
"KL/std": 168.90745544433594,
"epoch": 0.604992657856094,
"fcm_dpo/beta": 0.0046454789116978645,
"fcm_dpo/delta": 0.03075917437672615,
"fcm_dpo/margin": 166.09373474121094,
"fcm_dpo/q_t": 0.34237197041511536,
"grad_norm": 45.43087387084961,
"learning_rate": 2.0406262054585738e-07,
"logits/chosen": 0.2442624866962433,
"logits/rejected": 0.2392767369747162,
"logps/chosen": -400.7671813964844,
"logps/ref_chosen": -53.144126892089844,
"logps/ref_rejected": -100.0608139038086,
"logps/rejected": -613.777587890625,
"loss": 0.9166,
"margin_dpo/margin_mean": 166.09373474121094,
"margin_dpo/margin_std": 189.59783935546875,
"step": 412
},
{
"KL/chosen_KL_mean": -368.34149169921875,
"KL/mean": -449.5134582519531,
"KL/rejected_KL_mean": -530.6854248046875,
"KL/std": 179.1047821044922,
"epoch": 0.6064610866372981,
"fcm_dpo/beta": 0.004703960847109556,
"fcm_dpo/delta": 0.03838766738772392,
"fcm_dpo/margin": 162.3439178466797,
"fcm_dpo/q_t": 0.3409743309020996,
"grad_norm": 68.5807113647461,
"learning_rate": 2.0280175213768205e-07,
"logits/chosen": 0.14463230967521667,
"logits/rejected": 0.15965858101844788,
"logps/chosen": -429.9234619140625,
"logps/ref_chosen": -61.58196258544922,
"logps/ref_rejected": -99.47340393066406,
"logps/rejected": -630.1588134765625,
"loss": 0.929,
"margin_dpo/margin_mean": 162.3439178466797,
"margin_dpo/margin_std": 190.46279907226562,
"step": 413
},
{
"KL/chosen_KL_mean": -328.48309326171875,
"KL/mean": -433.293212890625,
"KL/rejected_KL_mean": -538.1033935546875,
"KL/std": 182.9401092529297,
"epoch": 0.6079295154185022,
"fcm_dpo/beta": 0.00460551492869854,
"fcm_dpo/delta": -0.1832694113254547,
"fcm_dpo/margin": 209.62030029296875,
"fcm_dpo/q_t": 0.2981165647506714,
"grad_norm": 48.85835266113281,
"learning_rate": 2.0154212744723247e-07,
"logits/chosen": 0.28938111662864685,
"logits/rejected": 0.30956315994262695,
"logps/chosen": -375.11456298828125,
"logps/ref_chosen": -46.63148498535156,
"logps/ref_rejected": -87.64653015136719,
"logps/rejected": -625.7498779296875,
"loss": 0.781,
"margin_dpo/margin_mean": 209.62030029296875,
"margin_dpo/margin_std": 176.873291015625,
"step": 414
},
{
"KL/chosen_KL_mean": -363.9873046875,
"KL/mean": -444.49053955078125,
"KL/rejected_KL_mean": -524.9938354492188,
"KL/std": 190.98814392089844,
"epoch": 0.6093979441997063,
"fcm_dpo/beta": 0.00454542925581336,
"fcm_dpo/delta": 0.07343503087759018,
"fcm_dpo/margin": 161.0065155029297,
"fcm_dpo/q_t": 0.3501279354095459,
"grad_norm": 53.78864288330078,
"learning_rate": 2.002837796667909e-07,
"logits/chosen": 0.15784013271331787,
"logits/rejected": 0.17198491096496582,
"logps/chosen": -442.6055908203125,
"logps/ref_chosen": -78.6182861328125,
"logps/ref_rejected": -100.47752380371094,
"logps/rejected": -625.4713134765625,
"loss": 0.9377,
"margin_dpo/margin_mean": 161.00653076171875,
"margin_dpo/margin_std": 189.8529052734375,
"step": 415
},
{
"KL/chosen_KL_mean": -329.3747253417969,
"KL/mean": -443.10009765625,
"KL/rejected_KL_mean": -556.8255004882812,
"KL/std": 199.22900390625,
"epoch": 0.6108663729809104,
"fcm_dpo/beta": 0.0044495644979178905,
"fcm_dpo/delta": -0.23528949916362762,
"fcm_dpo/margin": 227.45074462890625,
"fcm_dpo/q_t": 0.28878968954086304,
"grad_norm": 37.943359375,
"learning_rate": 1.990267419549914e-07,
"logits/chosen": 0.18898329138755798,
"logits/rejected": 0.20634178817272186,
"logps/chosen": -387.65386962890625,
"logps/ref_chosen": -58.27912521362305,
"logps/ref_rejected": -90.56871795654297,
"logps/rejected": -647.3942260742188,
"loss": 0.7443,
"margin_dpo/margin_mean": 227.45074462890625,
"margin_dpo/margin_std": 177.98248291015625,
"step": 416
},
{
"KL/chosen_KL_mean": -336.8597412109375,
"KL/mean": -438.7206726074219,
"KL/rejected_KL_mean": -540.5816650390625,
"KL/std": 184.04443359375,
"epoch": 0.6123348017621145,
"fcm_dpo/beta": 0.004322909750044346,
"fcm_dpo/delta": -0.08837594091892242,
"fcm_dpo/margin": 203.72186279296875,
"fcm_dpo/q_t": 0.31164246797561646,
"grad_norm": 35.816017150878906,
"learning_rate": 1.9777104743594686e-07,
"logits/chosen": 0.3776714503765106,
"logits/rejected": 0.4244968295097351,
"logps/chosen": -387.0584716796875,
"logps/ref_chosen": -50.1987190246582,
"logps/ref_rejected": -68.15184020996094,
"logps/rejected": -608.7334594726562,
"loss": 0.7978,
"margin_dpo/margin_mean": 203.72186279296875,
"margin_dpo/margin_std": 167.75634765625,
"step": 417
},
{
"KL/chosen_KL_mean": -322.0848388671875,
"KL/mean": -421.3946228027344,
"KL/rejected_KL_mean": -520.7044067382812,
"KL/std": 208.01580810546875,
"epoch": 0.6138032305433186,
"fcm_dpo/beta": 0.00430900976061821,
"fcm_dpo/delta": -0.06341703236103058,
"fcm_dpo/margin": 198.61956787109375,
"fcm_dpo/q_t": 0.3312340974807739,
"grad_norm": 37.52000427246094,
"learning_rate": 1.965167291983757e-07,
"logits/chosen": 0.09415233135223389,
"logits/rejected": 0.1325725018978119,
"logps/chosen": -404.0633239746094,
"logps/ref_chosen": -81.97846984863281,
"logps/ref_rejected": -104.69148254394531,
"logps/rejected": -625.3958740234375,
"loss": 0.8878,
"margin_dpo/margin_mean": 198.6195526123047,
"margin_dpo/margin_std": 221.0111083984375,
"step": 418
},
{
"KL/chosen_KL_mean": -281.81658935546875,
"KL/mean": -387.3044738769531,
"KL/rejected_KL_mean": -492.7923583984375,
"KL/std": 190.3030548095703,
"epoch": 0.6152716593245228,
"fcm_dpo/beta": 0.004175344482064247,
"fcm_dpo/delta": -0.08911103755235672,
"fcm_dpo/margin": 210.97579956054688,
"fcm_dpo/q_t": 0.3172077536582947,
"grad_norm": 30.190216064453125,
"learning_rate": 1.9526382029472988e-07,
"logits/chosen": 0.056289106607437134,
"logits/rejected": 0.07095717638731003,
"logps/chosen": -334.7652282714844,
"logps/ref_chosen": -52.948646545410156,
"logps/ref_rejected": -91.58309936523438,
"logps/rejected": -584.37548828125,
"loss": 0.8322,
"margin_dpo/margin_mean": 210.97579956054688,
"margin_dpo/margin_std": 200.0166015625,
"step": 419
},
{
"KL/chosen_KL_mean": -297.31671142578125,
"KL/mean": -375.4432678222656,
"KL/rejected_KL_mean": -453.5698547363281,
"KL/std": 171.12130737304688,
"epoch": 0.6167400881057269,
"fcm_dpo/beta": 0.004269109107553959,
"fcm_dpo/delta": 0.14179980754852295,
"fcm_dpo/margin": 156.25314331054688,
"fcm_dpo/q_t": 0.3652932941913605,
"grad_norm": 44.23408126831055,
"learning_rate": 1.9401235374032425e-07,
"logits/chosen": 0.11774572730064392,
"logits/rejected": 0.1661440134048462,
"logps/chosen": -375.08660888671875,
"logps/ref_chosen": -77.7699203491211,
"logps/ref_rejected": -69.31985473632812,
"logps/rejected": -522.8897094726562,
"loss": 0.9903,
"margin_dpo/margin_mean": 156.25314331054688,
"margin_dpo/margin_std": 211.12954711914062,
"step": 420
},
{
"KL/chosen_KL_mean": -253.75180053710938,
"KL/mean": -329.0003662109375,
"KL/rejected_KL_mean": -404.2489013671875,
"KL/std": 165.86834716796875,
"epoch": 0.618208516886931,
"fcm_dpo/beta": 0.004399363417178392,
"fcm_dpo/delta": 0.14674726128578186,
"fcm_dpo/margin": 150.49708557128906,
"fcm_dpo/q_t": 0.358406126499176,
"grad_norm": 47.58359146118164,
"learning_rate": 1.9276236251246653e-07,
"logits/chosen": 0.04012005031108856,
"logits/rejected": 0.05817845091223717,
"logps/chosen": -307.51763916015625,
"logps/ref_chosen": -53.765865325927734,
"logps/ref_rejected": -89.28144836425781,
"logps/rejected": -493.5303649902344,
"loss": 0.9651,
"margin_dpo/margin_mean": 150.49708557128906,
"margin_dpo/margin_std": 175.0691680908203,
"step": 421
},
{
"KL/chosen_KL_mean": -280.9953308105469,
"KL/mean": -360.83282470703125,
"KL/rejected_KL_mean": -440.67034912109375,
"KL/std": 170.18374633789062,
"epoch": 0.6196769456681351,
"fcm_dpo/beta": 0.004488244652748108,
"fcm_dpo/delta": 0.08926088362932205,
"fcm_dpo/margin": 159.67501831054688,
"fcm_dpo/q_t": 0.348935067653656,
"grad_norm": 43.716209411621094,
"learning_rate": 1.9151387954958792e-07,
"logits/chosen": 0.12674188613891602,
"logits/rejected": 0.15268999338150024,
"logps/chosen": -349.62908935546875,
"logps/ref_chosen": -68.6337661743164,
"logps/ref_rejected": -87.86351013183594,
"logps/rejected": -528.5338745117188,
"loss": 0.9463,
"margin_dpo/margin_mean": 159.6750030517578,
"margin_dpo/margin_std": 188.54898071289062,
"step": 422
},
{
"KL/chosen_KL_mean": -255.04405212402344,
"KL/mean": -343.0213928222656,
"KL/rejected_KL_mean": -430.998779296875,
"KL/std": 158.25143432617188,
"epoch": 0.6211453744493393,
"fcm_dpo/beta": 0.004496719688177109,
"fcm_dpo/delta": 0.009527413174510002,
"fcm_dpo/margin": 175.95469665527344,
"fcm_dpo/q_t": 0.3338441848754883,
"grad_norm": 42.53166198730469,
"learning_rate": 1.902669377503756e-07,
"logits/chosen": 0.10014916211366653,
"logits/rejected": 0.10393344610929489,
"logps/chosen": -310.03436279296875,
"logps/ref_chosen": -54.99030303955078,
"logps/ref_rejected": -86.30654907226562,
"logps/rejected": -517.3052978515625,
"loss": 0.8742,
"margin_dpo/margin_mean": 175.95469665527344,
"margin_dpo/margin_std": 175.76312255859375,
"step": 423
},
{
"KL/chosen_KL_mean": -227.72738647460938,
"KL/mean": -312.34954833984375,
"KL/rejected_KL_mean": -396.9716796875,
"KL/std": 165.03700256347656,
"epoch": 0.6226138032305433,
"fcm_dpo/beta": 0.004506932571530342,
"fcm_dpo/delta": 0.03997078165411949,
"fcm_dpo/margin": 169.24429321289062,
"fcm_dpo/q_t": 0.34591084718704224,
"grad_norm": 58.1711311340332,
"learning_rate": 1.890215699729057e-07,
"logits/chosen": 0.023241115733981133,
"logits/rejected": 0.05266202241182327,
"logps/chosen": -283.73931884765625,
"logps/ref_chosen": -56.01192092895508,
"logps/ref_rejected": -66.47896575927734,
"logps/rejected": -463.45062255859375,
"loss": 0.9243,
"margin_dpo/margin_mean": 169.24429321289062,
"margin_dpo/margin_std": 199.03924560546875,
"step": 424
},
{
"KL/chosen_KL_mean": -243.14434814453125,
"KL/mean": -318.24798583984375,
"KL/rejected_KL_mean": -393.3516540527344,
"KL/std": 154.16336059570312,
"epoch": 0.6240822320117474,
"fcm_dpo/beta": 0.004639962688088417,
"fcm_dpo/delta": 0.10944442451000214,
"fcm_dpo/margin": 150.20730590820312,
"fcm_dpo/q_t": 0.35073035955429077,
"grad_norm": 32.66258239746094,
"learning_rate": 1.8777780903377732e-07,
"logits/chosen": 0.09410110861063004,
"logits/rejected": 0.09336138516664505,
"logps/chosen": -290.0133361816406,
"logps/ref_chosen": -46.86899948120117,
"logps/ref_rejected": -95.92545318603516,
"logps/rejected": -489.277099609375,
"loss": 0.9379,
"margin_dpo/margin_mean": 150.20730590820312,
"margin_dpo/margin_std": 164.84654235839844,
"step": 425
},
{
"KL/chosen_KL_mean": -237.71083068847656,
"KL/mean": -332.9663391113281,
"KL/rejected_KL_mean": -428.22186279296875,
"KL/std": 173.30087280273438,
"epoch": 0.6255506607929515,
"fcm_dpo/beta": 0.0046137673780322075,
"fcm_dpo/delta": -0.0870470330119133,
"fcm_dpo/margin": 190.51104736328125,
"fcm_dpo/q_t": 0.31865280866622925,
"grad_norm": 66.36375427246094,
"learning_rate": 1.8653568770724803e-07,
"logits/chosen": 0.0031908173114061356,
"logits/rejected": 0.057025354355573654,
"logps/chosen": -314.29437255859375,
"logps/ref_chosen": -76.58354187011719,
"logps/ref_rejected": -81.26658630371094,
"logps/rejected": -509.48846435546875,
"loss": 0.841,
"margin_dpo/margin_mean": 190.51101684570312,
"margin_dpo/margin_std": 184.35989379882812,
"step": 426
},
{
"KL/chosen_KL_mean": -248.94723510742188,
"KL/mean": -313.7269287109375,
"KL/rejected_KL_mean": -378.50665283203125,
"KL/std": 144.7581787109375,
"epoch": 0.6270190895741556,
"fcm_dpo/beta": 0.0046956404112279415,
"fcm_dpo/delta": 0.20422431826591492,
"fcm_dpo/margin": 129.55941772460938,
"fcm_dpo/q_t": 0.3734450936317444,
"grad_norm": 40.48114776611328,
"learning_rate": 1.8529523872436977e-07,
"logits/chosen": 0.058118224143981934,
"logits/rejected": 0.08482979238033295,
"logps/chosen": -313.8011169433594,
"logps/ref_chosen": -64.8538818359375,
"logps/ref_rejected": -78.5660171508789,
"logps/rejected": -457.07269287109375,
"loss": 0.9929,
"margin_dpo/margin_mean": 129.55941772460938,
"margin_dpo/margin_std": 163.17843627929688,
"step": 427
},
{
"KL/chosen_KL_mean": -307.9185791015625,
"KL/mean": -392.6045837402344,
"KL/rejected_KL_mean": -477.29058837890625,
"KL/std": 186.76837158203125,
"epoch": 0.6284875183553598,
"fcm_dpo/beta": 0.004741638898849487,
"fcm_dpo/delta": -0.003403865732252598,
"fcm_dpo/margin": 169.37196350097656,
"fcm_dpo/q_t": 0.3380899727344513,
"grad_norm": 40.947059631347656,
"learning_rate": 1.8405649477212697e-07,
"logits/chosen": 0.03091556765139103,
"logits/rejected": 0.043592870235443115,
"logps/chosen": -370.5552673339844,
"logps/ref_chosen": -62.63666534423828,
"logps/ref_rejected": -103.28181457519531,
"logps/rejected": -580.5723876953125,
"loss": 0.9307,
"margin_dpo/margin_mean": 169.37197875976562,
"margin_dpo/margin_std": 204.71743774414062,
"step": 428
},
{
"KL/chosen_KL_mean": -319.4748840332031,
"KL/mean": -398.7829284667969,
"KL/rejected_KL_mean": -478.0909423828125,
"KL/std": 163.31729125976562,
"epoch": 0.6299559471365639,
"fcm_dpo/beta": 0.004819995723664761,
"fcm_dpo/delta": 0.03594332933425903,
"fcm_dpo/margin": 158.6160430908203,
"fcm_dpo/q_t": 0.3449682593345642,
"grad_norm": 59.84564971923828,
"learning_rate": 1.828194884925749e-07,
"logits/chosen": 0.09432405978441238,
"logits/rejected": 0.1432572454214096,
"logps/chosen": -400.70892333984375,
"logps/ref_chosen": -81.23401641845703,
"logps/ref_rejected": -91.79493713378906,
"logps/rejected": -569.8858642578125,
"loss": 0.9474,
"margin_dpo/margin_mean": 158.6160430908203,
"margin_dpo/margin_std": 191.10890197753906,
"step": 429
},
{
"KL/chosen_KL_mean": -291.04022216796875,
"KL/mean": -372.7721862792969,
"KL/rejected_KL_mean": -454.504150390625,
"KL/std": 185.45314025878906,
"epoch": 0.631424375917768,
"fcm_dpo/beta": 0.004818527959287167,
"fcm_dpo/delta": 0.012564368546009064,
"fcm_dpo/margin": 163.4639434814453,
"fcm_dpo/q_t": 0.33801186084747314,
"grad_norm": 33.16270446777344,
"learning_rate": 1.8158425248197928e-07,
"logits/chosen": 0.02384110540151596,
"logits/rejected": 0.03353766351938248,
"logps/chosen": -351.96051025390625,
"logps/ref_chosen": -60.920326232910156,
"logps/ref_rejected": -104.42280578613281,
"logps/rejected": -558.927001953125,
"loss": 0.8826,
"margin_dpo/margin_mean": 163.4639434814453,
"margin_dpo/margin_std": 169.9510955810547,
"step": 430
},
{
"KL/chosen_KL_mean": -307.9400329589844,
"KL/mean": -417.70831298828125,
"KL/rejected_KL_mean": -527.4765625,
"KL/std": 187.70150756835938,
"epoch": 0.6328928046989721,
"fcm_dpo/beta": 0.004604096990078688,
"fcm_dpo/delta": -0.23632526397705078,
"fcm_dpo/margin": 219.53659057617188,
"fcm_dpo/q_t": 0.2936067581176758,
"grad_norm": 48.98324203491211,
"learning_rate": 1.8035081928995788e-07,
"logits/chosen": 0.19879499077796936,
"logits/rejected": 0.2163088172674179,
"logps/chosen": -365.28875732421875,
"logps/ref_chosen": -57.34874725341797,
"logps/ref_rejected": -92.84022521972656,
"logps/rejected": -620.3167724609375,
"loss": 0.7763,
"margin_dpo/margin_mean": 219.53659057617188,
"margin_dpo/margin_std": 192.73171997070312,
"step": 431
},
{
"KL/chosen_KL_mean": -305.25067138671875,
"KL/mean": -402.177490234375,
"KL/rejected_KL_mean": -499.10430908203125,
"KL/std": 181.79977416992188,
"epoch": 0.6343612334801763,
"fcm_dpo/beta": 0.004561593756079674,
"fcm_dpo/delta": -0.09420053660869598,
"fcm_dpo/margin": 193.8536376953125,
"fcm_dpo/q_t": 0.316600501537323,
"grad_norm": 32.64216995239258,
"learning_rate": 1.791192214186223e-07,
"logits/chosen": 0.0449921078979969,
"logits/rejected": 0.07461453974246979,
"logps/chosen": -376.325439453125,
"logps/ref_chosen": -71.07479095458984,
"logps/ref_rejected": -98.57952880859375,
"logps/rejected": -597.683837890625,
"loss": 0.8207,
"margin_dpo/margin_mean": 193.8536376953125,
"margin_dpo/margin_std": 174.2122039794922,
"step": 432
},
{
"KL/chosen_KL_mean": -363.0186767578125,
"KL/mean": -432.42047119140625,
"KL/rejected_KL_mean": -501.82232666015625,
"KL/std": 174.90367126464844,
"epoch": 0.6358296622613803,
"fcm_dpo/beta": 0.004609774332493544,
"fcm_dpo/delta": 0.17070356011390686,
"fcm_dpo/margin": 138.8036651611328,
"fcm_dpo/q_t": 0.365825891494751,
"grad_norm": 56.30537796020508,
"learning_rate": 1.7788949132172193e-07,
"logits/chosen": 0.16686058044433594,
"logits/rejected": 0.1833561360836029,
"logps/chosen": -421.2918701171875,
"logps/ref_chosen": -58.273193359375,
"logps/ref_rejected": -95.95089721679688,
"logps/rejected": -597.773193359375,
"loss": 1.0208,
"margin_dpo/margin_mean": 138.8036651611328,
"margin_dpo/margin_std": 193.2645721435547,
"step": 433
},
{
"KL/chosen_KL_mean": -309.8509521484375,
"KL/mean": -392.6344299316406,
"KL/rejected_KL_mean": -475.41790771484375,
"KL/std": 162.32717895507812,
"epoch": 0.6372980910425844,
"fcm_dpo/beta": 0.00465200562030077,
"fcm_dpo/delta": 0.032210834324359894,
"fcm_dpo/margin": 165.56695556640625,
"fcm_dpo/q_t": 0.34336385130882263,
"grad_norm": 43.65876007080078,
"learning_rate": 1.7666166140378853e-07,
"logits/chosen": 0.15230190753936768,
"logits/rejected": 0.17373520135879517,
"logps/chosen": -371.82464599609375,
"logps/ref_chosen": -61.97370147705078,
"logps/ref_rejected": -78.49861145019531,
"logps/rejected": -553.91650390625,
"loss": 0.8959,
"margin_dpo/margin_mean": 165.56695556640625,
"margin_dpo/margin_std": 182.33963012695312,
"step": 434
},
{
"KL/chosen_KL_mean": -277.0976867675781,
"KL/mean": -369.7054748535156,
"KL/rejected_KL_mean": -462.3132629394531,
"KL/std": 176.11752319335938,
"epoch": 0.6387665198237885,
"fcm_dpo/beta": 0.004619380459189415,
"fcm_dpo/delta": -0.06102815270423889,
"fcm_dpo/margin": 185.21556091308594,
"fcm_dpo/q_t": 0.32273826003074646,
"grad_norm": 27.393489837646484,
"learning_rate": 1.7543576401928218e-07,
"logits/chosen": 0.22029942274093628,
"logits/rejected": 0.24756871163845062,
"logps/chosen": -328.5997314453125,
"logps/ref_chosen": -51.502052307128906,
"logps/ref_rejected": -87.56689453125,
"logps/rejected": -549.880126953125,
"loss": 0.8636,
"margin_dpo/margin_mean": 185.21554565429688,
"margin_dpo/margin_std": 187.09512329101562,
"step": 435
},
{
"KL/chosen_KL_mean": -269.60888671875,
"KL/mean": -361.96820068359375,
"KL/rejected_KL_mean": -454.3275146484375,
"KL/std": 168.82907104492188,
"epoch": 0.6402349486049926,
"fcm_dpo/beta": 0.004574541002511978,
"fcm_dpo/delta": -0.049284037202596664,
"fcm_dpo/margin": 184.71865844726562,
"fcm_dpo/q_t": 0.3233397603034973,
"grad_norm": 41.69789505004883,
"learning_rate": 1.742118314717391e-07,
"logits/chosen": 0.021279800683259964,
"logits/rejected": 0.06457970291376114,
"logps/chosen": -341.0125732421875,
"logps/ref_chosen": -71.40371704101562,
"logps/ref_rejected": -82.72775268554688,
"logps/rejected": -537.0552978515625,
"loss": 0.8396,
"margin_dpo/margin_mean": 184.71865844726562,
"margin_dpo/margin_std": 174.83840942382812,
"step": 436
},
{
"KL/chosen_KL_mean": -287.47174072265625,
"KL/mean": -368.12969970703125,
"KL/rejected_KL_mean": -448.78765869140625,
"KL/std": 144.98049926757812,
"epoch": 0.6417033773861968,
"fcm_dpo/beta": 0.004624016582965851,
"fcm_dpo/delta": 0.058035340160131454,
"fcm_dpo/margin": 161.31593322753906,
"fcm_dpo/q_t": 0.34285643696784973,
"grad_norm": 31.177148818969727,
"learning_rate": 1.7298989601292036e-07,
"logits/chosen": 0.14124414324760437,
"logits/rejected": 0.1843366026878357,
"logps/chosen": -352.21600341796875,
"logps/ref_chosen": -64.7442626953125,
"logps/ref_rejected": -82.04356384277344,
"logps/rejected": -530.8312377929688,
"loss": 0.903,
"margin_dpo/margin_mean": 161.31593322753906,
"margin_dpo/margin_std": 168.4124755859375,
"step": 437
},
{
"KL/chosen_KL_mean": -271.91680908203125,
"KL/mean": -367.9270935058594,
"KL/rejected_KL_mean": -463.9373779296875,
"KL/std": 161.41934204101562,
"epoch": 0.6431718061674009,
"fcm_dpo/beta": 0.004532171413302422,
"fcm_dpo/delta": -0.07903608679771423,
"fcm_dpo/margin": 192.02059936523438,
"fcm_dpo/q_t": 0.315254807472229,
"grad_norm": 40.19487762451172,
"learning_rate": 1.7176998984196144e-07,
"logits/chosen": 0.16977214813232422,
"logits/rejected": 0.21760720014572144,
"logps/chosen": -330.93548583984375,
"logps/ref_chosen": -59.0186653137207,
"logps/ref_rejected": -83.07682800292969,
"logps/rejected": -547.0142211914062,
"loss": 0.8096,
"margin_dpo/margin_mean": 192.02059936523438,
"margin_dpo/margin_std": 160.93798828125,
"step": 438
},
{
"KL/chosen_KL_mean": -286.5731201171875,
"KL/mean": -363.4912109375,
"KL/rejected_KL_mean": -440.4093322753906,
"KL/std": 169.75950622558594,
"epoch": 0.644640234948605,
"fcm_dpo/beta": 0.00456035602837801,
"fcm_dpo/delta": 0.10353709012269974,
"fcm_dpo/margin": 153.836181640625,
"fcm_dpo/q_t": 0.3557645082473755,
"grad_norm": 33.6142463684082,
"learning_rate": 1.7055214510452458e-07,
"logits/chosen": 0.15101350843906403,
"logits/rejected": 0.15643706917762756,
"logps/chosen": -340.3572082519531,
"logps/ref_chosen": -53.78407669067383,
"logps/ref_rejected": -83.98545837402344,
"logps/rejected": -524.394775390625,
"loss": 0.9447,
"margin_dpo/margin_mean": 153.836181640625,
"margin_dpo/margin_std": 170.22177124023438,
"step": 439
},
{
"KL/chosen_KL_mean": -303.6018981933594,
"KL/mean": -376.49786376953125,
"KL/rejected_KL_mean": -449.39385986328125,
"KL/std": 186.78781127929688,
"epoch": 0.6461086637298091,
"fcm_dpo/beta": 0.004747721366584301,
"fcm_dpo/delta": 0.11485690623521805,
"fcm_dpo/margin": 145.7919464111328,
"fcm_dpo/q_t": 0.3601565361022949,
"grad_norm": 40.849365234375,
"learning_rate": 1.6933639389195134e-07,
"logits/chosen": 0.10528746247291565,
"logits/rejected": 0.14277926087379456,
"logps/chosen": -382.16864013671875,
"logps/ref_chosen": -78.56671905517578,
"logps/ref_rejected": -96.49775695800781,
"logps/rejected": -545.8916015625,
"loss": 0.9642,
"margin_dpo/margin_mean": 145.79196166992188,
"margin_dpo/margin_std": 182.39181518554688,
"step": 440
},
{
"KL/chosen_KL_mean": -312.75555419921875,
"KL/mean": -401.1040344238281,
"KL/rejected_KL_mean": -489.45245361328125,
"KL/std": 192.77627563476562,
"epoch": 0.6475770925110133,
"fcm_dpo/beta": 0.0047426121309399605,
"fcm_dpo/delta": -0.04161972180008888,
"fcm_dpo/margin": 176.6968994140625,
"fcm_dpo/q_t": 0.33483031392097473,
"grad_norm": 37.40399932861328,
"learning_rate": 1.681227682404166e-07,
"logits/chosen": 0.1145353838801384,
"logits/rejected": 0.13276338577270508,
"logps/chosen": -373.58001708984375,
"logps/ref_chosen": -60.824440002441406,
"logps/ref_rejected": -96.47080993652344,
"logps/rejected": -585.9232788085938,
"loss": 0.9214,
"margin_dpo/margin_mean": 176.6968994140625,
"margin_dpo/margin_std": 215.09902954101562,
"step": 441
},
{
"KL/chosen_KL_mean": -285.8372802734375,
"KL/mean": -389.34747314453125,
"KL/rejected_KL_mean": -492.857666015625,
"KL/std": 188.40069580078125,
"epoch": 0.6490455212922174,
"fcm_dpo/beta": 0.004635802935808897,
"fcm_dpo/delta": -0.17678695917129517,
"fcm_dpo/margin": 207.02035522460938,
"fcm_dpo/q_t": 0.30531585216522217,
"grad_norm": 27.13920783996582,
"learning_rate": 1.669113001300851e-07,
"logits/chosen": 0.11083649098873138,
"logits/rejected": 0.13127142190933228,
"logps/chosen": -332.8485107421875,
"logps/ref_chosen": -47.01121520996094,
"logps/ref_rejected": -76.53926086425781,
"logps/rejected": -569.3969116210938,
"loss": 0.7884,
"margin_dpo/margin_mean": 207.02035522460938,
"margin_dpo/margin_std": 183.5563507080078,
"step": 442
},
{
"KL/chosen_KL_mean": -326.73773193359375,
"KL/mean": -398.07366943359375,
"KL/rejected_KL_mean": -469.4096374511719,
"KL/std": 170.14450073242188,
"epoch": 0.6505139500734214,
"fcm_dpo/beta": 0.004615455400198698,
"fcm_dpo/delta": 0.15127958357334137,
"fcm_dpo/margin": 142.67193603515625,
"fcm_dpo/q_t": 0.3653959035873413,
"grad_norm": 64.4756851196289,
"learning_rate": 1.6570202148426815e-07,
"logits/chosen": 0.12177616357803345,
"logits/rejected": 0.13952183723449707,
"logps/chosen": -398.0107421875,
"logps/ref_chosen": -71.27301788330078,
"logps/ref_rejected": -86.679931640625,
"logps/rejected": -556.089599609375,
"loss": 1.0129,
"margin_dpo/margin_mean": 142.6719207763672,
"margin_dpo/margin_std": 198.99505615234375,
"step": 443
},
{
"KL/chosen_KL_mean": -347.0489501953125,
"KL/mean": -446.3876953125,
"KL/rejected_KL_mean": -545.7264404296875,
"KL/std": 186.73049926757812,
"epoch": 0.6519823788546255,
"fcm_dpo/beta": 0.004596185404807329,
"fcm_dpo/delta": -0.12484229356050491,
"fcm_dpo/margin": 198.67745971679688,
"fcm_dpo/q_t": 0.31873539090156555,
"grad_norm": 42.87467575073242,
"learning_rate": 1.6449496416858282e-07,
"logits/chosen": 0.24206379055976868,
"logits/rejected": 0.2568224370479584,
"logps/chosen": -404.2626647949219,
"logps/ref_chosen": -57.213706970214844,
"logps/ref_rejected": -97.25489807128906,
"logps/rejected": -642.9813232421875,
"loss": 0.8479,
"margin_dpo/margin_mean": 198.67747497558594,
"margin_dpo/margin_std": 211.93258666992188,
"step": 444
},
{
"KL/chosen_KL_mean": -325.6207580566406,
"KL/mean": -424.31805419921875,
"KL/rejected_KL_mean": -523.0152587890625,
"KL/std": 176.32916259765625,
"epoch": 0.6534508076358296,
"fcm_dpo/beta": 0.004501967690885067,
"fcm_dpo/delta": -0.0975315123796463,
"fcm_dpo/margin": 197.39456176757812,
"fcm_dpo/q_t": 0.319426029920578,
"grad_norm": 37.24625015258789,
"learning_rate": 1.6329015999011182e-07,
"logits/chosen": 0.16529598832130432,
"logits/rejected": 0.19533869624137878,
"logps/chosen": -392.9205322265625,
"logps/ref_chosen": -67.29979705810547,
"logps/ref_rejected": -92.68267059326172,
"logps/rejected": -615.697998046875,
"loss": 0.8327,
"margin_dpo/margin_mean": 197.39456176757812,
"margin_dpo/margin_std": 190.868896484375,
"step": 445
},
{
"KL/chosen_KL_mean": -309.30706787109375,
"KL/mean": -408.1361999511719,
"KL/rejected_KL_mean": -506.96533203125,
"KL/std": 178.90072631835938,
"epoch": 0.6549192364170338,
"fcm_dpo/beta": 0.004462126176804304,
"fcm_dpo/delta": -0.09048521518707275,
"fcm_dpo/margin": 197.65823364257812,
"fcm_dpo/q_t": 0.31490451097488403,
"grad_norm": 40.9002799987793,
"learning_rate": 1.6208764069656578e-07,
"logits/chosen": 0.2029511034488678,
"logits/rejected": 0.20954206585884094,
"logps/chosen": -368.40557861328125,
"logps/ref_chosen": -59.098487854003906,
"logps/ref_rejected": -101.26419067382812,
"logps/rejected": -608.2294921875,
"loss": 0.8093,
"margin_dpo/margin_mean": 197.65823364257812,
"margin_dpo/margin_std": 169.7216796875,
"step": 446
},
{
"KL/chosen_KL_mean": -334.24090576171875,
"KL/mean": -437.32513427734375,
"KL/rejected_KL_mean": -540.4093627929688,
"KL/std": 209.1260986328125,
"epoch": 0.6563876651982379,
"fcm_dpo/beta": 0.004316662438213825,
"fcm_dpo/delta": -0.10000540316104889,
"fcm_dpo/margin": 206.16847229003906,
"fcm_dpo/q_t": 0.3189159333705902,
"grad_norm": 46.66408157348633,
"learning_rate": 1.608874379754465e-07,
"logits/chosen": 0.19859518110752106,
"logits/rejected": 0.20376473665237427,
"logps/chosen": -390.31622314453125,
"logps/ref_chosen": -56.07533264160156,
"logps/ref_rejected": -98.69475555419922,
"logps/rejected": -639.1041259765625,
"loss": 0.8258,
"margin_dpo/margin_mean": 206.16847229003906,
"margin_dpo/margin_std": 204.78793334960938,
"step": 447
},
{
"KL/chosen_KL_mean": -352.8293762207031,
"KL/mean": -461.22027587890625,
"KL/rejected_KL_mean": -569.6112060546875,
"KL/std": 188.87646484375,
"epoch": 0.657856093979442,
"fcm_dpo/beta": 0.004251912701874971,
"fcm_dpo/delta": -0.1339014321565628,
"fcm_dpo/margin": 216.7818145751953,
"fcm_dpo/q_t": 0.30998510122299194,
"grad_norm": 50.98678970336914,
"learning_rate": 1.5968958345321177e-07,
"logits/chosen": 0.1657218635082245,
"logits/rejected": 0.1783699095249176,
"logps/chosen": -412.8332214355469,
"logps/ref_chosen": -60.00384521484375,
"logps/ref_rejected": -102.26465606689453,
"logps/rejected": -671.8758544921875,
"loss": 0.7985,
"margin_dpo/margin_mean": 216.78179931640625,
"margin_dpo/margin_std": 192.1991424560547,
"step": 448
},
{
"KL/chosen_KL_mean": -352.67364501953125,
"KL/mean": -451.06048583984375,
"KL/rejected_KL_mean": -549.4473266601562,
"KL/std": 202.05987548828125,
"epoch": 0.6593245227606461,
"fcm_dpo/beta": 0.004183411132544279,
"fcm_dpo/delta": -0.025334613397717476,
"fcm_dpo/margin": 196.7737274169922,
"fcm_dpo/q_t": 0.33970075845718384,
"grad_norm": 35.54926300048828,
"learning_rate": 1.584941086944423e-07,
"logits/chosen": 0.14113399386405945,
"logits/rejected": 0.17219001054763794,
"logps/chosen": -420.20025634765625,
"logps/ref_chosen": -67.52661895751953,
"logps/ref_rejected": -88.59690856933594,
"logps/rejected": -638.0442504882812,
"loss": 0.9209,
"margin_dpo/margin_mean": 196.77371215820312,
"margin_dpo/margin_std": 245.24896240234375,
"step": 449
},
{
"KL/chosen_KL_mean": -308.78033447265625,
"KL/mean": -417.45819091796875,
"KL/rejected_KL_mean": -526.1360473632812,
"KL/std": 202.91213989257812,
"epoch": 0.6607929515418502,
"fcm_dpo/beta": 0.004141074605286121,
"fcm_dpo/delta": -0.11008300632238388,
"fcm_dpo/margin": 217.355712890625,
"fcm_dpo/q_t": 0.30950620770454407,
"grad_norm": 32.117591857910156,
"learning_rate": 1.573010452010098e-07,
"logits/chosen": 0.22345803678035736,
"logits/rejected": 0.23786935210227966,
"logps/chosen": -365.8884582519531,
"logps/ref_chosen": -57.10811996459961,
"logps/ref_rejected": -102.75494384765625,
"logps/rejected": -628.8909912109375,
"loss": 0.8082,
"margin_dpo/margin_mean": 217.355712890625,
"margin_dpo/margin_std": 193.65924072265625,
"step": 450
},
{
"KL/chosen_KL_mean": -373.90386962890625,
"KL/mean": -466.65606689453125,
"KL/rejected_KL_mean": -559.4083251953125,
"KL/std": 207.1577606201172,
"epoch": 0.6622613803230544,
"fcm_dpo/beta": 0.004139560274779797,
"fcm_dpo/delta": 0.033666130155324936,
"fcm_dpo/margin": 185.50445556640625,
"fcm_dpo/q_t": 0.33645984530448914,
"grad_norm": 49.498046875,
"learning_rate": 1.5611042441124687e-07,
"logits/chosen": 0.3563879132270813,
"logits/rejected": 0.3978639543056488,
"logps/chosen": -432.3726806640625,
"logps/ref_chosen": -58.46883010864258,
"logps/ref_rejected": -72.92941284179688,
"logps/rejected": -632.3377075195312,
"loss": 0.9322,
"margin_dpo/margin_mean": 185.50445556640625,
"margin_dpo/margin_std": 218.109130859375,
"step": 451
},
{
"KL/chosen_KL_mean": -291.71258544921875,
"KL/mean": -394.4192199707031,
"KL/rejected_KL_mean": -497.1258544921875,
"KL/std": 187.85025024414062,
"epoch": 0.6637298091042585,
"fcm_dpo/beta": 0.004082968458533287,
"fcm_dpo/delta": -0.04251035302877426,
"fcm_dpo/margin": 205.4132843017578,
"fcm_dpo/q_t": 0.3228408098220825,
"grad_norm": 27.555904388427734,
"learning_rate": 1.549222776991186e-07,
"logits/chosen": 0.2295912206172943,
"logits/rejected": 0.2270086109638214,
"logps/chosen": -342.1031494140625,
"logps/ref_chosen": -50.39055252075195,
"logps/ref_rejected": -97.77142333984375,
"logps/rejected": -594.8972778320312,
"loss": 0.8281,
"margin_dpo/margin_mean": 205.4132843017578,
"margin_dpo/margin_std": 183.20945739746094,
"step": 452
},
{
"KL/chosen_KL_mean": -329.4156799316406,
"KL/mean": -421.95611572265625,
"KL/rejected_KL_mean": -514.4965209960938,
"KL/std": 187.1441650390625,
"epoch": 0.6651982378854625,
"fcm_dpo/beta": 0.0040916576981544495,
"fcm_dpo/delta": 0.04588655009865761,
"fcm_dpo/margin": 185.08084106445312,
"fcm_dpo/q_t": 0.3440532088279724,
"grad_norm": 33.83127975463867,
"learning_rate": 1.5373663637339584e-07,
"logits/chosen": 0.14756467938423157,
"logits/rejected": 0.19183680415153503,
"logps/chosen": -387.13055419921875,
"logps/ref_chosen": -57.71485137939453,
"logps/ref_rejected": -82.20741271972656,
"logps/rejected": -596.7039794921875,
"loss": 0.8993,
"margin_dpo/margin_mean": 185.08082580566406,
"margin_dpo/margin_std": 200.801025390625,
"step": 453
},
{
"KL/chosen_KL_mean": -362.6598815917969,
"KL/mean": -471.7563171386719,
"KL/rejected_KL_mean": -580.852783203125,
"KL/std": 216.17813110351562,
"epoch": 0.6666666666666666,
"fcm_dpo/beta": 0.004048048052936792,
"fcm_dpo/delta": -0.09220601618289948,
"fcm_dpo/margin": 218.19287109375,
"fcm_dpo/q_t": 0.32163387537002563,
"grad_norm": 33.92574691772461,
"learning_rate": 1.5255353167683017e-07,
"logits/chosen": 0.24445435404777527,
"logits/rejected": 0.27604761719703674,
"logps/chosen": -423.60552978515625,
"logps/ref_chosen": -60.945648193359375,
"logps/ref_rejected": -84.95079040527344,
"logps/rejected": -665.8035888671875,
"loss": 0.8446,
"margin_dpo/margin_mean": 218.19287109375,
"margin_dpo/margin_std": 226.92523193359375,
"step": 454
},
{
"KL/chosen_KL_mean": -319.8667297363281,
"KL/mean": -438.78204345703125,
"KL/rejected_KL_mean": -557.6973876953125,
"KL/std": 214.24429321289062,
"epoch": 0.6681350954478708,
"fcm_dpo/beta": 0.003983162343502045,
"fcm_dpo/delta": -0.16275690495967865,
"fcm_dpo/margin": 237.83067321777344,
"fcm_dpo/q_t": 0.3083864152431488,
"grad_norm": 38.621768951416016,
"learning_rate": 1.5137299478533064e-07,
"logits/chosen": 0.14638906717300415,
"logits/rejected": 0.15352049469947815,
"logps/chosen": -364.75341796875,
"logps/ref_chosen": -44.88671112060547,
"logps/ref_rejected": -115.30147552490234,
"logps/rejected": -672.9989013671875,
"loss": 0.8024,
"margin_dpo/margin_mean": 237.83065795898438,
"margin_dpo/margin_std": 217.93719482421875,
"step": 455
},
{
"KL/chosen_KL_mean": -313.37225341796875,
"KL/mean": -429.8499755859375,
"KL/rejected_KL_mean": -546.32763671875,
"KL/std": 212.5858154296875,
"epoch": 0.6696035242290749,
"fcm_dpo/beta": 0.0038606026209890842,
"fcm_dpo/delta": -0.10905548930168152,
"fcm_dpo/margin": 232.95541381835938,
"fcm_dpo/q_t": 0.31055253744125366,
"grad_norm": 32.60857009887695,
"learning_rate": 1.5019505680714232e-07,
"logits/chosen": 0.1347317397594452,
"logits/rejected": 0.13765022158622742,
"logps/chosen": -370.4090270996094,
"logps/ref_chosen": -57.036781311035156,
"logps/ref_rejected": -105.21784210205078,
"logps/rejected": -651.5455322265625,
"loss": 0.7867,
"margin_dpo/margin_mean": 232.9553985595703,
"margin_dpo/margin_std": 192.2313232421875,
"step": 456
},
{
"KL/chosen_KL_mean": -303.17724609375,
"KL/mean": -408.58001708984375,
"KL/rejected_KL_mean": -513.9827880859375,
"KL/std": 202.3392333984375,
"epoch": 0.671071953010279,
"fcm_dpo/beta": 0.0038118790835142136,
"fcm_dpo/delta": -0.004409227520227432,
"fcm_dpo/margin": 210.80557250976562,
"fcm_dpo/q_t": 0.32770806550979614,
"grad_norm": 39.375999450683594,
"learning_rate": 1.4901974878202627e-07,
"logits/chosen": 0.1512976586818695,
"logits/rejected": 0.1942361295223236,
"logps/chosen": -357.4197998046875,
"logps/ref_chosen": -54.24253845214844,
"logps/ref_rejected": -85.10956573486328,
"logps/rejected": -599.0924072265625,
"loss": 0.8532,
"margin_dpo/margin_mean": 210.80555725097656,
"margin_dpo/margin_std": 192.32322692871094,
"step": 457
},
{
"KL/chosen_KL_mean": -301.6228332519531,
"KL/mean": -418.083251953125,
"KL/rejected_KL_mean": -534.543701171875,
"KL/std": 199.40939331054688,
"epoch": 0.6725403817914831,
"fcm_dpo/beta": 0.003761657979339361,
"fcm_dpo/delta": -0.08457423001527786,
"fcm_dpo/margin": 232.92083740234375,
"fcm_dpo/q_t": 0.3179261386394501,
"grad_norm": 29.335128784179688,
"learning_rate": 1.4784710168044212e-07,
"logits/chosen": 0.19247975945472717,
"logits/rejected": 0.2139834612607956,
"logps/chosen": -357.0317077636719,
"logps/ref_chosen": -55.40888214111328,
"logps/ref_rejected": -97.68325805664062,
"logps/rejected": -632.2269287109375,
"loss": 0.8221,
"margin_dpo/margin_mean": 232.92083740234375,
"margin_dpo/margin_std": 219.72933959960938,
"step": 458
},
{
"KL/chosen_KL_mean": -328.488037109375,
"KL/mean": -435.84149169921875,
"KL/rejected_KL_mean": -543.1949462890625,
"KL/std": 195.81832885742188,
"epoch": 0.6740088105726872,
"fcm_dpo/beta": 0.0037542409263551235,
"fcm_dpo/delta": -0.006693059578537941,
"fcm_dpo/margin": 214.70687866210938,
"fcm_dpo/q_t": 0.3288930654525757,
"grad_norm": 31.528162002563477,
"learning_rate": 1.466771464027316e-07,
"logits/chosen": 0.19821825623512268,
"logits/rejected": 0.21426156163215637,
"logps/chosen": -375.0455322265625,
"logps/ref_chosen": -46.55748748779297,
"logps/ref_rejected": -86.16854095458984,
"logps/rejected": -629.363525390625,
"loss": 0.8524,
"margin_dpo/margin_mean": 214.70687866210938,
"margin_dpo/margin_std": 196.4297332763672,
"step": 459
},
{
"KL/chosen_KL_mean": -353.2541809082031,
"KL/mean": -470.1562805175781,
"KL/rejected_KL_mean": -587.058349609375,
"KL/std": 202.15203857421875,
"epoch": 0.6754772393538914,
"fcm_dpo/beta": 0.0037221387028694153,
"fcm_dpo/delta": -0.07688962668180466,
"fcm_dpo/margin": 233.80418395996094,
"fcm_dpo/q_t": 0.31593120098114014,
"grad_norm": 36.09447479248047,
"learning_rate": 1.4550991377830423e-07,
"logits/chosen": 0.11949439346790314,
"logits/rejected": 0.1151965856552124,
"logps/chosen": -404.88909912109375,
"logps/ref_chosen": -51.63489532470703,
"logps/ref_rejected": -104.11935424804688,
"logps/rejected": -691.177734375,
"loss": 0.8139,
"margin_dpo/margin_mean": 233.80418395996094,
"margin_dpo/margin_std": 204.37997436523438,
"step": 460
},
{
"KL/chosen_KL_mean": -376.63531494140625,
"KL/mean": -465.3780517578125,
"KL/rejected_KL_mean": -554.1207885742188,
"KL/std": 202.32070922851562,
"epoch": 0.6769456681350955,
"fcm_dpo/beta": 0.0037951446138322353,
"fcm_dpo/delta": 0.13449496030807495,
"fcm_dpo/margin": 177.4854736328125,
"fcm_dpo/q_t": 0.3620460033416748,
"grad_norm": 37.02994155883789,
"learning_rate": 1.4434543456482518e-07,
"logits/chosen": 0.14143106341362,
"logits/rejected": 0.15454688668251038,
"logps/chosen": -431.8172607421875,
"logps/ref_chosen": -55.18195724487305,
"logps/ref_rejected": -86.47689819335938,
"logps/rejected": -640.59765625,
"loss": 0.962,
"margin_dpo/margin_mean": 177.48548889160156,
"margin_dpo/margin_std": 217.33816528320312,
"step": 461
},
{
"KL/chosen_KL_mean": -361.41998291015625,
"KL/mean": -443.12005615234375,
"KL/rejected_KL_mean": -524.820068359375,
"KL/std": 192.26971435546875,
"epoch": 0.6784140969162996,
"fcm_dpo/beta": 0.0038944468833506107,
"fcm_dpo/delta": 0.1750825047492981,
"fcm_dpo/margin": 163.40008544921875,
"fcm_dpo/q_t": 0.36596518754959106,
"grad_norm": 50.11738967895508,
"learning_rate": 1.4318373944740484e-07,
"logits/chosen": 0.20698338747024536,
"logits/rejected": 0.23005270957946777,
"logps/chosen": -431.3480224609375,
"logps/ref_chosen": -69.92803192138672,
"logps/ref_rejected": -78.84111022949219,
"logps/rejected": -603.6611938476562,
"loss": 0.9594,
"margin_dpo/margin_mean": 163.40008544921875,
"margin_dpo/margin_std": 186.82803344726562,
"step": 462
},
{
"KL/chosen_KL_mean": -377.38116455078125,
"KL/mean": -470.04058837890625,
"KL/rejected_KL_mean": -562.7000732421875,
"KL/std": 201.7800750732422,
"epoch": 0.6798825256975036,
"fcm_dpo/beta": 0.004008530639111996,
"fcm_dpo/delta": 0.05973725765943527,
"fcm_dpo/margin": 185.31890869140625,
"fcm_dpo/q_t": 0.3500801920890808,
"grad_norm": 59.69301223754883,
"learning_rate": 1.4202485903778976e-07,
"logits/chosen": 0.18849530816078186,
"logits/rejected": 0.20773401856422424,
"logps/chosen": -432.655517578125,
"logps/ref_chosen": -55.27437210083008,
"logps/ref_rejected": -89.02497863769531,
"logps/rejected": -651.7250366210938,
"loss": 0.9511,
"margin_dpo/margin_mean": 185.31890869140625,
"margin_dpo/margin_std": 225.99717712402344,
"step": 463
},
{
"KL/chosen_KL_mean": -368.3976135253906,
"KL/mean": -504.92755126953125,
"KL/rejected_KL_mean": -641.4574584960938,
"KL/std": 232.31454467773438,
"epoch": 0.6813509544787077,
"fcm_dpo/beta": 0.0038046562112867832,
"fcm_dpo/delta": -0.2696046829223633,
"fcm_dpo/margin": 273.0598449707031,
"fcm_dpo/q_t": 0.28859981894493103,
"grad_norm": 39.533355712890625,
"learning_rate": 1.4086882387355658e-07,
"logits/chosen": 0.23187042772769928,
"logits/rejected": 0.23417802155017853,
"logps/chosen": -419.3099365234375,
"logps/ref_chosen": -50.91230010986328,
"logps/ref_rejected": -102.4893798828125,
"logps/rejected": -743.9468383789062,
"loss": 0.7533,
"margin_dpo/margin_mean": 273.0598449707031,
"margin_dpo/margin_std": 236.38284301757812,
"step": 464
},
{
"KL/chosen_KL_mean": -363.36859130859375,
"KL/mean": -487.6932067871094,
"KL/rejected_KL_mean": -612.017822265625,
"KL/std": 230.75079345703125,
"epoch": 0.6828193832599119,
"fcm_dpo/beta": 0.0037093700375407934,
"fcm_dpo/delta": -0.1346028596162796,
"fcm_dpo/margin": 248.6492156982422,
"fcm_dpo/q_t": 0.30622804164886475,
"grad_norm": 36.44754409790039,
"learning_rate": 1.3971566441730714e-07,
"logits/chosen": 0.21581992506980896,
"logits/rejected": 0.22753633558750153,
"logps/chosen": -423.4854431152344,
"logps/ref_chosen": -60.116851806640625,
"logps/ref_rejected": -113.94602966308594,
"logps/rejected": -725.9638671875,
"loss": 0.7884,
"margin_dpo/margin_mean": 248.64923095703125,
"margin_dpo/margin_std": 211.38388061523438,
"step": 465
},
{
"KL/chosen_KL_mean": -404.96728515625,
"KL/mean": -510.61505126953125,
"KL/rejected_KL_mean": -616.2628173828125,
"KL/std": 214.00537109375,
"epoch": 0.684287812041116,
"fcm_dpo/beta": 0.00367511878721416,
"fcm_dpo/delta": 0.025151047855615616,
"fcm_dpo/margin": 211.29556274414062,
"fcm_dpo/q_t": 0.33769696950912476,
"grad_norm": 48.69867706298828,
"learning_rate": 1.3856541105586545e-07,
"logits/chosen": 0.19872266054153442,
"logits/rejected": 0.20927634835243225,
"logps/chosen": -457.88818359375,
"logps/ref_chosen": -52.920921325683594,
"logps/ref_rejected": -90.3154296875,
"logps/rejected": -706.5782470703125,
"loss": 0.8961,
"margin_dpo/margin_mean": 211.29556274414062,
"margin_dpo/margin_std": 224.05996704101562,
"step": 466
},
{
"KL/chosen_KL_mean": -474.77044677734375,
"KL/mean": -590.2333984375,
"KL/rejected_KL_mean": -705.6962890625,
"KL/std": 236.89700317382812,
"epoch": 0.6857562408223201,
"fcm_dpo/beta": 0.0036290897987782955,
"fcm_dpo/delta": -0.044904135167598724,
"fcm_dpo/margin": 230.92593383789062,
"fcm_dpo/q_t": 0.3338162899017334,
"grad_norm": 70.50183868408203,
"learning_rate": 1.3741809409947729e-07,
"logits/chosen": 0.24021601676940918,
"logits/rejected": 0.2654044032096863,
"logps/chosen": -553.4862670898438,
"logps/ref_chosen": -78.7158203125,
"logps/ref_rejected": -102.86019897460938,
"logps/rejected": -808.5565185546875,
"loss": 0.9236,
"margin_dpo/margin_mean": 230.92593383789062,
"margin_dpo/margin_std": 273.190673828125,
"step": 467
},
{
"KL/chosen_KL_mean": -392.1413269042969,
"KL/mean": -511.47100830078125,
"KL/rejected_KL_mean": -630.8007202148438,
"KL/std": 235.54798889160156,
"epoch": 0.6872246696035242,
"fcm_dpo/beta": 0.003602225799113512,
"fcm_dpo/delta": -0.06676232814788818,
"fcm_dpo/margin": 238.6593780517578,
"fcm_dpo/q_t": 0.32230842113494873,
"grad_norm": 40.31625747680664,
"learning_rate": 1.362737437810114e-07,
"logits/chosen": 0.16735252737998962,
"logits/rejected": 0.20658326148986816,
"logps/chosen": -462.0766906738281,
"logps/ref_chosen": -69.93536376953125,
"logps/ref_rejected": -101.02880859375,
"logps/rejected": -731.8295288085938,
"loss": 0.8509,
"margin_dpo/margin_mean": 238.6593780517578,
"margin_dpo/margin_std": 236.20262145996094,
"step": 468
},
{
"KL/chosen_KL_mean": -438.883056640625,
"KL/mean": -564.942138671875,
"KL/rejected_KL_mean": -691.0013427734375,
"KL/std": 231.7757568359375,
"epoch": 0.6886930983847284,
"fcm_dpo/beta": 0.0035334881395101547,
"fcm_dpo/delta": -0.10173042863607407,
"fcm_dpo/margin": 252.11825561523438,
"fcm_dpo/q_t": 0.3151817321777344,
"grad_norm": 39.68192672729492,
"learning_rate": 1.351323902551631e-07,
"logits/chosen": 0.17181815207004547,
"logits/rejected": 0.21328867971897125,
"logps/chosen": -507.00775146484375,
"logps/ref_chosen": -68.12469482421875,
"logps/ref_rejected": -104.78640747070312,
"logps/rejected": -795.7877197265625,
"loss": 0.8161,
"margin_dpo/margin_mean": 252.11825561523438,
"margin_dpo/margin_std": 229.35256958007812,
"step": 469
},
{
"KL/chosen_KL_mean": -379.2088928222656,
"KL/mean": -501.94903564453125,
"KL/rejected_KL_mean": -624.6891479492188,
"KL/std": 232.1168670654297,
"epoch": 0.6901615271659325,
"fcm_dpo/beta": 0.0035080176312476397,
"fcm_dpo/delta": -0.06688985973596573,
"fcm_dpo/margin": 245.4802703857422,
"fcm_dpo/q_t": 0.31790733337402344,
"grad_norm": 41.94991683959961,
"learning_rate": 1.339940635976592e-07,
"logits/chosen": 0.24481558799743652,
"logits/rejected": 0.26332205533981323,
"logps/chosen": -423.0008239746094,
"logps/ref_chosen": -43.791927337646484,
"logps/ref_rejected": -82.70285034179688,
"logps/rejected": -707.3919677734375,
"loss": 0.8406,
"margin_dpo/margin_mean": 245.48028564453125,
"margin_dpo/margin_std": 234.01971435546875,
"step": 470
},
{
"KL/chosen_KL_mean": -424.17352294921875,
"KL/mean": -526.1630859375,
"KL/rejected_KL_mean": -628.1526489257812,
"KL/std": 224.7677764892578,
"epoch": 0.6916299559471366,
"fcm_dpo/beta": 0.0035170000046491623,
"fcm_dpo/delta": 0.08863566070795059,
"fcm_dpo/margin": 203.97914123535156,
"fcm_dpo/q_t": 0.3468366861343384,
"grad_norm": 40.956668853759766,
"learning_rate": 1.3285879380446563e-07,
"logits/chosen": 0.22680866718292236,
"logits/rejected": 0.24914070963859558,
"logps/chosen": -487.5130310058594,
"logps/ref_chosen": -63.33952331542969,
"logps/ref_rejected": -83.61048126220703,
"logps/rejected": -711.7631225585938,
"loss": 0.9042,
"margin_dpo/margin_mean": 203.9791259765625,
"margin_dpo/margin_std": 201.23226928710938,
"step": 471
},
{
"KL/chosen_KL_mean": -403.254150390625,
"KL/mean": -534.051513671875,
"KL/rejected_KL_mean": -664.848876953125,
"KL/std": 286.0874328613281,
"epoch": 0.6930983847283406,
"fcm_dpo/beta": 0.0034922936465591192,
"fcm_dpo/delta": -0.1249755322933197,
"fcm_dpo/margin": 261.59478759765625,
"fcm_dpo/q_t": 0.32117411494255066,
"grad_norm": 30.76375389099121,
"learning_rate": 1.317266107909975e-07,
"logits/chosen": 0.0545220784842968,
"logits/rejected": 0.09548470377922058,
"logps/chosen": -486.92022705078125,
"logps/ref_chosen": -83.66610717773438,
"logps/ref_rejected": -117.20919799804688,
"logps/rejected": -782.05810546875,
"loss": 0.8339,
"margin_dpo/margin_mean": 261.59478759765625,
"margin_dpo/margin_std": 272.6410827636719,
"step": 472
},
{
"KL/chosen_KL_mean": -475.15594482421875,
"KL/mean": -565.5804443359375,
"KL/rejected_KL_mean": -656.0048828125,
"KL/std": 236.00418090820312,
"epoch": 0.6945668135095447,
"fcm_dpo/beta": 0.0035597169771790504,
"fcm_dpo/delta": 0.16679316759109497,
"fcm_dpo/margin": 180.84896850585938,
"fcm_dpo/q_t": 0.3708559274673462,
"grad_norm": 72.96520233154297,
"learning_rate": 1.3059754439133002e-07,
"logits/chosen": 0.2742480933666229,
"logits/rejected": 0.3015744984149933,
"logps/chosen": -538.6528930664062,
"logps/ref_chosen": -63.49696731567383,
"logps/ref_rejected": -81.14657592773438,
"logps/rejected": -737.1514892578125,
"loss": 1.0307,
"margin_dpo/margin_mean": 180.84896850585938,
"margin_dpo/margin_std": 274.80029296875,
"step": 473
},
{
"KL/chosen_KL_mean": -417.2640075683594,
"KL/mean": -509.0142822265625,
"KL/rejected_KL_mean": -600.7645874023438,
"KL/std": 221.151611328125,
"epoch": 0.6960352422907489,
"fcm_dpo/beta": 0.003691623918712139,
"fcm_dpo/delta": 0.12826226651668549,
"fcm_dpo/margin": 183.5005645751953,
"fcm_dpo/q_t": 0.35935598611831665,
"grad_norm": 61.98820114135742,
"learning_rate": 1.2947162435741277e-07,
"logits/chosen": 0.18188732862472534,
"logits/rejected": 0.20040690898895264,
"logps/chosen": -469.8759460449219,
"logps/ref_chosen": -52.6119384765625,
"logps/ref_rejected": -90.08041381835938,
"logps/rejected": -690.844970703125,
"loss": 0.9928,
"margin_dpo/margin_mean": 183.50054931640625,
"margin_dpo/margin_std": 239.28359985351562,
"step": 474
},
{
"KL/chosen_KL_mean": -372.036865234375,
"KL/mean": -496.2984619140625,
"KL/rejected_KL_mean": -620.56005859375,
"KL/std": 217.1239013671875,
"epoch": 0.697503671071953,
"fcm_dpo/beta": 0.003616760950535536,
"fcm_dpo/delta": -0.10878843069076538,
"fcm_dpo/margin": 248.52316284179688,
"fcm_dpo/q_t": 0.313048779964447,
"grad_norm": 60.8675651550293,
"learning_rate": 1.2834888035828596e-07,
"logits/chosen": 0.27965113520622253,
"logits/rejected": 0.28925400972366333,
"logps/chosen": -414.5320739746094,
"logps/ref_chosen": -42.49519348144531,
"logps/ref_rejected": -90.06294250488281,
"logps/rejected": -710.623046875,
"loss": 0.8097,
"margin_dpo/margin_mean": 248.52317810058594,
"margin_dpo/margin_std": 225.99652099609375,
"step": 475
},
{
"KL/chosen_KL_mean": -366.5478515625,
"KL/mean": -471.92510986328125,
"KL/rejected_KL_mean": -577.3023681640625,
"KL/std": 207.46705627441406,
"epoch": 0.6989720998531571,
"fcm_dpo/beta": 0.003618250833824277,
"fcm_dpo/delta": 0.040546517819166183,
"fcm_dpo/margin": 210.7545166015625,
"fcm_dpo/q_t": 0.33902594447135925,
"grad_norm": 44.247074127197266,
"learning_rate": 1.2722934197929802e-07,
"logits/chosen": 0.23911496996879578,
"logits/rejected": 0.2586338520050049,
"logps/chosen": -409.49725341796875,
"logps/ref_chosen": -42.94938278198242,
"logps/ref_rejected": -73.71023559570312,
"logps/rejected": -651.0125732421875,
"loss": 0.878,
"margin_dpo/margin_mean": 210.7545166015625,
"margin_dpo/margin_std": 205.98703002929688,
"step": 476
},
{
"KL/chosen_KL_mean": -377.1195983886719,
"KL/mean": -477.4259948730469,
"KL/rejected_KL_mean": -577.7323608398438,
"KL/std": 214.20526123046875,
"epoch": 0.7004405286343612,
"fcm_dpo/beta": 0.0036724847741425037,
"fcm_dpo/delta": 0.06813560426235199,
"fcm_dpo/margin": 200.6127471923828,
"fcm_dpo/q_t": 0.34649232029914856,
"grad_norm": 49.856998443603516,
"learning_rate": 1.2611303872132631e-07,
"logits/chosen": 0.13930974900722504,
"logits/rejected": 0.1919388622045517,
"logps/chosen": -447.8922119140625,
"logps/ref_chosen": -70.77261352539062,
"logps/ref_rejected": -76.13737487792969,
"logps/rejected": -653.8697509765625,
"loss": 0.9433,
"margin_dpo/margin_mean": 200.61273193359375,
"margin_dpo/margin_std": 242.2550048828125,
"step": 477
},
{
"KL/chosen_KL_mean": -317.3870544433594,
"KL/mean": -434.1727294921875,
"KL/rejected_KL_mean": -550.9583740234375,
"KL/std": 205.1141357421875,
"epoch": 0.7019089574155654,
"fcm_dpo/beta": 0.0036650304682552814,
"fcm_dpo/delta": -0.06150561571121216,
"fcm_dpo/margin": 233.57131958007812,
"fcm_dpo/q_t": 0.3226568102836609,
"grad_norm": 36.77776336669922,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": 0.15008513629436493,
"logits/rejected": 0.16725000739097595,
"logps/chosen": -358.82757568359375,
"logps/ref_chosen": -41.440513610839844,
"logps/ref_rejected": -85.36196899414062,
"logps/rejected": -636.3203125,
"loss": 0.824,
"margin_dpo/margin_mean": 233.57131958007812,
"margin_dpo/margin_std": 212.3719024658203,
"step": 478
},
{
"KL/chosen_KL_mean": -379.9550476074219,
"KL/mean": -474.99371337890625,
"KL/rejected_KL_mean": -570.0324096679688,
"KL/std": 183.4116668701172,
"epoch": 0.7033773861967695,
"fcm_dpo/beta": 0.003735472448170185,
"fcm_dpo/delta": 0.09173765778541565,
"fcm_dpo/margin": 190.0773468017578,
"fcm_dpo/q_t": 0.34823429584503174,
"grad_norm": 43.84068298339844,
"learning_rate": 1.2389025514492456e-07,
"logits/chosen": 0.1662050038576126,
"logits/rejected": 0.1779286414384842,
"logps/chosen": -433.86297607421875,
"logps/ref_chosen": -53.907920837402344,
"logps/ref_rejected": -95.1163330078125,
"logps/rejected": -665.1487426757812,
"loss": 0.9327,
"margin_dpo/margin_mean": 190.07733154296875,
"margin_dpo/margin_std": 199.5963897705078,
"step": 479
},
{
"KL/chosen_KL_mean": -391.65753173828125,
"KL/mean": -476.282470703125,
"KL/rejected_KL_mean": -560.9075317382812,
"KL/std": 195.97769165039062,
"epoch": 0.7048458149779736,
"fcm_dpo/beta": 0.0038283944595605135,
"fcm_dpo/delta": 0.15897266566753387,
"fcm_dpo/margin": 169.2499542236328,
"fcm_dpo/q_t": 0.36624574661254883,
"grad_norm": 58.622314453125,
"learning_rate": 1.227838333989088e-07,
"logits/chosen": 0.20244859158992767,
"logits/rejected": 0.23983803391456604,
"logps/chosen": -450.3402099609375,
"logps/ref_chosen": -58.682701110839844,
"logps/ref_rejected": -82.93248748779297,
"logps/rejected": -643.8399658203125,
"loss": 0.9996,
"margin_dpo/margin_mean": 169.2499542236328,
"margin_dpo/margin_std": 217.9320068359375,
"step": 480
},
{
"KL/chosen_KL_mean": -344.9322509765625,
"KL/mean": -453.0932312011719,
"KL/rejected_KL_mean": -561.254150390625,
"KL/std": 219.3949737548828,
"epoch": 0.7063142437591777,
"fcm_dpo/beta": 0.003801655489951372,
"fcm_dpo/delta": -0.024488603696227074,
"fcm_dpo/margin": 216.32192993164062,
"fcm_dpo/q_t": 0.3276780843734741,
"grad_norm": 44.4178352355957,
"learning_rate": 1.2168076391719489e-07,
"logits/chosen": 0.1561613380908966,
"logits/rejected": 0.18021619319915771,
"logps/chosen": -399.89654541015625,
"logps/ref_chosen": -54.964271545410156,
"logps/ref_rejected": -92.42044067382812,
"logps/rejected": -653.6746215820312,
"loss": 0.8579,
"margin_dpo/margin_mean": 216.32192993164062,
"margin_dpo/margin_std": 211.07113647460938,
"step": 481
},
{
"KL/chosen_KL_mean": -358.85565185546875,
"KL/mean": -441.71246337890625,
"KL/rejected_KL_mean": -524.5692749023438,
"KL/std": 197.96920776367188,
"epoch": 0.7077826725403817,
"fcm_dpo/beta": 0.0038895574398338795,
"fcm_dpo/delta": 0.16643041372299194,
"fcm_dpo/margin": 165.71365356445312,
"fcm_dpo/q_t": 0.3640594482421875,
"grad_norm": 45.240779876708984,
"learning_rate": 1.2058107576668938e-07,
"logits/chosen": 0.04070214927196503,
"logits/rejected": 0.056692514568567276,
"logps/chosen": -426.40911865234375,
"logps/ref_chosen": -67.553466796875,
"logps/ref_rejected": -87.58953857421875,
"logps/rejected": -612.1588134765625,
"loss": 0.9918,
"margin_dpo/margin_mean": 165.71365356445312,
"margin_dpo/margin_std": 216.013916015625,
"step": 482
},
{
"KL/chosen_KL_mean": -339.46856689453125,
"KL/mean": -464.3380126953125,
"KL/rejected_KL_mean": -589.2074584960938,
"KL/std": 215.98431396484375,
"epoch": 0.7092511013215859,
"fcm_dpo/beta": 0.003823434002697468,
"fcm_dpo/delta": -0.17181670665740967,
"fcm_dpo/margin": 249.7388916015625,
"fcm_dpo/q_t": 0.3055006265640259,
"grad_norm": 33.2778434753418,
"learning_rate": 1.194847979251979e-07,
"logits/chosen": 0.10603588819503784,
"logits/rejected": 0.14284950494766235,
"logps/chosen": -402.79840087890625,
"logps/ref_chosen": -63.32981872558594,
"logps/ref_rejected": -95.78697204589844,
"logps/rejected": -684.994384765625,
"loss": 0.7874,
"margin_dpo/margin_mean": 249.7388916015625,
"margin_dpo/margin_std": 225.68191528320312,
"step": 483
},
{
"KL/chosen_KL_mean": -308.18743896484375,
"KL/mean": -417.3006591796875,
"KL/rejected_KL_mean": -526.4138793945312,
"KL/std": 197.73350524902344,
"epoch": 0.71071953010279,
"fcm_dpo/beta": 0.0038092981558293104,
"fcm_dpo/delta": -0.035128235816955566,
"fcm_dpo/margin": 218.22645568847656,
"fcm_dpo/q_t": 0.3240478038787842,
"grad_norm": 34.91592788696289,
"learning_rate": 1.1839195928066101e-07,
"logits/chosen": 0.17551764845848083,
"logits/rejected": 0.20704582333564758,
"logps/chosen": -367.3255615234375,
"logps/ref_chosen": -59.13812255859375,
"logps/ref_rejected": -84.37144470214844,
"logps/rejected": -610.7853393554688,
"loss": 0.8259,
"margin_dpo/margin_mean": 218.2264404296875,
"margin_dpo/margin_std": 186.5542449951172,
"step": 484
},
{
"KL/chosen_KL_mean": -285.511474609375,
"KL/mean": -398.1817626953125,
"KL/rejected_KL_mean": -510.85205078125,
"KL/std": 202.4456024169922,
"epoch": 0.7121879588839941,
"fcm_dpo/beta": 0.003755971323698759,
"fcm_dpo/delta": -0.05070215463638306,
"fcm_dpo/margin": 225.34059143066406,
"fcm_dpo/q_t": 0.32257890701293945,
"grad_norm": 31.80185317993164,
"learning_rate": 1.1730258863039347e-07,
"logits/chosen": 0.06667602062225342,
"logits/rejected": 0.08204781264066696,
"logps/chosen": -344.36102294921875,
"logps/ref_chosen": -58.849571228027344,
"logps/ref_rejected": -103.36408233642578,
"logps/rejected": -614.2161865234375,
"loss": 0.842,
"margin_dpo/margin_mean": 225.34059143066406,
"margin_dpo/margin_std": 212.36538696289062,
"step": 485
},
{
"KL/chosen_KL_mean": -327.9944763183594,
"KL/mean": -442.83380126953125,
"KL/rejected_KL_mean": -557.673095703125,
"KL/std": 211.04153442382812,
"epoch": 0.7136563876651982,
"fcm_dpo/beta": 0.003716058097779751,
"fcm_dpo/delta": -0.05855964869260788,
"fcm_dpo/margin": 229.6786651611328,
"fcm_dpo/q_t": 0.3270586133003235,
"grad_norm": 57.901912689208984,
"learning_rate": 1.1621671468032493e-07,
"logits/chosen": 0.14983290433883667,
"logits/rejected": 0.17763462662696838,
"logps/chosen": -383.254150390625,
"logps/ref_chosen": -55.25966262817383,
"logps/ref_rejected": -92.13936614990234,
"logps/rejected": -649.8125,
"loss": 0.8833,
"margin_dpo/margin_mean": 229.6786651611328,
"margin_dpo/margin_std": 250.62548828125,
"step": 486
},
{
"KL/chosen_KL_mean": -321.1956481933594,
"KL/mean": -420.681640625,
"KL/rejected_KL_mean": -520.1676025390625,
"KL/std": 171.1717529296875,
"epoch": 0.7151248164464024,
"fcm_dpo/beta": 0.003711356781423092,
"fcm_dpo/delta": 0.0663451999425888,
"fcm_dpo/margin": 198.97198486328125,
"fcm_dpo/q_t": 0.3368815779685974,
"grad_norm": 44.26673126220703,
"learning_rate": 1.1513436604424378e-07,
"logits/chosen": 0.10643830895423889,
"logits/rejected": 0.1315649002790451,
"logps/chosen": -374.25897216796875,
"logps/ref_chosen": -53.06330871582031,
"logps/ref_rejected": -92.41883087158203,
"logps/rejected": -612.58642578125,
"loss": 0.8648,
"margin_dpo/margin_mean": 198.97198486328125,
"margin_dpo/margin_std": 166.13626098632812,
"step": 487
},
{
"KL/chosen_KL_mean": -299.201416015625,
"KL/mean": -398.70916748046875,
"KL/rejected_KL_mean": -498.2169189453125,
"KL/std": 195.89431762695312,
"epoch": 0.7165932452276065,
"fcm_dpo/beta": 0.003781733103096485,
"fcm_dpo/delta": 0.050800926983356476,
"fcm_dpo/margin": 199.01553344726562,
"fcm_dpo/q_t": 0.3352172374725342,
"grad_norm": 42.39677047729492,
"learning_rate": 1.1405557124304335e-07,
"logits/chosen": 0.13744409382343292,
"logits/rejected": 0.16995671391487122,
"logps/chosen": -351.4295654296875,
"logps/ref_chosen": -52.22815704345703,
"logps/ref_rejected": -84.00656127929688,
"logps/rejected": -582.2235107421875,
"loss": 0.8553,
"margin_dpo/margin_mean": 199.01553344726562,
"margin_dpo/margin_std": 166.0457305908203,
"step": 488
},
{
"KL/chosen_KL_mean": -295.97509765625,
"KL/mean": -381.76287841796875,
"KL/rejected_KL_mean": -467.5506896972656,
"KL/std": 193.10520935058594,
"epoch": 0.7180616740088106,
"fcm_dpo/beta": 0.0038628973998129368,
"fcm_dpo/delta": 0.1469191014766693,
"fcm_dpo/margin": 171.57559204101562,
"fcm_dpo/q_t": 0.36090078949928284,
"grad_norm": 44.343040466308594,
"learning_rate": 1.1298035870396985e-07,
"logits/chosen": 0.03998090326786041,
"logits/rejected": 0.06467624753713608,
"logps/chosen": -351.9647216796875,
"logps/ref_chosen": -55.989627838134766,
"logps/ref_rejected": -79.39812469482422,
"logps/rejected": -546.9488525390625,
"loss": 0.9379,
"margin_dpo/margin_mean": 171.5756072998047,
"margin_dpo/margin_std": 188.32510375976562,
"step": 489
},
{
"KL/chosen_KL_mean": -306.4906005859375,
"KL/mean": -406.5208435058594,
"KL/rejected_KL_mean": -506.55108642578125,
"KL/std": 221.122802734375,
"epoch": 0.7195301027900147,
"fcm_dpo/beta": 0.003920567687600851,
"fcm_dpo/delta": 0.01635659858584404,
"fcm_dpo/margin": 200.06051635742188,
"fcm_dpo/q_t": 0.3403034508228302,
"grad_norm": 34.440757751464844,
"learning_rate": 1.1190875675987355e-07,
"logits/chosen": 0.07778964936733246,
"logits/rejected": 0.08467268198728561,
"logps/chosen": -358.85699462890625,
"logps/ref_chosen": -52.36639404296875,
"logps/ref_rejected": -110.4090576171875,
"logps/rejected": -616.9601440429688,
"loss": 0.9101,
"margin_dpo/margin_mean": 200.06051635742188,
"margin_dpo/margin_std": 225.04501342773438,
"step": 490
},
{
"KL/chosen_KL_mean": -340.3194580078125,
"KL/mean": -423.6872253417969,
"KL/rejected_KL_mean": -507.05499267578125,
"KL/std": 196.24343872070312,
"epoch": 0.7209985315712188,
"fcm_dpo/beta": 0.003996577113866806,
"fcm_dpo/delta": 0.14273640513420105,
"fcm_dpo/margin": 166.73556518554688,
"fcm_dpo/q_t": 0.36313116550445557,
"grad_norm": 44.4641227722168,
"learning_rate": 1.1084079364846241e-07,
"logits/chosen": 0.1599426120519638,
"logits/rejected": 0.18364211916923523,
"logps/chosen": -400.4356994628906,
"logps/ref_chosen": -60.11626434326172,
"logps/ref_rejected": -73.27278900146484,
"logps/rejected": -580.3277587890625,
"loss": 0.9682,
"margin_dpo/margin_mean": 166.73556518554688,
"margin_dpo/margin_std": 207.41067504882812,
"step": 491
},
{
"KL/chosen_KL_mean": -371.9100341796875,
"KL/mean": -440.6224060058594,
"KL/rejected_KL_mean": -509.33477783203125,
"KL/std": 191.6051025390625,
"epoch": 0.7224669603524229,
"fcm_dpo/beta": 0.0041659013368189335,
"fcm_dpo/delta": 0.24118927121162415,
"fcm_dpo/margin": 137.42471313476562,
"fcm_dpo/q_t": 0.38607728481292725,
"grad_norm": 58.81159591674805,
"learning_rate": 1.097764975115576e-07,
"logits/chosen": 0.24060788750648499,
"logits/rejected": 0.2635948956012726,
"logps/chosen": -425.90423583984375,
"logps/ref_chosen": -53.994178771972656,
"logps/ref_rejected": -72.65962219238281,
"logps/rejected": -581.994384765625,
"loss": 1.1029,
"margin_dpo/margin_mean": 137.42471313476562,
"margin_dpo/margin_std": 237.78509521484375,
"step": 492
},
{
"KL/chosen_KL_mean": -365.27728271484375,
"KL/mean": -450.06695556640625,
"KL/rejected_KL_mean": -534.8565673828125,
"KL/std": 205.73464965820312,
"epoch": 0.723935389133627,
"fcm_dpo/beta": 0.0042484961450099945,
"fcm_dpo/delta": 0.08563549071550369,
"fcm_dpo/margin": 169.57925415039062,
"fcm_dpo/q_t": 0.3462260067462921,
"grad_norm": 44.68215560913086,
"learning_rate": 1.0871589639435203e-07,
"logits/chosen": 0.22852273285388947,
"logits/rejected": 0.2710832357406616,
"logps/chosen": -440.7745361328125,
"logps/ref_chosen": -75.49723815917969,
"logps/ref_rejected": -87.32301330566406,
"logps/rejected": -622.1795654296875,
"loss": 0.9105,
"margin_dpo/margin_mean": 169.5792236328125,
"margin_dpo/margin_std": 173.59259033203125,
"step": 493
},
{
"KL/chosen_KL_mean": -329.7471008300781,
"KL/mean": -448.144775390625,
"KL/rejected_KL_mean": -566.5425415039062,
"KL/std": 210.83590698242188,
"epoch": 0.7254038179148311,
"fcm_dpo/beta": 0.004156805574893951,
"fcm_dpo/delta": -0.20466670393943787,
"fcm_dpo/margin": 236.79537963867188,
"fcm_dpo/q_t": 0.2935987114906311,
"grad_norm": 42.584938049316406,
"learning_rate": 1.0765901824467166e-07,
"logits/chosen": 0.22543635964393616,
"logits/rejected": 0.2317037582397461,
"logps/chosen": -371.10638427734375,
"logps/ref_chosen": -41.35926818847656,
"logps/ref_rejected": -86.09136962890625,
"logps/rejected": -652.6339111328125,
"loss": 0.7692,
"margin_dpo/margin_mean": 236.79537963867188,
"margin_dpo/margin_std": 198.07933044433594,
"step": 494
},
{
"KL/chosen_KL_mean": -370.0741882324219,
"KL/mean": -475.6034851074219,
"KL/rejected_KL_mean": -581.1327514648438,
"KL/std": 207.8790283203125,
"epoch": 0.7268722466960352,
"fcm_dpo/beta": 0.004097241908311844,
"fcm_dpo/delta": -0.07118328660726547,
"fcm_dpo/margin": 211.05856323242188,
"fcm_dpo/q_t": 0.3244919180870056,
"grad_norm": 42.264686584472656,
"learning_rate": 1.0660589091223854e-07,
"logits/chosen": 0.1444653421640396,
"logits/rejected": 0.16225489974021912,
"logps/chosen": -433.6092529296875,
"logps/ref_chosen": -63.53507995605469,
"logps/ref_rejected": -91.42443084716797,
"logps/rejected": -672.5571899414062,
"loss": 0.8478,
"margin_dpo/margin_mean": 211.05856323242188,
"margin_dpo/margin_std": 213.89817810058594,
"step": 495
},
{
"KL/chosen_KL_mean": -429.86212158203125,
"KL/mean": -512.4937744140625,
"KL/rejected_KL_mean": -595.12548828125,
"KL/std": 178.7728729248047,
"epoch": 0.7283406754772394,
"fcm_dpo/beta": 0.004128463100641966,
"fcm_dpo/delta": 0.1264854371547699,
"fcm_dpo/margin": 165.26336669921875,
"fcm_dpo/q_t": 0.3561249375343323,
"grad_norm": 46.71969223022461,
"learning_rate": 1.0555654214793722e-07,
"logits/chosen": 0.20871032774448395,
"logits/rejected": 0.24310749769210815,
"logps/chosen": -502.45404052734375,
"logps/ref_chosen": -72.5919189453125,
"logps/ref_rejected": -84.32933807373047,
"logps/rejected": -679.454833984375,
"loss": 0.9337,
"margin_dpo/margin_mean": 165.26336669921875,
"margin_dpo/margin_std": 178.31752014160156,
"step": 496
},
{
"KL/chosen_KL_mean": -427.16143798828125,
"KL/mean": -507.173828125,
"KL/rejected_KL_mean": -587.1861572265625,
"KL/std": 211.1545867919922,
"epoch": 0.7298091042584435,
"fcm_dpo/beta": 0.004193710163235664,
"fcm_dpo/delta": 0.13649700582027435,
"fcm_dpo/margin": 160.02471923828125,
"fcm_dpo/q_t": 0.3586384654045105,
"grad_norm": 39.61761474609375,
"learning_rate": 1.0451099960308374e-07,
"logits/chosen": 0.22612471878528595,
"logits/rejected": 0.2511892318725586,
"logps/chosen": -485.75543212890625,
"logps/ref_chosen": -58.59397506713867,
"logps/ref_rejected": -76.28836822509766,
"logps/rejected": -663.4745483398438,
"loss": 0.9579,
"margin_dpo/margin_mean": 160.02471923828125,
"margin_dpo/margin_std": 175.66616821289062,
"step": 497
},
{
"KL/chosen_KL_mean": -437.52252197265625,
"KL/mean": -525.6541137695312,
"KL/rejected_KL_mean": -613.7857055664062,
"KL/std": 198.28433227539062,
"epoch": 0.7312775330396476,
"fcm_dpo/beta": 0.00430191308259964,
"fcm_dpo/delta": 0.04517921805381775,
"fcm_dpo/margin": 176.26318359375,
"fcm_dpo/q_t": 0.3466299772262573,
"grad_norm": 46.89978790283203,
"learning_rate": 1.0346929082869641e-07,
"logits/chosen": 0.22263531386852264,
"logits/rejected": 0.24923723936080933,
"logps/chosen": -508.7281494140625,
"logps/ref_chosen": -71.20565795898438,
"logps/ref_rejected": -83.95803833007812,
"logps/rejected": -697.7437744140625,
"loss": 0.9419,
"margin_dpo/margin_mean": 176.26318359375,
"margin_dpo/margin_std": 214.70091247558594,
"step": 498
},
{
"KL/chosen_KL_mean": -423.108642578125,
"KL/mean": -536.3960571289062,
"KL/rejected_KL_mean": -649.6835327148438,
"KL/std": 213.3042449951172,
"epoch": 0.7327459618208517,
"fcm_dpo/beta": 0.004202335141599178,
"fcm_dpo/delta": -0.16854572296142578,
"fcm_dpo/margin": 226.5749053955078,
"fcm_dpo/q_t": 0.3053300976753235,
"grad_norm": 44.01908493041992,
"learning_rate": 1.0243144327477013e-07,
"logits/chosen": 0.3921792805194855,
"logits/rejected": 0.40327873826026917,
"logps/chosen": -474.36383056640625,
"logps/ref_chosen": -51.25519561767578,
"logps/ref_rejected": -101.07870483398438,
"logps/rejected": -750.76220703125,
"loss": 0.8274,
"margin_dpo/margin_mean": 226.5749053955078,
"margin_dpo/margin_std": 223.81814575195312,
"step": 499
},
{
"KL/chosen_KL_mean": -462.30181884765625,
"KL/mean": -562.2794189453125,
"KL/rejected_KL_mean": -662.257080078125,
"KL/std": 209.902587890625,
"epoch": 0.7342143906020558,
"fcm_dpo/beta": 0.004142479971051216,
"fcm_dpo/delta": -0.031111618503928185,
"fcm_dpo/margin": 199.95518493652344,
"fcm_dpo/q_t": 0.3318169414997101,
"grad_norm": 54.924827575683594,
"learning_rate": 1.0139748428955333e-07,
"logits/chosen": 0.2692311108112335,
"logits/rejected": 0.2694999873638153,
"logps/chosen": -519.3292846679688,
"logps/ref_chosen": -57.027442932128906,
"logps/ref_rejected": -93.93421173095703,
"logps/rejected": -756.1912841796875,
"loss": 0.8904,
"margin_dpo/margin_mean": 199.95518493652344,
"margin_dpo/margin_std": 222.5806884765625,
"step": 500
},
{
"KL/chosen_KL_mean": -428.00701904296875,
"KL/mean": -532.467529296875,
"KL/rejected_KL_mean": -636.9281005859375,
"KL/std": 207.19921875,
"epoch": 0.73568281938326,
"fcm_dpo/beta": 0.0041295429691672325,
"fcm_dpo/delta": -0.0689932331442833,
"fcm_dpo/margin": 208.921142578125,
"fcm_dpo/q_t": 0.3278210759162903,
"grad_norm": 55.09239959716797,
"learning_rate": 1.0036744111882672e-07,
"logits/chosen": 0.2993459403514862,
"logits/rejected": 0.3334452509880066,
"logps/chosen": -482.36651611328125,
"logps/ref_chosen": -54.359527587890625,
"logps/ref_rejected": -80.15670013427734,
"logps/rejected": -717.0848388671875,
"loss": 0.8729,
"margin_dpo/margin_mean": 208.921142578125,
"margin_dpo/margin_std": 227.49624633789062,
"step": 501
},
{
"KL/chosen_KL_mean": -392.55389404296875,
"KL/mean": -508.1488342285156,
"KL/rejected_KL_mean": -623.7437744140625,
"KL/std": 193.1407470703125,
"epoch": 0.737151248164464,
"fcm_dpo/beta": 0.00403004651889205,
"fcm_dpo/delta": -0.14517813920974731,
"fcm_dpo/margin": 231.1898956298828,
"fcm_dpo/q_t": 0.29726549983024597,
"grad_norm": 32.64988327026367,
"learning_rate": 9.934134090518592e-08,
"logits/chosen": 0.20329806208610535,
"logits/rejected": 0.23956874012947083,
"logps/chosen": -460.1544189453125,
"logps/ref_chosen": -67.60050964355469,
"logps/ref_rejected": -82.94876098632812,
"logps/rejected": -706.6925048828125,
"loss": 0.7467,
"margin_dpo/margin_mean": 231.1898956298828,
"margin_dpo/margin_std": 155.42315673828125,
"step": 502
},
{
"KL/chosen_KL_mean": -414.6917419433594,
"KL/mean": -511.0799560546875,
"KL/rejected_KL_mean": -607.4681396484375,
"KL/std": 182.72433471679688,
"epoch": 0.7386196769456681,
"fcm_dpo/beta": 0.0039795683696866035,
"fcm_dpo/delta": 0.03540978953242302,
"fcm_dpo/margin": 192.7764129638672,
"fcm_dpo/q_t": 0.335806667804718,
"grad_norm": 36.449703216552734,
"learning_rate": 9.831921068732571e-08,
"logits/chosen": 0.30815067887306213,
"logits/rejected": 0.33968621492385864,
"logps/chosen": -469.7701416015625,
"logps/ref_chosen": -55.078407287597656,
"logps/ref_rejected": -82.50544738769531,
"logps/rejected": -689.9736328125,
"loss": 0.8623,
"margin_dpo/margin_mean": 192.7764129638672,
"margin_dpo/margin_std": 176.17752075195312,
"step": 503
},
{
"KL/chosen_KL_mean": -423.18780517578125,
"KL/mean": -543.5272216796875,
"KL/rejected_KL_mean": -663.86669921875,
"KL/std": 212.20285034179688,
"epoch": 0.7400881057268722,
"fcm_dpo/beta": 0.00390539038926363,
"fcm_dpo/delta": -0.154744952917099,
"fcm_dpo/margin": 240.67892456054688,
"fcm_dpo/q_t": 0.3052040934562683,
"grad_norm": 37.20085906982422,
"learning_rate": 9.730107739932805e-08,
"logits/chosen": 0.29225870966911316,
"logits/rejected": 0.29667824506759644,
"logps/chosen": -483.1535339355469,
"logps/ref_chosen": -59.96575164794922,
"logps/ref_rejected": -103.76212310791016,
"logps/rejected": -767.6288452148438,
"loss": 0.7907,
"margin_dpo/margin_mean": 240.67892456054688,
"margin_dpo/margin_std": 207.40602111816406,
"step": 504
},
{
"KL/chosen_KL_mean": -443.1572570800781,
"KL/mean": -517.259521484375,
"KL/rejected_KL_mean": -591.3617553710938,
"KL/std": 220.9525146484375,
"epoch": 0.7415565345080763,
"fcm_dpo/beta": 0.00401611253619194,
"fcm_dpo/delta": 0.2171427607536316,
"fcm_dpo/margin": 148.2044677734375,
"fcm_dpo/q_t": 0.3818475008010864,
"grad_norm": 64.7453384399414,
"learning_rate": 9.628696786995188e-08,
"logits/chosen": 0.24049341678619385,
"logits/rejected": 0.2687991261482239,
"logps/chosen": -519.3121337890625,
"logps/ref_chosen": -76.1549072265625,
"logps/ref_rejected": -88.58537292480469,
"logps/rejected": -679.9471435546875,
"loss": 1.0594,
"margin_dpo/margin_mean": 148.20448303222656,
"margin_dpo/margin_std": 231.49600219726562,
"step": 505
},
{
"KL/chosen_KL_mean": -386.2157287597656,
"KL/mean": -494.36199951171875,
"KL/rejected_KL_mean": -602.50830078125,
"KL/std": 219.51278686523438,
"epoch": 0.7430249632892805,
"fcm_dpo/beta": 0.0039803218096494675,
"fcm_dpo/delta": -0.06841699779033661,
"fcm_dpo/margin": 216.2925567626953,
"fcm_dpo/q_t": 0.3228600323200226,
"grad_norm": 34.961204528808594,
"learning_rate": 9.527690882192635e-08,
"logits/chosen": 0.281488299369812,
"logits/rejected": 0.30117908120155334,
"logps/chosen": -435.17620849609375,
"logps/ref_chosen": -48.96050262451172,
"logps/ref_rejected": -78.41505432128906,
"logps/rejected": -680.92333984375,
"loss": 0.8502,
"margin_dpo/margin_mean": 216.2925567626953,
"margin_dpo/margin_std": 212.88848876953125,
"step": 506
},
{
"KL/chosen_KL_mean": -417.7148132324219,
"KL/mean": -510.52874755859375,
"KL/rejected_KL_mean": -603.3427124023438,
"KL/std": 215.5636749267578,
"epoch": 0.7444933920704846,
"fcm_dpo/beta": 0.004029689822345972,
"fcm_dpo/delta": 0.056185171008110046,
"fcm_dpo/margin": 185.62789916992188,
"fcm_dpo/q_t": 0.3467479944229126,
"grad_norm": 33.984588623046875,
"learning_rate": 9.427092687124691e-08,
"logits/chosen": 0.32121342420578003,
"logits/rejected": 0.34334227442741394,
"logps/chosen": -484.51629638671875,
"logps/ref_chosen": -66.80149841308594,
"logps/ref_rejected": -95.37289428710938,
"logps/rejected": -698.715576171875,
"loss": 0.9057,
"margin_dpo/margin_mean": 185.62791442871094,
"margin_dpo/margin_std": 205.94578552246094,
"step": 507
},
{
"KL/chosen_KL_mean": -395.92645263671875,
"KL/mean": -485.9989013671875,
"KL/rejected_KL_mean": -576.0712890625,
"KL/std": 222.60186767578125,
"epoch": 0.7459618208516887,
"fcm_dpo/beta": 0.0040952954441308975,
"fcm_dpo/delta": 0.06677697598934174,
"fcm_dpo/margin": 180.14486694335938,
"fcm_dpo/q_t": 0.3528652787208557,
"grad_norm": 45.727481842041016,
"learning_rate": 9.326904852647344e-08,
"logits/chosen": 0.2253168672323227,
"logits/rejected": 0.23925551772117615,
"logps/chosen": -467.22991943359375,
"logps/ref_chosen": -71.303466796875,
"logps/ref_rejected": -95.6275405883789,
"logps/rejected": -671.6988525390625,
"loss": 0.9525,
"margin_dpo/margin_mean": 180.14486694335938,
"margin_dpo/margin_std": 225.4059295654297,
"step": 508
},
{
"KL/chosen_KL_mean": -346.7314453125,
"KL/mean": -436.817626953125,
"KL/rejected_KL_mean": -526.90380859375,
"KL/std": 187.9528045654297,
"epoch": 0.7474302496328928,
"fcm_dpo/beta": 0.0041460758075118065,
"fcm_dpo/delta": 0.05677645653486252,
"fcm_dpo/margin": 180.17236328125,
"fcm_dpo/q_t": 0.34532153606414795,
"grad_norm": 53.75666427612305,
"learning_rate": 9.227130018803195e-08,
"logits/chosen": 0.09170372784137726,
"logits/rejected": 0.10787695646286011,
"logps/chosen": -410.5503845214844,
"logps/ref_chosen": -63.81895065307617,
"logps/ref_rejected": -83.25643920898438,
"logps/rejected": -610.1602783203125,
"loss": 0.9047,
"margin_dpo/margin_mean": 180.17236328125,
"margin_dpo/margin_std": 193.12432861328125,
"step": 509
},
{
"KL/chosen_KL_mean": -383.3047180175781,
"KL/mean": -495.38372802734375,
"KL/rejected_KL_mean": -607.4627685546875,
"KL/std": 199.19956970214844,
"epoch": 0.748898678414097,
"fcm_dpo/beta": 0.004071595147252083,
"fcm_dpo/delta": -0.12390496581792831,
"fcm_dpo/margin": 224.15802001953125,
"fcm_dpo/q_t": 0.3062693178653717,
"grad_norm": 41.95302200317383,
"learning_rate": 9.127770814751932e-08,
"logits/chosen": 0.1895131915807724,
"logits/rejected": 0.1943446695804596,
"logps/chosen": -435.18316650390625,
"logps/ref_chosen": -51.878448486328125,
"logps/ref_rejected": -102.7651596069336,
"logps/rejected": -710.2279052734375,
"loss": 0.7903,
"margin_dpo/margin_mean": 224.1580352783203,
"margin_dpo/margin_std": 186.15345764160156,
"step": 510
},
{
"KL/chosen_KL_mean": -357.55572509765625,
"KL/mean": -462.718017578125,
"KL/rejected_KL_mean": -567.88037109375,
"KL/std": 215.96746826171875,
"epoch": 0.750367107195301,
"fcm_dpo/beta": 0.004031065851449966,
"fcm_dpo/delta": -0.052543915808200836,
"fcm_dpo/margin": 210.32461547851562,
"fcm_dpo/q_t": 0.3308233618736267,
"grad_norm": 35.133419036865234,
"learning_rate": 9.028829858700973e-08,
"logits/chosen": 0.2895551919937134,
"logits/rejected": 0.3053359091281891,
"logps/chosen": -417.7938537597656,
"logps/ref_chosen": -60.23811721801758,
"logps/ref_rejected": -92.85676574707031,
"logps/rejected": -660.7371215820312,
"loss": 0.8985,
"margin_dpo/margin_mean": 210.32460021972656,
"margin_dpo/margin_std": 240.78411865234375,
"step": 511
},
{
"KL/chosen_KL_mean": -330.9661865234375,
"KL/mean": -445.86968994140625,
"KL/rejected_KL_mean": -560.773193359375,
"KL/std": 198.74737548828125,
"epoch": 0.7518355359765051,
"fcm_dpo/beta": 0.003930510953068733,
"fcm_dpo/delta": -0.11358829587697983,
"fcm_dpo/margin": 229.80703735351562,
"fcm_dpo/q_t": 0.3084445595741272,
"grad_norm": 35.95514678955078,
"learning_rate": 8.930309757836516e-08,
"logits/chosen": 0.2970372438430786,
"logits/rejected": 0.3260389566421509,
"logps/chosen": -385.8717041015625,
"logps/ref_chosen": -54.905494689941406,
"logps/ref_rejected": -81.87586975097656,
"logps/rejected": -642.6491088867188,
"loss": 0.7866,
"margin_dpo/margin_mean": 229.80703735351562,
"margin_dpo/margin_std": 188.13687133789062,
"step": 512
},
{
"KL/chosen_KL_mean": -364.7303466796875,
"KL/mean": -452.98175048828125,
"KL/rejected_KL_mean": -541.2332153320312,
"KL/std": 184.21725463867188,
"epoch": 0.7533039647577092,
"fcm_dpo/beta": 0.0039430903270840645,
"fcm_dpo/delta": 0.1113673523068428,
"fcm_dpo/margin": 176.50286865234375,
"fcm_dpo/q_t": 0.3536303639411926,
"grad_norm": 54.2635498046875,
"learning_rate": 8.832213108254863e-08,
"logits/chosen": 0.24457916617393494,
"logits/rejected": 0.28037741780281067,
"logps/chosen": -429.64678955078125,
"logps/ref_chosen": -64.91644287109375,
"logps/ref_rejected": -76.06245422363281,
"logps/rejected": -617.295654296875,
"loss": 0.9551,
"margin_dpo/margin_mean": 176.5028533935547,
"margin_dpo/margin_std": 205.36679077148438,
"step": 513
},
{
"KL/chosen_KL_mean": -346.7003479003906,
"KL/mean": -434.6519775390625,
"KL/rejected_KL_mean": -522.6036376953125,
"KL/std": 185.1610870361328,
"epoch": 0.7547723935389133,
"fcm_dpo/beta": 0.0040502483025193214,
"fcm_dpo/delta": 0.0943315327167511,
"fcm_dpo/margin": 175.90335083007812,
"fcm_dpo/q_t": 0.35776999592781067,
"grad_norm": 48.51445770263672,
"learning_rate": 8.734542494893954e-08,
"logits/chosen": 0.15230363607406616,
"logits/rejected": 0.1823866069316864,
"logps/chosen": -420.929931640625,
"logps/ref_chosen": -74.22957611083984,
"logps/ref_rejected": -78.945556640625,
"logps/rejected": -601.5491943359375,
"loss": 0.9599,
"margin_dpo/margin_mean": 175.90333557128906,
"margin_dpo/margin_std": 223.0550079345703,
"step": 514
},
{
"KL/chosen_KL_mean": -307.29766845703125,
"KL/mean": -386.40625,
"KL/rejected_KL_mean": -465.5148010253906,
"KL/std": 185.33645629882812,
"epoch": 0.7562408223201175,
"fcm_dpo/beta": 0.00415542908012867,
"fcm_dpo/delta": 0.15283825993537903,
"fcm_dpo/margin": 158.21713256835938,
"fcm_dpo/q_t": 0.36064812541007996,
"grad_norm": 34.504180908203125,
"learning_rate": 8.637300491465272e-08,
"logits/chosen": 0.12813155353069305,
"logits/rejected": 0.13620558381080627,
"logps/chosen": -357.6992492675781,
"logps/ref_chosen": -50.40156555175781,
"logps/ref_rejected": -87.09774780273438,
"logps/rejected": -552.612548828125,
"loss": 0.9722,
"margin_dpo/margin_mean": 158.21713256835938,
"margin_dpo/margin_std": 191.54144287109375,
"step": 515
},
{
"KL/chosen_KL_mean": -318.70489501953125,
"KL/mean": -427.0130920410156,
"KL/rejected_KL_mean": -535.3212890625,
"KL/std": 196.55706787109375,
"epoch": 0.7577092511013216,
"fcm_dpo/beta": 0.004153442569077015,
"fcm_dpo/delta": -0.10953111201524734,
"fcm_dpo/margin": 216.61639404296875,
"fcm_dpo/q_t": 0.3109952211380005,
"grad_norm": 40.440147399902344,
"learning_rate": 8.540489660386064e-08,
"logits/chosen": 0.19949766993522644,
"logits/rejected": 0.20106914639472961,
"logps/chosen": -383.3544921875,
"logps/ref_chosen": -64.64956665039062,
"logps/ref_rejected": -111.72237396240234,
"logps/rejected": -647.043701171875,
"loss": 0.7982,
"margin_dpo/margin_mean": 216.61639404296875,
"margin_dpo/margin_std": 184.9787139892578,
"step": 516
},
{
"KL/chosen_KL_mean": -329.6126708984375,
"KL/mean": -439.53631591796875,
"KL/rejected_KL_mean": -549.4599609375,
"KL/std": 210.18545532226562,
"epoch": 0.7591776798825257,
"fcm_dpo/beta": 0.0040242038667202,
"fcm_dpo/delta": -0.09458979964256287,
"fcm_dpo/margin": 219.8473358154297,
"fcm_dpo/q_t": 0.31907662749290466,
"grad_norm": 38.87144088745117,
"learning_rate": 8.444112552711752e-08,
"logits/chosen": 0.13860376179218292,
"logits/rejected": 0.1707230806350708,
"logps/chosen": -390.5262145996094,
"logps/ref_chosen": -60.913551330566406,
"logps/ref_rejected": -89.08308410644531,
"logps/rejected": -638.5430908203125,
"loss": 0.8339,
"margin_dpo/margin_mean": 219.8473358154297,
"margin_dpo/margin_std": 214.79222106933594,
"step": 517
},
{
"KL/chosen_KL_mean": -329.9923095703125,
"KL/mean": -424.657958984375,
"KL/rejected_KL_mean": -519.3236083984375,
"KL/std": 184.806884765625,
"epoch": 0.7606461086637298,
"fcm_dpo/beta": 0.00404348922893405,
"fcm_dpo/delta": 0.03724416345357895,
"fcm_dpo/margin": 189.33128356933594,
"fcm_dpo/q_t": 0.3417090177536011,
"grad_norm": 34.10972595214844,
"learning_rate": 8.348171708068747e-08,
"logits/chosen": 0.1555401086807251,
"logits/rejected": 0.1683722883462906,
"logps/chosen": -387.44818115234375,
"logps/ref_chosen": -57.45589065551758,
"logps/ref_rejected": -85.31269836425781,
"logps/rejected": -604.63623046875,
"loss": 0.9226,
"margin_dpo/margin_mean": 189.331298828125,
"margin_dpo/margin_std": 217.88580322265625,
"step": 518
},
{
"KL/chosen_KL_mean": -317.1443786621094,
"KL/mean": -401.89031982421875,
"KL/rejected_KL_mean": -486.63629150390625,
"KL/std": 192.45364379882812,
"epoch": 0.762114537444934,
"fcm_dpo/beta": 0.004131737165153027,
"fcm_dpo/delta": 0.10707257688045502,
"fcm_dpo/margin": 169.49188232421875,
"fcm_dpo/q_t": 0.35672521591186523,
"grad_norm": 38.87397384643555,
"learning_rate": 8.25266965458755e-08,
"logits/chosen": 0.10519933700561523,
"logits/rejected": 0.12509751319885254,
"logps/chosen": -391.20770263671875,
"logps/ref_chosen": -74.06331634521484,
"logps/ref_rejected": -104.44416809082031,
"logps/rejected": -591.0804443359375,
"loss": 0.954,
"margin_dpo/margin_mean": 169.49188232421875,
"margin_dpo/margin_std": 207.0226287841797,
"step": 519
},
{
"KL/chosen_KL_mean": -316.388916015625,
"KL/mean": -407.5377502441406,
"KL/rejected_KL_mean": -498.68658447265625,
"KL/std": 183.60214233398438,
"epoch": 0.7635829662261381,
"fcm_dpo/beta": 0.0041408720426261425,
"fcm_dpo/delta": 0.047438718378543854,
"fcm_dpo/margin": 182.29766845703125,
"fcm_dpo/q_t": 0.3425585627555847,
"grad_norm": 35.9736213684082,
"learning_rate": 8.15760890883607e-08,
"logits/chosen": 0.13645502924919128,
"logits/rejected": 0.1466117650270462,
"logps/chosen": -386.688720703125,
"logps/ref_chosen": -70.2998275756836,
"logps/ref_rejected": -99.98133850097656,
"logps/rejected": -598.66796875,
"loss": 0.8873,
"margin_dpo/margin_mean": 182.29766845703125,
"margin_dpo/margin_std": 177.412841796875,
"step": 520
},
{
"KL/chosen_KL_mean": -308.92608642578125,
"KL/mean": -396.453125,
"KL/rejected_KL_mean": -483.9801330566406,
"KL/std": 180.9889678955078,
"epoch": 0.7650513950073421,
"fcm_dpo/beta": 0.004272553138434887,
"fcm_dpo/delta": 0.0528116375207901,
"fcm_dpo/margin": 175.0540771484375,
"fcm_dpo/q_t": 0.3469804525375366,
"grad_norm": 47.978233337402344,
"learning_rate": 8.062991975753378e-08,
"logits/chosen": 0.2017364799976349,
"logits/rejected": 0.22750027477741241,
"logps/chosen": -367.0690002441406,
"logps/ref_chosen": -58.14292526245117,
"logps/ref_rejected": -83.28060913085938,
"logps/rejected": -567.2607421875,
"loss": 0.9271,
"margin_dpo/margin_mean": 175.05406188964844,
"margin_dpo/margin_std": 195.4500732421875,
"step": 521
},
{
"KL/chosen_KL_mean": -332.21148681640625,
"KL/mean": -413.32086181640625,
"KL/rejected_KL_mean": -494.43017578125,
"KL/std": 178.46389770507812,
"epoch": 0.7665198237885462,
"fcm_dpo/beta": 0.004303772002458572,
"fcm_dpo/delta": 0.10948702692985535,
"fcm_dpo/margin": 162.21871948242188,
"fcm_dpo/q_t": 0.35133394598960876,
"grad_norm": 61.93113327026367,
"learning_rate": 7.968821348583643e-08,
"logits/chosen": 0.21870219707489014,
"logits/rejected": 0.2350131869316101,
"logps/chosen": -378.7591552734375,
"logps/ref_chosen": -46.54766845703125,
"logps/ref_rejected": -66.01388549804688,
"logps/rejected": -560.444091796875,
"loss": 0.9312,
"margin_dpo/margin_mean": 162.21871948242188,
"margin_dpo/margin_std": 176.44223022460938,
"step": 522
},
{
"KL/chosen_KL_mean": -325.970703125,
"KL/mean": -411.48406982421875,
"KL/rejected_KL_mean": -496.9974670410156,
"KL/std": 194.84500122070312,
"epoch": 0.7679882525697503,
"fcm_dpo/beta": 0.004355207085609436,
"fcm_dpo/delta": 0.05959649384021759,
"fcm_dpo/margin": 171.02676391601562,
"fcm_dpo/q_t": 0.34175705909729004,
"grad_norm": 45.9669189453125,
"learning_rate": 7.875099508810484e-08,
"logits/chosen": 0.1334128975868225,
"logits/rejected": 0.1688709706068039,
"logps/chosen": -387.74029541015625,
"logps/ref_chosen": -61.76960372924805,
"logps/ref_rejected": -83.76141357421875,
"logps/rejected": -580.7589111328125,
"loss": 0.9417,
"margin_dpo/margin_mean": 171.02676391601562,
"margin_dpo/margin_std": 201.88861083984375,
"step": 523
},
{
"KL/chosen_KL_mean": -332.4698791503906,
"KL/mean": -422.786865234375,
"KL/rejected_KL_mean": -513.1038818359375,
"KL/std": 188.12332153320312,
"epoch": 0.7694566813509545,
"fcm_dpo/beta": 0.0043691061437129974,
"fcm_dpo/delta": 0.011401128023862839,
"fcm_dpo/margin": 180.63401794433594,
"fcm_dpo/q_t": 0.3315002918243408,
"grad_norm": 45.90804672241211,
"learning_rate": 7.781828926091535e-08,
"logits/chosen": 0.15166175365447998,
"logits/rejected": 0.19428567588329315,
"logps/chosen": -410.54193115234375,
"logps/ref_chosen": -78.0720443725586,
"logps/ref_rejected": -81.30198669433594,
"logps/rejected": -594.4058837890625,
"loss": 0.877,
"margin_dpo/margin_mean": 180.63400268554688,
"margin_dpo/margin_std": 177.1727294921875,
"step": 524
},
{
"KL/chosen_KL_mean": -329.3613586425781,
"KL/mean": -443.7110290527344,
"KL/rejected_KL_mean": -558.0606689453125,
"KL/std": 204.1719970703125,
"epoch": 0.7709251101321586,
"fcm_dpo/beta": 0.004234597086906433,
"fcm_dpo/delta": -0.18942071497440338,
"fcm_dpo/margin": 228.69935607910156,
"fcm_dpo/q_t": 0.2971675992012024,
"grad_norm": 40.713829040527344,
"learning_rate": 7.689012058193384e-08,
"logits/chosen": 0.23026111721992493,
"logits/rejected": 0.23547565937042236,
"logps/chosen": -380.189208984375,
"logps/ref_chosen": -50.827857971191406,
"logps/ref_rejected": -100.05294036865234,
"logps/rejected": -658.1136474609375,
"loss": 0.7668,
"margin_dpo/margin_mean": 228.69935607910156,
"margin_dpo/margin_std": 187.14825439453125,
"step": 525
},
{
"KL/chosen_KL_mean": -326.16119384765625,
"KL/mean": -438.5784606933594,
"KL/rejected_KL_mean": -550.9957275390625,
"KL/std": 197.4532470703125,
"epoch": 0.7723935389133627,
"fcm_dpo/beta": 0.004164829384535551,
"fcm_dpo/delta": -0.1505172997713089,
"fcm_dpo/margin": 224.83453369140625,
"fcm_dpo/q_t": 0.3045281171798706,
"grad_norm": 35.237789154052734,
"learning_rate": 7.596651350926836e-08,
"logits/chosen": 0.11118023097515106,
"logits/rejected": 0.14706848561763763,
"logps/chosen": -389.32843017578125,
"logps/ref_chosen": -63.167236328125,
"logps/ref_rejected": -86.30934143066406,
"logps/rejected": -637.3050537109375,
"loss": 0.7863,
"margin_dpo/margin_mean": 224.83456420898438,
"margin_dpo/margin_std": 190.66650390625,
"step": 526
},
{
"KL/chosen_KL_mean": -355.33746337890625,
"KL/mean": -443.8697509765625,
"KL/rejected_KL_mean": -532.4020385742188,
"KL/std": 185.5819091796875,
"epoch": 0.7738619676945668,
"fcm_dpo/beta": 0.0041229259222745895,
"fcm_dpo/delta": 0.07519938796758652,
"fcm_dpo/margin": 177.0645751953125,
"fcm_dpo/q_t": 0.3446522355079651,
"grad_norm": 32.963722229003906,
"learning_rate": 7.504749238082414e-08,
"logits/chosen": 0.27513420581817627,
"logits/rejected": 0.3138313889503479,
"logps/chosen": -426.46612548828125,
"logps/ref_chosen": -71.12867736816406,
"logps/ref_rejected": -78.3425521850586,
"logps/rejected": -610.74462890625,
"loss": 0.8913,
"margin_dpo/margin_mean": 177.0645751953125,
"margin_dpo/margin_std": 170.20639038085938,
"step": 527
},
{
"KL/chosen_KL_mean": -355.3000183105469,
"KL/mean": -449.8873291015625,
"KL/rejected_KL_mean": -544.474609375,
"KL/std": 202.4794921875,
"epoch": 0.775330396475771,
"fcm_dpo/beta": 0.004161626100540161,
"fcm_dpo/delta": 0.013718700036406517,
"fcm_dpo/margin": 189.17462158203125,
"fcm_dpo/q_t": 0.34284335374832153,
"grad_norm": 35.8046760559082,
"learning_rate": 7.413308141366254e-08,
"logits/chosen": 0.22551241517066956,
"logits/rejected": 0.2533000111579895,
"logps/chosen": -423.38946533203125,
"logps/ref_chosen": -68.0894546508789,
"logps/ref_rejected": -93.91006469726562,
"logps/rejected": -638.3847045898438,
"loss": 0.9099,
"margin_dpo/margin_mean": 189.17462158203125,
"margin_dpo/margin_std": 217.96946716308594,
"step": 528
},
{
"KL/chosen_KL_mean": -412.2044372558594,
"KL/mean": -486.0581970214844,
"KL/rejected_KL_mean": -559.9119262695312,
"KL/std": 171.0134735107422,
"epoch": 0.7767988252569751,
"fcm_dpo/beta": 0.004288380034267902,
"fcm_dpo/delta": 0.1777651607990265,
"fcm_dpo/margin": 147.70750427246094,
"fcm_dpo/q_t": 0.3686566948890686,
"grad_norm": 50.290096282958984,
"learning_rate": 7.322330470336313e-08,
"logits/chosen": 0.32002973556518555,
"logits/rejected": 0.32282739877700806,
"logps/chosen": -467.7794189453125,
"logps/ref_chosen": -55.57495880126953,
"logps/ref_rejected": -89.20909118652344,
"logps/rejected": -649.1210327148438,
"loss": 1.0074,
"margin_dpo/margin_mean": 147.70748901367188,
"margin_dpo/margin_std": 195.08078002929688,
"step": 529
},
{
"KL/chosen_KL_mean": -339.9566650390625,
"KL/mean": -456.3387451171875,
"KL/rejected_KL_mean": -572.7208251953125,
"KL/std": 200.80197143554688,
"epoch": 0.7782672540381792,
"fcm_dpo/beta": 0.004182647448033094,
"fcm_dpo/delta": -0.19384393095970154,
"fcm_dpo/margin": 232.76409912109375,
"fcm_dpo/q_t": 0.3022667169570923,
"grad_norm": 34.06055450439453,
"learning_rate": 7.231818622338822e-08,
"logits/chosen": 0.2138541340827942,
"logits/rejected": 0.2278527319431305,
"logps/chosen": -387.55810546875,
"logps/ref_chosen": -47.601417541503906,
"logps/ref_rejected": -87.2845230102539,
"logps/rejected": -660.00537109375,
"loss": 0.8102,
"margin_dpo/margin_mean": 232.7640838623047,
"margin_dpo/margin_std": 222.93911743164062,
"step": 530
},
{
"KL/chosen_KL_mean": -359.77294921875,
"KL/mean": -463.82427978515625,
"KL/rejected_KL_mean": -567.8756103515625,
"KL/std": 231.09474182128906,
"epoch": 0.7797356828193832,
"fcm_dpo/beta": 0.004124460741877556,
"fcm_dpo/delta": -0.06376082450151443,
"fcm_dpo/margin": 208.1026153564453,
"fcm_dpo/q_t": 0.3246777057647705,
"grad_norm": 45.138309478759766,
"learning_rate": 7.141774982445147e-08,
"logits/chosen": 0.2456667125225067,
"logits/rejected": 0.27481505274772644,
"logps/chosen": -415.01904296875,
"logps/ref_chosen": -55.246063232421875,
"logps/ref_rejected": -70.60598754882812,
"logps/rejected": -638.4815673828125,
"loss": 0.847,
"margin_dpo/margin_mean": 208.10260009765625,
"margin_dpo/margin_std": 207.6148681640625,
"step": 531
},
{
"KL/chosen_KL_mean": -383.8913269042969,
"KL/mean": -493.2384033203125,
"KL/rejected_KL_mean": -602.5855102539062,
"KL/std": 211.597900390625,
"epoch": 0.7812041116005873,
"fcm_dpo/beta": 0.0039962041191756725,
"fcm_dpo/delta": -0.08703669160604477,
"fcm_dpo/margin": 218.69418334960938,
"fcm_dpo/q_t": 0.32373446226119995,
"grad_norm": 54.648067474365234,
"learning_rate": 7.052201923388953e-08,
"logits/chosen": 0.2216191440820694,
"logits/rejected": 0.25669944286346436,
"logps/chosen": -454.1773376464844,
"logps/ref_chosen": -70.28601837158203,
"logps/ref_rejected": -86.5913314819336,
"logps/rejected": -689.1768798828125,
"loss": 0.8813,
"margin_dpo/margin_mean": 218.69418334960938,
"margin_dpo/margin_std": 233.58529663085938,
"step": 532
},
{
"KL/chosen_KL_mean": -381.64031982421875,
"KL/mean": -468.139892578125,
"KL/rejected_KL_mean": -554.6394653320312,
"KL/std": 193.63104248046875,
"epoch": 0.7826725403817915,
"fcm_dpo/beta": 0.00412764260545373,
"fcm_dpo/delta": 0.09021273255348206,
"fcm_dpo/margin": 172.99917602539062,
"fcm_dpo/q_t": 0.3548873960971832,
"grad_norm": 38.81024932861328,
"learning_rate": 6.963101805503646e-08,
"logits/chosen": 0.34045305848121643,
"logits/rejected": 0.37642914056777954,
"logps/chosen": -446.49542236328125,
"logps/ref_chosen": -64.8551025390625,
"logps/ref_rejected": -76.58805847167969,
"logps/rejected": -631.2275390625,
"loss": 0.9619,
"margin_dpo/margin_mean": 172.99917602539062,
"margin_dpo/margin_std": 212.29209899902344,
"step": 533
},
{
"KL/chosen_KL_mean": -384.55780029296875,
"KL/mean": -476.9858093261719,
"KL/rejected_KL_mean": -569.413818359375,
"KL/std": 202.78414916992188,
"epoch": 0.7841409691629956,
"fcm_dpo/beta": 0.004138133954256773,
"fcm_dpo/delta": 0.03789476677775383,
"fcm_dpo/margin": 184.8560791015625,
"fcm_dpo/q_t": 0.3369339108467102,
"grad_norm": 42.43622970581055,
"learning_rate": 6.874476976660184e-08,
"logits/chosen": 0.2626418173313141,
"logits/rejected": 0.2821497619152069,
"logps/chosen": -444.67718505859375,
"logps/ref_chosen": -60.119388580322266,
"logps/ref_rejected": -78.54347229003906,
"logps/rejected": -647.957275390625,
"loss": 0.8917,
"margin_dpo/margin_mean": 184.8560791015625,
"margin_dpo/margin_std": 190.45816040039062,
"step": 534
},
{
"KL/chosen_KL_mean": -347.71710205078125,
"KL/mean": -462.66558837890625,
"KL/rejected_KL_mean": -577.6140747070312,
"KL/std": 206.4551544189453,
"epoch": 0.7856093979441997,
"fcm_dpo/beta": 0.004087292123585939,
"fcm_dpo/delta": -0.15449239313602448,
"fcm_dpo/margin": 229.89695739746094,
"fcm_dpo/q_t": 0.3038337826728821,
"grad_norm": 39.14911651611328,
"learning_rate": 6.786329772205246e-08,
"logits/chosen": 0.2347976267337799,
"logits/rejected": 0.2542041838169098,
"logps/chosen": -402.0473327636719,
"logps/ref_chosen": -54.330238342285156,
"logps/ref_rejected": -96.30763244628906,
"logps/rejected": -673.9216918945312,
"loss": 0.7864,
"margin_dpo/margin_mean": 229.89694213867188,
"margin_dpo/margin_std": 190.98971557617188,
"step": 535
},
{
"KL/chosen_KL_mean": -333.5712890625,
"KL/mean": -463.6264953613281,
"KL/rejected_KL_mean": -593.681640625,
"KL/std": 235.50570678710938,
"epoch": 0.7870778267254038,
"fcm_dpo/beta": 0.0038786856457591057,
"fcm_dpo/delta": -0.23251280188560486,
"fcm_dpo/margin": 260.1103515625,
"fcm_dpo/q_t": 0.29672110080718994,
"grad_norm": 32.708805084228516,
"learning_rate": 6.698662514899638e-08,
"logits/chosen": 0.28418147563934326,
"logits/rejected": 0.30755770206451416,
"logps/chosen": -380.65185546875,
"logps/ref_chosen": -47.08053207397461,
"logps/ref_rejected": -89.09783935546875,
"logps/rejected": -682.7794799804688,
"loss": 0.76,
"margin_dpo/margin_mean": 260.1103515625,
"margin_dpo/margin_std": 234.55047607421875,
"step": 536
},
{
"KL/chosen_KL_mean": -354.51861572265625,
"KL/mean": -460.71697998046875,
"KL/rejected_KL_mean": -566.9154052734375,
"KL/std": 200.9781494140625,
"epoch": 0.788546255506608,
"fcm_dpo/beta": 0.003838079050183296,
"fcm_dpo/delta": -0.01674051769077778,
"fcm_dpo/margin": 212.39678955078125,
"fcm_dpo/q_t": 0.3316698670387268,
"grad_norm": 39.343170166015625,
"learning_rate": 6.611477514857114e-08,
"logits/chosen": 0.33055514097213745,
"logits/rejected": 0.3605949878692627,
"logps/chosen": -412.2660827636719,
"logps/ref_chosen": -57.747467041015625,
"logps/ref_rejected": -70.43838500976562,
"logps/rejected": -637.353759765625,
"loss": 0.8782,
"margin_dpo/margin_mean": 212.39678955078125,
"margin_dpo/margin_std": 221.20095825195312,
"step": 537
},
{
"KL/chosen_KL_mean": -386.23681640625,
"KL/mean": -490.32037353515625,
"KL/rejected_KL_mean": -594.4039306640625,
"KL/std": 197.8646697998047,
"epoch": 0.7900146842878121,
"fcm_dpo/beta": 0.003815419040620327,
"fcm_dpo/delta": 0.006140880286693573,
"fcm_dpo/margin": 208.16705322265625,
"fcm_dpo/q_t": 0.3316478431224823,
"grad_norm": 46.62859344482422,
"learning_rate": 6.524777069483525e-08,
"logits/chosen": 0.1873122751712799,
"logits/rejected": 0.21275225281715393,
"logps/chosen": -452.65277099609375,
"logps/ref_chosen": -66.41594696044922,
"logps/ref_rejected": -84.22808837890625,
"logps/rejected": -678.6319580078125,
"loss": 0.863,
"margin_dpo/margin_mean": 208.16705322265625,
"margin_dpo/margin_std": 199.23483276367188,
"step": 538
},
{
"KL/chosen_KL_mean": -381.7893981933594,
"KL/mean": -473.711669921875,
"KL/rejected_KL_mean": -565.6339721679688,
"KL/std": 190.73289489746094,
"epoch": 0.7914831130690162,
"fcm_dpo/beta": 0.003870626911520958,
"fcm_dpo/delta": 0.09528693556785583,
"fcm_dpo/margin": 183.84457397460938,
"fcm_dpo/q_t": 0.3515859842300415,
"grad_norm": 36.88890838623047,
"learning_rate": 6.438563463416221e-08,
"logits/chosen": 0.2974720001220703,
"logits/rejected": 0.335232138633728,
"logps/chosen": -440.2822570800781,
"logps/ref_chosen": -58.492855072021484,
"logps/ref_rejected": -91.85395050048828,
"logps/rejected": -657.4879150390625,
"loss": 0.9384,
"margin_dpo/margin_mean": 183.84457397460938,
"margin_dpo/margin_std": 208.98944091796875,
"step": 539
},
{
"KL/chosen_KL_mean": -350.2906494140625,
"KL/mean": -476.7803955078125,
"KL/rejected_KL_mean": -603.2701416015625,
"KL/std": 221.31947326660156,
"epoch": 0.7929515418502202,
"fcm_dpo/beta": 0.003814155701547861,
"fcm_dpo/delta": -0.18212105333805084,
"fcm_dpo/margin": 252.97952270507812,
"fcm_dpo/q_t": 0.30182313919067383,
"grad_norm": 34.22950744628906,
"learning_rate": 6.352838968463919e-08,
"logits/chosen": 0.20235419273376465,
"logits/rejected": 0.2219226211309433,
"logps/chosen": -413.77313232421875,
"logps/ref_chosen": -63.482513427734375,
"logps/ref_rejected": -116.42999267578125,
"logps/rejected": -719.7001342773438,
"loss": 0.7934,
"margin_dpo/margin_mean": 252.97952270507812,
"margin_dpo/margin_std": 223.89288330078125,
"step": 540
},
{
"KL/chosen_KL_mean": -401.22979736328125,
"KL/mean": -487.3087158203125,
"KL/rejected_KL_mean": -573.3875732421875,
"KL/std": 208.10055541992188,
"epoch": 0.7944199706314243,
"fcm_dpo/beta": 0.0038143517449498177,
"fcm_dpo/delta": 0.1531994491815567,
"fcm_dpo/margin": 172.1578369140625,
"fcm_dpo/q_t": 0.3660222291946411,
"grad_norm": 36.54274368286133,
"learning_rate": 6.267605843546767e-08,
"logits/chosen": 0.24667702615261078,
"logits/rejected": 0.2629375755786896,
"logps/chosen": -479.5101623535156,
"logps/ref_chosen": -78.28036499023438,
"logps/ref_rejected": -103.273681640625,
"logps/rejected": -676.6612548828125,
"loss": 0.9814,
"margin_dpo/margin_mean": 172.1578369140625,
"margin_dpo/margin_std": 218.95196533203125,
"step": 541
},
{
"KL/chosen_KL_mean": -390.4757080078125,
"KL/mean": -500.078369140625,
"KL/rejected_KL_mean": -609.6810302734375,
"KL/std": 207.84117126464844,
"epoch": 0.7958883994126285,
"fcm_dpo/beta": 0.003827635897323489,
"fcm_dpo/delta": -0.043748270720243454,
"fcm_dpo/margin": 219.20529174804688,
"fcm_dpo/q_t": 0.3255137801170349,
"grad_norm": 54.06191635131836,
"learning_rate": 6.182866334636888e-08,
"logits/chosen": 0.33256542682647705,
"logits/rejected": 0.34963661432266235,
"logps/chosen": -447.960693359375,
"logps/ref_chosen": -57.48497009277344,
"logps/ref_rejected": -96.47506713867188,
"logps/rejected": -706.1560668945312,
"loss": 0.8597,
"margin_dpo/margin_mean": 219.20529174804688,
"margin_dpo/margin_std": 215.65493774414062,
"step": 542
},
{
"KL/chosen_KL_mean": -401.6968994140625,
"KL/mean": -475.1959533691406,
"KL/rejected_KL_mean": -548.6950073242188,
"KL/std": 224.97775268554688,
"epoch": 0.7973568281938326,
"fcm_dpo/beta": 0.003976443782448769,
"fcm_dpo/delta": 0.22884975373744965,
"fcm_dpo/margin": 146.99813842773438,
"fcm_dpo/q_t": 0.38513779640197754,
"grad_norm": 41.66843032836914,
"learning_rate": 6.098622674699147e-08,
"logits/chosen": 0.216482013463974,
"logits/rejected": 0.2214020937681198,
"logps/chosen": -462.31439208984375,
"logps/ref_chosen": -60.61750793457031,
"logps/ref_rejected": -105.59896850585938,
"logps/rejected": -654.2939453125,
"loss": 1.0618,
"margin_dpo/margin_mean": 146.99813842773438,
"margin_dpo/margin_std": 232.58026123046875,
"step": 543
},
{
"KL/chosen_KL_mean": -414.082275390625,
"KL/mean": -527.2254028320312,
"KL/rejected_KL_mean": -640.3685302734375,
"KL/std": 213.15147399902344,
"epoch": 0.7988252569750367,
"fcm_dpo/beta": 0.00395975261926651,
"fcm_dpo/delta": -0.1055314838886261,
"fcm_dpo/margin": 226.28622436523438,
"fcm_dpo/q_t": 0.3122187852859497,
"grad_norm": 40.56058883666992,
"learning_rate": 6.01487708363232e-08,
"logits/chosen": 0.2870156764984131,
"logits/rejected": 0.3034914433956146,
"logps/chosen": -473.724609375,
"logps/ref_chosen": -59.642303466796875,
"logps/ref_rejected": -100.95469665527344,
"logps/rejected": -741.3232421875,
"loss": 0.8012,
"margin_dpo/margin_mean": 226.2862091064453,
"margin_dpo/margin_std": 201.3153076171875,
"step": 544
},
{
"KL/chosen_KL_mean": -384.6066589355469,
"KL/mean": -493.9320068359375,
"KL/rejected_KL_mean": -603.25732421875,
"KL/std": 203.6180877685547,
"epoch": 0.8002936857562408,
"fcm_dpo/beta": 0.0039124698378145695,
"fcm_dpo/delta": -0.060617730021476746,
"fcm_dpo/margin": 218.65066528320312,
"fcm_dpo/q_t": 0.32227370142936707,
"grad_norm": 41.63042068481445,
"learning_rate": 5.9316317682106294e-08,
"logits/chosen": 0.24290139973163605,
"logits/rejected": 0.2708423137664795,
"logps/chosen": -452.2552490234375,
"logps/ref_chosen": -67.64859771728516,
"logps/ref_rejected": -95.90800476074219,
"logps/rejected": -699.1653442382812,
"loss": 0.8485,
"margin_dpo/margin_mean": 218.65066528320312,
"margin_dpo/margin_std": 211.84780883789062,
"step": 545
},
{
"KL/chosen_KL_mean": -358.6064453125,
"KL/mean": -456.1077575683594,
"KL/rejected_KL_mean": -553.6090698242188,
"KL/std": 187.7569580078125,
"epoch": 0.801762114537445,
"fcm_dpo/beta": 0.0038997926749289036,
"fcm_dpo/delta": 0.04258295148611069,
"fcm_dpo/margin": 195.0026397705078,
"fcm_dpo/q_t": 0.334590345621109,
"grad_norm": 35.64754867553711,
"learning_rate": 5.848888922025552e-08,
"logits/chosen": 0.24750055372714996,
"logits/rejected": 0.25986167788505554,
"logps/chosen": -409.3506774902344,
"logps/ref_chosen": -50.744232177734375,
"logps/ref_rejected": -81.86622619628906,
"logps/rejected": -635.475341796875,
"loss": 0.8731,
"margin_dpo/margin_mean": 195.0026397705078,
"margin_dpo/margin_std": 179.05099487304688,
"step": 546
},
{
"KL/chosen_KL_mean": -405.56817626953125,
"KL/mean": -502.1980285644531,
"KL/rejected_KL_mean": -598.827880859375,
"KL/std": 210.65838623046875,
"epoch": 0.8032305433186491,
"fcm_dpo/beta": 0.00398046150803566,
"fcm_dpo/delta": 0.031187400221824646,
"fcm_dpo/margin": 193.25973510742188,
"fcm_dpo/q_t": 0.3425300419330597,
"grad_norm": 67.2081069946289,
"learning_rate": 5.7666507254280265e-08,
"logits/chosen": 0.15230880677700043,
"logits/rejected": 0.18068407475948334,
"logps/chosen": -479.2558898925781,
"logps/ref_chosen": -73.6877212524414,
"logps/ref_rejected": -90.76136779785156,
"logps/rejected": -689.5892333984375,
"loss": 0.9202,
"margin_dpo/margin_mean": 193.25973510742188,
"margin_dpo/margin_std": 217.03179931640625,
"step": 547
},
{
"KL/chosen_KL_mean": -402.8782958984375,
"KL/mean": -503.20465087890625,
"KL/rejected_KL_mean": -603.531005859375,
"KL/std": 230.92819213867188,
"epoch": 0.8046989720998532,
"fcm_dpo/beta": 0.00395785178989172,
"fcm_dpo/delta": 0.006317374296486378,
"fcm_dpo/margin": 200.65269470214844,
"fcm_dpo/q_t": 0.337555468082428,
"grad_norm": 41.28041076660156,
"learning_rate": 5.684919345471029e-08,
"logits/chosen": 0.2283911406993866,
"logits/rejected": 0.25433415174484253,
"logps/chosen": -468.1246337890625,
"logps/ref_chosen": -65.24634552001953,
"logps/ref_rejected": -94.11807250976562,
"logps/rejected": -697.6490478515625,
"loss": 0.8981,
"margin_dpo/margin_mean": 200.65267944335938,
"margin_dpo/margin_std": 227.65338134765625,
"step": 548
},
{
"KL/chosen_KL_mean": -392.39306640625,
"KL/mean": -486.2154846191406,
"KL/rejected_KL_mean": -580.037841796875,
"KL/std": 183.3447723388672,
"epoch": 0.8061674008810573,
"fcm_dpo/beta": 0.003991181496530771,
"fcm_dpo/delta": 0.05513293296098709,
"fcm_dpo/margin": 187.64480590820312,
"fcm_dpo/q_t": 0.34387335181236267,
"grad_norm": 46.40816879272461,
"learning_rate": 5.603696935852426e-08,
"logits/chosen": 0.38207992911338806,
"logits/rejected": 0.4072762727737427,
"logps/chosen": -441.6054382324219,
"logps/ref_chosen": -49.21235656738281,
"logps/ref_rejected": -73.91031646728516,
"logps/rejected": -653.9481811523438,
"loss": 0.9046,
"margin_dpo/margin_mean": 187.64480590820312,
"margin_dpo/margin_std": 201.8494415283203,
"step": 549
},
{
"KL/chosen_KL_mean": -383.96148681640625,
"KL/mean": -468.65374755859375,
"KL/rejected_KL_mean": -553.3460083007812,
"KL/std": 199.1809539794922,
"epoch": 0.8076358296622613,
"fcm_dpo/beta": 0.00405759084969759,
"fcm_dpo/delta": 0.12119770795106888,
"fcm_dpo/margin": 169.38455200195312,
"fcm_dpo/q_t": 0.35117456316947937,
"grad_norm": 39.510982513427734,
"learning_rate": 5.5229856368582376e-08,
"logits/chosen": 0.21509867906570435,
"logits/rejected": 0.2270488440990448,
"logps/chosen": -440.7684326171875,
"logps/ref_chosen": -56.80695343017578,
"logps/ref_rejected": -95.12580871582031,
"logps/rejected": -648.4718017578125,
"loss": 0.9164,
"margin_dpo/margin_mean": 169.38455200195312,
"margin_dpo/margin_std": 168.8013458251953,
"step": 550
},
{
"KL/chosen_KL_mean": -375.353515625,
"KL/mean": -500.36590576171875,
"KL/rejected_KL_mean": -625.3782958984375,
"KL/std": 219.81362915039062,
"epoch": 0.8091042584434655,
"fcm_dpo/beta": 0.0039620669558644295,
"fcm_dpo/delta": -0.21213532984256744,
"fcm_dpo/margin": 250.02468872070312,
"fcm_dpo/q_t": 0.29334428906440735,
"grad_norm": 48.182064056396484,
"learning_rate": 5.4427875753062734e-08,
"logits/chosen": 0.23227638006210327,
"logits/rejected": 0.24997326731681824,
"logps/chosen": -434.4598693847656,
"logps/ref_chosen": -59.10633087158203,
"logps/ref_rejected": -111.67280578613281,
"logps/rejected": -737.051025390625,
"loss": 0.7635,
"margin_dpo/margin_mean": 250.02468872070312,
"margin_dpo/margin_std": 206.69105529785156,
"step": 551
},
{
"KL/chosen_KL_mean": -364.43963623046875,
"KL/mean": -506.6065979003906,
"KL/rejected_KL_mean": -648.7735595703125,
"KL/std": 241.85960388183594,
"epoch": 0.8105726872246696,
"fcm_dpo/beta": 0.0037087902892380953,
"fcm_dpo/delta": -0.29273247718811035,
"fcm_dpo/margin": 284.33392333984375,
"fcm_dpo/q_t": 0.2889665365219116,
"grad_norm": 31.593692779541016,
"learning_rate": 5.363104864490034e-08,
"logits/chosen": 0.28846144676208496,
"logits/rejected": 0.3311532139778137,
"logps/chosen": -426.79425048828125,
"logps/ref_chosen": -62.35459899902344,
"logps/ref_rejected": -104.56210327148438,
"logps/rejected": -753.335693359375,
"loss": 0.7566,
"margin_dpo/margin_mean": 284.33392333984375,
"margin_dpo/margin_std": 249.8412322998047,
"step": 552
},
{
"KL/chosen_KL_mean": -394.02691650390625,
"KL/mean": -487.235107421875,
"KL/rejected_KL_mean": -580.4432983398438,
"KL/std": 202.1296844482422,
"epoch": 0.8120411160058737,
"fcm_dpo/beta": 0.0037544944789260626,
"fcm_dpo/delta": 0.10777129977941513,
"fcm_dpo/margin": 186.4163818359375,
"fcm_dpo/q_t": 0.3546206057071686,
"grad_norm": 40.13036346435547,
"learning_rate": 5.2839396041230415e-08,
"logits/chosen": 0.2658536434173584,
"logits/rejected": 0.2780281603336334,
"logps/chosen": -462.28570556640625,
"logps/ref_chosen": -68.25881958007812,
"logps/ref_rejected": -98.0971450805664,
"logps/rejected": -678.5404663085938,
"loss": 0.9369,
"margin_dpo/margin_mean": 186.4163818359375,
"margin_dpo/margin_std": 208.78411865234375,
"step": 553
},
{
"KL/chosen_KL_mean": -389.50079345703125,
"KL/mean": -510.76385498046875,
"KL/rejected_KL_mean": -632.02685546875,
"KL/std": 229.28463745117188,
"epoch": 0.8135095447870778,
"fcm_dpo/beta": 0.0037528942339122295,
"fcm_dpo/delta": -0.12198401987552643,
"fcm_dpo/margin": 242.52606201171875,
"fcm_dpo/q_t": 0.31189385056495667,
"grad_norm": 38.72057342529297,
"learning_rate": 5.205293880283551e-08,
"logits/chosen": 0.222869411110878,
"logits/rejected": 0.27173179388046265,
"logps/chosen": -457.448486328125,
"logps/ref_chosen": -67.94767761230469,
"logps/ref_rejected": -89.78272247314453,
"logps/rejected": -721.8095703125,
"loss": 0.838,
"margin_dpo/margin_mean": 242.52606201171875,
"margin_dpo/margin_std": 234.7236328125,
"step": 554
},
{
"KL/chosen_KL_mean": -394.62530517578125,
"KL/mean": -514.3497924804688,
"KL/rejected_KL_mean": -634.0742797851562,
"KL/std": 226.64907836914062,
"epoch": 0.8149779735682819,
"fcm_dpo/beta": 0.0036291368305683136,
"fcm_dpo/delta": -0.07678112387657166,
"fcm_dpo/margin": 239.44900512695312,
"fcm_dpo/q_t": 0.32688435912132263,
"grad_norm": 41.123992919921875,
"learning_rate": 5.127169765359515e-08,
"logits/chosen": 0.24900421500205994,
"logits/rejected": 0.2614419162273407,
"logps/chosen": -447.9557800292969,
"logps/ref_chosen": -53.33049011230469,
"logps/ref_rejected": -108.47937774658203,
"logps/rejected": -742.5537109375,
"loss": 0.8651,
"margin_dpo/margin_mean": 239.44900512695312,
"margin_dpo/margin_std": 255.90615844726562,
"step": 555
},
{
"KL/chosen_KL_mean": -399.20941162109375,
"KL/mean": -492.0675048828125,
"KL/rejected_KL_mean": -584.925537109375,
"KL/std": 213.84945678710938,
"epoch": 0.8164464023494861,
"fcm_dpo/beta": 0.0037066666409373283,
"fcm_dpo/delta": 0.11965786665678024,
"fcm_dpo/margin": 185.71615600585938,
"fcm_dpo/q_t": 0.3593837022781372,
"grad_norm": 39.725162506103516,
"learning_rate": 5.049569317994012e-08,
"logits/chosen": 0.331043541431427,
"logits/rejected": 0.3502947688102722,
"logps/chosen": -457.8538818359375,
"logps/ref_chosen": -58.64447021484375,
"logps/ref_rejected": -101.34040832519531,
"logps/rejected": -686.2659912109375,
"loss": 0.9503,
"margin_dpo/margin_mean": 185.71615600585938,
"margin_dpo/margin_std": 223.22207641601562,
"step": 556
},
{
"KL/chosen_KL_mean": -410.8165588378906,
"KL/mean": -513.6693115234375,
"KL/rejected_KL_mean": -616.5220947265625,
"KL/std": 224.26837158203125,
"epoch": 0.8179148311306902,
"fcm_dpo/beta": 0.003730146214365959,
"fcm_dpo/delta": 0.035248152911663055,
"fcm_dpo/margin": 205.70550537109375,
"fcm_dpo/q_t": 0.3421016335487366,
"grad_norm": 39.09975051879883,
"learning_rate": 4.9724945830310144e-08,
"logits/chosen": 0.2737029194831848,
"logits/rejected": 0.29351887106895447,
"logps/chosen": -478.6572265625,
"logps/ref_chosen": -67.84066009521484,
"logps/ref_rejected": -109.93965911865234,
"logps/rejected": -726.4617309570312,
"loss": 0.9037,
"margin_dpo/margin_mean": 205.70550537109375,
"margin_dpo/margin_std": 222.4257354736328,
"step": 557
},
{
"KL/chosen_KL_mean": -386.15411376953125,
"KL/mean": -531.5560913085938,
"KL/rejected_KL_mean": -676.9581298828125,
"KL/std": 238.99404907226562,
"epoch": 0.8193832599118943,
"fcm_dpo/beta": 0.0035815369337797165,
"fcm_dpo/delta": -0.2720809578895569,
"fcm_dpo/margin": 290.80401611328125,
"fcm_dpo/q_t": 0.2847321629524231,
"grad_norm": 30.991840362548828,
"learning_rate": 4.8959475914614554e-08,
"logits/chosen": 0.3357253074645996,
"logits/rejected": 0.3656435012817383,
"logps/chosen": -448.5223388671875,
"logps/ref_chosen": -62.36824035644531,
"logps/ref_rejected": -102.16102600097656,
"logps/rejected": -779.119140625,
"loss": 0.7381,
"margin_dpo/margin_mean": 290.8039855957031,
"margin_dpo/margin_std": 235.93875122070312,
"step": 558
},
{
"KL/chosen_KL_mean": -433.37945556640625,
"KL/mean": -544.3544921875,
"KL/rejected_KL_mean": -655.32958984375,
"KL/std": 226.74810791015625,
"epoch": 0.8208516886930984,
"fcm_dpo/beta": 0.0035474528558552265,
"fcm_dpo/delta": 0.013534316793084145,
"fcm_dpo/margin": 221.95016479492188,
"fcm_dpo/q_t": 0.3346388638019562,
"grad_norm": 42.02165603637695,
"learning_rate": 4.8199303603697614e-08,
"logits/chosen": 0.40246227383613586,
"logits/rejected": 0.45247191190719604,
"logps/chosen": -494.13177490234375,
"logps/ref_chosen": -60.752323150634766,
"logps/ref_rejected": -93.44229125976562,
"logps/rejected": -748.7718505859375,
"loss": 0.8745,
"margin_dpo/margin_mean": 221.95016479492188,
"margin_dpo/margin_std": 219.32046508789062,
"step": 559
},
{
"KL/chosen_KL_mean": -384.0647277832031,
"KL/mean": -466.8363342285156,
"KL/rejected_KL_mean": -549.60791015625,
"KL/std": 207.88125610351562,
"epoch": 0.8223201174743024,
"fcm_dpo/beta": 0.0036635929718613625,
"fcm_dpo/delta": 0.20640337467193604,
"fcm_dpo/margin": 165.54315185546875,
"fcm_dpo/q_t": 0.3670238256454468,
"grad_norm": 40.4533805847168,
"learning_rate": 4.7444448928806615e-08,
"logits/chosen": 0.19835129380226135,
"logits/rejected": 0.23476439714431763,
"logps/chosen": -442.1685485839844,
"logps/ref_chosen": -58.10382080078125,
"logps/ref_rejected": -79.99122619628906,
"logps/rejected": -629.59912109375,
"loss": 0.9562,
"margin_dpo/margin_mean": 165.54315185546875,
"margin_dpo/margin_std": 169.37548828125,
"step": 560
},
{
"KL/chosen_KL_mean": -452.8956298828125,
"KL/mean": -540.2266845703125,
"KL/rejected_KL_mean": -627.5577392578125,
"KL/std": 207.67529296875,
"epoch": 0.8237885462555066,
"fcm_dpo/beta": 0.0038133093621581793,
"fcm_dpo/delta": 0.14196228981018066,
"fcm_dpo/margin": 174.6620635986328,
"fcm_dpo/q_t": 0.3567246198654175,
"grad_norm": 46.923065185546875,
"learning_rate": 4.669493178106432e-08,
"logits/chosen": 0.39849230647087097,
"logits/rejected": 0.41054990887641907,
"logps/chosen": -503.8085021972656,
"logps/ref_chosen": -50.912879943847656,
"logps/ref_rejected": -99.06856536865234,
"logps/rejected": -726.6262817382812,
"loss": 0.9425,
"margin_dpo/margin_mean": 174.6620635986328,
"margin_dpo/margin_std": 190.61865234375,
"step": 561
},
{
"KL/chosen_KL_mean": -404.7358703613281,
"KL/mean": -527.0230712890625,
"KL/rejected_KL_mean": -649.310302734375,
"KL/std": 235.49441528320312,
"epoch": 0.8252569750367107,
"fcm_dpo/beta": 0.003720635548233986,
"fcm_dpo/delta": -0.12324625253677368,
"fcm_dpo/margin": 244.5744171142578,
"fcm_dpo/q_t": 0.31406521797180176,
"grad_norm": 39.70796585083008,
"learning_rate": 4.5950771910944596e-08,
"logits/chosen": 0.339399516582489,
"logits/rejected": 0.3695017397403717,
"logps/chosen": -464.20025634765625,
"logps/ref_chosen": -59.46440124511719,
"logps/ref_rejected": -96.54266357421875,
"logps/rejected": -745.8529663085938,
"loss": 0.812,
"margin_dpo/margin_mean": 244.57443237304688,
"margin_dpo/margin_std": 231.22760009765625,
"step": 562
},
{
"KL/chosen_KL_mean": -439.4453430175781,
"KL/mean": -530.522216796875,
"KL/rejected_KL_mean": -621.59912109375,
"KL/std": 201.02435302734375,
"epoch": 0.8267254038179148,
"fcm_dpo/beta": 0.003821668680757284,
"fcm_dpo/delta": 0.10885676741600037,
"fcm_dpo/margin": 182.15380859375,
"fcm_dpo/q_t": 0.35326603055000305,
"grad_norm": 45.26503372192383,
"learning_rate": 4.521198892775202e-08,
"logits/chosen": 0.4035786986351013,
"logits/rejected": 0.41392847895622253,
"logps/chosen": -500.05352783203125,
"logps/ref_chosen": -60.60819625854492,
"logps/ref_rejected": -94.56770324707031,
"logps/rejected": -716.1668701171875,
"loss": 0.9582,
"margin_dpo/margin_mean": 182.15380859375,
"margin_dpo/margin_std": 211.79025268554688,
"step": 563
},
{
"KL/chosen_KL_mean": -413.4257507324219,
"KL/mean": -526.880126953125,
"KL/rejected_KL_mean": -640.33447265625,
"KL/std": 224.15956115722656,
"epoch": 0.8281938325991189,
"fcm_dpo/beta": 0.0037801191210746765,
"fcm_dpo/delta": -0.0631168857216835,
"fcm_dpo/margin": 226.90872192382812,
"fcm_dpo/q_t": 0.31227725744247437,
"grad_norm": 34.521514892578125,
"learning_rate": 4.447860229910544e-08,
"logits/chosen": 0.3029387593269348,
"logits/rejected": 0.3524191975593567,
"logps/chosen": -487.6941223144531,
"logps/ref_chosen": -74.26837921142578,
"logps/ref_rejected": -93.23818969726562,
"logps/rejected": -733.5726318359375,
"loss": 0.7874,
"margin_dpo/margin_mean": 226.90870666503906,
"margin_dpo/margin_std": 165.97833251953125,
"step": 564
},
{
"KL/chosen_KL_mean": -429.74359130859375,
"KL/mean": -546.124267578125,
"KL/rejected_KL_mean": -662.5048828125,
"KL/std": 252.56912231445312,
"epoch": 0.8296622613803231,
"fcm_dpo/beta": 0.0037043162155896425,
"fcm_dpo/delta": -0.06889684498310089,
"fcm_dpo/margin": 232.76126098632812,
"fcm_dpo/q_t": 0.33259573578834534,
"grad_norm": 33.25062942504883,
"learning_rate": 4.375063135042445e-08,
"logits/chosen": 0.2388344705104828,
"logits/rejected": 0.26381534337997437,
"logps/chosen": -498.7635192871094,
"logps/ref_chosen": -69.0199203491211,
"logps/ref_rejected": -85.7789306640625,
"logps/rejected": -748.2838134765625,
"loss": 0.8844,
"margin_dpo/margin_mean": 232.76126098632812,
"margin_dpo/margin_std": 266.616455078125,
"step": 565
},
{
"KL/chosen_KL_mean": -427.2159423828125,
"KL/mean": -536.328857421875,
"KL/rejected_KL_mean": -645.4417724609375,
"KL/std": 245.62149047851562,
"epoch": 0.8311306901615272,
"fcm_dpo/beta": 0.003733118064701557,
"fcm_dpo/delta": -0.017540939152240753,
"fcm_dpo/margin": 218.225830078125,
"fcm_dpo/q_t": 0.3383902311325073,
"grad_norm": 40.47699737548828,
"learning_rate": 4.3028095264420525e-08,
"logits/chosen": 0.3286312520503998,
"logits/rejected": 0.3403201997280121,
"logps/chosen": -493.76123046875,
"logps/ref_chosen": -66.5453109741211,
"logps/ref_rejected": -103.86932373046875,
"logps/rejected": -749.31103515625,
"loss": 0.9307,
"margin_dpo/margin_mean": 218.22581481933594,
"margin_dpo/margin_std": 258.7934265136719,
"step": 566
},
{
"KL/chosen_KL_mean": -430.94976806640625,
"KL/mean": -536.2676391601562,
"KL/rejected_KL_mean": -641.58544921875,
"KL/std": 210.7244873046875,
"epoch": 0.8325991189427313,
"fcm_dpo/beta": 0.0036982346791774035,
"fcm_dpo/delta": 0.022707507014274597,
"fcm_dpo/margin": 210.63571166992188,
"fcm_dpo/q_t": 0.33780205249786377,
"grad_norm": 41.74602508544922,
"learning_rate": 4.231101308059165e-08,
"logits/chosen": 0.4672779440879822,
"logits/rejected": 0.4847617745399475,
"logps/chosen": -483.80804443359375,
"logps/ref_chosen": -52.85829544067383,
"logps/ref_rejected": -85.37095642089844,
"logps/rejected": -726.9564208984375,
"loss": 0.889,
"margin_dpo/margin_mean": 210.63571166992188,
"margin_dpo/margin_std": 218.34823608398438,
"step": 567
},
{
"KL/chosen_KL_mean": -407.42236328125,
"KL/mean": -529.507080078125,
"KL/rejected_KL_mean": -651.591796875,
"KL/std": 220.82595825195312,
"epoch": 0.8340675477239354,
"fcm_dpo/beta": 0.0036355347838252783,
"fcm_dpo/delta": -0.09775268286466599,
"fcm_dpo/margin": 244.16946411132812,
"fcm_dpo/q_t": 0.3122249245643616,
"grad_norm": 37.94232940673828,
"learning_rate": 4.1599403694720145e-08,
"logits/chosen": 0.4218721389770508,
"logits/rejected": 0.4371316432952881,
"logps/chosen": -452.6147155761719,
"logps/ref_chosen": -45.1923828125,
"logps/ref_rejected": -89.09236907958984,
"logps/rejected": -740.6842041015625,
"loss": 0.8192,
"margin_dpo/margin_mean": 244.16946411132812,
"margin_dpo/margin_std": 215.54330444335938,
"step": 568
},
{
"KL/chosen_KL_mean": -464.3712158203125,
"KL/mean": -566.7379760742188,
"KL/rejected_KL_mean": -669.104736328125,
"KL/std": 238.54409790039062,
"epoch": 0.8355359765051396,
"fcm_dpo/beta": 0.0036928458139300346,
"fcm_dpo/delta": 0.04605846479535103,
"fcm_dpo/margin": 204.73341369628906,
"fcm_dpo/q_t": 0.34364962577819824,
"grad_norm": 44.00316619873047,
"learning_rate": 4.089328585837512e-08,
"logits/chosen": 0.38823315501213074,
"logits/rejected": 0.41456738114356995,
"logps/chosen": -528.091796875,
"logps/ref_chosen": -63.72056198120117,
"logps/ref_rejected": -79.10325622558594,
"logps/rejected": -748.2079467773438,
"loss": 0.9298,
"margin_dpo/margin_mean": 204.73342895507812,
"margin_dpo/margin_std": 236.36395263671875,
"step": 569
},
{
"KL/chosen_KL_mean": -430.80474853515625,
"KL/mean": -545.889404296875,
"KL/rejected_KL_mean": -660.97412109375,
"KL/std": 216.6941680908203,
"epoch": 0.8370044052863436,
"fcm_dpo/beta": 0.003635650733485818,
"fcm_dpo/delta": -0.04079482704401016,
"fcm_dpo/margin": 230.1693572998047,
"fcm_dpo/q_t": 0.32262638211250305,
"grad_norm": 45.467594146728516,
"learning_rate": 4.019267817841834e-08,
"logits/chosen": 0.40148618817329407,
"logits/rejected": 0.43486684560775757,
"logps/chosen": -492.4193115234375,
"logps/ref_chosen": -61.61454391479492,
"logps/ref_rejected": -82.14186096191406,
"logps/rejected": -743.115966796875,
"loss": 0.8298,
"margin_dpo/margin_mean": 230.16937255859375,
"margin_dpo/margin_std": 200.99362182617188,
"step": 570
},
{
"KL/chosen_KL_mean": -439.533447265625,
"KL/mean": -559.2398681640625,
"KL/rejected_KL_mean": -678.9462890625,
"KL/std": 225.64910888671875,
"epoch": 0.8384728340675477,
"fcm_dpo/beta": 0.003588983789086342,
"fcm_dpo/delta": -0.06584499031305313,
"fcm_dpo/margin": 239.4127960205078,
"fcm_dpo/q_t": 0.3223009705543518,
"grad_norm": 41.48381805419922,
"learning_rate": 3.9497599116513705e-08,
"logits/chosen": 0.4067568778991699,
"logits/rejected": 0.4149426817893982,
"logps/chosen": -492.5875244140625,
"logps/ref_chosen": -53.05406188964844,
"logps/ref_rejected": -91.33682250976562,
"logps/rejected": -770.2830810546875,
"loss": 0.8685,
"margin_dpo/margin_mean": 239.4127960205078,
"margin_dpo/margin_std": 250.37957763671875,
"step": 571
},
{
"KL/chosen_KL_mean": -467.0171203613281,
"KL/mean": -577.5743408203125,
"KL/rejected_KL_mean": -688.1314086914062,
"KL/std": 266.3624572753906,
"epoch": 0.8399412628487518,
"fcm_dpo/beta": 0.003590911626815796,
"fcm_dpo/delta": 0.006393725052475929,
"fcm_dpo/margin": 221.11422729492188,
"fcm_dpo/q_t": 0.34955984354019165,
"grad_norm": 40.90983200073242,
"learning_rate": 3.880806698864086e-08,
"logits/chosen": 0.4752965569496155,
"logits/rejected": 0.4910871684551239,
"logps/chosen": -515.4764404296875,
"logps/ref_chosen": -48.45928955078125,
"logps/ref_rejected": -83.55703735351562,
"logps/rejected": -771.6884155273438,
"loss": 0.9482,
"margin_dpo/margin_mean": 221.11422729492188,
"margin_dpo/margin_std": 291.041748046875,
"step": 572
},
{
"KL/chosen_KL_mean": -441.55621337890625,
"KL/mean": -551.5286254882812,
"KL/rejected_KL_mean": -661.5009765625,
"KL/std": 213.51548767089844,
"epoch": 0.8414096916299559,
"fcm_dpo/beta": 0.0036352514289319515,
"fcm_dpo/delta": -0.0013968870043754578,
"fcm_dpo/margin": 219.9447784423828,
"fcm_dpo/q_t": 0.3341352939605713,
"grad_norm": 35.48988342285156,
"learning_rate": 3.812409996461275e-08,
"logits/chosen": 0.468715637922287,
"logits/rejected": 0.4946897029876709,
"logps/chosen": -493.1788330078125,
"logps/ref_chosen": -51.62262725830078,
"logps/ref_rejected": -85.32499694824219,
"logps/rejected": -746.8259887695312,
"loss": 0.8768,
"margin_dpo/margin_mean": 219.94479370117188,
"margin_dpo/margin_std": 218.58367919921875,
"step": 573
},
{
"KL/chosen_KL_mean": -424.98175048828125,
"KL/mean": -547.060546875,
"KL/rejected_KL_mean": -669.1392822265625,
"KL/std": 229.44229125976562,
"epoch": 0.8428781204111601,
"fcm_dpo/beta": 0.0035688234493136406,
"fcm_dpo/delta": -0.07810860127210617,
"fcm_dpo/margin": 244.15753173828125,
"fcm_dpo/q_t": 0.3205257058143616,
"grad_norm": 35.287593841552734,
"learning_rate": 3.74457160675965e-08,
"logits/chosen": 0.33639222383499146,
"logits/rejected": 0.35209882259368896,
"logps/chosen": -476.0262145996094,
"logps/ref_chosen": -51.04446029663086,
"logps/ref_rejected": -92.80640411376953,
"logps/rejected": -761.9456787109375,
"loss": 0.8361,
"margin_dpo/margin_mean": 244.15753173828125,
"margin_dpo/margin_std": 235.37338256835938,
"step": 574
},
{
"KL/chosen_KL_mean": -473.6583557128906,
"KL/mean": -580.267578125,
"KL/rejected_KL_mean": -686.8767700195312,
"KL/std": 237.57046508789062,
"epoch": 0.8443465491923642,
"fcm_dpo/beta": 0.0035233558155596256,
"fcm_dpo/delta": 0.04960758984088898,
"fcm_dpo/margin": 213.21844482421875,
"fcm_dpo/q_t": 0.3446178436279297,
"grad_norm": 44.265228271484375,
"learning_rate": 3.677293317363864e-08,
"logits/chosen": 0.35401681065559387,
"logits/rejected": 0.3639416992664337,
"logps/chosen": -545.448486328125,
"logps/ref_chosen": -71.7901382446289,
"logps/ref_rejected": -95.38619995117188,
"logps/rejected": -782.262939453125,
"loss": 0.9677,
"margin_dpo/margin_mean": 213.2184295654297,
"margin_dpo/margin_std": 257.09466552734375,
"step": 575
},
{
"KL/chosen_KL_mean": -453.58380126953125,
"KL/mean": -559.41845703125,
"KL/rejected_KL_mean": -665.2530517578125,
"KL/std": 233.61068725585938,
"epoch": 0.8458149779735683,
"fcm_dpo/beta": 0.003570101223886013,
"fcm_dpo/delta": 0.04643898457288742,
"fcm_dpo/margin": 211.66921997070312,
"fcm_dpo/q_t": 0.3475932478904724,
"grad_norm": 49.50276565551758,
"learning_rate": 3.6105769011194224e-08,
"logits/chosen": 0.4114004373550415,
"logits/rejected": 0.4138645529747009,
"logps/chosen": -507.8467712402344,
"logps/ref_chosen": -54.262962341308594,
"logps/ref_rejected": -100.75428009033203,
"logps/rejected": -766.00732421875,
"loss": 0.9458,
"margin_dpo/margin_mean": 211.66921997070312,
"margin_dpo/margin_std": 250.59912109375,
"step": 576
},
{
"KL/chosen_KL_mean": -411.8608703613281,
"KL/mean": -526.9452514648438,
"KL/rejected_KL_mean": -642.0296630859375,
"KL/std": 223.98553466796875,
"epoch": 0.8472834067547724,
"fcm_dpo/beta": 0.0035911118611693382,
"fcm_dpo/delta": -0.029081393033266068,
"fcm_dpo/margin": 230.16879272460938,
"fcm_dpo/q_t": 0.3222889304161072,
"grad_norm": 39.094970703125,
"learning_rate": 3.5444241160659304e-08,
"logits/chosen": 0.36829674243927,
"logits/rejected": 0.39512234926223755,
"logps/chosen": -473.77056884765625,
"logps/ref_chosen": -61.909706115722656,
"logps/ref_rejected": -84.07069396972656,
"logps/rejected": -726.100341796875,
"loss": 0.8506,
"margin_dpo/margin_mean": 230.16879272460938,
"margin_dpo/margin_std": 211.40219116210938,
"step": 577
},
{
"KL/chosen_KL_mean": -424.4698486328125,
"KL/mean": -533.7640380859375,
"KL/rejected_KL_mean": -643.0581665039062,
"KL/std": 227.083251953125,
"epoch": 0.8487518355359766,
"fcm_dpo/beta": 0.0035750193055719137,
"fcm_dpo/delta": 0.019029080867767334,
"fcm_dpo/margin": 218.58831787109375,
"fcm_dpo/q_t": 0.3325645327568054,
"grad_norm": 35.93294143676758,
"learning_rate": 3.478836705390808e-08,
"logits/chosen": 0.313589870929718,
"logits/rejected": 0.33140918612480164,
"logps/chosen": -473.7335205078125,
"logps/ref_chosen": -49.26368713378906,
"logps/ref_rejected": -83.4362564086914,
"logps/rejected": -726.494384765625,
"loss": 0.859,
"margin_dpo/margin_mean": 218.58831787109375,
"margin_dpo/margin_std": 195.58734130859375,
"step": 578
},
{
"KL/chosen_KL_mean": -435.37188720703125,
"KL/mean": -529.50390625,
"KL/rejected_KL_mean": -623.6358642578125,
"KL/std": 218.8971405029297,
"epoch": 0.8502202643171806,
"fcm_dpo/beta": 0.0036643778439611197,
"fcm_dpo/delta": 0.11842280626296997,
"fcm_dpo/margin": 188.2639617919922,
"fcm_dpo/q_t": 0.3539060950279236,
"grad_norm": 43.979103088378906,
"learning_rate": 3.41381639738331e-08,
"logits/chosen": 0.35747650265693665,
"logits/rejected": 0.36031123995780945,
"logps/chosen": -494.2576904296875,
"logps/ref_chosen": -58.88581848144531,
"logps/ref_rejected": -94.78762817382812,
"logps/rejected": -718.4234619140625,
"loss": 0.9431,
"margin_dpo/margin_mean": 188.26394653320312,
"margin_dpo/margin_std": 212.978271484375,
"step": 579
},
{
"KL/chosen_KL_mean": -360.24114990234375,
"KL/mean": -488.2108154296875,
"KL/rejected_KL_mean": -616.1805419921875,
"KL/std": 242.11322021484375,
"epoch": 0.8516886930983847,
"fcm_dpo/beta": 0.0035925463307648897,
"fcm_dpo/delta": -0.13383767008781433,
"fcm_dpo/margin": 255.93939208984375,
"fcm_dpo/q_t": 0.3108557462692261,
"grad_norm": 30.20220375061035,
"learning_rate": 3.349364905389032e-08,
"logits/chosen": 0.23727968335151672,
"logits/rejected": 0.25300416350364685,
"logps/chosen": -408.9479675292969,
"logps/ref_chosen": -48.70683670043945,
"logps/ref_rejected": -81.7583999633789,
"logps/rejected": -697.93896484375,
"loss": 0.8221,
"margin_dpo/margin_mean": 255.9394073486328,
"margin_dpo/margin_std": 249.52294921875,
"step": 580
},
{
"KL/chosen_KL_mean": -450.68023681640625,
"KL/mean": -544.3995361328125,
"KL/rejected_KL_mean": -638.118896484375,
"KL/std": 226.23956298828125,
"epoch": 0.8531571218795888,
"fcm_dpo/beta": 0.003664699150249362,
"fcm_dpo/delta": 0.12125371396541595,
"fcm_dpo/margin": 187.4386444091797,
"fcm_dpo/q_t": 0.35620471835136414,
"grad_norm": 39.6314582824707,
"learning_rate": 3.285483927764726e-08,
"logits/chosen": 0.4068590998649597,
"logits/rejected": 0.4267165958881378,
"logps/chosen": -512.902587890625,
"logps/ref_chosen": -62.22235107421875,
"logps/ref_rejected": -91.73568725585938,
"logps/rejected": -729.8546142578125,
"loss": 0.9458,
"margin_dpo/margin_mean": 187.43865966796875,
"margin_dpo/margin_std": 216.36355590820312,
"step": 581
},
{
"KL/chosen_KL_mean": -409.5931396484375,
"KL/mean": -532.977294921875,
"KL/rejected_KL_mean": -656.3615112304688,
"KL/std": 232.1181182861328,
"epoch": 0.8546255506607929,
"fcm_dpo/beta": 0.0036610260140150785,
"fcm_dpo/delta": -0.11489932984113693,
"fcm_dpo/margin": 246.76834106445312,
"fcm_dpo/q_t": 0.3156302571296692,
"grad_norm": 32.382896423339844,
"learning_rate": 3.222175147833556e-08,
"logits/chosen": 0.37507086992263794,
"logits/rejected": 0.3699257969856262,
"logps/chosen": -467.82183837890625,
"logps/ref_chosen": -58.228660583496094,
"logps/ref_rejected": -110.06959533691406,
"logps/rejected": -766.4310913085938,
"loss": 0.8497,
"margin_dpo/margin_mean": 246.76832580566406,
"margin_dpo/margin_std": 242.81654357910156,
"step": 582
},
{
"KL/chosen_KL_mean": -448.87646484375,
"KL/mean": -538.7371215820312,
"KL/rejected_KL_mean": -628.5977783203125,
"KL/std": 230.92794799804688,
"epoch": 0.856093979441997,
"fcm_dpo/beta": 0.0036516801919788122,
"fcm_dpo/delta": 0.15361306071281433,
"fcm_dpo/margin": 179.72134399414062,
"fcm_dpo/q_t": 0.3652193546295166,
"grad_norm": 43.001060485839844,
"learning_rate": 3.159440233840763e-08,
"logits/chosen": 0.3155054450035095,
"logits/rejected": 0.3352203071117401,
"logps/chosen": -505.73931884765625,
"logps/ref_chosen": -56.86286163330078,
"logps/ref_rejected": -88.4039306640625,
"logps/rejected": -717.001708984375,
"loss": 0.9814,
"margin_dpo/margin_mean": 179.72134399414062,
"margin_dpo/margin_std": 224.05186462402344,
"step": 583
},
{
"KL/chosen_KL_mean": -398.5378112792969,
"KL/mean": -541.896240234375,
"KL/rejected_KL_mean": -685.254638671875,
"KL/std": 245.92782592773438,
"epoch": 0.8575624082232012,
"fcm_dpo/beta": 0.0035745359491556883,
"fcm_dpo/delta": -0.2507776916027069,
"fcm_dpo/margin": 286.71685791015625,
"fcm_dpo/q_t": 0.2880155146121979,
"grad_norm": 34.256526947021484,
"learning_rate": 3.0972808389096635e-08,
"logits/chosen": 0.39042460918426514,
"logits/rejected": 0.42286020517349243,
"logps/chosen": -455.4384765625,
"logps/ref_chosen": -56.90068054199219,
"logps/ref_rejected": -97.63606262207031,
"logps/rejected": -782.8907470703125,
"loss": 0.7289,
"margin_dpo/margin_mean": 286.71685791015625,
"margin_dpo/margin_std": 226.08956909179688,
"step": 584
},
{
"KL/chosen_KL_mean": -460.065185546875,
"KL/mean": -570.1961669921875,
"KL/rejected_KL_mean": -680.3271484375,
"KL/std": 234.60592651367188,
"epoch": 0.8590308370044053,
"fcm_dpo/beta": 0.0035448740236461163,
"fcm_dpo/delta": 0.02077949233353138,
"fcm_dpo/margin": 220.26193237304688,
"fcm_dpo/q_t": 0.3401145339012146,
"grad_norm": 36.705047607421875,
"learning_rate": 3.035698600998121e-08,
"logits/chosen": 0.3827165961265564,
"logits/rejected": 0.4175301790237427,
"logps/chosen": -521.0391845703125,
"logps/ref_chosen": -60.973968505859375,
"logps/ref_rejected": -84.16952514648438,
"logps/rejected": -764.4967041015625,
"loss": 0.9029,
"margin_dpo/margin_mean": 220.26193237304688,
"margin_dpo/margin_std": 237.73251342773438,
"step": 585
},
{
"KL/chosen_KL_mean": -461.75762939453125,
"KL/mean": -558.2662353515625,
"KL/rejected_KL_mean": -654.7747802734375,
"KL/std": 208.19444274902344,
"epoch": 0.8604992657856094,
"fcm_dpo/beta": 0.0036112640518695116,
"fcm_dpo/delta": 0.11033973097801208,
"fcm_dpo/margin": 193.01708984375,
"fcm_dpo/q_t": 0.3578340709209442,
"grad_norm": 36.1845588684082,
"learning_rate": 2.974695142855388e-08,
"logits/chosen": 0.41748034954071045,
"logits/rejected": 0.4262012243270874,
"logps/chosen": -518.61328125,
"logps/ref_chosen": -56.85559844970703,
"logps/ref_rejected": -91.80261993408203,
"logps/rejected": -746.577392578125,
"loss": 0.971,
"margin_dpo/margin_mean": 193.01708984375,
"margin_dpo/margin_std": 244.68115234375,
"step": 586
},
{
"KL/chosen_KL_mean": -383.22210693359375,
"KL/mean": -492.18524169921875,
"KL/rejected_KL_mean": -601.1483154296875,
"KL/std": 240.5768585205078,
"epoch": 0.8619676945668135,
"fcm_dpo/beta": 0.003636622568592429,
"fcm_dpo/delta": 0.008001847192645073,
"fcm_dpo/margin": 217.92623901367188,
"fcm_dpo/q_t": 0.33623430132865906,
"grad_norm": 48.267826080322266,
"learning_rate": 2.9142720719793122e-08,
"logits/chosen": 0.463498592376709,
"logits/rejected": 0.46634483337402344,
"logps/chosen": -427.9136962890625,
"logps/ref_chosen": -44.69159698486328,
"logps/ref_rejected": -82.62385559082031,
"logps/rejected": -683.772216796875,
"loss": 0.8911,
"margin_dpo/margin_mean": 217.9262237548828,
"margin_dpo/margin_std": 232.04367065429688,
"step": 587
},
{
"KL/chosen_KL_mean": -446.12030029296875,
"KL/mean": -551.8760986328125,
"KL/rejected_KL_mean": -657.6319580078125,
"KL/std": 213.5641632080078,
"epoch": 0.8634361233480177,
"fcm_dpo/beta": 0.003631887724623084,
"fcm_dpo/delta": 0.03419031202793121,
"fcm_dpo/margin": 211.51162719726562,
"fcm_dpo/q_t": 0.34210073947906494,
"grad_norm": 43.596290588378906,
"learning_rate": 2.8544309805740018e-08,
"logits/chosen": 0.37401342391967773,
"logits/rejected": 0.3757128119468689,
"logps/chosen": -496.415283203125,
"logps/ref_chosen": -50.29494857788086,
"logps/ref_rejected": -107.36988067626953,
"logps/rejected": -765.0018310546875,
"loss": 0.899,
"margin_dpo/margin_mean": 211.51162719726562,
"margin_dpo/margin_std": 227.99298095703125,
"step": 588
},
{
"KL/chosen_KL_mean": -417.7206726074219,
"KL/mean": -556.0487670898438,
"KL/rejected_KL_mean": -694.3768920898438,
"KL/std": 251.08468627929688,
"epoch": 0.8649045521292217,
"fcm_dpo/beta": 0.003547674510627985,
"fcm_dpo/delta": -0.20126084983348846,
"fcm_dpo/margin": 276.6562194824219,
"fcm_dpo/q_t": 0.29785215854644775,
"grad_norm": 36.69344711303711,
"learning_rate": 2.7951734455078786e-08,
"logits/chosen": 0.36783045530319214,
"logits/rejected": 0.3880786597728729,
"logps/chosen": -477.65057373046875,
"logps/ref_chosen": -59.929908752441406,
"logps/ref_rejected": -111.65534973144531,
"logps/rejected": -806.0322265625,
"loss": 0.7685,
"margin_dpo/margin_mean": 276.6562194824219,
"margin_dpo/margin_std": 234.46910095214844,
"step": 589
},
{
"KL/chosen_KL_mean": -402.7647705078125,
"KL/mean": -522.7323608398438,
"KL/rejected_KL_mean": -642.699951171875,
"KL/std": 220.0315399169922,
"epoch": 0.8663729809104258,
"fcm_dpo/beta": 0.003481434192508459,
"fcm_dpo/delta": -0.03877948969602585,
"fcm_dpo/margin": 239.93521118164062,
"fcm_dpo/q_t": 0.32213735580444336,
"grad_norm": 31.813249588012695,
"learning_rate": 2.736501028272095e-08,
"logits/chosen": 0.2651674449443817,
"logits/rejected": 0.26474422216415405,
"logps/chosen": -458.5745849609375,
"logps/ref_chosen": -55.80979537963867,
"logps/ref_rejected": -106.06282043457031,
"logps/rejected": -748.7627563476562,
"loss": 0.8322,
"margin_dpo/margin_mean": 239.93521118164062,
"margin_dpo/margin_std": 213.3310546875,
"step": 590
},
{
"KL/chosen_KL_mean": -433.9034729003906,
"KL/mean": -554.28173828125,
"KL/rejected_KL_mean": -674.659912109375,
"KL/std": 218.40057373046875,
"epoch": 0.8678414096916299,
"fcm_dpo/beta": 0.0034645451232790947,
"fcm_dpo/delta": -0.0372135229408741,
"fcm_dpo/margin": 240.75643920898438,
"fcm_dpo/q_t": 0.32035619020462036,
"grad_norm": 36.15827560424805,
"learning_rate": 2.678415274939408e-08,
"logits/chosen": 0.47813642024993896,
"logits/rejected": 0.5184578895568848,
"logps/chosen": -490.14410400390625,
"logps/ref_chosen": -56.24061965942383,
"logps/ref_rejected": -83.78629302978516,
"logps/rejected": -758.4462280273438,
"loss": 0.8311,
"margin_dpo/margin_mean": 240.75643920898438,
"margin_dpo/margin_std": 211.91455078125,
"step": 591
},
{
"KL/chosen_KL_mean": -464.30401611328125,
"KL/mean": -573.9903564453125,
"KL/rejected_KL_mean": -683.6767578125,
"KL/std": 229.73587036132812,
"epoch": 0.869309838472834,
"fcm_dpo/beta": 0.003464614972472191,
"fcm_dpo/delta": 0.043035559356212616,
"fcm_dpo/margin": 219.3727569580078,
"fcm_dpo/q_t": 0.3421498239040375,
"grad_norm": 41.69553756713867,
"learning_rate": 2.6209177161234442e-08,
"logits/chosen": 0.5036705732345581,
"logits/rejected": 0.5235416889190674,
"logps/chosen": -512.2442626953125,
"logps/ref_chosen": -47.94025421142578,
"logps/ref_rejected": -75.73287963867188,
"logps/rejected": -759.40966796875,
"loss": 0.9679,
"margin_dpo/margin_mean": 219.37274169921875,
"margin_dpo/margin_std": 278.5716552734375,
"step": 592
},
{
"KL/chosen_KL_mean": -416.8182373046875,
"KL/mean": -500.542236328125,
"KL/rejected_KL_mean": -584.2662353515625,
"KL/std": 228.785400390625,
"epoch": 0.8707782672540382,
"fcm_dpo/beta": 0.0035965959541499615,
"fcm_dpo/delta": 0.21053200960159302,
"fcm_dpo/margin": 167.44802856445312,
"fcm_dpo/q_t": 0.3748427629470825,
"grad_norm": 50.23680114746094,
"learning_rate": 2.564009866938349e-08,
"logits/chosen": 0.31660762429237366,
"logits/rejected": 0.3367197811603546,
"logps/chosen": -465.50897216796875,
"logps/ref_chosen": -48.690757751464844,
"logps/ref_rejected": -60.90800094604492,
"logps/rejected": -645.1742553710938,
"loss": 1.0208,
"margin_dpo/margin_mean": 167.44802856445312,
"margin_dpo/margin_std": 228.37478637695312,
"step": 593
},
{
"KL/chosen_KL_mean": -397.3114013671875,
"KL/mean": -511.14520263671875,
"KL/rejected_KL_mean": -624.97900390625,
"KL/std": 272.0648193359375,
"epoch": 0.8722466960352423,
"fcm_dpo/beta": 0.0036704763770103455,
"fcm_dpo/delta": -0.04310472309589386,
"fcm_dpo/margin": 227.66758728027344,
"fcm_dpo/q_t": 0.3338211178779602,
"grad_norm": 39.49517059326172,
"learning_rate": 2.5076932269588708e-08,
"logits/chosen": 0.3315795361995697,
"logits/rejected": 0.3542073965072632,
"logps/chosen": -452.2463073730469,
"logps/ref_chosen": -54.93488693237305,
"logps/ref_rejected": -86.09967803955078,
"logps/rejected": -711.0787353515625,
"loss": 0.9104,
"margin_dpo/margin_mean": 227.6676025390625,
"margin_dpo/margin_std": 253.65826416015625,
"step": 594
},
{
"KL/chosen_KL_mean": -421.3223876953125,
"KL/mean": -522.7353515625,
"KL/rejected_KL_mean": -624.1483154296875,
"KL/std": 207.23240661621094,
"epoch": 0.8737151248164464,
"fcm_dpo/beta": 0.003634433727711439,
"fcm_dpo/delta": 0.0678904801607132,
"fcm_dpo/margin": 202.82586669921875,
"fcm_dpo/q_t": 0.34680286049842834,
"grad_norm": 36.4913444519043,
"learning_rate": 2.451969280180849e-08,
"logits/chosen": 0.37026911973953247,
"logits/rejected": 0.39209526777267456,
"logps/chosen": -470.7427978515625,
"logps/ref_chosen": -49.4204216003418,
"logps/ref_rejected": -80.62731170654297,
"logps/rejected": -704.7755737304688,
"loss": 0.9031,
"margin_dpo/margin_mean": 202.82586669921875,
"margin_dpo/margin_std": 217.4996337890625,
"step": 595
},
{
"KL/chosen_KL_mean": -426.61724853515625,
"KL/mean": -521.0478515625,
"KL/rejected_KL_mean": -615.478515625,
"KL/std": 226.96453857421875,
"epoch": 0.8751835535976505,
"fcm_dpo/beta": 0.003726873081177473,
"fcm_dpo/delta": 0.10264364629983902,
"fcm_dpo/margin": 188.86123657226562,
"fcm_dpo/q_t": 0.3537423014640808,
"grad_norm": 52.47248077392578,
"learning_rate": 2.396839494982103e-08,
"logits/chosen": 0.35756534337997437,
"logits/rejected": 0.38035476207733154,
"logps/chosen": -486.408935546875,
"logps/ref_chosen": -59.791683197021484,
"logps/ref_rejected": -80.09111785888672,
"logps/rejected": -695.569580078125,
"loss": 0.9556,
"margin_dpo/margin_mean": 188.8612518310547,
"margin_dpo/margin_std": 228.7149200439453,
"step": 596
},
{
"KL/chosen_KL_mean": -430.0118713378906,
"KL/mean": -549.8414916992188,
"KL/rejected_KL_mean": -669.671142578125,
"KL/std": 246.92037963867188,
"epoch": 0.8766519823788547,
"fcm_dpo/beta": 0.003649506950750947,
"fcm_dpo/delta": -0.0845845490694046,
"fcm_dpo/margin": 239.65924072265625,
"fcm_dpo/q_t": 0.3215242922306061,
"grad_norm": 36.12924575805664,
"learning_rate": 2.3423053240837514e-08,
"logits/chosen": 0.2778972089290619,
"logits/rejected": 0.2754002809524536,
"logps/chosen": -487.27264404296875,
"logps/ref_chosen": -57.26078796386719,
"logps/ref_rejected": -100.6937255859375,
"logps/rejected": -770.3648681640625,
"loss": 0.8523,
"margin_dpo/margin_mean": 239.65924072265625,
"margin_dpo/margin_std": 241.89382934570312,
"step": 597
},
{
"KL/chosen_KL_mean": -445.42364501953125,
"KL/mean": -549.445068359375,
"KL/rejected_KL_mean": -653.4664916992188,
"KL/std": 218.0029754638672,
"epoch": 0.8781204111600588,
"fcm_dpo/beta": 0.0036473223008215427,
"fcm_dpo/delta": 0.042003557085990906,
"fcm_dpo/margin": 208.04281616210938,
"fcm_dpo/q_t": 0.33893561363220215,
"grad_norm": 56.39809036254883,
"learning_rate": 2.2883682045119062e-08,
"logits/chosen": 0.4461902976036072,
"logits/rejected": 0.46808603405952454,
"logps/chosen": -497.9421691894531,
"logps/ref_chosen": -52.51850509643555,
"logps/ref_rejected": -89.44385528564453,
"logps/rejected": -742.9103393554688,
"loss": 0.9324,
"margin_dpo/margin_mean": 208.04281616210938,
"margin_dpo/margin_std": 228.42031860351562,
"step": 598
},
{
"KL/chosen_KL_mean": -409.59747314453125,
"KL/mean": -512.0952758789062,
"KL/rejected_KL_mean": -614.5930786132812,
"KL/std": 232.806884765625,
"epoch": 0.8795888399412628,
"fcm_dpo/beta": 0.0037043686024844646,
"fcm_dpo/delta": 0.043617166578769684,
"fcm_dpo/margin": 204.99563598632812,
"fcm_dpo/q_t": 0.33468934893608093,
"grad_norm": 44.148353576660156,
"learning_rate": 2.2350295575598367e-08,
"logits/chosen": 0.3408563733100891,
"logits/rejected": 0.3492530882358551,
"logps/chosen": -459.400146484375,
"logps/ref_chosen": -49.802677154541016,
"logps/ref_rejected": -82.978515625,
"logps/rejected": -697.5715942382812,
"loss": 0.8645,
"margin_dpo/margin_mean": 204.99563598632812,
"margin_dpo/margin_std": 180.45458984375,
"step": 599
},
{
"KL/chosen_KL_mean": -465.63177490234375,
"KL/mean": -566.748779296875,
"KL/rejected_KL_mean": -667.86572265625,
"KL/std": 217.93067932128906,
"epoch": 0.8810572687224669,
"fcm_dpo/beta": 0.0037527671083807945,
"fcm_dpo/delta": 0.044459305703639984,
"fcm_dpo/margin": 202.23391723632812,
"fcm_dpo/q_t": 0.3429974317550659,
"grad_norm": 52.01202392578125,
"learning_rate": 2.1822907887504932e-08,
"logits/chosen": 0.39914214611053467,
"logits/rejected": 0.41855812072753906,
"logps/chosen": -532.066650390625,
"logps/ref_chosen": -66.43487548828125,
"logps/ref_rejected": -85.45649719238281,
"logps/rejected": -753.3222045898438,
"loss": 0.9215,
"margin_dpo/margin_mean": 202.2339324951172,
"margin_dpo/margin_std": 229.5970458984375,
"step": 600
},
{
"KL/chosen_KL_mean": -471.54205322265625,
"KL/mean": -581.1104736328125,
"KL/rejected_KL_mean": -690.678955078125,
"KL/std": 239.7049102783203,
"epoch": 0.882525697503671,
"fcm_dpo/beta": 0.0037345641758292913,
"fcm_dpo/delta": -0.02068949118256569,
"fcm_dpo/margin": 219.13687133789062,
"fcm_dpo/q_t": 0.33221566677093506,
"grad_norm": 43.92999267578125,
"learning_rate": 2.1301532877994742e-08,
"logits/chosen": 0.35876715183258057,
"logits/rejected": 0.3796877861022949,
"logps/chosen": -530.6756591796875,
"logps/ref_chosen": -59.13361358642578,
"logps/ref_rejected": -94.69093322753906,
"logps/rejected": -785.369873046875,
"loss": 0.8863,
"margin_dpo/margin_mean": 219.13687133789062,
"margin_dpo/margin_std": 236.77896118164062,
"step": 601
},
{
"KL/chosen_KL_mean": -375.32904052734375,
"KL/mean": -511.0220947265625,
"KL/rejected_KL_mean": -646.715087890625,
"KL/std": 236.4164581298828,
"epoch": 0.8839941262848752,
"fcm_dpo/beta": 0.0036473888903856277,
"fcm_dpo/delta": -0.2102523148059845,
"fcm_dpo/margin": 271.3861083984375,
"fcm_dpo/q_t": 0.2962424159049988,
"grad_norm": 41.782928466796875,
"learning_rate": 2.0786184285784298e-08,
"logits/chosen": 0.37781381607055664,
"logits/rejected": 0.3983898162841797,
"logps/chosen": -423.92254638671875,
"logps/ref_chosen": -48.59352111816406,
"logps/ref_rejected": -87.6685562133789,
"logps/rejected": -734.3836669921875,
"loss": 0.7675,
"margin_dpo/margin_mean": 271.3861083984375,
"margin_dpo/margin_std": 233.6563262939453,
"step": 602
},
{
"KL/chosen_KL_mean": -410.86407470703125,
"KL/mean": -521.83203125,
"KL/rejected_KL_mean": -632.7999877929688,
"KL/std": 244.81134033203125,
"epoch": 0.8854625550660793,
"fcm_dpo/beta": 0.0036042386200278997,
"fcm_dpo/delta": -2.483464777469635e-05,
"fcm_dpo/margin": 221.93588256835938,
"fcm_dpo/q_t": 0.33811676502227783,
"grad_norm": 44.7195930480957,
"learning_rate": 2.0276875690788204e-08,
"logits/chosen": 0.28510865569114685,
"logits/rejected": 0.3141903281211853,
"logps/chosen": -481.2786865234375,
"logps/ref_chosen": -70.41461944580078,
"logps/ref_rejected": -100.32559967041016,
"logps/rejected": -733.1256103515625,
"loss": 0.8952,
"margin_dpo/margin_mean": 221.93589782714844,
"margin_dpo/margin_std": 245.03948974609375,
"step": 603
},
{
"KL/chosen_KL_mean": -377.646240234375,
"KL/mean": -506.044677734375,
"KL/rejected_KL_mean": -634.4429931640625,
"KL/std": 218.4007568359375,
"epoch": 0.8869309838472834,
"fcm_dpo/beta": 0.0035292627289891243,
"fcm_dpo/delta": -0.11708834767341614,
"fcm_dpo/margin": 256.7967834472656,
"fcm_dpo/q_t": 0.30936452746391296,
"grad_norm": 39.38010787963867,
"learning_rate": 1.977362051376158e-08,
"logits/chosen": 0.3584352433681488,
"logits/rejected": 0.3700353503227234,
"logps/chosen": -424.1043395996094,
"logps/ref_chosen": -46.45808029174805,
"logps/ref_rejected": -91.8544921875,
"logps/rejected": -726.2974853515625,
"loss": 0.824,
"margin_dpo/margin_mean": 256.7967834472656,
"margin_dpo/margin_std": 237.95303344726562,
"step": 604
},
{
"KL/chosen_KL_mean": -433.8162841796875,
"KL/mean": -544.268310546875,
"KL/rejected_KL_mean": -654.7203369140625,
"KL/std": 223.2996063232422,
"epoch": 0.8883994126284875,
"fcm_dpo/beta": 0.0035091633908450603,
"fcm_dpo/delta": 0.026538338512182236,
"fcm_dpo/margin": 220.90402221679688,
"fcm_dpo/q_t": 0.34067851305007935,
"grad_norm": 29.942068099975586,
"learning_rate": 1.9276432015946446e-08,
"logits/chosen": 0.30465924739837646,
"logits/rejected": 0.3171878755092621,
"logps/chosen": -500.06561279296875,
"logps/ref_chosen": -66.24933624267578,
"logps/ref_rejected": -102.30496978759766,
"logps/rejected": -757.0252685546875,
"loss": 0.8932,
"margin_dpo/margin_mean": 220.90403747558594,
"margin_dpo/margin_std": 245.03860473632812,
"step": 605
},
{
"KL/chosen_KL_mean": -403.8315734863281,
"KL/mean": -526.64990234375,
"KL/rejected_KL_mean": -649.4683837890625,
"KL/std": 218.93609619140625,
"epoch": 0.8898678414096917,
"fcm_dpo/beta": 0.0035280329175293446,
"fcm_dpo/delta": -0.07430367916822433,
"fcm_dpo/margin": 245.6367645263672,
"fcm_dpo/q_t": 0.3186294436454773,
"grad_norm": 45.23887252807617,
"learning_rate": 1.8785323298722093e-08,
"logits/chosen": 0.354251503944397,
"logits/rejected": 0.3615856170654297,
"logps/chosen": -458.65069580078125,
"logps/ref_chosen": -54.819122314453125,
"logps/ref_rejected": -98.37146759033203,
"logps/rejected": -747.83984375,
"loss": 0.8256,
"margin_dpo/margin_mean": 245.6367645263672,
"margin_dpo/margin_std": 220.42266845703125,
"step": 606
},
{
"KL/chosen_KL_mean": -425.9730224609375,
"KL/mean": -522.0699462890625,
"KL/rejected_KL_mean": -618.1668090820312,
"KL/std": 217.41937255859375,
"epoch": 0.8913362701908958,
"fcm_dpo/beta": 0.003552454523742199,
"fcm_dpo/delta": 0.1254272311925888,
"fcm_dpo/margin": 192.19375610351562,
"fcm_dpo/q_t": 0.35280275344848633,
"grad_norm": 35.883914947509766,
"learning_rate": 1.8300307303259904e-08,
"logits/chosen": 0.3167957663536072,
"logits/rejected": 0.3372369408607483,
"logps/chosen": -484.05706787109375,
"logps/ref_chosen": -58.08403778076172,
"logps/ref_rejected": -79.777099609375,
"logps/rejected": -697.9439086914062,
"loss": 0.9203,
"margin_dpo/margin_mean": 192.19375610351562,
"margin_dpo/margin_std": 190.5302734375,
"step": 607
},
{
"KL/chosen_KL_mean": -415.585693359375,
"KL/mean": -521.9092407226562,
"KL/rejected_KL_mean": -628.2327880859375,
"KL/std": 218.6051025390625,
"epoch": 0.8928046989720999,
"fcm_dpo/beta": 0.003595003392547369,
"fcm_dpo/delta": 0.03830421715974808,
"fcm_dpo/margin": 212.6471405029297,
"fcm_dpo/q_t": 0.3348754942417145,
"grad_norm": 34.83126449584961,
"learning_rate": 1.7821396810182437e-08,
"logits/chosen": 0.30306464433670044,
"logits/rejected": 0.33365947008132935,
"logps/chosen": -473.0364990234375,
"logps/ref_chosen": -57.450836181640625,
"logps/ref_rejected": -94.77339172363281,
"logps/rejected": -723.0062255859375,
"loss": 0.8616,
"margin_dpo/margin_mean": 212.6471405029297,
"margin_dpo/margin_std": 190.2625732421875,
"step": 608
},
{
"KL/chosen_KL_mean": -408.29541015625,
"KL/mean": -530.133056640625,
"KL/rejected_KL_mean": -651.970703125,
"KL/std": 240.07913208007812,
"epoch": 0.8942731277533039,
"fcm_dpo/beta": 0.0035578757524490356,
"fcm_dpo/delta": -0.07329061627388,
"fcm_dpo/margin": 243.67532348632812,
"fcm_dpo/q_t": 0.31681567430496216,
"grad_norm": 38.81436538696289,
"learning_rate": 1.7348604439226617e-08,
"logits/chosen": 0.3799295425415039,
"logits/rejected": 0.4069666266441345,
"logps/chosen": -467.10076904296875,
"logps/ref_chosen": -58.805355072021484,
"logps/ref_rejected": -88.81600952148438,
"logps/rejected": -740.7867431640625,
"loss": 0.8195,
"margin_dpo/margin_mean": 243.67532348632812,
"margin_dpo/margin_std": 222.36453247070312,
"step": 609
},
{
"KL/chosen_KL_mean": -399.6932373046875,
"KL/mean": -503.40771484375,
"KL/rejected_KL_mean": -607.1221923828125,
"KL/std": 210.8145751953125,
"epoch": 0.895741556534508,
"fcm_dpo/beta": 0.0035813814029097557,
"fcm_dpo/delta": 0.06151764839887619,
"fcm_dpo/margin": 207.42897033691406,
"fcm_dpo/q_t": 0.3418347239494324,
"grad_norm": 38.237361907958984,
"learning_rate": 1.6881942648911074e-08,
"logits/chosen": 0.3074260950088501,
"logits/rejected": 0.3430824875831604,
"logps/chosen": -465.3882751464844,
"logps/ref_chosen": -65.69503784179688,
"logps/ref_rejected": -83.40538787841797,
"logps/rejected": -690.527587890625,
"loss": 0.8948,
"margin_dpo/margin_mean": 207.42898559570312,
"margin_dpo/margin_std": 206.7069549560547,
"step": 610
},
{
"KL/chosen_KL_mean": -426.5433044433594,
"KL/mean": -542.793701171875,
"KL/rejected_KL_mean": -659.0440673828125,
"KL/std": 228.12405395507812,
"epoch": 0.8972099853157122,
"fcm_dpo/beta": 0.003594658337533474,
"fcm_dpo/delta": -0.04010578989982605,
"fcm_dpo/margin": 232.50071716308594,
"fcm_dpo/q_t": 0.3268967270851135,
"grad_norm": 36.26604461669922,
"learning_rate": 1.6421423736208e-08,
"logits/chosen": 0.3895799517631531,
"logits/rejected": 0.4121781587600708,
"logps/chosen": -479.14276123046875,
"logps/ref_chosen": -52.59946823120117,
"logps/ref_rejected": -86.33099365234375,
"logps/rejected": -745.375,
"loss": 0.8602,
"margin_dpo/margin_mean": 232.50071716308594,
"margin_dpo/margin_std": 227.5194854736328,
"step": 611
},
{
"KL/chosen_KL_mean": -422.2787780761719,
"KL/mean": -545.01708984375,
"KL/rejected_KL_mean": -667.7554321289062,
"KL/std": 208.62973022460938,
"epoch": 0.8986784140969163,
"fcm_dpo/beta": 0.003525904379785061,
"fcm_dpo/delta": -0.07167483866214752,
"fcm_dpo/margin": 245.47662353515625,
"fcm_dpo/q_t": 0.31289637088775635,
"grad_norm": 41.34052658081055,
"learning_rate": 1.5967059836219042e-08,
"logits/chosen": 0.3888121247291565,
"logits/rejected": 0.4153617024421692,
"logps/chosen": -481.6025085449219,
"logps/ref_chosen": -59.32372283935547,
"logps/ref_rejected": -88.31239318847656,
"logps/rejected": -756.0678100585938,
"loss": 0.7975,
"margin_dpo/margin_mean": 245.47662353515625,
"margin_dpo/margin_std": 193.85101318359375,
"step": 612
},
{
"KL/chosen_KL_mean": -387.6116638183594,
"KL/mean": -502.537353515625,
"KL/rejected_KL_mean": -617.4630126953125,
"KL/std": 222.54290771484375,
"epoch": 0.9001468428781204,
"fcm_dpo/beta": 0.0035072462633252144,
"fcm_dpo/delta": -0.006700858939439058,
"fcm_dpo/margin": 229.85130310058594,
"fcm_dpo/q_t": 0.3251153230667114,
"grad_norm": 40.078956604003906,
"learning_rate": 1.551886292185553e-08,
"logits/chosen": 0.2332531213760376,
"logits/rejected": 0.23660428822040558,
"logps/chosen": -447.34161376953125,
"logps/ref_chosen": -59.72996520996094,
"logps/ref_rejected": -105.10752868652344,
"logps/rejected": -722.570556640625,
"loss": 0.8226,
"margin_dpo/margin_mean": 229.85128784179688,
"margin_dpo/margin_std": 184.74441528320312,
"step": 613
},
{
"KL/chosen_KL_mean": -423.49017333984375,
"KL/mean": -545.447021484375,
"KL/rejected_KL_mean": -667.4038696289062,
"KL/std": 228.83831787109375,
"epoch": 0.9016152716593245,
"fcm_dpo/beta": 0.003461125772446394,
"fcm_dpo/delta": -0.04895541071891785,
"fcm_dpo/margin": 243.9136962890625,
"fcm_dpo/q_t": 0.3211820125579834,
"grad_norm": 47.83527755737305,
"learning_rate": 1.507684480352292e-08,
"logits/chosen": 0.43596264719963074,
"logits/rejected": 0.44089680910110474,
"logps/chosen": -476.4291687011719,
"logps/ref_chosen": -52.93898010253906,
"logps/ref_rejected": -104.67938232421875,
"logps/rejected": -772.083251953125,
"loss": 0.8376,
"margin_dpo/margin_mean": 243.91368103027344,
"margin_dpo/margin_std": 221.93887329101562,
"step": 614
},
{
"KL/chosen_KL_mean": -399.7521667480469,
"KL/mean": -496.95867919921875,
"KL/rejected_KL_mean": -594.1651611328125,
"KL/std": 232.88909912109375,
"epoch": 0.9030837004405287,
"fcm_dpo/beta": 0.0035465422552078962,
"fcm_dpo/delta": 0.11760629713535309,
"fcm_dpo/margin": 194.41302490234375,
"fcm_dpo/q_t": 0.3538597524166107,
"grad_norm": 35.42436599731445,
"learning_rate": 1.4641017128809801e-08,
"logits/chosen": 0.23408935964107513,
"logits/rejected": 0.25060975551605225,
"logps/chosen": -465.5694580078125,
"logps/ref_chosen": -65.81727600097656,
"logps/ref_rejected": -95.17749786376953,
"logps/rejected": -689.3426513671875,
"loss": 0.9149,
"margin_dpo/margin_mean": 194.41302490234375,
"margin_dpo/margin_std": 198.14556884765625,
"step": 615
},
{
"KL/chosen_KL_mean": -471.95989990234375,
"KL/mean": -557.6803588867188,
"KL/rejected_KL_mean": -643.40087890625,
"KL/std": 202.3949432373047,
"epoch": 0.9045521292217328,
"fcm_dpo/beta": 0.0036708577536046505,
"fcm_dpo/delta": 0.18005023896694183,
"fcm_dpo/margin": 171.44093322753906,
"fcm_dpo/q_t": 0.36842331290245056,
"grad_norm": 45.34204864501953,
"learning_rate": 1.4211391382180637e-08,
"logits/chosen": 0.34676775336265564,
"logits/rejected": 0.37802496552467346,
"logps/chosen": -537.0927734375,
"logps/ref_chosen": -65.13285827636719,
"logps/ref_rejected": -74.70050048828125,
"logps/rejected": -718.101318359375,
"loss": 0.9901,
"margin_dpo/margin_mean": 171.44094848632812,
"margin_dpo/margin_std": 215.03761291503906,
"step": 616
},
{
"KL/chosen_KL_mean": -433.5853271484375,
"KL/mean": -518.7471923828125,
"KL/rejected_KL_mean": -603.9089965820312,
"KL/std": 193.24517822265625,
"epoch": 0.9060205580029369,
"fcm_dpo/beta": 0.0037929839454591274,
"fcm_dpo/delta": 0.16294321417808533,
"fcm_dpo/margin": 170.32363891601562,
"fcm_dpo/q_t": 0.3635261654853821,
"grad_norm": 44.73681640625,
"learning_rate": 1.378797888467345e-08,
"logits/chosen": 0.30464887619018555,
"logits/rejected": 0.329974889755249,
"logps/chosen": -496.59088134765625,
"logps/ref_chosen": -63.005550384521484,
"logps/ref_rejected": -64.234130859375,
"logps/rejected": -668.1431274414062,
"loss": 0.9621,
"margin_dpo/margin_mean": 170.32363891601562,
"margin_dpo/margin_std": 197.97344970703125,
"step": 617
},
{
"KL/chosen_KL_mean": -440.02435302734375,
"KL/mean": -560.755859375,
"KL/rejected_KL_mean": -681.4873046875,
"KL/std": 222.37545776367188,
"epoch": 0.9074889867841409,
"fcm_dpo/beta": 0.003762473352253437,
"fcm_dpo/delta": -0.1195143535733223,
"fcm_dpo/margin": 241.4630126953125,
"fcm_dpo/q_t": 0.31400012969970703,
"grad_norm": 43.647186279296875,
"learning_rate": 1.3370790793601371e-08,
"logits/chosen": 0.3098023533821106,
"logits/rejected": 0.35203438997268677,
"logps/chosen": -507.1257019042969,
"logps/ref_chosen": -67.10134887695312,
"logps/ref_rejected": -92.15340423583984,
"logps/rejected": -773.6407470703125,
"loss": 0.8398,
"margin_dpo/margin_mean": 241.46299743652344,
"margin_dpo/margin_std": 234.85427856445312,
"step": 618
},
{
"KL/chosen_KL_mean": -433.3082580566406,
"KL/mean": -527.0829467773438,
"KL/rejected_KL_mean": -620.8576049804688,
"KL/std": 218.11892700195312,
"epoch": 0.908957415565345,
"fcm_dpo/beta": 0.003781287930905819,
"fcm_dpo/delta": 0.09707266837358475,
"fcm_dpo/margin": 187.5493621826172,
"fcm_dpo/q_t": 0.35968726873397827,
"grad_norm": 39.4860725402832,
"learning_rate": 1.2959838102258535e-08,
"logits/chosen": 0.35403043031692505,
"logits/rejected": 0.3776950538158417,
"logps/chosen": -489.2864990234375,
"logps/ref_chosen": -55.978233337402344,
"logps/ref_rejected": -93.1854019165039,
"logps/rejected": -714.04296875,
"loss": 0.9901,
"margin_dpo/margin_mean": 187.5493621826172,
"margin_dpo/margin_std": 255.41448974609375,
"step": 619
},
{
"KL/chosen_KL_mean": -406.3300476074219,
"KL/mean": -509.80792236328125,
"KL/rejected_KL_mean": -613.2857666015625,
"KL/std": 210.70660400390625,
"epoch": 0.9104258443465492,
"fcm_dpo/beta": 0.0037673478946089745,
"fcm_dpo/delta": 0.020875804126262665,
"fcm_dpo/margin": 206.95571899414062,
"fcm_dpo/q_t": 0.3405061960220337,
"grad_norm": 50.14612579345703,
"learning_rate": 1.2555131639630567e-08,
"logits/chosen": 0.3730146288871765,
"logits/rejected": 0.40417569875717163,
"logps/chosen": -466.1275634765625,
"logps/ref_chosen": -59.79750061035156,
"logps/ref_rejected": -78.41075134277344,
"logps/rejected": -691.696533203125,
"loss": 0.9001,
"margin_dpo/margin_mean": 206.95571899414062,
"margin_dpo/margin_std": 227.1011505126953,
"step": 620
},
{
"KL/chosen_KL_mean": -416.3944091796875,
"KL/mean": -542.18994140625,
"KL/rejected_KL_mean": -667.9854736328125,
"KL/std": 217.998291015625,
"epoch": 0.9118942731277533,
"fcm_dpo/beta": 0.003742808010429144,
"fcm_dpo/delta": -0.15637120604515076,
"fcm_dpo/margin": 251.5911102294922,
"fcm_dpo/q_t": 0.3048107326030731,
"grad_norm": 43.15443801879883,
"learning_rate": 1.2156682070109086e-08,
"logits/chosen": 0.47899624705314636,
"logits/rejected": 0.49514925479888916,
"logps/chosen": -470.3281555175781,
"logps/ref_chosen": -53.93375778198242,
"logps/ref_rejected": -88.36951446533203,
"logps/rejected": -756.35498046875,
"loss": 0.7993,
"margin_dpo/margin_mean": 251.5911102294922,
"margin_dpo/margin_std": 224.11578369140625,
"step": 621
},
{
"KL/chosen_KL_mean": -408.5030212402344,
"KL/mean": -518.6573486328125,
"KL/rejected_KL_mean": -628.8116455078125,
"KL/std": 197.64805603027344,
"epoch": 0.9133627019089574,
"fcm_dpo/beta": 0.0036497246474027634,
"fcm_dpo/delta": -0.006030656397342682,
"fcm_dpo/margin": 220.30859375,
"fcm_dpo/q_t": 0.33241087198257446,
"grad_norm": 36.527130126953125,
"learning_rate": 1.1764499893210878e-08,
"logits/chosen": 0.2730959951877594,
"logits/rejected": 0.28664201498031616,
"logps/chosen": -468.7888488769531,
"logps/ref_chosen": -60.28582000732422,
"logps/ref_rejected": -85.51873779296875,
"logps/rejected": -714.330322265625,
"loss": 0.8572,
"margin_dpo/margin_mean": 220.30859375,
"margin_dpo/margin_std": 212.906005859375,
"step": 622
},
{
"KL/chosen_KL_mean": -440.00592041015625,
"KL/mean": -529.0611572265625,
"KL/rejected_KL_mean": -618.116455078125,
"KL/std": 196.42318725585938,
"epoch": 0.9148311306901615,
"fcm_dpo/beta": 0.0037519452162086964,
"fcm_dpo/delta": 0.14135098457336426,
"fcm_dpo/margin": 178.11048889160156,
"fcm_dpo/q_t": 0.36387038230895996,
"grad_norm": 47.815452575683594,
"learning_rate": 1.1378595443300998e-08,
"logits/chosen": 0.45722734928131104,
"logits/rejected": 0.4867405891418457,
"logps/chosen": -504.16290283203125,
"logps/ref_chosen": -64.1569595336914,
"logps/ref_rejected": -85.08304595947266,
"logps/rejected": -703.199462890625,
"loss": 0.9863,
"margin_dpo/margin_mean": 178.11048889160156,
"margin_dpo/margin_std": 237.95982360839844,
"step": 623
},
{
"KL/chosen_KL_mean": -433.9539794921875,
"KL/mean": -552.2816162109375,
"KL/rejected_KL_mean": -670.609375,
"KL/std": 225.03269958496094,
"epoch": 0.9162995594713657,
"fcm_dpo/beta": 0.0037298626266419888,
"fcm_dpo/delta": -0.0908697247505188,
"fcm_dpo/margin": 236.65536499023438,
"fcm_dpo/q_t": 0.31607311964035034,
"grad_norm": 42.4611930847168,
"learning_rate": 1.0998978889320582e-08,
"logits/chosen": 0.272026389837265,
"logits/rejected": 0.31344348192214966,
"logps/chosen": -505.8725891113281,
"logps/ref_chosen": -71.91862487792969,
"logps/ref_rejected": -97.13203430175781,
"logps/rejected": -767.7413330078125,
"loss": 0.8393,
"margin_dpo/margin_mean": 236.65538024902344,
"margin_dpo/margin_std": 227.3984375,
"step": 624
},
{
"KL/chosen_KL_mean": -404.8118896484375,
"KL/mean": -517.1644287109375,
"KL/rejected_KL_mean": -629.5169067382812,
"KL/std": 228.5667266845703,
"epoch": 0.9177679882525698,
"fcm_dpo/beta": 0.003692234866321087,
"fcm_dpo/delta": -0.032496195286512375,
"fcm_dpo/margin": 224.70501708984375,
"fcm_dpo/q_t": 0.3303322196006775,
"grad_norm": 46.905677795410156,
"learning_rate": 1.0625660234518913e-08,
"logits/chosen": 0.34732866287231445,
"logits/rejected": 0.3889002799987793,
"logps/chosen": -463.1539611816406,
"logps/ref_chosen": -58.342071533203125,
"logps/ref_rejected": -86.09038543701172,
"logps/rejected": -715.6072998046875,
"loss": 0.8505,
"margin_dpo/margin_mean": 224.7050323486328,
"margin_dpo/margin_std": 226.28468322753906,
"step": 625
},
{
"KL/chosen_KL_mean": -436.82305908203125,
"KL/mean": -520.375732421875,
"KL/rejected_KL_mean": -603.928466796875,
"KL/std": 223.38800048828125,
"epoch": 0.9192364170337739,
"fcm_dpo/beta": 0.003800017526373267,
"fcm_dpo/delta": 0.1756470501422882,
"fcm_dpo/margin": 167.10537719726562,
"fcm_dpo/q_t": 0.36899447441101074,
"grad_norm": 52.31256866455078,
"learning_rate": 1.0258649316189721e-08,
"logits/chosen": 0.21514081954956055,
"logits/rejected": 0.23527026176452637,
"logps/chosen": -511.9356689453125,
"logps/ref_chosen": -75.11260986328125,
"logps/ref_rejected": -99.188720703125,
"logps/rejected": -703.1171875,
"loss": 0.9835,
"margin_dpo/margin_mean": 167.10537719726562,
"margin_dpo/margin_std": 206.45814514160156,
"step": 626
},
{
"KL/chosen_KL_mean": -373.7752380371094,
"KL/mean": -493.3340759277344,
"KL/rejected_KL_mean": -612.8929443359375,
"KL/std": 255.04714965820312,
"epoch": 0.920704845814978,
"fcm_dpo/beta": 0.003763319458812475,
"fcm_dpo/delta": -0.10977232456207275,
"fcm_dpo/margin": 239.11770629882812,
"fcm_dpo/q_t": 0.3324761986732483,
"grad_norm": 54.27702331542969,
"learning_rate": 9.897955805412e-09,
"logits/chosen": 0.2599024772644043,
"logits/rejected": 0.2572460472583771,
"logps/chosen": -421.51837158203125,
"logps/ref_chosen": -47.74314880371094,
"logps/ref_rejected": -106.75448608398438,
"logps/rejected": -719.6474609375,
"loss": 0.9026,
"margin_dpo/margin_mean": 239.11770629882812,
"margin_dpo/margin_std": 296.06634521484375,
"step": 627
},
{
"KL/chosen_KL_mean": -424.67303466796875,
"KL/mean": -548.8368530273438,
"KL/rejected_KL_mean": -673.0006713867188,
"KL/std": 218.74835205078125,
"epoch": 0.922173274596182,
"fcm_dpo/beta": 0.0036470850463956594,
"fcm_dpo/delta": -0.11861401796340942,
"fcm_dpo/margin": 248.32763671875,
"fcm_dpo/q_t": 0.3126278221607208,
"grad_norm": 31.617103576660156,
"learning_rate": 9.543589206795238e-09,
"logits/chosen": 0.3635599613189697,
"logits/rejected": 0.3939652144908905,
"logps/chosen": -484.8559875488281,
"logps/ref_chosen": -60.182945251464844,
"logps/ref_rejected": -101.55467224121094,
"logps/rejected": -774.5553588867188,
"loss": 0.8208,
"margin_dpo/margin_mean": 248.32762145996094,
"margin_dpo/margin_std": 228.94168090820312,
"step": 628
},
{
"KL/chosen_KL_mean": -443.83966064453125,
"KL/mean": -550.1680908203125,
"KL/rejected_KL_mean": -656.4964599609375,
"KL/std": 213.83009338378906,
"epoch": 0.9236417033773862,
"fcm_dpo/beta": 0.00366116501390934,
"fcm_dpo/delta": 0.023219341412186623,
"fcm_dpo/margin": 212.65676879882812,
"fcm_dpo/q_t": 0.3320544362068176,
"grad_norm": 44.99949645996094,
"learning_rate": 9.19555885822887e-09,
"logits/chosen": 0.3668147921562195,
"logits/rejected": 0.39162763953208923,
"logps/chosen": -508.0531921386719,
"logps/ref_chosen": -64.21354675292969,
"logps/ref_rejected": -91.65367126464844,
"logps/rejected": -748.150146484375,
"loss": 0.8466,
"margin_dpo/margin_mean": 212.65676879882812,
"margin_dpo/margin_std": 182.10867309570312,
"step": 629
},
{
"KL/chosen_KL_mean": -376.22467041015625,
"KL/mean": -463.4542236328125,
"KL/rejected_KL_mean": -550.683837890625,
"KL/std": 224.61175537109375,
"epoch": 0.9251101321585903,
"fcm_dpo/beta": 0.003765605855733156,
"fcm_dpo/delta": 0.15256844460964203,
"fcm_dpo/margin": 174.45912170410156,
"fcm_dpo/q_t": 0.36428147554397583,
"grad_norm": 54.26115417480469,
"learning_rate": 8.85387393063622e-09,
"logits/chosen": 0.2349189817905426,
"logits/rejected": 0.25930139422416687,
"logps/chosen": -435.51568603515625,
"logps/ref_chosen": -59.29100036621094,
"logps/ref_rejected": -83.59829711914062,
"logps/rejected": -634.2821044921875,
"loss": 0.9658,
"margin_dpo/margin_mean": 174.45912170410156,
"margin_dpo/margin_std": 213.33807373046875,
"step": 630
},
{
"KL/chosen_KL_mean": -453.9466552734375,
"KL/mean": -548.2926635742188,
"KL/rejected_KL_mean": -642.638671875,
"KL/std": 223.95266723632812,
"epoch": 0.9265785609397944,
"fcm_dpo/beta": 0.003838051576167345,
"fcm_dpo/delta": 0.08150237053632736,
"fcm_dpo/margin": 188.6920623779297,
"fcm_dpo/q_t": 0.3542540371417999,
"grad_norm": 41.3115348815918,
"learning_rate": 8.518543427732949e-09,
"logits/chosen": 0.4632750153541565,
"logits/rejected": 0.49574044346809387,
"logps/chosen": -513.4002685546875,
"logps/ref_chosen": -59.45360565185547,
"logps/ref_rejected": -80.95156860351562,
"logps/rejected": -723.5902709960938,
"loss": 0.984,
"margin_dpo/margin_mean": 188.69207763671875,
"margin_dpo/margin_std": 250.87442016601562,
"step": 631
},
{
"KL/chosen_KL_mean": -445.6044616699219,
"KL/mean": -548.7745361328125,
"KL/rejected_KL_mean": -651.944580078125,
"KL/std": 197.17782592773438,
"epoch": 0.9280469897209985,
"fcm_dpo/beta": 0.0038299935404211283,
"fcm_dpo/delta": 0.00984569638967514,
"fcm_dpo/margin": 206.340087890625,
"fcm_dpo/q_t": 0.33502843976020813,
"grad_norm": 51.954689025878906,
"learning_rate": 8.189576185789637e-09,
"logits/chosen": 0.4514876902103424,
"logits/rejected": 0.48781538009643555,
"logps/chosen": -506.95599365234375,
"logps/ref_chosen": -61.35155487060547,
"logps/ref_rejected": -86.16017150878906,
"logps/rejected": -738.104736328125,
"loss": 0.9268,
"margin_dpo/margin_mean": 206.340087890625,
"margin_dpo/margin_std": 238.1063232421875,
"step": 632
},
{
"KL/chosen_KL_mean": -445.6147155761719,
"KL/mean": -528.477783203125,
"KL/rejected_KL_mean": -611.3408203125,
"KL/std": 218.96871948242188,
"epoch": 0.9295154185022027,
"fcm_dpo/beta": 0.003975407220423222,
"fcm_dpo/delta": 0.14825844764709473,
"fcm_dpo/margin": 165.72608947753906,
"fcm_dpo/q_t": 0.3641470968723297,
"grad_norm": 43.61115646362305,
"learning_rate": 7.866980873399015e-09,
"logits/chosen": 0.39295196533203125,
"logits/rejected": 0.39339303970336914,
"logps/chosen": -502.89288330078125,
"logps/ref_chosen": -57.27816390991211,
"logps/ref_rejected": -91.58395385742188,
"logps/rejected": -702.9247436523438,
"loss": 1.0067,
"margin_dpo/margin_mean": 165.72608947753906,
"margin_dpo/margin_std": 223.81736755371094,
"step": 633
},
{
"KL/chosen_KL_mean": -447.18731689453125,
"KL/mean": -530.3380126953125,
"KL/rejected_KL_mean": -613.48876953125,
"KL/std": 233.11648559570312,
"epoch": 0.9309838472834068,
"fcm_dpo/beta": 0.004039407707750797,
"fcm_dpo/delta": 0.13770422339439392,
"fcm_dpo/margin": 166.3013916015625,
"fcm_dpo/q_t": 0.36527395248413086,
"grad_norm": 50.8237190246582,
"learning_rate": 7.550765991247654e-09,
"logits/chosen": 0.2625778913497925,
"logits/rejected": 0.26742881536483765,
"logps/chosen": -513.8062744140625,
"logps/ref_chosen": -66.61896514892578,
"logps/ref_rejected": -107.12564849853516,
"logps/rejected": -720.6143798828125,
"loss": 0.9847,
"margin_dpo/margin_mean": 166.3013916015625,
"margin_dpo/margin_std": 215.0738525390625,
"step": 634
},
{
"KL/chosen_KL_mean": -420.28643798828125,
"KL/mean": -511.7974548339844,
"KL/rejected_KL_mean": -603.3084716796875,
"KL/std": 232.0856475830078,
"epoch": 0.9324522760646109,
"fcm_dpo/beta": 0.004115741699934006,
"fcm_dpo/delta": 0.05048708617687225,
"fcm_dpo/margin": 183.02200317382812,
"fcm_dpo/q_t": 0.3472629189491272,
"grad_norm": 42.469337463378906,
"learning_rate": 7.240939871891699e-09,
"logits/chosen": 0.23344659805297852,
"logits/rejected": 0.2846734821796417,
"logps/chosen": -494.241943359375,
"logps/ref_chosen": -73.95551300048828,
"logps/ref_rejected": -82.50045776367188,
"logps/rejected": -685.8089599609375,
"loss": 0.9132,
"margin_dpo/margin_mean": 183.0220184326172,
"margin_dpo/margin_std": 207.57760620117188,
"step": 635
},
{
"KL/chosen_KL_mean": -409.06329345703125,
"KL/mean": -514.981689453125,
"KL/rejected_KL_mean": -620.900146484375,
"KL/std": 233.29795837402344,
"epoch": 0.933920704845815,
"fcm_dpo/beta": 0.004116167314350605,
"fcm_dpo/delta": -0.08010595291852951,
"fcm_dpo/margin": 211.83688354492188,
"fcm_dpo/q_t": 0.31917688250541687,
"grad_norm": 41.35598373413086,
"learning_rate": 6.937510679537628e-09,
"logits/chosen": 0.2601292133331299,
"logits/rejected": 0.2906838655471802,
"logps/chosen": -468.69219970703125,
"logps/ref_chosen": -59.628910064697266,
"logps/ref_rejected": -81.97883605957031,
"logps/rejected": -702.8790283203125,
"loss": 0.8391,
"margin_dpo/margin_mean": 211.83688354492188,
"margin_dpo/margin_std": 202.292724609375,
"step": 636
},
{
"KL/chosen_KL_mean": -432.2290344238281,
"KL/mean": -548.3233642578125,
"KL/rejected_KL_mean": -664.4176635742188,
"KL/std": 238.66481018066406,
"epoch": 0.9353891336270191,
"fcm_dpo/beta": 0.003974968567490578,
"fcm_dpo/delta": -0.13554571568965912,
"fcm_dpo/margin": 232.18862915039062,
"fcm_dpo/q_t": 0.31292521953582764,
"grad_norm": 50.888275146484375,
"learning_rate": 6.640486409826785e-09,
"logits/chosen": 0.45192140340805054,
"logits/rejected": 0.47448259592056274,
"logps/chosen": -481.8817138671875,
"logps/ref_chosen": -49.652687072753906,
"logps/ref_rejected": -98.40513610839844,
"logps/rejected": -762.8228149414062,
"loss": 0.8169,
"margin_dpo/margin_mean": 232.1886444091797,
"margin_dpo/margin_std": 223.49786376953125,
"step": 637
},
{
"KL/chosen_KL_mean": -392.0146179199219,
"KL/mean": -504.70819091796875,
"KL/rejected_KL_mean": -617.4017333984375,
"KL/std": 214.28329467773438,
"epoch": 0.9368575624082232,
"fcm_dpo/beta": 0.003946408629417419,
"fcm_dpo/delta": -0.10087276995182037,
"fcm_dpo/margin": 225.3870849609375,
"fcm_dpo/q_t": 0.312773197889328,
"grad_norm": 45.85356140136719,
"learning_rate": 6.349874889624962e-09,
"logits/chosen": 0.3099361062049866,
"logits/rejected": 0.3372785747051239,
"logps/chosen": -450.1712646484375,
"logps/ref_chosen": -58.156639099121094,
"logps/ref_rejected": -79.3014907836914,
"logps/rejected": -696.7032470703125,
"loss": 0.7993,
"margin_dpo/margin_mean": 225.3870849609375,
"margin_dpo/margin_std": 178.96417236328125,
"step": 638
},
{
"KL/chosen_KL_mean": -464.5148620605469,
"KL/mean": -549.45556640625,
"KL/rejected_KL_mean": -634.396240234375,
"KL/std": 196.27197265625,
"epoch": 0.9383259911894273,
"fcm_dpo/beta": 0.003963670693337917,
"fcm_dpo/delta": 0.13499195873737335,
"fcm_dpo/margin": 169.88137817382812,
"fcm_dpo/q_t": 0.3594561815261841,
"grad_norm": 48.99491500854492,
"learning_rate": 6.065683776815933e-09,
"logits/chosen": 0.2308313548564911,
"logits/rejected": 0.26190468668937683,
"logps/chosen": -536.8380126953125,
"logps/ref_chosen": -72.32319641113281,
"logps/ref_rejected": -74.2749252319336,
"logps/rejected": -708.671142578125,
"loss": 0.9754,
"margin_dpo/margin_mean": 169.88137817382812,
"margin_dpo/margin_std": 211.46803283691406,
"step": 639
},
{
"KL/chosen_KL_mean": -420.45050048828125,
"KL/mean": -549.830078125,
"KL/rejected_KL_mean": -679.209716796875,
"KL/std": 223.74356079101562,
"epoch": 0.9397944199706314,
"fcm_dpo/beta": 0.0038788788951933384,
"fcm_dpo/delta": -0.22599047422409058,
"fcm_dpo/margin": 258.75921630859375,
"fcm_dpo/q_t": 0.30272677540779114,
"grad_norm": 39.78026580810547,
"learning_rate": 5.7879205600998296e-09,
"logits/chosen": 0.29272258281707764,
"logits/rejected": 0.326360285282135,
"logps/chosen": -476.58489990234375,
"logps/ref_chosen": -56.13436508178711,
"logps/ref_rejected": -108.60014343261719,
"logps/rejected": -787.8098754882812,
"loss": 0.7921,
"margin_dpo/margin_mean": 258.75921630859375,
"margin_dpo/margin_std": 251.56118774414062,
"step": 640
},
{
"KL/chosen_KL_mean": -481.4560241699219,
"KL/mean": -571.785400390625,
"KL/rejected_KL_mean": -662.1146850585938,
"KL/std": 199.70594787597656,
"epoch": 0.9412628487518355,
"fcm_dpo/beta": 0.0038441240321844816,
"fcm_dpo/delta": 0.11339370161294937,
"fcm_dpo/margin": 180.65867614746094,
"fcm_dpo/q_t": 0.3584289252758026,
"grad_norm": 49.26630401611328,
"learning_rate": 5.516592558795746e-09,
"logits/chosen": 0.3979315161705017,
"logits/rejected": 0.42757904529571533,
"logps/chosen": -546.452880859375,
"logps/ref_chosen": -64.99689483642578,
"logps/ref_rejected": -86.99232482910156,
"logps/rejected": -749.1070556640625,
"loss": 0.9607,
"margin_dpo/margin_mean": 180.65866088867188,
"margin_dpo/margin_std": 226.99839782714844,
"step": 641
},
{
"KL/chosen_KL_mean": -422.64208984375,
"KL/mean": -533.4561157226562,
"KL/rejected_KL_mean": -644.2701416015625,
"KL/std": 238.713623046875,
"epoch": 0.9427312775330396,
"fcm_dpo/beta": 0.0038909269496798515,
"fcm_dpo/delta": -0.06998465955257416,
"fcm_dpo/margin": 221.62806701660156,
"fcm_dpo/q_t": 0.33400386571884155,
"grad_norm": 46.33919906616211,
"learning_rate": 5.251706922648868e-09,
"logits/chosen": 0.2569606304168701,
"logits/rejected": 0.2733567953109741,
"logps/chosen": -488.33135986328125,
"logps/ref_chosen": -65.68924713134766,
"logps/ref_rejected": -110.24205017089844,
"logps/rejected": -754.51220703125,
"loss": 0.9007,
"margin_dpo/margin_mean": 221.6280517578125,
"margin_dpo/margin_std": 262.16290283203125,
"step": 642
},
{
"KL/chosen_KL_mean": -417.83306884765625,
"KL/mean": -518.9586181640625,
"KL/rejected_KL_mean": -620.084228515625,
"KL/std": 239.10421752929688,
"epoch": 0.9441997063142438,
"fcm_dpo/beta": 0.0038129761815071106,
"fcm_dpo/delta": 0.029478274285793304,
"fcm_dpo/margin": 202.25112915039062,
"fcm_dpo/q_t": 0.33728426694869995,
"grad_norm": 35.140663146972656,
"learning_rate": 4.993270631642038e-09,
"logits/chosen": 0.4179457128047943,
"logits/rejected": 0.4211399555206299,
"logps/chosen": -469.7830810546875,
"logps/ref_chosen": -51.94999694824219,
"logps/ref_rejected": -87.46833801269531,
"logps/rejected": -707.5525512695312,
"loss": 0.8858,
"margin_dpo/margin_mean": 202.25112915039062,
"margin_dpo/margin_std": 198.2943115234375,
"step": 643
},
{
"KL/chosen_KL_mean": -431.06512451171875,
"KL/mean": -527.8743896484375,
"KL/rejected_KL_mean": -624.68359375,
"KL/std": 257.38726806640625,
"epoch": 0.9456681350954479,
"fcm_dpo/beta": 0.00388428196310997,
"fcm_dpo/delta": 0.05185030400753021,
"fcm_dpo/margin": 193.61854553222656,
"fcm_dpo/q_t": 0.3501347303390503,
"grad_norm": 50.662540435791016,
"learning_rate": 4.741290495811873e-09,
"logits/chosen": 0.3143874704837799,
"logits/rejected": 0.33662909269332886,
"logps/chosen": -490.0827941894531,
"logps/ref_chosen": -59.017662048339844,
"logps/ref_rejected": -87.13668823242188,
"logps/rejected": -711.8203125,
"loss": 0.9535,
"margin_dpo/margin_mean": 193.6185302734375,
"margin_dpo/margin_std": 244.5896759033203,
"step": 644
},
{
"KL/chosen_KL_mean": -429.000244140625,
"KL/mean": -505.9991760253906,
"KL/rejected_KL_mean": -582.9981079101562,
"KL/std": 221.11386108398438,
"epoch": 0.947136563876652,
"fcm_dpo/beta": 0.0039842319674789906,
"fcm_dpo/delta": 0.19895383715629578,
"fcm_dpo/margin": 153.99783325195312,
"fcm_dpo/q_t": 0.3828478455543518,
"grad_norm": 52.093482971191406,
"learning_rate": 4.495773155069299e-09,
"logits/chosen": 0.27324581146240234,
"logits/rejected": 0.2675897479057312,
"logps/chosen": -484.87628173828125,
"logps/ref_chosen": -55.87602233886719,
"logps/ref_rejected": -97.78080749511719,
"logps/rejected": -680.7789306640625,
"loss": 1.0637,
"margin_dpo/margin_mean": 153.99783325195312,
"margin_dpo/margin_std": 245.86737060546875,
"step": 645
},
{
"KL/chosen_KL_mean": -431.00897216796875,
"KL/mean": -530.1956176757812,
"KL/rejected_KL_mean": -629.3822021484375,
"KL/std": 180.67453002929688,
"epoch": 0.9486049926578561,
"fcm_dpo/beta": 0.004018571227788925,
"fcm_dpo/delta": 0.001294448971748352,
"fcm_dpo/margin": 198.37326049804688,
"fcm_dpo/q_t": 0.33232998847961426,
"grad_norm": 46.75684356689453,
"learning_rate": 4.256725079024553e-09,
"logits/chosen": 0.405788779258728,
"logits/rejected": 0.43029361963272095,
"logps/chosen": -492.2847595214844,
"logps/ref_chosen": -61.275787353515625,
"logps/ref_rejected": -77.50580596923828,
"logps/rejected": -706.8880615234375,
"loss": 0.8649,
"margin_dpo/margin_mean": 198.37326049804688,
"margin_dpo/margin_std": 189.44442749023438,
"step": 646
},
{
"KL/chosen_KL_mean": -395.1177673339844,
"KL/mean": -494.07708740234375,
"KL/rejected_KL_mean": -593.036376953125,
"KL/std": 234.684326171875,
"epoch": 0.9500734214390602,
"fcm_dpo/beta": 0.0040655904449522495,
"fcm_dpo/delta": -0.005251521244645119,
"fcm_dpo/margin": 197.91859436035156,
"fcm_dpo/q_t": 0.3365859389305115,
"grad_norm": 46.11881637573242,
"learning_rate": 4.024152566816791e-09,
"logits/chosen": 0.3098648488521576,
"logits/rejected": 0.32036569714546204,
"logps/chosen": -449.9701843261719,
"logps/ref_chosen": -54.8524169921875,
"logps/ref_rejected": -93.5194091796875,
"logps/rejected": -686.5557861328125,
"loss": 0.9117,
"margin_dpo/margin_mean": 197.9185791015625,
"margin_dpo/margin_std": 225.00868225097656,
"step": 647
},
{
"KL/chosen_KL_mean": -425.3743896484375,
"KL/mean": -558.19775390625,
"KL/rejected_KL_mean": -691.0211181640625,
"KL/std": 252.74656677246094,
"epoch": 0.9515418502202643,
"fcm_dpo/beta": 0.003904239274561405,
"fcm_dpo/delta": -0.26420071721076965,
"fcm_dpo/margin": 265.6467590332031,
"fcm_dpo/q_t": 0.30838528275489807,
"grad_norm": 40.06119918823242,
"learning_rate": 3.798061746947995e-09,
"logits/chosen": 0.40732091665267944,
"logits/rejected": 0.4274319112300873,
"logps/chosen": -479.54583740234375,
"logps/ref_chosen": -54.17146682739258,
"logps/ref_rejected": -98.7127914428711,
"logps/rejected": -789.7339477539062,
"loss": 0.8385,
"margin_dpo/margin_mean": 265.64678955078125,
"margin_dpo/margin_std": 304.36572265625,
"step": 648
},
{
"KL/chosen_KL_mean": -435.12518310546875,
"KL/mean": -517.2813110351562,
"KL/rejected_KL_mean": -599.4374389648438,
"KL/std": 208.06198120117188,
"epoch": 0.9530102790014684,
"fcm_dpo/beta": 0.0039540440775454044,
"fcm_dpo/delta": 0.15951567888259888,
"fcm_dpo/margin": 164.31224060058594,
"fcm_dpo/q_t": 0.3723710775375366,
"grad_norm": 54.13444137573242,
"learning_rate": 3.5784585771215235e-09,
"logits/chosen": 0.3933088779449463,
"logits/rejected": 0.4235169291496277,
"logps/chosen": -497.60552978515625,
"logps/ref_chosen": -62.480350494384766,
"logps/ref_rejected": -80.07717895507812,
"logps/rejected": -679.5146484375,
"loss": 1.0082,
"margin_dpo/margin_mean": 164.312255859375,
"margin_dpo/margin_std": 231.4293212890625,
"step": 649
},
{
"KL/chosen_KL_mean": -433.6247253417969,
"KL/mean": -552.1237182617188,
"KL/rejected_KL_mean": -670.6226806640625,
"KL/std": 234.92752075195312,
"epoch": 0.9544787077826725,
"fcm_dpo/beta": 0.003866554470732808,
"fcm_dpo/delta": -0.12990686297416687,
"fcm_dpo/margin": 236.99801635742188,
"fcm_dpo/q_t": 0.31889447569847107,
"grad_norm": 50.74138641357422,
"learning_rate": 3.3653488440851253e-09,
"logits/chosen": 0.3859984278678894,
"logits/rejected": 0.4003593921661377,
"logps/chosen": -489.717529296875,
"logps/ref_chosen": -56.09281921386719,
"logps/ref_rejected": -98.26483917236328,
"logps/rejected": -768.8875732421875,
"loss": 0.8666,
"margin_dpo/margin_mean": 236.99801635742188,
"margin_dpo/margin_std": 258.2767639160156,
"step": 650
},
{
"KL/chosen_KL_mean": -375.603271484375,
"KL/mean": -498.3697814941406,
"KL/rejected_KL_mean": -621.1362915039062,
"KL/std": 243.66693115234375,
"epoch": 0.9559471365638766,
"fcm_dpo/beta": 0.0037730569019913673,
"fcm_dpo/delta": -0.14011479914188385,
"fcm_dpo/margin": 245.53298950195312,
"fcm_dpo/q_t": 0.3070628046989441,
"grad_norm": 38.228172302246094,
"learning_rate": 3.158738163478475e-09,
"logits/chosen": 0.3418081998825073,
"logits/rejected": 0.34055066108703613,
"logps/chosen": -419.02874755859375,
"logps/ref_chosen": -43.42544937133789,
"logps/ref_rejected": -99.95791625976562,
"logps/rejected": -721.0941772460938,
"loss": 0.7932,
"margin_dpo/margin_mean": 245.53298950195312,
"margin_dpo/margin_std": 212.47959899902344,
"step": 651
},
{
"KL/chosen_KL_mean": -399.320556640625,
"KL/mean": -506.40570068359375,
"KL/rejected_KL_mean": -613.4908447265625,
"KL/std": 232.77484130859375,
"epoch": 0.9574155653450808,
"fcm_dpo/beta": 0.0037672333419322968,
"fcm_dpo/delta": -0.007625289261341095,
"fcm_dpo/margin": 214.17034912109375,
"fcm_dpo/q_t": 0.3373221755027771,
"grad_norm": 51.426910400390625,
"learning_rate": 2.9586319796851555e-09,
"logits/chosen": 0.3437264561653137,
"logits/rejected": 0.36428600549697876,
"logps/chosen": -461.8973388671875,
"logps/ref_chosen": -62.57680892944336,
"logps/ref_rejected": -111.76779174804688,
"logps/rejected": -725.2586669921875,
"loss": 0.8841,
"margin_dpo/margin_mean": 214.17034912109375,
"margin_dpo/margin_std": 232.0792236328125,
"step": 652
},
{
"KL/chosen_KL_mean": -456.14044189453125,
"KL/mean": -569.4014892578125,
"KL/rejected_KL_mean": -682.6625366210938,
"KL/std": 250.85052490234375,
"epoch": 0.9588839941262849,
"fcm_dpo/beta": 0.0037269643507897854,
"fcm_dpo/delta": -0.048316843807697296,
"fcm_dpo/margin": 226.52212524414062,
"fcm_dpo/q_t": 0.33045560121536255,
"grad_norm": 45.9043083190918,
"learning_rate": 2.7650355656892166e-09,
"logits/chosen": 0.39898934960365295,
"logits/rejected": 0.40452104806900024,
"logps/chosen": -517.25341796875,
"logps/ref_chosen": -61.11295700073242,
"logps/ref_rejected": -103.24960327148438,
"logps/rejected": -785.912109375,
"loss": 0.8724,
"margin_dpo/margin_mean": 226.52212524414062,
"margin_dpo/margin_std": 243.38613891601562,
"step": 653
},
{
"KL/chosen_KL_mean": -455.92962646484375,
"KL/mean": -552.9722900390625,
"KL/rejected_KL_mean": -650.0150146484375,
"KL/std": 228.19635009765625,
"epoch": 0.960352422907489,
"fcm_dpo/beta": 0.00373679306358099,
"fcm_dpo/delta": 0.08018621057271957,
"fcm_dpo/margin": 194.08538818359375,
"fcm_dpo/q_t": 0.3584059476852417,
"grad_norm": 45.61049270629883,
"learning_rate": 2.577954022936174e-09,
"logits/chosen": 0.3774251341819763,
"logits/rejected": 0.3957637548446655,
"logps/chosen": -517.6577758789062,
"logps/ref_chosen": -61.7281379699707,
"logps/ref_rejected": -98.7738037109375,
"logps/rejected": -748.788818359375,
"loss": 0.9826,
"margin_dpo/margin_mean": 194.08538818359375,
"margin_dpo/margin_std": 264.8636474609375,
"step": 654
},
{
"KL/chosen_KL_mean": -419.35821533203125,
"KL/mean": -520.3919677734375,
"KL/rejected_KL_mean": -621.4256591796875,
"KL/std": 227.86204528808594,
"epoch": 0.9618208516886931,
"fcm_dpo/beta": 0.0037737099919468164,
"fcm_dpo/delta": 0.03982290253043175,
"fcm_dpo/margin": 202.06744384765625,
"fcm_dpo/q_t": 0.35060518980026245,
"grad_norm": 53.30228042602539,
"learning_rate": 2.397392281198729e-09,
"logits/chosen": 0.43709027767181396,
"logits/rejected": 0.43980515003204346,
"logps/chosen": -468.9350280761719,
"logps/ref_chosen": -49.576812744140625,
"logps/ref_rejected": -98.29183197021484,
"logps/rejected": -719.717529296875,
"loss": 0.944,
"margin_dpo/margin_mean": 202.0674591064453,
"margin_dpo/margin_std": 251.4522705078125,
"step": 655
},
{
"KL/chosen_KL_mean": -409.84423828125,
"KL/mean": -561.7591552734375,
"KL/rejected_KL_mean": -713.674072265625,
"KL/std": 255.30087280273438,
"epoch": 0.9632892804698973,
"fcm_dpo/beta": 0.0036293421871960163,
"fcm_dpo/delta": -0.3396731913089752,
"fcm_dpo/margin": 303.82989501953125,
"fcm_dpo/q_t": 0.2718742787837982,
"grad_norm": 53.86192321777344,
"learning_rate": 2.223355098446622e-09,
"logits/chosen": 0.35994166135787964,
"logits/rejected": 0.3799811601638794,
"logps/chosen": -462.3936767578125,
"logps/ref_chosen": -52.54943084716797,
"logps/ref_rejected": -113.67464447021484,
"logps/rejected": -827.3487548828125,
"loss": 0.6877,
"margin_dpo/margin_mean": 303.82989501953125,
"margin_dpo/margin_std": 221.92645263671875,
"step": 656
},
{
"KL/chosen_KL_mean": -405.48822021484375,
"KL/mean": -541.8997192382812,
"KL/rejected_KL_mean": -678.311279296875,
"KL/std": 256.29046630859375,
"epoch": 0.9647577092511013,
"fcm_dpo/beta": 0.003453074023127556,
"fcm_dpo/delta": -0.15873435139656067,
"fcm_dpo/margin": 272.82305908203125,
"fcm_dpo/q_t": 0.30371803045272827,
"grad_norm": 35.55537796020508,
"learning_rate": 2.055847060721566e-09,
"logits/chosen": 0.4056025743484497,
"logits/rejected": 0.41361987590789795,
"logps/chosen": -452.1887512207031,
"logps/ref_chosen": -46.700538635253906,
"logps/ref_rejected": -97.91487121582031,
"logps/rejected": -776.2261352539062,
"loss": 0.776,
"margin_dpo/margin_mean": 272.82305908203125,
"margin_dpo/margin_std": 231.46688842773438,
"step": 657
},
{
"KL/chosen_KL_mean": -450.57080078125,
"KL/mean": -557.7039794921875,
"KL/rejected_KL_mean": -664.8372192382812,
"KL/std": 209.4986572265625,
"epoch": 0.9662261380323054,
"fcm_dpo/beta": 0.0034792963415384293,
"fcm_dpo/delta": 0.05883553624153137,
"fcm_dpo/margin": 214.26651000976562,
"fcm_dpo/q_t": 0.3380606174468994,
"grad_norm": 33.780548095703125,
"learning_rate": 1.8948725820160662e-09,
"logits/chosen": 0.4116886258125305,
"logits/rejected": 0.43388843536376953,
"logps/chosen": -511.52899169921875,
"logps/ref_chosen": -60.95820999145508,
"logps/ref_rejected": -95.93949127197266,
"logps/rejected": -760.7767333984375,
"loss": 0.8741,
"margin_dpo/margin_mean": 214.26651000976562,
"margin_dpo/margin_std": 196.41368103027344,
"step": 658
},
{
"KL/chosen_KL_mean": -411.34381103515625,
"KL/mean": -527.1859741210938,
"KL/rejected_KL_mean": -643.0281372070312,
"KL/std": 235.82553100585938,
"epoch": 0.9676945668135095,
"fcm_dpo/beta": 0.0034661423414945602,
"fcm_dpo/delta": -0.003968711942434311,
"fcm_dpo/margin": 231.68438720703125,
"fcm_dpo/q_t": 0.3321956992149353,
"grad_norm": 31.871097564697266,
"learning_rate": 1.7404359041573723e-09,
"logits/chosen": 0.22371786832809448,
"logits/rejected": 0.26776695251464844,
"logps/chosen": -488.0867919921875,
"logps/ref_chosen": -76.74298095703125,
"logps/ref_rejected": -87.4709701538086,
"logps/rejected": -730.4991455078125,
"loss": 0.8695,
"margin_dpo/margin_mean": 231.68435668945312,
"margin_dpo/margin_std": 228.46832275390625,
"step": 659
},
{
"KL/chosen_KL_mean": -431.14703369140625,
"KL/mean": -555.50244140625,
"KL/rejected_KL_mean": -679.8578491210938,
"KL/std": 207.2142333984375,
"epoch": 0.9691629955947136,
"fcm_dpo/beta": 0.0034529143013060093,
"fcm_dpo/delta": -0.0642768144607544,
"fcm_dpo/margin": 248.7108154296875,
"fcm_dpo/q_t": 0.315839558839798,
"grad_norm": 30.965335845947266,
"learning_rate": 1.592541096695571e-09,
"logits/chosen": 0.4715263247489929,
"logits/rejected": 0.5011571645736694,
"logps/chosen": -490.1949462890625,
"logps/ref_chosen": -59.04788589477539,
"logps/ref_rejected": -75.96005249023438,
"logps/rejected": -755.81787109375,
"loss": 0.8116,
"margin_dpo/margin_mean": 248.71083068847656,
"margin_dpo/margin_std": 211.64910888671875,
"step": 660
},
{
"KL/chosen_KL_mean": -400.5611877441406,
"KL/mean": -514.7567749023438,
"KL/rejected_KL_mean": -628.952392578125,
"KL/std": 235.23724365234375,
"epoch": 0.9706314243759178,
"fcm_dpo/beta": 0.00347909121774137,
"fcm_dpo/delta": 0.00342472642660141,
"fcm_dpo/margin": 228.3911895751953,
"fcm_dpo/q_t": 0.3365651071071625,
"grad_norm": 35.08213424682617,
"learning_rate": 1.4511920567963908e-09,
"logits/chosen": 0.344051718711853,
"logits/rejected": 0.3751180171966553,
"logps/chosen": -451.23516845703125,
"logps/ref_chosen": -50.673973083496094,
"logps/ref_rejected": -86.00569152832031,
"logps/rejected": -714.9581298828125,
"loss": 0.893,
"margin_dpo/margin_mean": 228.39120483398438,
"margin_dpo/margin_std": 240.8126678466797,
"step": 661
},
{
"KL/chosen_KL_mean": -426.64874267578125,
"KL/mean": -528.4208984375,
"KL/rejected_KL_mean": -630.193115234375,
"KL/std": 217.40945434570312,
"epoch": 0.9720998531571219,
"fcm_dpo/beta": 0.0034924065694212914,
"fcm_dpo/delta": 0.09593937546014786,
"fcm_dpo/margin": 203.54437255859375,
"fcm_dpo/q_t": 0.3504369258880615,
"grad_norm": 31.930891036987305,
"learning_rate": 1.3163925091384532e-09,
"logits/chosen": 0.2393924593925476,
"logits/rejected": 0.2724754810333252,
"logps/chosen": -495.9097900390625,
"logps/ref_chosen": -69.26106262207031,
"logps/ref_rejected": -89.05593872070312,
"logps/rejected": -719.2490234375,
"loss": 0.9342,
"margin_dpo/margin_mean": 203.54437255859375,
"margin_dpo/margin_std": 228.84689331054688,
"step": 662
},
{
"KL/chosen_KL_mean": -405.4971923828125,
"KL/mean": -521.6410522460938,
"KL/rejected_KL_mean": -637.784912109375,
"KL/std": 248.77371215820312,
"epoch": 0.973568281938326,
"fcm_dpo/beta": 0.0034925497602671385,
"fcm_dpo/delta": -0.012477856129407883,
"fcm_dpo/margin": 232.28768920898438,
"fcm_dpo/q_t": 0.33259835839271545,
"grad_norm": 34.18339538574219,
"learning_rate": 1.1881460058152382e-09,
"logits/chosen": 0.26970383524894714,
"logits/rejected": 0.28010663390159607,
"logps/chosen": -470.3760986328125,
"logps/ref_chosen": -64.87890625,
"logps/ref_rejected": -113.92536926269531,
"logps/rejected": -751.7102661132812,
"loss": 0.8659,
"margin_dpo/margin_mean": 232.28768920898438,
"margin_dpo/margin_std": 239.78091430664062,
"step": 663
},
{
"KL/chosen_KL_mean": -440.453369140625,
"KL/mean": -567.512939453125,
"KL/rejected_KL_mean": -694.572509765625,
"KL/std": 230.0228729248047,
"epoch": 0.9750367107195301,
"fcm_dpo/beta": 0.003445860929787159,
"fcm_dpo/delta": -0.08343236148357391,
"fcm_dpo/margin": 254.11920166015625,
"fcm_dpo/q_t": 0.3171752691268921,
"grad_norm": 51.455936431884766,
"learning_rate": 1.066455926241383e-09,
"logits/chosen": 0.371050089597702,
"logits/rejected": 0.3953966796398163,
"logps/chosen": -501.3418273925781,
"logps/ref_chosen": -60.88847351074219,
"logps/ref_rejected": -105.521728515625,
"logps/rejected": -800.09423828125,
"loss": 0.8572,
"margin_dpo/margin_mean": 254.11920166015625,
"margin_dpo/margin_std": 262.84967041015625,
"step": 664
},
{
"KL/chosen_KL_mean": -427.70672607421875,
"KL/mean": -538.66796875,
"KL/rejected_KL_mean": -649.629150390625,
"KL/std": 203.9910888671875,
"epoch": 0.9765051395007343,
"fcm_dpo/beta": 0.003454443998634815,
"fcm_dpo/delta": 0.03614869341254234,
"fcm_dpo/margin": 221.92245483398438,
"fcm_dpo/q_t": 0.3373722434043884,
"grad_norm": 51.79203796386719,
"learning_rate": 9.513254770636137e-10,
"logits/chosen": 0.5427942276000977,
"logits/rejected": 0.5816524028778076,
"logps/chosen": -488.2708435058594,
"logps/ref_chosen": -60.56413269042969,
"logps/ref_rejected": -84.80882263183594,
"logps/rejected": -734.43798828125,
"loss": 0.8702,
"margin_dpo/margin_mean": 221.9224395751953,
"margin_dpo/margin_std": 207.96560668945312,
"step": 665
},
{
"KL/chosen_KL_mean": -430.86529541015625,
"KL/mean": -541.441650390625,
"KL/rejected_KL_mean": -652.01806640625,
"KL/std": 219.16168212890625,
"epoch": 0.9779735682819384,
"fcm_dpo/beta": 0.0034718806855380535,
"fcm_dpo/delta": 0.03468765318393707,
"fcm_dpo/margin": 221.15277099609375,
"fcm_dpo/q_t": 0.33385133743286133,
"grad_norm": 41.26150894165039,
"learning_rate": 8.427576920763956e-10,
"logits/chosen": 0.2883135974407196,
"logits/rejected": 0.31811410188674927,
"logps/chosen": -495.2852783203125,
"logps/ref_chosen": -64.41996002197266,
"logps/ref_rejected": -95.8916244506836,
"logps/rejected": -747.90966796875,
"loss": 0.8674,
"margin_dpo/margin_mean": 221.15277099609375,
"margin_dpo/margin_std": 200.27261352539062,
"step": 666
},
{
"KL/chosen_KL_mean": -439.869384765625,
"KL/mean": -552.4962158203125,
"KL/rejected_KL_mean": -665.123046875,
"KL/std": 228.28411865234375,
"epoch": 0.9794419970631424,
"fcm_dpo/beta": 0.003500557504594326,
"fcm_dpo/delta": 0.012448148801922798,
"fcm_dpo/margin": 225.253662109375,
"fcm_dpo/q_t": 0.328310489654541,
"grad_norm": 37.877227783203125,
"learning_rate": 7.407554321417764e-10,
"logits/chosen": 0.3518103361129761,
"logits/rejected": 0.4001040458679199,
"logps/chosen": -509.14642333984375,
"logps/ref_chosen": -69.27702331542969,
"logps/ref_rejected": -87.83549499511719,
"logps/rejected": -752.95849609375,
"loss": 0.8385,
"margin_dpo/margin_mean": 225.253662109375,
"margin_dpo/margin_std": 187.47291564941406,
"step": 667
},
{
"KL/chosen_KL_mean": -464.80975341796875,
"KL/mean": -560.3359375,
"KL/rejected_KL_mean": -655.8621826171875,
"KL/std": 241.27587890625,
"epoch": 0.9809104258443465,
"fcm_dpo/beta": 0.0035862771328538656,
"fcm_dpo/delta": 0.12170104682445526,
"fcm_dpo/margin": 191.05247497558594,
"fcm_dpo/q_t": 0.3601748049259186,
"grad_norm": 59.456485748291016,
"learning_rate": 6.453213851142225e-10,
"logits/chosen": 0.3165743947029114,
"logits/rejected": 0.3272818326950073,
"logps/chosen": -537.4137573242188,
"logps/ref_chosen": -72.60400390625,
"logps/ref_rejected": -103.73905944824219,
"logps/rejected": -759.601318359375,
"loss": 0.969,
"margin_dpo/margin_mean": 191.05245971679688,
"margin_dpo/margin_std": 241.36276245117188,
"step": 668
},
{
"KL/chosen_KL_mean": -378.2247314453125,
"KL/mean": -504.9932861328125,
"KL/rejected_KL_mean": -631.7618408203125,
"KL/std": 232.52206420898438,
"epoch": 0.9823788546255506,
"fcm_dpo/beta": 0.0035265935584902763,
"fcm_dpo/delta": -0.10357140004634857,
"fcm_dpo/margin": 253.537109375,
"fcm_dpo/q_t": 0.31180116534233093,
"grad_norm": 32.91140365600586,
"learning_rate": 5.564580657695939e-10,
"logits/chosen": 0.37182021141052246,
"logits/rejected": 0.4036600589752197,
"logps/chosen": -424.34112548828125,
"logps/ref_chosen": -46.116416931152344,
"logps/ref_rejected": -77.92434692382812,
"logps/rejected": -709.6861572265625,
"loss": 0.823,
"margin_dpo/margin_mean": 253.537109375,
"margin_dpo/margin_std": 236.66348266601562,
"step": 669
},
{
"KL/chosen_KL_mean": -376.5809020996094,
"KL/mean": -510.88238525390625,
"KL/rejected_KL_mean": -645.1838989257812,
"KL/std": 223.031494140625,
"epoch": 0.9838472834067548,
"fcm_dpo/beta": 0.003458178834989667,
"fcm_dpo/delta": -0.141901895403862,
"fcm_dpo/margin": 268.60302734375,
"fcm_dpo/q_t": 0.3073880076408386,
"grad_norm": 31.727798461914062,
"learning_rate": 4.741678157389739e-10,
"logits/chosen": 0.3487994074821472,
"logits/rejected": 0.36743998527526855,
"logps/chosen": -438.9266357421875,
"logps/ref_chosen": -62.34575271606445,
"logps/ref_rejected": -96.9405517578125,
"logps/rejected": -742.1244506835938,
"loss": 0.799,
"margin_dpo/margin_mean": 268.60302734375,
"margin_dpo/margin_std": 237.01303100585938,
"step": 670
},
{
"KL/chosen_KL_mean": -434.69549560546875,
"KL/mean": -543.616943359375,
"KL/rejected_KL_mean": -652.538330078125,
"KL/std": 201.36114501953125,
"epoch": 0.9853157121879589,
"fcm_dpo/beta": 0.0034685127902776003,
"fcm_dpo/delta": 0.04600398987531662,
"fcm_dpo/margin": 217.8428192138672,
"fcm_dpo/q_t": 0.34018510580062866,
"grad_norm": 39.71592712402344,
"learning_rate": 3.9845280344705245e-10,
"logits/chosen": 0.40200304985046387,
"logits/rejected": 0.421750009059906,
"logps/chosen": -482.69561767578125,
"logps/ref_chosen": -48.00010681152344,
"logps/ref_rejected": -83.81932067871094,
"logps/rejected": -736.357666015625,
"loss": 0.9002,
"margin_dpo/margin_mean": 217.84280395507812,
"margin_dpo/margin_std": 222.2292022705078,
"step": 671
},
{
"KL/chosen_KL_mean": -462.762939453125,
"KL/mean": -562.1483764648438,
"KL/rejected_KL_mean": -661.5338134765625,
"KL/std": 223.07664489746094,
"epoch": 0.986784140969163,
"fcm_dpo/beta": 0.003505786880850792,
"fcm_dpo/delta": 0.11078579723834991,
"fcm_dpo/margin": 198.7708740234375,
"fcm_dpo/q_t": 0.3557296097278595,
"grad_norm": 42.18230438232422,
"learning_rate": 3.293150240547549e-10,
"logits/chosen": 0.40828007459640503,
"logits/rejected": 0.43640410900115967,
"logps/chosen": -521.34619140625,
"logps/ref_chosen": -58.58328628540039,
"logps/ref_rejected": -93.14015197753906,
"logps/rejected": -754.6739501953125,
"loss": 0.9532,
"margin_dpo/margin_mean": 198.7708740234375,
"margin_dpo/margin_std": 234.2972412109375,
"step": 672
},
{
"KL/chosen_KL_mean": -421.04833984375,
"KL/mean": -542.5941772460938,
"KL/rejected_KL_mean": -664.1400756835938,
"KL/std": 201.49720764160156,
"epoch": 0.9882525697503671,
"fcm_dpo/beta": 0.0034946207888424397,
"fcm_dpo/delta": -0.054121218621730804,
"fcm_dpo/margin": 243.0917510986328,
"fcm_dpo/q_t": 0.31589680910110474,
"grad_norm": 34.77309799194336,
"learning_rate": 2.6675629940689504e-10,
"logits/chosen": 0.45647066831588745,
"logits/rejected": 0.4740258753299713,
"logps/chosen": -467.77154541015625,
"logps/ref_chosen": -46.72320556640625,
"logps/ref_rejected": -85.29623413085938,
"logps/rejected": -749.436279296875,
"loss": 0.8136,
"margin_dpo/margin_mean": 243.0917510986328,
"margin_dpo/margin_std": 200.75131225585938,
"step": 673
},
{
"KL/chosen_KL_mean": -399.45220947265625,
"KL/mean": -528.6614990234375,
"KL/rejected_KL_mean": -657.8707885742188,
"KL/std": 229.89468383789062,
"epoch": 0.9897209985315712,
"fcm_dpo/beta": 0.0034479242749512196,
"fcm_dpo/delta": -0.09983845800161362,
"fcm_dpo/margin": 258.4185791015625,
"fcm_dpo/q_t": 0.3155099153518677,
"grad_norm": 30.887535095214844,
"learning_rate": 2.1077827798404725e-10,
"logits/chosen": 0.422026664018631,
"logits/rejected": 0.44703778624534607,
"logps/chosen": -444.897705078125,
"logps/ref_chosen": -45.445526123046875,
"logps/ref_rejected": -70.04593658447266,
"logps/rejected": -727.916748046875,
"loss": 0.8132,
"margin_dpo/margin_mean": 258.4185791015625,
"margin_dpo/margin_std": 237.52337646484375,
"step": 674
},
{
"KL/chosen_KL_mean": -390.8414306640625,
"KL/mean": -529.8683471679688,
"KL/rejected_KL_mean": -668.895263671875,
"KL/std": 247.90399169921875,
"epoch": 0.9911894273127754,
"fcm_dpo/beta": 0.003326475154608488,
"fcm_dpo/delta": -0.13932178914546967,
"fcm_dpo/margin": 278.0538330078125,
"fcm_dpo/q_t": 0.3059813976287842,
"grad_norm": 27.469951629638672,
"learning_rate": 1.6138243485910863e-10,
"logits/chosen": 0.3832669258117676,
"logits/rejected": 0.4059588313102722,
"logps/chosen": -435.0177001953125,
"logps/ref_chosen": -44.17628479003906,
"logps/ref_rejected": -74.09197998046875,
"logps/rejected": -742.9872436523438,
"loss": 0.7809,
"margin_dpo/margin_mean": 278.0538635253906,
"margin_dpo/margin_std": 232.23904418945312,
"step": 675
},
{
"KL/chosen_KL_mean": -423.63348388671875,
"KL/mean": -543.1539306640625,
"KL/rejected_KL_mean": -662.6744384765625,
"KL/std": 235.30764770507812,
"epoch": 0.9926578560939795,
"fcm_dpo/beta": 0.0033134541008621454,
"fcm_dpo/delta": 0.008331548422574997,
"fcm_dpo/margin": 239.04092407226562,
"fcm_dpo/q_t": 0.32824352383613586,
"grad_norm": 35.013919830322266,
"learning_rate": 1.1857007165852472e-10,
"logits/chosen": 0.2349829077720642,
"logits/rejected": 0.2648879289627075,
"logps/chosen": -495.03204345703125,
"logps/ref_chosen": -71.39852905273438,
"logps/ref_rejected": -88.3587646484375,
"logps/rejected": -751.033203125,
"loss": 0.8388,
"margin_dpo/margin_mean": 239.04090881347656,
"margin_dpo/margin_std": 200.05181884765625,
"step": 676
},
{
"KL/chosen_KL_mean": -425.65728759765625,
"KL/mean": -543.5009155273438,
"KL/rejected_KL_mean": -661.3445434570312,
"KL/std": 198.14552307128906,
"epoch": 0.9941262848751835,
"fcm_dpo/beta": 0.0033442298881709576,
"fcm_dpo/delta": 0.012598922476172447,
"fcm_dpo/margin": 235.68731689453125,
"fcm_dpo/q_t": 0.3280433714389801,
"grad_norm": 32.42127990722656,
"learning_rate": 8.23423165278725e-11,
"logits/chosen": 0.4454345703125,
"logits/rejected": 0.48518845438957214,
"logps/chosen": -482.1846923828125,
"logps/ref_chosen": -56.527435302734375,
"logps/ref_rejected": -78.22654724121094,
"logps/rejected": -739.5711059570312,
"loss": 0.8385,
"margin_dpo/margin_mean": 235.68731689453125,
"margin_dpo/margin_std": 192.02017211914062,
"step": 677
},
{
"KL/chosen_KL_mean": -405.45452880859375,
"KL/mean": -529.14453125,
"KL/rejected_KL_mean": -652.8345947265625,
"KL/std": 241.40565490722656,
"epoch": 0.9955947136563876,
"fcm_dpo/beta": 0.003331228159368038,
"fcm_dpo/delta": -0.026290910318493843,
"fcm_dpo/margin": 247.3800811767578,
"fcm_dpo/q_t": 0.32946473360061646,
"grad_norm": 31.699295043945312,
"learning_rate": 5.270012410216185e-11,
"logits/chosen": 0.3945927619934082,
"logits/rejected": 0.4203825891017914,
"logps/chosen": -451.5890197753906,
"logps/ref_chosen": -46.13447570800781,
"logps/ref_rejected": -80.60462951660156,
"logps/rejected": -733.439208984375,
"loss": 0.8689,
"margin_dpo/margin_mean": 247.3800811767578,
"margin_dpo/margin_std": 250.53334045410156,
"step": 678
},
{
"KL/chosen_KL_mean": -416.02294921875,
"KL/mean": -518.5464477539062,
"KL/rejected_KL_mean": -621.0698852539062,
"KL/std": 217.73243713378906,
"epoch": 0.9970631424375918,
"fcm_dpo/beta": 0.0033760610967874527,
"fcm_dpo/delta": 0.11577928066253662,
"fcm_dpo/margin": 205.04698181152344,
"fcm_dpo/q_t": 0.3537669777870178,
"grad_norm": 43.124732971191406,
"learning_rate": 2.9644275480772416e-11,
"logits/chosen": 0.3546045124530792,
"logits/rejected": 0.3747688829898834,
"logps/chosen": -466.31787109375,
"logps/ref_chosen": -50.294921875,
"logps/ref_rejected": -76.59813690185547,
"logps/rejected": -697.6680297851562,
"loss": 0.9293,
"margin_dpo/margin_mean": 205.04696655273438,
"margin_dpo/margin_std": 223.11904907226562,
"step": 679
},
{
"KL/chosen_KL_mean": -422.438232421875,
"KL/mean": -542.1629028320312,
"KL/rejected_KL_mean": -661.8876342773438,
"KL/std": 242.85482788085938,
"epoch": 0.9985315712187959,
"fcm_dpo/beta": 0.00338840764015913,
"fcm_dpo/delta": -0.012371369637548923,
"fcm_dpo/margin": 239.4494171142578,
"fcm_dpo/q_t": 0.33219584822654724,
"grad_norm": 35.54345703125,
"learning_rate": 1.31753782067201e-11,
"logits/chosen": 0.28841865062713623,
"logits/rejected": 0.3221368193626404,
"logps/chosen": -499.35394287109375,
"logps/ref_chosen": -76.91569519042969,
"logps/ref_rejected": -112.384765625,
"logps/rejected": -774.2723388671875,
"loss": 0.8816,
"margin_dpo/margin_mean": 239.44940185546875,
"margin_dpo/margin_std": 254.77374267578125,
"step": 680
},
{
"KL/chosen_KL_mean": -436.93084716796875,
"KL/mean": -549.1124267578125,
"KL/rejected_KL_mean": -661.2940673828125,
"KL/std": 232.23263549804688,
"epoch": 1.0,
"fcm_dpo/beta": 0.003436029888689518,
"fcm_dpo/delta": 0.029315777122974396,
"fcm_dpo/margin": 224.3632354736328,
"fcm_dpo/q_t": 0.33981072902679443,
"grad_norm": 44.08296585083008,
"learning_rate": 3.2938662507808745e-12,
"logits/chosen": 0.33995503187179565,
"logits/rejected": 0.36489707231521606,
"logps/chosen": -497.8881530761719,
"logps/ref_chosen": -60.957279205322266,
"logps/ref_rejected": -88.55797576904297,
"logps/rejected": -749.85205078125,
"loss": 0.8963,
"margin_dpo/margin_mean": 224.36322021484375,
"margin_dpo/margin_std": 229.5245361328125,
"step": 681
},
{
"epoch": 1.0,
"step": 681,
"total_flos": 0.0,
"train_loss": 0.9160819085158965,
"train_runtime": 1729.8503,
"train_samples_per_second": 25.203,
"train_steps_per_second": 0.394
}
],
"logging_steps": 1,
"max_steps": 681,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue View Git Blame Copy Permalink